diff --git a/README.md b/README.md index 7969ff617..bf4fc04ac 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,14 @@ And you'll get an ironfunctions.com host for your app: myapp.USER_ID.ironfunctions.com/hello ``` -## API Reference +## More Documentation -https://swaggerhub.com/api/iron/functions +* [IronFunctions Options](api.md) +* [Running in Production](docs/production.md) +* [Triggers](docs/triggers.md) +* [Metrics](docs/metrics.md) +* [Extending IronFunctions](docs/extending.md) +* [API Reference](https://swaggerhub.com/api/iron/functions) ## Join Our Community diff --git a/api/mqs/bolt.go b/api/mqs/bolt.go index ff2f46824..beb00db0c 100644 --- a/api/mqs/bolt.go +++ b/api/mqs/bolt.go @@ -42,14 +42,14 @@ func timeoutToIDKey(timeout []byte) []byte { return b } -var delayQueueName = []byte("titan_delay") +var delayQueueName = []byte("functions_delay") func queueName(i int) []byte { - return []byte(fmt.Sprintf("titan_%d_queue", i)) + return []byte(fmt.Sprintf("functions_%d_queue", i)) } func timeoutName(i int) []byte { - return []byte(fmt.Sprintf("titan_%d_timeout", i)) + return []byte(fmt.Sprintf("functions_%d_timeout", i)) } func NewBoltMQ(url *url.URL) (*BoltDbMQ, error) { diff --git a/api/mqs/ironmq.go b/api/mqs/ironmq.go new file mode 100644 index 000000000..93ef75dcc --- /dev/null +++ b/api/mqs/ironmq.go @@ -0,0 +1,165 @@ +package mqs + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/Sirupsen/logrus" + "github.com/iron-io/functions/api/models" + mq_config "github.com/iron-io/iron_go3/config" + ironmq "github.com/iron-io/iron_go3/mq" +) + +type assoc struct { + msgId string + reservationId string +} + +type IronMQ struct { + queues []ironmq.Queue + // Protects the map + sync.Mutex + // job id to {msgid, reservationid} + msgAssoc map[string]*assoc +} + +type IronMQConfig struct { + Token string `mapstructure:"token"` + ProjectId string `mapstructure:"project_id"` + Host string `mapstructure:"host"` + Scheme string `mapstructure:"scheme"` + Port uint16 `mapstructure:"port"` + QueuePrefix string `mapstructure:"queue_prefix"` +} + +func NewIronMQ(url *url.URL) *IronMQ { + + if url.User == nil || url.User.Username() == "" { + logrus.Fatal("IronMQ requires PROJECT_ID and TOKEN") + } + p, ok := url.User.Password() + if !ok { + logrus.Fatal("IronMQ requires PROJECT_ID and TOKEN") + } + settings := &mq_config.Settings{ + Token: p, + ProjectId: url.User.Username(), + Host: url.Host, + Scheme: "https", + } + + if url.Scheme == "ironmq+http" { + settings.Scheme = "http" + } + + parts := strings.Split(url.Host, ":") + if len(parts) > 1 { + settings.Host = parts[0] + p, err := strconv.Atoi(parts[1]) + if err != nil { + logrus.WithFields(logrus.Fields{"host_port": url.Host}).Fatal("Invalid host+port combination") + } + settings.Port = uint16(p) + } + + var queueName string + if url.Path != "" { + queueName = url.Path + } else { + queueName = "titan" + } + mq := &IronMQ{ + queues: make([]ironmq.Queue, 3), + msgAssoc: make(map[string]*assoc), + } + + // Check we can connect by trying to create one of the queues. Create is + // idempotent, so this is fine. + _, err := ironmq.ConfigCreateQueue(ironmq.QueueInfo{Name: fmt.Sprintf("%s_%d", queueName, 0)}, settings) + if err != nil { + logrus.WithError(err).Fatal("Could not connect to IronMQ") + } + + for i := 0; i < 3; i++ { + mq.queues[i] = ironmq.ConfigNew(fmt.Sprintf("%s_%d", queueName, i), settings) + } + + logrus.WithFields(logrus.Fields{"base_queue": queueName}).Info("IronMQ initialized") + return mq +} + +func (mq *IronMQ) Push(ctx context.Context, job *models.Task) (*models.Task, error) { + if job.Priority == nil || *job.Priority < 0 || *job.Priority > 2 { + return nil, fmt.Errorf("IronMQ Push job %s: Bad priority", job.ID) + } + + // Push the work onto the queue. + buf, err := json.Marshal(job) + if err != nil { + return nil, err + } + _, err = mq.queues[*job.Priority].PushMessage(ironmq.Message{Body: string(buf), Delay: int64(job.Delay)}) + return job, err +} + +func (mq *IronMQ) Reserve(ctx context.Context) (*models.Task, error) { + var job models.Task + + var messages []ironmq.Message + var err error + for i := 2; i >= 0; i-- { + messages, err = mq.queues[i].LongPoll(1, int(time.Minute), 0 /* wait */, false /* delete */) + if err != nil { + // It is OK if the queue does not exist, it will be created when a message is queued. + if !strings.Contains(err.Error(), "404 Not Found") { + return nil, err + } + } + + if len(messages) == 0 { + // Try next priority. + if i == 0 { + return nil, nil + } + continue + } + + // Found a message! + break + } + + message := messages[0] + if message.Body == "" { + return nil, nil + } + + err = json.Unmarshal([]byte(message.Body), &job) + if err != nil { + return nil, err + } + mq.Lock() + mq.msgAssoc[job.ID] = &assoc{message.Id, message.ReservationId} + mq.Unlock() + return &job, nil +} + +func (mq *IronMQ) Delete(ctx context.Context, job *models.Task) error { + if job.Priority == nil || *job.Priority < 0 || *job.Priority > 2 { + return fmt.Errorf("IronMQ Delete job %s: Bad priority", job.ID) + } + mq.Lock() + assoc, exists := mq.msgAssoc[job.ID] + delete(mq.msgAssoc, job.ID) + mq.Unlock() + + if exists { + return mq.queues[*job.Priority].DeleteMessage(assoc.msgId, assoc.reservationId) + } + return nil +} diff --git a/api/mqs/new.go b/api/mqs/new.go index 33f16c246..9b82fa210 100644 --- a/api/mqs/new.go +++ b/api/mqs/new.go @@ -3,6 +3,7 @@ package mqs import ( "fmt" "net/url" + "strings" "github.com/Sirupsen/logrus" "github.com/iron-io/functions/api/models" @@ -24,6 +25,9 @@ func New(mqURL string) (models.MessageQueue, error) { case "bolt": return NewBoltMQ(u) } + if strings.HasPrefix(u.Scheme, "ironmq") { + return NewIronMQ(u), nil + } return nil, fmt.Errorf("mq type not supported %v", u.Scheme) } diff --git a/api/runner/runner.go b/api/runner/runner.go index 059b7e6f3..5484c5347 100644 --- a/api/runner/runner.go +++ b/api/runner/runner.go @@ -51,8 +51,7 @@ var ( ) func New(metricLogger Logger) (*Runner, error) { - // TODO: Is this really required for Titan's driver? - // Can we remove it? + // TODO: Is this really required for the container drivers? Can we remove it? env := common.NewEnvironment(func(e *common.Environment) {}) // TODO: Create a drivers.New(runnerConfig) in Titan diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..958331961 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,10 @@ + +# IronFunctions Documentation + +* [Running IronFunctions](api.md) +* [Databases](databases/README.md) +* [Message Queues](mqs/README.md) +* [Running in Production](production.md) +* [Triggers](triggers.md) +* [Metrics](metrics.md) +* [Extending IronFunctions](extending.md) diff --git a/docs/api.md b/docs/api.md index 0c3ce81e9..0b1137b08 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,7 +1,14 @@ -## API Options -#### Env Variables + +## IronFunctions Config Options + +When starting IronFunctions, you can pass in the following configuration variables as environment variables. Use `-e VAR_NAME=VALUE` in +docker run. For example: + +``` +docker run -e VAR_NAME=VALUE ... +``` @@ -13,10 +20,6 @@ - - - - @@ -25,79 +28,16 @@ + + + + + + + + +
The database URL to use in URL format. See Databases below for more information. Default: BoltDB in current working directory `bolt.db`.
PORTDefault (8080), sets the port to run on.
MQ The message queue to use in URL format. See Message Queues below for more information. Default: BoltDB in current working directory `queue.db`.
The primary functions api URL to pull tasks from (the address is that of another running functions process).
PORTDefault (8080), sets the port to run on.
NUM_ASYNC The number of async runners in the functions process (default 1).
LOG_LEVELSet to `DEBUG` to enable debugging. Default is INFO.
- -## Databases - -We currently support the following databases and they are passed in via the `DB` environment variable. For example: - -```sh -docker run -v /titan/data:/titan/data -e "DB=postgres://user:pass@localhost:6212/mydb" ... -``` - -### Memory - -URL: `memory:///` - -Stores all data in memory. Fast and easy, but you'll lose all your data when it stops! NEVER use this in production. - -### [Bolt](https://github.com/boltdb/bolt) - -URL: `bolt:///titan/data/bolt.db` - -Bolt is an embedded database which stores to disk. If you want to use this, be sure you don't lose the data directory by mounting -the directory on your host. eg: `docker run -v $PWD/data:/titan/data -e DB=bolt:///titan/data/bolt.db ...` - -### [Redis](http://redis.io/) - -URL: `redis://localhost:6379/` - -Uses any Redis instance. Be sure to enable [peristence](http://redis.io/topics/persistence). - -### [PostgreSQL](http://www.postgresql.org/) - -URL: `postgres://user3123:passkja83kd8@ec2-117-21-174-214.compute-1.amazonaws.com:6212/db982398` - -If you're using Titan in production, you should probably start here. - -### What about database X? - -We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point. - -## Message Queues - -A message queue is used to coordinate the jobs that run through Titan. - -We currently support the following message queues and they are passed in via the `MQ` environment variable. For example: - -```sh -docker run -v /titan/data:/titan/data -e "MQ=redis://localhost:6379/" ... -``` - -### Memory - -See memory in databases above. - -### Bolt - -URL: `bolt:///titan/data/bolt-mq.db` - -See Bolt in databases above. The Bolt database is locked at the file level, so -the file cannot be the same as the one used for the Bolt Datastore. - -### Redis - -See Redis in databases above. - -### What about message queue X? - -We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point. - -## Troubleshooting - -Enable debugging by passing in the `LOG_LEVEL` env var with DEBUG level. - - diff --git a/docs/architecture.png b/docs/architecture.png new file mode 100644 index 000000000..5d5e305bf Binary files /dev/null and b/docs/architecture.png differ diff --git a/docs/architecture.svg b/docs/architecture.svg new file mode 100644 index 000000000..d437a2b89 --- /dev/null +++ b/docs/architecture.svg @@ -0,0 +1,4 @@ + + + + diff --git a/docs/databases/README.md b/docs/databases/README.md new file mode 100644 index 000000000..183cfe497 --- /dev/null +++ b/docs/databases/README.md @@ -0,0 +1,38 @@ + +# Databases + +We currently support the following databases and they are passed in via the `DB` environment variable. For example: + +```sh +docker run -e "DB=postgres://user:pass@localhost:6212/mydb" ... +``` + +## [Bolt](https://github.com/boltdb/bolt) (default) + +URL: `bolt:///functions/data/functions.db` + +Bolt is an embedded database which stores to disk. If you want to use this, be sure you don't lose the data directory by mounting +the directory on your host. eg: `docker run -v $PWD/data:/functions/data -e DB=bolt:///functions/data/bolt.db ...` + +[More on BoltDB](databases/boltdb.md) + +## [Redis](http://redis.io/) + +URL: `redis://localhost:6379/` + +Use a Redis instance as your database. Be sure to enable [peristence](http://redis.io/topics/persistence). + +[More on Redis](databases/redis.md) + +## [PostgreSQL](http://www.postgresql.org/) + +URL: `postgres://user123:pass456@ec2-117-21-174-214.compute-1.amazonaws.com:6212/db982398` + +Use a PostgreSQL database. If you're using IronFunctions in production, you should probably start here. + +[More on Postgres](databases/postgres.md) + +## What about database X? + +We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point. + diff --git a/docs/database/boltdb.md b/docs/databases/boltdb.md similarity index 77% rename from docs/database/boltdb.md rename to docs/databases/boltdb.md index d42fda40d..979ca9c7e 100644 --- a/docs/database/boltdb.md +++ b/docs/databases/boltdb.md @@ -4,7 +4,7 @@ BoltDB is the default database, you just need to run the API. ## Persistent -To keep it persistent you add a volume flag to the command: +To keep it persistent, add a volume flag to the command: ``` docker run --rm -it --privileged -v $PWD/bolt.db:/app/bolt.db -p 8080:8080 iron/functions diff --git a/docs/database/postgres.md b/docs/databases/postgres.md similarity index 100% rename from docs/database/postgres.md rename to docs/databases/postgres.md diff --git a/docs/extending.md b/docs/extending.md index 8ab65c9a5..884c0dcb6 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -1,8 +1,13 @@ -IronFunctions is extensible so you can add custom functionality and extend the project without needing to modify the core. +# Extending IronFunctions + +IronFunctions is extensible so you can add custom functionality and extend the project without needing to modify the core. ## Listeners -This is the main way to do it. To add listeners, copy main.go and use one of the following functions on the Server. +Listeners are the main way to extend IronFunctions. + +To add listeners, copy `main.go` into your own repo and add your own listener implementations. When ready, +compile your main package to create your extended version of IronFunctions. ### AppListener diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 000000000..16d727f5f --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,10 @@ +# Metrics + +TODO: Explain metrics format. Log Metric Format - LMF(AO) + +``` +metric=someevent value=1 type=count +metric=somegauge value=50 type=gauge +``` + +TODO: List all metrics we emit to logs. diff --git a/docs/mqs/README.md b/docs/mqs/README.md new file mode 100644 index 000000000..f7857a76c --- /dev/null +++ b/docs/mqs/README.md @@ -0,0 +1,37 @@ + +# Message Queues + +A message queue is used to coordinate asynchronous function calls that run through IronFunctions. + +We currently support the following message queues and they are passed in via the `MQ` environment variable. For example: + +```sh +docker run -e "MQ=redis://localhost:6379/" ... +``` + +## [Bolt](https://github.com/boltdb/bolt) (default) + +URL: `bolt:///titan/data/functions-mq.db` + +See Bolt in databases above. The Bolt database is locked at the file level, so +the file cannot be the same as the one used for the Bolt Datastore. + +## [Redis](http://redis.io/) + +See Redis in databases above. + +## [IronMQ](https://www.iron.io/platform/ironmq/) + +URL: `ironmq://project_id:token@mq-aws-us-east-1.iron.io/queue_prefix` + +IronMQ is a hosted message queue service provided by [Iron.io](http://iron.io). If you're using IronFunctions in production and don't +want to manage a message queue, you should start here. + +The IronMQ connector uses HTTPS by default. To use HTTP set the scheme to +`ironmq+http`. You can also use a custom port. An example URL is: +`ironmq+http://project_id:token@localhost:8090/queue_prefix`. + +## What about message queue X? + +We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point. + diff --git a/docs/production.md b/docs/production.md new file mode 100644 index 000000000..af9eea725 --- /dev/null +++ b/docs/production.md @@ -0,0 +1,46 @@ + +# Running IronFunctions in Production + +The [QuickStart guide](/README.md) is intended to quickly get started and kick the tires. To run in production and be ready to scale, you need +to use more production ready components. + +* Put the IronFunctions API behind a load balancer and launch run several instances of them (the more the merrier). +* Run a database that can scale. +* Asynchronous functions requires a message queue (preferably one that scales). + +Here's a rough diagram of what a production deployment looks like: + +![IronFunctions Architecture Diagram](architecture.svg) + +## Load Balancer + +Any load balancer will work, put every instance of IronFunctions that you run behind the load balancer. + +**Note**: We will work on a smart load balancer that can direct traffic in a smarter way. See [#151](https://github.com/iron-io/functions/issues/151). + +## Database + +We've done our best to keep the database usage to a minimum. There are no writes during the request/response cycle which where most of the load will be. + +The database is pluggable and we currently support a few options that can be [found here](/docs/databases/). We welcome pull requests for more! + +## Message Queue + +The message queue is an important part of asynchronous functions, essentially buffering requests for processing when resources are available. The reliability and scale of the message queue will play an important part +in how well IronFunctions runs, in particular if you use a lot of asynchronous function calls. + +The message queue is pluggable and we currently support a few options that can be [found here](/docs/mqs/). We welcome pull requests for more! + +## Logging, Metrics and Monitoring + +Logging is a particularly important part of IronFunctions. It not only emits logs, but metrics are also emitted to the logs. Ops teams can then decide how they want +to use the logs and metrics without us prescribing a particular technology. For instance, you can [logspout-statsd](https://github.com/iron-io/logspout-statsd) to capture metrics +from the logs and forward them to statsd. + +[More about Metrics](metrics.md) + +## Scaling + +There are metrics emitted to the logs that can be used to notify you when to scale. The most important being the `wait_time` metrics for both the +synchronous and asynchronous functions. If `wait_time` increases, you'll want to start more IronFunctions instances. + diff --git a/docs/scaling.md b/docs/scaling.md index 9d1540e72..071f4d7c4 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -1,15 +1,2 @@ # Scaling IronFunctions - -The QuickStart guide is intended just to quickly get started and kick the tires. To run in production and be ready to scale, there are a few more steps. - -* Run a database that can scale, such as Postgres. -* Put the iron/functions API behind a load balancer and launch more than one machine. -* For asynchronous functions: - * Start a separate message queue (preferably one that scales) - * Start multiple iron/functions-runner containers, the more the merrier - -There are metrics emitted to the logs that can be used to notify you when to scale. The most important being the `wait_time` metrics for both the -synchronous and asynchronous functions. If `wait_time` increases, you'll want to start more servers with either the `iron/functions` image or the `iron/functions-runner` image. - - \ No newline at end of file diff --git a/docs/triggers.md b/docs/triggers.md new file mode 100644 index 000000000..b78a99431 --- /dev/null +++ b/docs/triggers.md @@ -0,0 +1,5 @@ +# Triggers + +Triggers are integrations that you can use in other systems to fire off functions in IronFunctions. + +TODO: \ No newline at end of file diff --git a/glide.lock b/glide.lock index c0a45ed4a..3bbde03e6 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 4acb4372011661fa4731e89ffd27714eb7ca6285a3d53c2f16f87c3a564d4d4e -updated: 2016-10-14T12:52:58.841636657+02:00 +updated: 2016-10-17T09:57:41.118174867-07:00 imports: - name: github.com/amir/raidman version: c74861fe6a7bb8ede0a010ce4485bdbb4fc4c985 @@ -28,7 +28,7 @@ imports: - name: github.com/dgrijalva/jwt-go version: 268038b363c7a8d7306b8e35bf77a1fde4b0c402 - name: github.com/docker/distribution - version: 04c8db562da407630075b2452daee09b894a070a + version: 717ac0337f312fc7ca0fc35279f00001caf6dd0b subpackages: - context - digest @@ -61,7 +61,7 @@ imports: - name: github.com/fsnotify/fsnotify version: f12c6236fe7b5cf6bcf30e5935d08cb079d78334 - name: github.com/fsouza/go-dockerclient - version: 5e508da6ac33603fcc7dc200b670ba295f24d975 + version: 6bb5d2ca867c937db2e4281f74ad0f4287b86437 - name: github.com/garyburd/redigo version: 4ed1111375cbeb698249ffe48dd463e9b0a63a7a subpackages: @@ -92,8 +92,6 @@ imports: version: 0e04f5e499b19bf51031c01a00f098f25067d8dc - name: github.com/go-openapi/validate version: e6da236c48ce621803fc5d3883d8739aad0ce318 -- name: github.com/go-resty/resty - version: 1a3bb60986d90e32c04575111b1ccb8eab24a3e5 - name: github.com/golang/protobuf version: 2402d76f3d41f928c7902a765dfc872356dd3aad subpackages: @@ -125,6 +123,12 @@ imports: version: 2a2e6b9e3eed0a98d438f111ba7469744c07281d subpackages: - registry +- name: github.com/iron-io/iron_go3 + version: cd9cc95ce2d2bb25d2e4e10cd62fff1d97ad1906 + subpackages: + - api + - config + - mq - name: github.com/iron-io/runner version: b953cea264b1a5a2db46beeaa73f495c19207e51 repo: https://github.com/iron-io/runner.git @@ -204,7 +208,6 @@ imports: - context/ctxhttp - idna - proxy - - publicsuffix - name: golang.org/x/sys version: 9eef40adf05b951699605195b829612bd7b69952 subpackages: