alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Docs related to running in production. (#174)

Browse Source 
* Fixed up api.md, removed Titan references.

* Adding more documentation on running in production.

* Update deps for ironmq.
This commit is contained in:
Travis Reeder
2016-10-17 11:31:58 -07:00
committed by GitHub
parent 42efb2ed6b
commit 41c06644d9
19 changed files with 365 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -120,9 +120,14 @@ And you'll get an ironfunctions.com host for your app:
myapp.USER_ID.ironfunctions.com/hello
```
## API Reference
## More Documentation
https://swaggerhub.com/api/iron/functions
* [IronFunctions Options](api.md)
* [Running in Production](docs/production.md)
* [Triggers](docs/triggers.md)
* [Metrics](docs/metrics.md)
* [Extending IronFunctions](docs/extending.md)
* [API Reference](https://swaggerhub.com/api/iron/functions)
## Join Our Community

6
api/mqs/bolt.go
View File

@@ -42,14 +42,14 @@ func timeoutToIDKey(timeout []byte) []byte {
return b
}
var delayQueueName = []byte("titan_delay")
var delayQueueName = []byte("functions_delay")
func queueName(i int) []byte {
return []byte(fmt.Sprintf("titan_%d_queue", i))
return []byte(fmt.Sprintf("functions_%d_queue", i))
}
func timeoutName(i int) []byte {
return []byte(fmt.Sprintf("titan_%d_timeout", i))
return []byte(fmt.Sprintf("functions_%d_timeout", i))
}
func NewBoltMQ(url *url.URL) (*BoltDbMQ, error) {

165
api/mqs/ironmq.go Normal file
View File

@@ -0,0 +1,165 @@
package mqs
import (
"context"
"encoding/json"
"fmt"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/iron-io/functions/api/models"
mq_config "github.com/iron-io/iron_go3/config"
ironmq "github.com/iron-io/iron_go3/mq"
)
type assoc struct {
msgId string
reservationId string
}
type IronMQ struct {
queues []ironmq.Queue
// Protects the map
sync.Mutex
// job id to {msgid, reservationid}
msgAssoc map[string]*assoc
}
type IronMQConfig struct {
Token string `mapstructure:"token"`
ProjectId string `mapstructure:"project_id"`
Host string `mapstructure:"host"`
Scheme string `mapstructure:"scheme"`
Port uint16 `mapstructure:"port"`
QueuePrefix string `mapstructure:"queue_prefix"`
}
func NewIronMQ(url *url.URL) *IronMQ {
if url.User == nil || url.User.Username() == "" {
logrus.Fatal("IronMQ requires PROJECT_ID and TOKEN")
}
p, ok := url.User.Password()
if !ok {
logrus.Fatal("IronMQ requires PROJECT_ID and TOKEN")
}
settings := &mq_config.Settings{
Token: p,
ProjectId: url.User.Username(),
Host: url.Host,
Scheme: "https",
}
if url.Scheme == "ironmq+http" {
settings.Scheme = "http"
}
parts := strings.Split(url.Host, ":")
if len(parts) > 1 {
settings.Host = parts[0]
p, err := strconv.Atoi(parts[1])
if err != nil {
logrus.WithFields(logrus.Fields{"host_port": url.Host}).Fatal("Invalid host+port combination")
}
settings.Port = uint16(p)
}
var queueName string
if url.Path != "" {
queueName = url.Path
} else {
queueName = "titan"
}
mq := &IronMQ{
queues: make([]ironmq.Queue, 3),
msgAssoc: make(map[string]*assoc),
}
// Check we can connect by trying to create one of the queues. Create is
// idempotent, so this is fine.
_, err := ironmq.ConfigCreateQueue(ironmq.QueueInfo{Name: fmt.Sprintf("%s_%d", queueName, 0)}, settings)
if err != nil {
logrus.WithError(err).Fatal("Could not connect to IronMQ")
}
for i := 0; i < 3; i++ {
mq.queues[i] = ironmq.ConfigNew(fmt.Sprintf("%s_%d", queueName, i), settings)
}
logrus.WithFields(logrus.Fields{"base_queue": queueName}).Info("IronMQ initialized")
return mq
}
func (mq *IronMQ) Push(ctx context.Context, job *models.Task) (*models.Task, error) {
if job.Priority == nil || *job.Priority < 0 || *job.Priority > 2 {
return nil, fmt.Errorf("IronMQ Push job %s: Bad priority", job.ID)
}
// Push the work onto the queue.
buf, err := json.Marshal(job)
if err != nil {
return nil, err
}
_, err = mq.queues[*job.Priority].PushMessage(ironmq.Message{Body: string(buf), Delay: int64(job.Delay)})
return job, err
}
func (mq *IronMQ) Reserve(ctx context.Context) (*models.Task, error) {
var job models.Task
var messages []ironmq.Message
var err error
for i := 2; i >= 0; i-- {
messages, err = mq.queues[i].LongPoll(1, int(time.Minute), 0 /* wait */, false /* delete */)
if err != nil {
// It is OK if the queue does not exist, it will be created when a message is queued.
if !strings.Contains(err.Error(), "404 Not Found") {
return nil, err
}
}
if len(messages) == 0 {
// Try next priority.
if i == 0 {
return nil, nil
}
continue
}
// Found a message!
break
}
message := messages[0]
if message.Body == "" {
return nil, nil
}
err = json.Unmarshal([]byte(message.Body), &job)
if err != nil {
return nil, err
}
mq.Lock()
mq.msgAssoc[job.ID] = &assoc{message.Id, message.ReservationId}
mq.Unlock()
return &job, nil
}
func (mq *IronMQ) Delete(ctx context.Context, job *models.Task) error {
if job.Priority == nil || *job.Priority < 0 || *job.Priority > 2 {
return fmt.Errorf("IronMQ Delete job %s: Bad priority", job.ID)
}
mq.Lock()
assoc, exists := mq.msgAssoc[job.ID]
delete(mq.msgAssoc, job.ID)
mq.Unlock()
if exists {
return mq.queues[*job.Priority].DeleteMessage(assoc.msgId, assoc.reservationId)
}
return nil
}

4
api/mqs/new.go
View File

@@ -3,6 +3,7 @@ package mqs
import (
"fmt"
"net/url"
"strings"
"github.com/Sirupsen/logrus"
"github.com/iron-io/functions/api/models"
@@ -24,6 +25,9 @@ func New(mqURL string) (models.MessageQueue, error) {
case "bolt":
return NewBoltMQ(u)
}
if strings.HasPrefix(u.Scheme, "ironmq") {
return NewIronMQ(u), nil
}
return nil, fmt.Errorf("mq type not supported %v", u.Scheme)
}

3
api/runner/runner.go
View File

@@ -51,8 +51,7 @@ var (
)
func New(metricLogger Logger) (*Runner, error) {
// TODO: Is this really required for Titan's driver?
// Can we remove it?
// TODO: Is this really required for the container drivers? Can we remove it?
env := common.NewEnvironment(func(e *common.Environment) {})
// TODO: Create a drivers.New(runnerConfig) in Titan

10
docs/README.md Normal file
View File

@@ -0,0 +1,10 @@
# IronFunctions Documentation
* [Running IronFunctions](api.md)
* [Databases](databases/README.md)
* [Message Queues](mqs/README.md)
* [Running in Production](production.md)
* [Triggers](triggers.md)
* [Metrics](metrics.md)
* [Extending IronFunctions](extending.md)

96
docs/api.md
View File

@@ -1,7 +1,14 @@
## API Options
#### Env Variables
## IronFunctions Config Options
When starting IronFunctions, you can pass in the following configuration variables as environment variables. Use `-e VAR_NAME=VALUE` in
docker run. For example:
```
docker run -e VAR_NAME=VALUE ...
```
<table>
<tr>
@@ -13,10 +20,6 @@
<td>The database URL to use in URL format. See Databases below for more information. Default: BoltDB in current working directory `bolt.db`.</td>
</tr>
<tr>
<td>PORT</td>
<td>Default (8080), sets the port to run on.</td>
</tr>
<tr>
<td>MQ</td>
<td>The message queue to use in URL format. See Message Queues below for more information. Default: BoltDB in current working directory `queue.db`.</td>
</tr>
@@ -25,79 +28,16 @@
<td>The primary functions api URL to pull tasks from (the address is that of another running functions process).</td>
</tr>
<tr>
<td>PORT</td>
<td>Default (8080), sets the port to run on.</td>
</tr>
<tr>
<td>NUM_ASYNC</td>
<td>The number of async runners in the functions process (default 1).</td>
</tr>
<tr>
<td>LOG_LEVEL</td>
<td>Set to `DEBUG` to enable debugging. Default is INFO.</td>
</tr>
</table>
## Databases
We currently support the following databases and they are passed in via the `DB` environment variable. For example:
```sh
docker run -v /titan/data:/titan/data -e "DB=postgres://user:pass@localhost:6212/mydb" ...
```
### Memory
URL: `memory:///`
Stores all data in memory. Fast and easy, but you'll lose all your data when it stops! NEVER use this in production.
### [Bolt](https://github.com/boltdb/bolt)
URL: `bolt:///titan/data/bolt.db`
Bolt is an embedded database which stores to disk. If you want to use this, be sure you don't lose the data directory by mounting
the directory on your host. eg: `docker run -v $PWD/data:/titan/data -e DB=bolt:///titan/data/bolt.db ...`
### [Redis](http://redis.io/)
URL: `redis://localhost:6379/`
Uses any Redis instance. Be sure to enable [peristence](http://redis.io/topics/persistence).
### [PostgreSQL](http://www.postgresql.org/)
URL: `postgres://user3123:passkja83kd8@ec2-117-21-174-214.compute-1.amazonaws.com:6212/db982398`
If you're using Titan in production, you should probably start here.
### What about database X?
We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point.
## Message Queues
A message queue is used to coordinate the jobs that run through Titan.
We currently support the following message queues and they are passed in via the `MQ` environment variable. For example:
```sh
docker run -v /titan/data:/titan/data -e "MQ=redis://localhost:6379/" ...
```
### Memory
See memory in databases above.
### Bolt
URL: `bolt:///titan/data/bolt-mq.db`
See Bolt in databases above. The Bolt database is locked at the file level, so
the file cannot be the same as the one used for the Bolt Datastore.
### Redis
See Redis in databases above.
### What about message queue X?
We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point.
## Troubleshooting
Enable debugging by passing in the `LOG_LEVEL` env var with DEBUG level.

BIN
docs/architecture.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 49 KiB

4
docs/architecture.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 384 KiB

38
docs/databases/README.md Normal file
View File

@@ -0,0 +1,38 @@
# Databases
We currently support the following databases and they are passed in via the `DB` environment variable. For example:
```sh
docker run -e "DB=postgres://user:pass@localhost:6212/mydb" ...
```
## [Bolt](https://github.com/boltdb/bolt) (default)
URL: `bolt:///functions/data/functions.db`
Bolt is an embedded database which stores to disk. If you want to use this, be sure you don't lose the data directory by mounting
the directory on your host. eg: `docker run -v $PWD/data:/functions/data -e DB=bolt:///functions/data/bolt.db ...`
[More on BoltDB](databases/boltdb.md)
## [Redis](http://redis.io/)
URL: `redis://localhost:6379/`
Use a Redis instance as your database. Be sure to enable [peristence](http://redis.io/topics/persistence).
[More on Redis](databases/redis.md)
## [PostgreSQL](http://www.postgresql.org/)
URL: `postgres://user123:pass456@ec2-117-21-174-214.compute-1.amazonaws.com:6212/db982398`
Use a PostgreSQL database. If you're using IronFunctions in production, you should probably start here.
[More on Postgres](databases/postgres.md)
## What about database X?
We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point.

2
docs/database/boltdb.md → docs/databases/boltdb.md
View File

@@ -4,7 +4,7 @@ BoltDB is the default database, you just need to run the API.
## Persistent
To keep it persistent you add a volume flag to the command:
To keep it persistent, add a volume flag to the command:
```
docker run --rm -it --privileged -v $PWD/bolt.db:/app/bolt.db -p 8080:8080 iron/functions

0
docs/database/postgres.md → docs/databases/postgres.md
View File

9
docs/extending.md
View File

@@ -1,8 +1,13 @@
IronFunctions is extensible so you can add custom functionality and extend the project without needing to modify the core.
# Extending IronFunctions
IronFunctions is extensible so you can add custom functionality and extend the project without needing to modify the core.
## Listeners
This is the main way to do it. To add listeners, copy main.go and use one of the following functions on the Server.
Listeners are the main way to extend IronFunctions.
To add listeners, copy `main.go` into your own repo and add your own listener implementations. When ready,
compile your main package to create your extended version of IronFunctions.
### AppListener

10
docs/metrics.md Normal file
View File

@@ -0,0 +1,10 @@
# Metrics
TODO: Explain metrics format. Log Metric Format - LMF(AO)
```
metric=someevent value=1 type=count
metric=somegauge value=50 type=gauge
```
TODO: List all metrics we emit to logs.

37
docs/mqs/README.md Normal file
View File

@@ -0,0 +1,37 @@
# Message Queues
A message queue is used to coordinate asynchronous function calls that run through IronFunctions.
We currently support the following message queues and they are passed in via the `MQ` environment variable. For example:
```sh
docker run -e "MQ=redis://localhost:6379/" ...
```
## [Bolt](https://github.com/boltdb/bolt) (default)
URL: `bolt:///titan/data/functions-mq.db`
See Bolt in databases above. The Bolt database is locked at the file level, so
the file cannot be the same as the one used for the Bolt Datastore.
## [Redis](http://redis.io/)
See Redis in databases above.
## [IronMQ](https://www.iron.io/platform/ironmq/)
URL: `ironmq://project_id:token@mq-aws-us-east-1.iron.io/queue_prefix`
IronMQ is a hosted message queue service provided by [Iron.io](http://iron.io). If you're using IronFunctions in production and don't
want to manage a message queue, you should start here.
The IronMQ connector uses HTTPS by default. To use HTTP set the scheme to
`ironmq+http`. You can also use a custom port. An example URL is:
`ironmq+http://project_id:token@localhost:8090/queue_prefix`.
## What about message queue X?
We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point.

46
docs/production.md Normal file
View File

@@ -0,0 +1,46 @@
# Running IronFunctions in Production
The [QuickStart guide](/README.md) is intended to quickly get started and kick the tires. To run in production and be ready to scale, you need
to use more production ready components.
* Put the IronFunctions API behind a load balancer and launch run several instances of them (the more the merrier).
* Run a database that can scale.
* Asynchronous functions requires a message queue (preferably one that scales).
Here's a rough diagram of what a production deployment looks like:
![IronFunctions Architecture Diagram](architecture.svg)
## Load Balancer
Any load balancer will work, put every instance of IronFunctions that you run behind the load balancer.
**Note**: We will work on a smart load balancer that can direct traffic in a smarter way. See [#151](https://github.com/iron-io/functions/issues/151).
## Database
We've done our best to keep the database usage to a minimum. There are no writes during the request/response cycle which where most of the load will be.
The database is pluggable and we currently support a few options that can be [found here](/docs/databases/). We welcome pull requests for more!
## Message Queue
The message queue is an important part of asynchronous functions, essentially buffering requests for processing when resources are available. The reliability and scale of the message queue will play an important part
in how well IronFunctions runs, in particular if you use a lot of asynchronous function calls.
The message queue is pluggable and we currently support a few options that can be [found here](/docs/mqs/). We welcome pull requests for more!
## Logging, Metrics and Monitoring
Logging is a particularly important part of IronFunctions. It not only emits logs, but metrics are also emitted to the logs. Ops teams can then decide how they want
to use the logs and metrics without us prescribing a particular technology. For instance, you can [logspout-statsd](https://github.com/iron-io/logspout-statsd) to capture metrics
from the logs and forward them to statsd.
[More about Metrics](metrics.md)
## Scaling
There are metrics emitted to the logs that can be used to notify you when to scale. The most important being the `wait_time` metrics for both the
synchronous and asynchronous functions. If `wait_time` increases, you'll want to start more IronFunctions instances.

13
docs/scaling.md
View File

@@ -1,15 +1,2 @@
# Scaling IronFunctions
The QuickStart guide is intended just to quickly get started and kick the tires. To run in production and be ready to scale, there are a few more steps.
* Run a database that can scale, such as Postgres.
* Put the iron/functions API behind a load balancer and launch more than one machine.
* For asynchronous functions:
* Start a separate message queue (preferably one that scales)
* Start multiple iron/functions-runner containers, the more the merrier
There are metrics emitted to the logs that can be used to notify you when to scale. The most important being the `wait_time` metrics for both the
synchronous and asynchronous functions. If `wait_time` increases, you'll want to start more servers with either the `iron/functions` image or the `iron/functions-runner` image.

5
docs/triggers.md Normal file
View File

@@ -0,0 +1,5 @@
# Triggers
Triggers are integrations that you can use in other systems to fire off functions in IronFunctions.
TODO:

15
glide.lock generated
View File

@@ -1,5 +1,5 @@
hash: 4acb4372011661fa4731e89ffd27714eb7ca6285a3d53c2f16f87c3a564d4d4e
updated: 2016-10-14T12:52:58.841636657+02:00
updated: 2016-10-17T09:57:41.118174867-07:00
imports:
- name: github.com/amir/raidman
version: c74861fe6a7bb8ede0a010ce4485bdbb4fc4c985
@@ -28,7 +28,7 @@ imports:
- name: github.com/dgrijalva/jwt-go
version: 268038b363c7a8d7306b8e35bf77a1fde4b0c402
- name: github.com/docker/distribution
version: 04c8db562da407630075b2452daee09b894a070a
version: 717ac0337f312fc7ca0fc35279f00001caf6dd0b
subpackages:
- context
- digest
@@ -61,7 +61,7 @@ imports:
- name: github.com/fsnotify/fsnotify
version: f12c6236fe7b5cf6bcf30e5935d08cb079d78334
- name: github.com/fsouza/go-dockerclient
version: 5e508da6ac33603fcc7dc200b670ba295f24d975
version: 6bb5d2ca867c937db2e4281f74ad0f4287b86437
- name: github.com/garyburd/redigo
version: 4ed1111375cbeb698249ffe48dd463e9b0a63a7a
subpackages:
@@ -92,8 +92,6 @@ imports:
version: 0e04f5e499b19bf51031c01a00f098f25067d8dc
- name: github.com/go-openapi/validate
version: e6da236c48ce621803fc5d3883d8739aad0ce318
- name: github.com/go-resty/resty
version: 1a3bb60986d90e32c04575111b1ccb8eab24a3e5
- name: github.com/golang/protobuf
version: 2402d76f3d41f928c7902a765dfc872356dd3aad
subpackages:
@@ -125,6 +123,12 @@ imports:
version: 2a2e6b9e3eed0a98d438f111ba7469744c07281d
subpackages:
- registry
- name: github.com/iron-io/iron_go3
version: cd9cc95ce2d2bb25d2e4e10cd62fff1d97ad1906
subpackages:
- api
- config
- mq
- name: github.com/iron-io/runner
version: b953cea264b1a5a2db46beeaa73f495c19207e51
repo: https://github.com/iron-io/runner.git
@@ -204,7 +208,6 @@ imports:
- context/ctxhttp
- idna
- proxy
- publicsuffix
- name: golang.org/x/sys
version: 9eef40adf05b951699605195b829612bd7b69952
subpackages: