From b7bf73f5d21be3ff9ae7910af724b7063ad4d762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Seif=20Lotfy=20=D8=B3=D9=8A=D9=81=20=D9=84=D8=B7=D9=81?= =?UTF-8?q?=D9=8A?= Date: Thu, 6 Oct 2016 20:46:29 +0200 Subject: [PATCH] Makefile (#122) * Update Readme and add Makefile * Skip stale tests (in wait for stdin support) * Revert "Skip stale tests (in wait for stdin support)" This reverts commit 228da3776503f40ca53df70a79a9e4a9c73fd8b5. --- CONTRIBUTING.md | 37 ++++++---- Makefile | 23 ++++++ README.md | 41 +++++------ api/runner/runner.go | 16 +++-- circle.yml | 2 +- docs/api.md | 104 +++++++++++++++++++++++++++- main.go | 4 +- build.sh => scripts/build-docker.sh | 0 release.sh => scripts/release.sh | 0 api.sh => scripts/run-docker.sh | 2 +- scripts/test.sh | 3 + test.sh | 5 -- 12 files changed, 182 insertions(+), 55 deletions(-) create mode 100644 Makefile rename build.sh => scripts/build-docker.sh (100%) rename release.sh => scripts/release.sh (100%) rename api.sh => scripts/run-docker.sh (83%) create mode 100755 scripts/test.sh delete mode 100755 test.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9747e8421..14f09d3ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,24 +1,35 @@ -## Building +## Contributing + +### Build -First time or when a dependency changes or when the API changes, run: ``` -glide install +make all ``` -To quick build and run (using default database): +### Run Functions service -```sh -api.sh +First let's start our IronFunctions API + +##### Run in Docker +``` +make run-docker ``` -To build the docker image: +will start Functions using an embedded `Bolt` database running on `:8080`. -```sh -build.sh +##### Running on Metal (recommended only on Linux) + +``` +./functions ``` -## Releasing +will start Functions with a default of 1 async runner -```sh -release.sh -``` +### Contributing + +##### Code +* Fork the repo +* Fix an issue +* Create a Pull Request +* Sign the CLA +* Good Job! Thanks for being awesome! diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ff21635b2 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +# Just builds +dep: + @ glide install + +build: + @ go build -o functions + +build-docker: + sh scripts/build-docker.sh + +release: + sh scripts/release.sh + +test: + sh scripts/test.sh + +run-docker: + sh scripts/run-docker.sh + +run-simple: + ./functions + +all: dep build \ No newline at end of file diff --git a/README.md b/README.md index 401f1ee30..e6ce3f96d 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,14 @@ # IronFunctions -## Quick Start +## Run functions -### Start the IronFunctions API +```docker run --rm --name functions --privileged -it -v $PWD/data:/app/data -p 8080:8080 iron/functions``` -First let's start our IronFunctions API +*Note: A list of configurations via env variables can be found [here](docs/api.md).* -```sh -docker run --rm --name functions --privileged -it -e "DB=bolt:///app/data/bolt.db" -v $PWD/data:/app/data -p 8080:8080 iron/functions -``` +## Using Functions -This command will quickly start IronFunctions using an embedded `Bolt` database running on `:8080`. - -### Create an Application +#### Create an Application An application is essentially a grouping of functions, that put together, form an API. Here's how to create an app. @@ -24,7 +20,7 @@ curl -H "Content-Type: application/json" -X POST -d '{ Now that we have an app, we can map routes to functions. -### Add a route to a Function +#### Add a route to a Function ```sh curl -H "Content-Type: application/json" -X POST -d '{ @@ -35,7 +31,7 @@ curl -H "Content-Type: application/json" -X POST -d '{ }' http://localhost:8080/v1/apps/myapp/routes ``` -### Calling your Function +#### Calling your Function Just hit the URL you got back from adding a route above: @@ -43,7 +39,7 @@ Just hit the URL you got back from adding a route above: curl http://localhost:8080/r/myapp/hello ``` -### To pass in data to your function +#### To pass in data to your function Your function will get the body of the request as is, and the headers of the request will be passed in as env vars. Try this: @@ -75,24 +71,19 @@ See the [Blog Example](https://github.com/iron-io/functions/blob/master/examples ## Adding Asynchronous Data Processing Support -Data processing is for functions that run in the background. This type of functionality is good for functions that are CPU heavy or take more than a few seconds to complete. +Data processing is for functions that run in the background. This type of functionality is good for functions +that are CPU heavy or take more than a few seconds to complete. Architecturally, the main difference between synchronous you tried above and asynchronous is that requests -to asynchronous functions are put in a queue and executed on separate `runner` machines so that they do not interfere with the fast synchronous responses required by an API. Also, since -it uses a queue, you can queue up millions of jobs without worrying about capacity as requests will just be queued up and run at some point in the future. +to asynchronous functions are put in a queue and executed on upon resource availablitiy on the same process +or a remote functions process so that they do not interfere with the fast synchronous responses required by an API. +Also, since it uses a queue, you can queue up millions of jobs without worrying about capacity as requests will +just be queued up and run at some point in the future. TODO: Add link to differences here in README.io docs here. -### Start Runner(s) +#### Running remote functions process -Start a runner: - -```sh -docker run --rm -it --link functions --privileged -e "API_URL=http://functions:8080" iron/functions-runner -``` - -You can start as many runners as you want. The more the merrier. - -For runner configuration, see the [Runner README](runner/README.md). +Coming soon... ## Using IronFunctions Hosted by Iron.io diff --git a/api/runner/runner.go b/api/runner/runner.go index 178e119ee..b8f33447a 100644 --- a/api/runner/runner.go +++ b/api/runner/runner.go @@ -8,6 +8,7 @@ import ( "io" "io/ioutil" "os" + "runtime" "strconv" "strings" "sync" @@ -238,10 +239,8 @@ func selectDriver(driver string, env *common.Environment, conf *driverscommon.Co func getAvailableMemory() int64 { const tooBig = 322122547200 // #300GB or 0, biggest aws instance is 244GB - var availableMemory uint64 - if os.Getenv("IGNORE_MEMORY") == "1" { - availableMemory = tooBig - } else { + var availableMemory uint64 = tooBig + if runtime.GOOS == "linux" { availableMemory, err := checkCgroup() if err != nil { logrus.WithError(err).Error("Error checking for cgroup memory limits, falling back to host memory available..") @@ -249,8 +248,9 @@ func getAvailableMemory() int64 { if availableMemory > tooBig || availableMemory == 0 { // Then -m flag probably wasn't set, so use max available on system availableMemory, err = checkProc() - if availableMemory > tooBig || availableMemory == 0 { - logrus.WithError(err).Fatal("Your Linux version is too old (<3.14) then we can't get the proper information to . You must specify the maximum available memory by passing the -m command with docker run when starting the runner via docker, eg: `docker run -m 2G ...`") + if err != errCantReadMemInfo && + (availableMemory > tooBig || availableMemory == 0) { + logrus.WithError(err).Fatal("Cannot get the proper information to. You must specify the maximum available memory by passing the -m command with docker run when starting the runner via docker, eg: `docker run -m 2G ...`") } } } @@ -273,6 +273,8 @@ func checkCgroup() (uint64, error) { return strconv.ParseUint(limBytes, 10, 64) } +var errCantReadMemInfo = errors.New("Didn't find MemAvailable in /proc/meminfo, kernel is probably < 3.14") + func checkProc() (uint64, error) { f, err := os.Open("/proc/meminfo") if err != nil { @@ -309,5 +311,5 @@ func checkProc() (uint64, error) { return c, nil } - return 0, fmt.Errorf("Didn't find MemAvailable in /proc/meminfo, kernel is probably < 3.14") + return 0, errCantReadMemInfo } diff --git a/circle.yml b/circle.yml index 77a80d3ea..b6b42fc86 100644 --- a/circle.yml +++ b/circle.yml @@ -29,5 +29,5 @@ dependencies: test: override: - - ./test.sh: + - scripts/test.sh: pwd: $GO_PROJECT diff --git a/docs/api.md b/docs/api.md index 1f9327fc4..0c3ce81e9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1 +1,103 @@ -TODO: Add swagger URL. + +## API Options + +#### Env Variables + + + + + + + + + + + + + + + + + + + + + + + + + + +
Env VariablesDescription
DBThe database URL to use in URL format. See Databases below for more information. Default: BoltDB in current working directory `bolt.db`.
PORTDefault (8080), sets the port to run on.
MQThe message queue to use in URL format. See Message Queues below for more information. Default: BoltDB in current working directory `queue.db`.
API_URLThe primary functions api URL to pull tasks from (the address is that of another running functions process).
NUM_ASYNCThe number of async runners in the functions process (default 1).
+ +## Databases + +We currently support the following databases and they are passed in via the `DB` environment variable. For example: + +```sh +docker run -v /titan/data:/titan/data -e "DB=postgres://user:pass@localhost:6212/mydb" ... +``` + +### Memory + +URL: `memory:///` + +Stores all data in memory. Fast and easy, but you'll lose all your data when it stops! NEVER use this in production. + +### [Bolt](https://github.com/boltdb/bolt) + +URL: `bolt:///titan/data/bolt.db` + +Bolt is an embedded database which stores to disk. If you want to use this, be sure you don't lose the data directory by mounting +the directory on your host. eg: `docker run -v $PWD/data:/titan/data -e DB=bolt:///titan/data/bolt.db ...` + +### [Redis](http://redis.io/) + +URL: `redis://localhost:6379/` + +Uses any Redis instance. Be sure to enable [peristence](http://redis.io/topics/persistence). + +### [PostgreSQL](http://www.postgresql.org/) + +URL: `postgres://user3123:passkja83kd8@ec2-117-21-174-214.compute-1.amazonaws.com:6212/db982398` + +If you're using Titan in production, you should probably start here. + +### What about database X? + +We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point. + +## Message Queues + +A message queue is used to coordinate the jobs that run through Titan. + +We currently support the following message queues and they are passed in via the `MQ` environment variable. For example: + +```sh +docker run -v /titan/data:/titan/data -e "MQ=redis://localhost:6379/" ... +``` + +### Memory + +See memory in databases above. + +### Bolt + +URL: `bolt:///titan/data/bolt-mq.db` + +See Bolt in databases above. The Bolt database is locked at the file level, so +the file cannot be the same as the one used for the Bolt Datastore. + +### Redis + +See Redis in databases above. + +### What about message queue X? + +We're happy to add more and we love pull requests, so feel free to add one! Copy one of the implementations above as a starting point. + +## Troubleshooting + +Enable debugging by passing in the `LOG_LEVEL` env var with DEBUG level. + + diff --git a/main.go b/main.go index 20a9bc5d8..8d25244c5 100644 --- a/main.go +++ b/main.go @@ -21,8 +21,8 @@ const ( envMQ = "mq" envDB = "db" envPort = "port" // be careful, Gin expects this variable to be "port" - envAPIURL = "tasks_url" - envNumAsync = "nasync" + envAPIURL = "api_url" + envNumAsync = "num_async" ) func init() { diff --git a/build.sh b/scripts/build-docker.sh similarity index 100% rename from build.sh rename to scripts/build-docker.sh diff --git a/release.sh b/scripts/release.sh similarity index 100% rename from release.sh rename to scripts/release.sh diff --git a/api.sh b/scripts/run-docker.sh similarity index 83% rename from api.sh rename to scripts/run-docker.sh index 3f57907e7..f78d06eba 100755 --- a/api.sh +++ b/scripts/run-docker.sh @@ -1,5 +1,5 @@ set -ex -./build.sh +sh ./scripts/build-docker.sh docker run --rm --privileged -it -e LOG_LEVEL=debug -e "DB=bolt:///app/data/bolt.db" -v $PWD/data:/app/data -p 8080:8080 iron/functions diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 000000000..8f8eea209 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker run -ti --privileged --rm -e LOG_LEVEL=debug -v /var/run/docker.sock:/var/run/docker.sock -v "$PWD":/go/src/github.com/iron-io/functions -w /go/src/github.com/iron-io/functions iron/go:dev go test -v $(glide nv) \ No newline at end of file diff --git a/test.sh b/test.sh deleted file mode 100755 index cfb658fe4..000000000 --- a/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -export GO15VENDOREXPERIMENT=1 -export LOG_LEVEL=debug -export IGNORE_MEMORY=1 - -go test -v $(go list ./... | grep -v /vendor/ | grep -v /examples/) \ No newline at end of file