alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Many updates, working ok at this point.

Browse Source
This commit is contained in:
Travis Reeder
2016-06-17 19:43:26 -07:00
parent 60db6e1d51
commit 40321a6bdb
10 changed files with 180 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -10,3 +10,5 @@ app.zip
vendor/ vendor/
/microgateway /microgateway
/gateway /gateway
private.sh

41
README.md
View File

@@ -1,31 +1,39 @@
Note: currently running at: http://gateway.iron.computer:8080/
# MicroServices Gateway / API Gateway # MicroServices Gateway / API Gateway
First things first, register an app: First things first, create an app/service:
TOOD: App or service??
```sh ```sh
curl -H "Content-Type: application/json" -X POST -d '{"name":"myapp","password":"xyz"}' http://localhost:8080/test/1/projects/123/apps iron create app
# OR
curl -H "Content-Type: application/json" -X POST -d '{"name":"myapp"}' http://localhost:8080/api/v1/apps
``` ```
Now add routes to the app. First we'll add a route to the output of a docker container: Now add routes to the app. First we'll add a route to the output of a docker container:
```sh ```sh
curl -H "Content-Type: application/json" -X POST -d '{"path":"/hello.rb","image":"treeder/hello.rb", "type":"run"}' http://localhost:8080/test/1/projects/123/apps/myapp/routes iron add route myapp /hello iron/hello:0.0.1
# OR
curl -H "Content-Type: application/json" -X POST -d '{"path":"/hello", "image":"iron/hello", "type":"run"}' http://localhost:8080/api/v1/apps/myapp/routes
``` ```
curl -H "Content-Type: application/json" -X POST -d '{"path":"/helloiron","image":"iron/hello", "type":"run"}' http://localhost:8080/test/1/projects/123/apps/myapp/routes And how about a [slackbot](https://github.com/treeder/slackbots/tree/master/guppy) endpoint:
```sh
curl -H "Content-Type: application/json" -X POST -d '{"path":"/guppy","image":"treeder/guppy:0.0.2"}' http://localhost:8080/api/v1/apps/myapp/routes
```
Test out the route: Test out the route:
```sh Surf to: http://localhost:8080/hello?app=myapp
curl -i -X GET http://localhost:8080/hello.rb?app=myapp
```
Now try mapping an app endpoint: Now try mapping an app endpoint:
```sh ```sh
curl -H "Content-Type: application/json" -X POST -d '{"path":"/sinatra","image":"treeder/hello-sinatra", "type":"app", "cpath":"/"}' http://localhost:8080/test/1/projects/123/apps/myapp/routes curl -H "Content-Type: application/json" -X POST -d '{"path":"/sinatra","image":"treeder/hello-sinatra", "type":"app", "cpath":"/"}' http://localhost:8080/api/v1/apps/myapp/routes
``` ```
And test it out: And test it out:
@@ -37,7 +45,7 @@ curl -i -X GET http://localhost:8080/sinatra?app=myapp
And another: And another:
```sh ```sh
curl -H "Content-Type: application/json" -X POST -d '{"path":"/sinatra/ping","image":"treeder/hello-sinatra", "type":"app", "cpath":"/ping"}' http://localhost:8080/test/1/projects/123/apps/myapp/routes curl -H "Content-Type: application/json" -X POST -d '{"path":"/sinatra/ping","image":"treeder/hello-sinatra", "type":"app", "cpath":"/ping"}' http://localhost:8080/api/v1/apps/myapp/routes
``` ```
And test it out: And test it out:
@@ -49,7 +57,7 @@ curl -i -X GET http://localhost:8080/sinatra?app=myapp
You'all also get a custom URL like this when in production. You'all also get a custom URL like this when in production.
``` ```
appname.projectid.iron.computer appname.iron.computer
``` ```
## Building/Testing ## Building/Testing
@@ -66,7 +74,7 @@ glide install
Test it, the iron token and project id are for cache. Test it, the iron token and project id are for cache.
```sh ```sh
docker run -e "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" --rm -it --privileged -p 8080:8080 iron/gateway docker run -e "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" -e "CLOUDFLARE_EMAIL=treeder@gmail.com" -e "CLOUDFLARE_API_KEY=X" --rm -it --privileged -p 8080:8080 iron/gateway
``` ```
Push it: Push it:
@@ -82,3 +90,12 @@ After deploying, running it with:
```sh ```sh
docker run -e "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" --name irongateway -it --privileged --net=host -p 8080:8080 -d --name irongateway iron/gateway docker run -e "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" --name irongateway -it --privileged --net=host -p 8080:8080 -d --name irongateway iron/gateway
``` ```
## TODOS
* [ ] Check if image exists when registering the endpoint, not at run time
* [ ] Put stats into influxdb or something to show to user (requests, errors)
* [ ] Store recent logs for user.
* [ ] Allow env vars for config on the app and routes (routes override apps).
* [ ] Provide a base url for each app, eg: appname.userid.iron.computer
* [ ] Allow setting content-type on a route, then use that when responding

12
dns.go
View File

@@ -7,8 +7,8 @@ import (
"net/http" "net/http"
"strings" "strings"
log "github.com/Sirupsen/logrus"
"github.com/iron-io/go/common" "github.com/iron-io/go/common"
"gopkg.in/inconshreveable/log15.v2"
) )
type CloudFlareResult struct { type CloudFlareResult struct {
@@ -30,7 +30,7 @@ func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool {
// WORKER_NAME.PROJECT_ID.iron.computer. // WORKER_NAME.PROJECT_ID.iron.computer.
dnsHost := fmt.Sprintf("%v.%v.iron.computer", app.Name, 123) dnsHost := fmt.Sprintf("%v.%v.iron.computer", app.Name, 123)
app.Dns = dnsHost app.Dns = dnsHost
log15.Info("registering dns", "dnsname", dnsHost) log.Info("registering dns", "dnsname", dnsHost)
if app.CloudFlareId == "" { if app.CloudFlareId == "" {
// Tad hacky, but their go lib is pretty confusing. // Tad hacky, but their go lib is pretty confusing.
@@ -53,7 +53,7 @@ func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool {
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
log15.Error("Could not register dns entry.", "err", err) log.Error("Could not register dns entry.", "err", err)
common.SendError(w, 500, fmt.Sprint("Could not register dns entry.", err)) common.SendError(w, 500, fmt.Sprint("Could not register dns entry.", err))
return false return false
} }
@@ -61,14 +61,14 @@ func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool {
// todo: get error message from body for bad status code // todo: get error message from body for bad status code
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
log15.Error("Could not register dns entry 2.", "code", resp.StatusCode, "body", string(body)) log.Error("Could not register dns entry 2.", "code", resp.StatusCode, "body", string(body))
common.SendError(w, 500, fmt.Sprint("Could not register dns entry 2. ", resp.StatusCode)) common.SendError(w, 500, fmt.Sprint("Could not register dns entry 2. ", resp.StatusCode))
return false return false
} }
cfResult := CloudFlareResponse{} cfResult := CloudFlareResponse{}
err = json.Unmarshal(body, &cfResult) err = json.Unmarshal(body, &cfResult)
if err != nil { if err != nil {
log15.Error("Could not parse DNS response.", "err", err, "code", resp.StatusCode, "body", string(body)) log.Error("Could not parse DNS response.", "err", err, "code", resp.StatusCode, "body", string(body))
common.SendError(w, 500, fmt.Sprint("Could not parse DNS response. ", resp.StatusCode)) common.SendError(w, 500, fmt.Sprint("Could not parse DNS response. ", resp.StatusCode))
return false return false
} }
@@ -76,6 +76,6 @@ func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool {
app.CloudFlareId = cfResult.Result.Id app.CloudFlareId = cfResult.Result.Id
} }
log15.Info("host registered successfully with cloudflare", "app", app) log.Info("host registered successfully with cloudflare", "app", app)
return true return true
} }

23
gateway.pem Normal file
View File

@@ -0,0 +1,23 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEAhof/Rgb5YUGUCynxYAkbkSRQqIaaZem7g6r/lyePBk6IKzjia6QA+Ut+1KZt
uGrNXFR0THtfEWCFjIPBJfYLGmkG1gaYocWJUu/b3be3rKGlLWEuSpfHsYyLh4803QNU79Uu0ft8
ODB4QJl54WImD1JKzAZyarDalyb+GKbnU5NAULBbTccbGFbNSwPwebvoK9G6Z8qWChPqsYAZxfyC
D2LBt0PANwB+haC6Rj0t99R6mtLRz/iKYaHz26d6UxSzNsXArJlhSCABHdQ71rbPkO0M9PvJrhfg
y+bLA4sMrHvOSjEDWGY+j1qqEYXSc/Rwe5SMd8kV7i902ks7PjcCZQIDAQABAoIBAHjvlHk9F71o
GE+Y2tV8Gn31aVS1++IVpW2NsMoO07HVsu836cLd4co5JcDAA+4+hHG1sf53AVU7sZJJdr5LWlvZ
gj2wHFGApBwcZ0f/OWxEu5n5vIVtwCRJtbyc7eaochhPShGVw2s3l0JrNXd4pcIsNfUG7qAeb8Jl
WRKMJ3OmoEMOz5M3scRypQKOulRjO6RMJCtbl4AntMYNF7cdWeuIJ3eaMD8HaYbkr1USrwGk65QC
mKdUcNl9k++Txuf7UtbRB0apFsMnAKRPUTU+9TPGwMsZSzszk8TClMNO1ALYKKY5mE+cPqrAl5gC
ZpPOf45oT+2lxktqq5u5N8XMFoECgYEA16eGE3inh23EVly5ARnowGgFtorZsI6XIJHkYsETRocf
GAvQrEMAuFWZy0n5TlNBfzPhHh9rUXWCKlbldgnDgWKpphPux3UXTqTUak+j6397rtEP85RfYqDm
QxtW0uOkKkZSGyXXEYBwTbCsQBH28VZsJEYVe+G3uXPcUu3WEZUCgYEAn7Mwgk/JP5wgYo4E3gga
fOi0/de11MV5ad337qfdUC1pf1ju9q2CyHaV3g6eo2OnynGZHYq5qlyLWoi/hTr6A+yMZSnQ15io
9ker4uyAX6DdVDmWK9uErwrqLAV+Q6HoVmxoyBbMihQW8TqX/5jZegxqipDW16+qOFxtPbZhmZEC
gYEAlD85UBFVOSggHC5Jj5Q8CGh55O62j0S2Z1Fjau/HTGh+24zjukelKxLNUo5br5hUIhmL26VF
pQ3emTR7MRWtLDii3uQ89ShtCUcOLrbovG86mwZkrNGGcMqi/+a/XOHYbKdCsh7lJcbhbMbS4oh2
9ZivZpA3HJ4iKn6XKvsMebECgYEAhNWXU8zpqG9EwLVAdy5mWd92LG5wYDqhct2ejHQ0MayUQ8jF
e4l3bya0IbAnY+BQgKNcqKXrKTkw8G0uYLNdokXvwXW2sJ3abH/RCT+Ox/wWHSiJMJG3G6IIhfVL
wRW7G6ewwD22hGORcbU7GO8addo+BGPVUDJdc+PtOZeqNwECgYEAvLJQas3PKLL+qVmO9asvDtCl
tvOuvPFAuZBk6hLm9SSrFt5cCW+rulL8Kx1PxUk0C8LiJV8uwZIqQxE+gY5MkCfvt+xG1yurkywd
SCHOFr0m4gi8+XHvL4fmGzYPgHRi10kepSta5G8USigbcf5fOmw+Upa3qVnwot3CQdxmSfU=
-----END RSA PRIVATE KEY-----

10
glide.lock generated
View File

@@ -1,5 +1,5 @@
hash: a104a522dc2ba982d25101330bbd5d44778565128f0cb09f34061c167921046c hash: a104a522dc2ba982d25101330bbd5d44778565128f0cb09f34061c167921046c
updated: 2016-06-15T17:17:05.21953321-07:00 updated: 2016-06-17T14:27:42.156301621-07:00
imports: imports:
- name: github.com/amir/raidman - name: github.com/amir/raidman
version: 91c20f3f475cab75bb40ad7951d9bbdde357ade7 version: 91c20f3f475cab75bb40ad7951d9bbdde357ade7
@@ -12,7 +12,7 @@ imports:
subpackages: subpackages:
- statsd - statsd
- name: github.com/dgrijalva/jwt-go - name: github.com/dgrijalva/jwt-go
version: c04502f106d7c5b3fae17c5da49a1bbdd3006b3c version: f0777076321ab64f6efc15a82d9d23b98539b943
- name: github.com/golang/protobuf - name: github.com/golang/protobuf
version: 0c1f6d65b5a189c2250d10e71a5506f06f9fa0a0 version: 0c1f6d65b5a189c2250d10e71a5506f06f9fa0a0
subpackages: subpackages:
@@ -29,7 +29,7 @@ imports:
- semaphore - semaphore
- serverutil - serverutil
- name: github.com/iron-io/go - name: github.com/iron-io/go
version: 277190779e20ad6bff386a5f7f6a8009d73fa475 version: 6b082f50f20cb0e0146bdf0ffbc6c07216eeea77
subpackages: subpackages:
- common - common
- common/httpshutdown - common/httpshutdown
@@ -50,12 +50,14 @@ imports:
version: 9056b7a9f2d1f2d96498d6d146acd1f9d5ed3d59 version: 9056b7a9f2d1f2d96498d6d146acd1f9d5ed3d59
- name: github.com/mattn/go-isatty - name: github.com/mattn/go-isatty
version: 56b76bdf51f7708750eac80fa38b952bb9f32639 version: 56b76bdf51f7708750eac80fa38b952bb9f32639
- name: github.com/Sirupsen/logrus
version: f3cfb454f4c209e6668c95216c4744b8fddb2356
- name: github.com/vmihailenco/bufio - name: github.com/vmihailenco/bufio
version: 24e7e48f60fc2d9e99e43c07485d9fff42051e66 version: 24e7e48f60fc2d9e99e43c07485d9fff42051e66
- name: github.com/vrischmann/envconfig - name: github.com/vrischmann/envconfig
version: 9e6e1c4d3b73427d03118518603bb904d9c55236 version: 9e6e1c4d3b73427d03118518603bb904d9c55236
- name: golang.org/x/net - name: golang.org/x/net
version: 3f122ce3dbbe488b7e6a8bdb26f41edec852a40b version: d7bf3545bb0dacf009c535b3d3fbf53ac0a339ab
subpackages: subpackages:
- proxy - proxy
- name: golang.org/x/sys - name: golang.org/x/sys

2
models.go
View File

@@ -21,7 +21,7 @@ type App struct {
ProjectId string `json:"project_id"` ProjectId string `json:"project_id"`
CloudFlareId string `json:"-"` CloudFlareId string `json:"-"`
Dns string `json:"dns"` Dns string `json:"dns"`
Routes []Route3 `json:"routes"` Routes map[string]*Route3 `json:"routes"`
} }
// this is for the new api gateway // this is for the new api gateway

140
router.go
View File

@@ -11,20 +11,19 @@ import (
"encoding/json" "encoding/json"
"flag" "flag"
"fmt" "fmt"
"github.com/gorilla/mux"
"github.com/iron-io/go/common"
"github.com/iron-io/golog"
"github.com/iron-io/iron_go/cache"
"github.com/iron-io/iron_go/worker"
"gopkg.in/inconshreveable/log15.v2"
"log"
"math/rand" "math/rand"
"net/http" "net/http"
"net/url" "net/url"
"os"
"reflect" "reflect"
"runtime"
"strings" "strings"
"time" "time"
log "github.com/Sirupsen/logrus"
"github.com/gorilla/mux"
"github.com/iron-io/go/common"
"github.com/iron-io/iron_go/cache"
"github.com/iron-io/iron_go/worker"
) )
var config struct { var config struct {
@@ -82,11 +81,12 @@ func main() {
common.LoadConfigFile(configFile, &config) common.LoadConfigFile(configFile, &config)
// common.SetLogging(common.LoggingConfig{To: config.Logging.To, Level: config.Logging.Level, Prefix: config.Logging.Prefix}) // common.SetLogging(common.LoggingConfig{To: config.Logging.To, Level: config.Logging.Level, Prefix: config.Logging.Prefix})
log.Println("config:", config) // TODO: validate inputs, iron tokens, cloudflare stuff, etc
golog.Infoln("Starting up router version", version) config.CloudFlare.Email = os.Getenv("CLOUDFLARE_EMAIL")
config.CloudFlare.AuthKey = os.Getenv("CLOUDFLARE_API_KEY")
runtime.GOMAXPROCS(runtime.NumCPU()) log.Println("config:", config)
log.Println("Running on", runtime.NumCPU(), "CPUs") log.Infoln("Starting up router version", version)
r := mux.NewRouter() r := mux.NewRouter()
@@ -95,35 +95,34 @@ func main() {
s2.Handle("/", &WorkerHandler{}) s2.Handle("/", &WorkerHandler{})
// dev: // dev:
fmt.Println("Setting /test subrouter") s := r.PathPrefix("/api").Subrouter()
s := r.PathPrefix("/test").Subrouter()
// production: // production:
// s := r.Host("router.irondns.info").Subrouter() // s := r.Host("router.irondns.info").Subrouter()
s.Handle("/1/projects/{project_id}/register", &Register{}) // s.Handle("/1/projects/{project_id}/register", &Register{})
s.Handle("/1/projects/{project_id}/apps", &NewApp{}) s.Handle("/v1/apps", &NewApp{})
s.HandleFunc("/1/projects/{project_id}/apps/{app_name}/routes", NewRoute) s.HandleFunc("/v1/apps/{app_name}/routes", NewRoute)
s.HandleFunc("/ping", Ping) s.HandleFunc("/ping", Ping)
s.HandleFunc("/version", Version) s.HandleFunc("/version", Version)
s.Handle("/addworker", &WorkerHandler{}) // s.Handle("/addworker", &WorkerHandler{})
s.HandleFunc("/", Ping) s.HandleFunc("/", Ping)
r.HandleFunc("/elb-ping-router", Ping) // for ELB health check r.HandleFunc("/elb-ping-router", Ping) // for ELB health check
// for testing app responses, pass in app name, can use localhost // for testing app responses, pass in app name, can use localhost
s4 := r.Queries("app", "").Subrouter() s4 := r.Queries("app", "").Subrouter()
s4.HandleFunc("/appsr", Ping) // s4.HandleFunc("/appsr", Ping)
s4.HandleFunc("/{rest:.*}", Run) s4.HandleFunc("/{rest:.*}", Run)
s4.NotFoundHandler = http.HandlerFunc(Run) s4.NotFoundHandler = http.HandlerFunc(Run)
s3 := r.Queries("rhost", "").Subrouter() // s3 := r.Queries("rhost", "").Subrouter()
s3.HandleFunc("/", ProxyFunc2) // s3.HandleFunc("/", ProxyFunc2)
// This is where all the main incoming traffic goes // This is where all the main incoming traffic goes
r.NotFoundHandler = http.HandlerFunc(Run) r.NotFoundHandler = http.HandlerFunc(Run)
http.Handle("/", r) http.Handle("/", r)
port := 8080 port := 8080
golog.Infoln("Router started, listening and serving on port", port) log.Infoln("Router started, listening and serving on port", port)
log.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%v", port), nil)) log.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%v", port), nil))
} }
@@ -133,10 +132,10 @@ func ProxyFunc2(w http.ResponseWriter, req *http.Request) {
} }
func ProxyFunc(w http.ResponseWriter, req *http.Request) { func ProxyFunc(w http.ResponseWriter, req *http.Request) {
golog.Infoln("HOST:", req.Host) log.Infoln("HOST:", req.Host)
host := strings.Split(req.Host, ":")[0] host := strings.Split(req.Host, ":")[0]
rhost := req.FormValue("rhost") rhost := req.FormValue("rhost")
golog.Infoln("rhost:", rhost) log.Infoln("rhost:", rhost)
if rhost != "" { if rhost != "" {
host = rhost host = rhost
} }
@@ -146,10 +145,10 @@ func ProxyFunc(w http.ResponseWriter, req *http.Request) {
// 2) This host has active workers so we do the proxy // 2) This host has active workers so we do the proxy
// 3) This host has no active workers so we queue one (or more) up and return a 503 or something with message that says "try again in a minute" // 3) This host has no active workers so we queue one (or more) up and return a 503 or something with message that says "try again in a minute"
// route := routingTable[host] // route := routingTable[host]
golog.Infoln("getting route for host:", host, "--") log.Infoln("getting route for host:", host, "--")
route, err := getRoute(host) route, err := getRoute(host)
golog.Infoln("route:", route) log.Infoln("route:", route)
golog.Infoln("err:", err) log.Infoln("err:", err)
if err != nil { if err != nil {
common.SendError(w, 400, fmt.Sprintln("Host not registered or error!", err)) common.SendError(w, 400, fmt.Sprintln("Host not registered or error!", err))
return return
@@ -165,13 +164,13 @@ func ProxyFunc(w http.ResponseWriter, req *http.Request) {
func serveEndpoint(w http.ResponseWriter, req *http.Request, route *Route) { func serveEndpoint(w http.ResponseWriter, req *http.Request, route *Route) {
dlen := len(route.Destinations) dlen := len(route.Destinations)
if dlen == 0 { if dlen == 0 {
golog.Infoln("No workers running, starting new task.") log.Infoln("No workers running, starting new task.")
startNewWorker(route) startNewWorker(route)
common.SendError(w, 500, fmt.Sprintln("No workers running, starting them up...")) common.SendError(w, 500, fmt.Sprintln("No workers running, starting them up..."))
return return
} }
if dlen < 3 { if dlen < 3 {
golog.Infoln("Less than three workers running, starting a new task.") log.Infoln("Less than three workers running, starting a new task.")
startNewWorker(route) startNewWorker(route)
} }
destIndex := rand.Intn(dlen) destIndex := rand.Intn(dlen)
@@ -181,21 +180,21 @@ func serveEndpoint(w http.ResponseWriter, req *http.Request, route *Route) {
destUrl, err := url.Parse(destUrlString2) destUrl, err := url.Parse(destUrlString2)
if err != nil { if err != nil {
removeDestination(route, destIndex, w) removeDestination(route, destIndex, w)
golog.Infoln("error!", err) log.Infoln("error!", err)
common.SendError(w, 500, fmt.Sprintln("Internal error occurred:", err)) common.SendError(w, 500, fmt.Sprintln("Internal error occurred:", err))
return return
} }
// todo: check destination runtime and remove it if it's expired so we don't send requests to an endpoint that is about to be killed // todo: check destination runtime and remove it if it's expired so we don't send requests to an endpoint that is about to be killed
golog.Infoln("proxying to", destUrl) log.Infoln("proxying to", destUrl)
proxy := NewSingleHostReverseProxy(destUrl) proxy := NewSingleHostReverseProxy(destUrl)
err = proxy.ServeHTTP(w, req) err = proxy.ServeHTTP(w, req)
if err != nil { if err != nil {
golog.Infoln("Error proxying!", err) log.Infoln("Error proxying!", err)
etype := reflect.TypeOf(err) etype := reflect.TypeOf(err)
golog.Infoln("err type:", etype) log.Infoln("err type:", etype)
// can't figure out how to compare types so comparing strings.... lame. // can't figure out how to compare types so comparing strings.... lame.
if true || strings.Contains(etype.String(), "net.OpError") { // == reflect.TypeOf(net.OpError{}) { // couldn't figure out a better way to do this if true || strings.Contains(etype.String(), "net.OpError") { // == reflect.TypeOf(net.OpError{}) { // couldn't figure out a better way to do this
golog.Infoln("It's a network error so we're going to remove destination.") log.Infoln("It's a network error so we're going to remove destination.")
removeDestination(route, destIndex, w) removeDestination(route, destIndex, w)
serveEndpoint(w, req, route) serveEndpoint(w, req, route)
return return
@@ -203,28 +202,28 @@ func serveEndpoint(w http.ResponseWriter, req *http.Request, route *Route) {
common.SendError(w, 500, fmt.Sprintln("Internal error occurred:", err)) common.SendError(w, 500, fmt.Sprintln("Internal error occurred:", err))
return return
} }
golog.Infoln("Served!") log.Infoln("Served!")
} }
func removeDestination(route *Route, destIndex int, w http.ResponseWriter) { func removeDestination(route *Route, destIndex int, w http.ResponseWriter) {
golog.Infoln("Removing destination", destIndex, "from route:", route) log.Infoln("Removing destination", destIndex, "from route:", route)
route.Destinations = append(route.Destinations[:destIndex], route.Destinations[destIndex+1:]...) route.Destinations = append(route.Destinations[:destIndex], route.Destinations[destIndex+1:]...)
err := putRoute(route) err := putRoute(route)
if err != nil { if err != nil {
golog.Infoln("Couldn't update routing table:", err) log.Infoln("Couldn't update routing table:", err)
common.SendError(w, 500, fmt.Sprintln("couldn't update routing table", err)) common.SendError(w, 500, fmt.Sprintln("couldn't update routing table", err))
return return
} }
golog.Infoln("New route after remove destination:", route) log.Infoln("New route after remove destination:", route)
// if len(route.Destinations) < 3 { // if len(route.Destinations) < 3 {
// golog.Infoln("After network error, there are less than three destinations, so starting a new one. ") // log.Infoln("After network error, there are less than three destinations, so starting a new one. ")
// // always want at least three running // // always want at least three running
// startNewWorker(route) // startNewWorker(route)
// } // }
} }
func startNewWorker(route *Route) error { func startNewWorker(route *Route) error {
golog.Infoln("Starting a new worker") log.Infoln("Starting a new worker")
// start new worker // start new worker
payload := map[string]interface{}{ payload := map[string]interface{}{
"token": config.Iron.SuperToken, "token": config.Iron.SuperToken,
@@ -238,7 +237,7 @@ func startNewWorker(route *Route) error {
workerapi.Settings.UseConfigMap(payload) workerapi.Settings.UseConfigMap(payload)
jsonPayload, err := json.Marshal(payload) jsonPayload, err := json.Marshal(payload)
if err != nil { if err != nil {
golog.Infoln("Couldn't marshal json!", err) log.Infoln("Couldn't marshal json!", err)
return err return err
} }
timeout := time.Second * time.Duration(1800+rand.Intn(600)) // a little random factor in here to spread out worker deaths timeout := time.Second * time.Duration(1800+rand.Intn(600)) // a little random factor in here to spread out worker deaths
@@ -250,9 +249,9 @@ func startNewWorker(route *Route) error {
tasks := make([]worker.Task, 1) tasks := make([]worker.Task, 1)
tasks[0] = task tasks[0] = task
taskIds, err := workerapi.TaskQueue(tasks...) taskIds, err := workerapi.TaskQueue(tasks...)
golog.Infoln("Tasks queued.", taskIds) log.Infoln("Tasks queued.", taskIds)
if err != nil { if err != nil {
golog.Infoln("Couldn't queue up worker!", err) log.Infoln("Couldn't queue up worker!", err)
return err return err
} }
return err return err
@@ -267,13 +266,13 @@ func (r *Register) ServeHTTP(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req) vars := mux.Vars(req)
projectId := vars["project_id"] projectId := vars["project_id"]
token := common.GetToken(req) token := common.GetToken(req)
golog.Infoln("project_id:", projectId, "token:", token.Token) log.Infoln("project_id:", projectId, "token:", token.Token)
route := Route{} route := Route{}
if !common.ReadJSON(w, req, &route) { if !common.ReadJSON(w, req, &route) {
return return
} }
golog.Infoln("body read into route:", route) log.Infoln("body read into route:", route)
route.ProjectId = projectId route.ProjectId = projectId
route.Token = token.Token route.Token = token.Token
@@ -287,11 +286,11 @@ func (r *Register) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// todo: do we need to close body? // todo: do we need to close body?
err = putRoute(&route) err = putRoute(&route)
if err != nil { if err != nil {
golog.Infoln("couldn't register host:", err) log.Infoln("couldn't register host:", err)
common.SendError(w, 400, fmt.Sprintln("Could not register host!", err)) common.SendError(w, 400, fmt.Sprintln("Could not register host!", err))
return return
} }
golog.Infoln("registered route:", route) log.Infoln("registered route:", route)
fmt.Fprintln(w, "Host registered successfully.") fmt.Fprintln(w, "Host registered successfully.")
} }
@@ -304,13 +303,13 @@ func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req) vars := mux.Vars(req)
projectId := vars["project_id"] projectId := vars["project_id"]
// token := common.GetToken(req) // token := common.GetToken(req)
golog.Infoln("project_id:", projectId) log.Infoln("project_id:", projectId)
app := App{} app := App{}
if !common.ReadJSON(w, req, &app) { if !common.ReadJSON(w, req, &app) {
return return
} }
golog.Infoln("body read into app:", app) log.Infoln("body read into app:", app)
app.ProjectId = projectId app.ProjectId = projectId
_, err := getApp(app.Name) _, err := getApp(app.Name)
@@ -319,9 +318,11 @@ func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
app.Routes = make(map[string]*Route3)
// create dns entry // create dns entry
// TODO: Add project id to this. eg: appname.projectid.iron.computer // TODO: Add project id to this. eg: appname.projectid.iron.computer
log15.Debug("Creating dns entry.") log.Debug("Creating dns entry.")
regOk := registerHost(w, req, &app) regOk := registerHost(w, req, &app)
if !regOk { if !regOk {
return return
@@ -330,11 +331,11 @@ func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// todo: do we need to close body? // todo: do we need to close body?
err = putApp(&app) err = putApp(&app)
if err != nil { if err != nil {
golog.Infoln("couldn't create app:", err) log.Infoln("couldn't create app:", err)
common.SendError(w, 400, fmt.Sprintln("Could not create app!", err)) common.SendError(w, 400, fmt.Sprintln("Could not create app!", err))
return return
} }
golog.Infoln("registered app:", app) log.Infoln("registered app:", app)
v := map[string]interface{}{"app": app} v := map[string]interface{}{"app": app}
common.SendSuccess(w, "App created successfully.", v) common.SendSuccess(w, "App created successfully.", v)
} }
@@ -344,13 +345,13 @@ func NewRoute(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req) vars := mux.Vars(req)
projectId := vars["project_id"] projectId := vars["project_id"]
appName := vars["app_name"] appName := vars["app_name"]
golog.Infoln("project_id:", projectId, "app_name", appName) log.Infoln("project_id:", projectId, "app_name", appName)
route := Route3{} route := &Route3{}
if !common.ReadJSON(w, req, &route) { if !common.ReadJSON(w, req, &route) {
return return
} }
golog.Infoln("body read into route:", route) log.Infoln("body read into route:", route)
// TODO: validate route // TODO: validate route
@@ -360,14 +361,19 @@ func NewRoute(w http.ResponseWriter, req *http.Request) {
return return
} }
app.Routes = append(app.Routes, route) if route.Type == "" {
route.Type = "run"
}
// app.Routes = append(app.Routes, route)
app.Routes[route.Path] = route
err = putApp(app) err = putApp(app)
if err != nil { if err != nil {
golog.Errorln("Couldn't create route!:", err) log.Errorln("Couldn't create route!:", err)
common.SendError(w, 400, fmt.Sprintln("Could not create route!", err)) common.SendError(w, 400, fmt.Sprintln("Could not create route!", err))
return return
} }
golog.Infoln("Route created:", route) log.Infoln("Route created:", route)
fmt.Fprintln(w, "Route created successfully.") fmt.Fprintln(w, "Route created successfully.")
} }
@@ -376,13 +382,13 @@ type WorkerHandler struct {
// When a worker starts up, it calls this // When a worker starts up, it calls this
func (wh *WorkerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { func (wh *WorkerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
golog.Infoln("AddWorker called!") log.Infoln("AddWorker called!")
// get project id and token // get project id and token
projectId := req.FormValue("project_id") projectId := req.FormValue("project_id")
token := req.FormValue("token") token := req.FormValue("token")
// codeName := req.FormValue("code_name") // codeName := req.FormValue("code_name")
golog.Infoln("project_id:", projectId, "token:", token) log.Infoln("project_id:", projectId, "token:", token)
// check header for what operation to perform // check header for what operation to perform
routerHeader := req.Header.Get("Iron-Router") routerHeader := req.Header.Get("Iron-Router")
@@ -394,28 +400,28 @@ func (wh *WorkerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
// todo: do we need to close body? // todo: do we need to close body?
golog.Infoln("Incoming body from worker:", r2) log.Infoln("Incoming body from worker:", r2)
route, err := getRoute(r2.Host) route, err := getRoute(r2.Host)
if err != nil { if err != nil {
common.SendError(w, 400, fmt.Sprintln("This host is not registered!", err)) common.SendError(w, 400, fmt.Sprintln("This host is not registered!", err))
return return
} }
golog.Infoln("ROUTE:", route) log.Infoln("ROUTE:", route)
route.Destinations = append(route.Destinations, r2.Dest) route.Destinations = append(route.Destinations, r2.Dest)
golog.Infoln("ROUTE new:", route) log.Infoln("ROUTE new:", route)
err = putRoute(route) err = putRoute(route)
if err != nil { if err != nil {
golog.Infoln("couldn't register host:", err) log.Infoln("couldn't register host:", err)
common.SendError(w, 400, fmt.Sprintln("Could not register host!", err)) common.SendError(w, 400, fmt.Sprintln("Could not register host!", err))
return return
} }
fmt.Fprintln(w, "Worker added") fmt.Fprintln(w, "Worker added")
golog.Infoln("Worked added.") log.Infoln("Worked added.")
} }
} }
func getRoute(host string) (*Route, error) { func getRoute(host string) (*Route, error) {
golog.Infoln("getRoute for host:", host) log.Infoln("getRoute for host:", host)
rx, err := icache.Get(host) rx, err := icache.Get(host)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -441,7 +447,7 @@ func putRoute(route *Route) error {
} }
func getApp(name string) (*App, error) { func getApp(name string) (*App, error) {
golog.Infoln("getapp:", name) log.Infoln("getapp:", name)
rx, err := icache.Get(name) rx, err := icache.Get(name)
if err != nil { if err != nil {
return nil, err return nil, err

0
run.sh Executable file
View File

48
runner.go
View File

@@ -4,8 +4,6 @@ import (
"bufio" "bufio"
"bytes" "bytes"
"fmt" "fmt"
"github.com/iron-io/go/common"
"github.com/iron-io/golog"
"io" "io"
"io/ioutil" "io/ioutil"
"math/rand" "math/rand"
@@ -13,7 +11,9 @@ import (
"os" "os"
"os/exec" "os/exec"
"strings" "strings"
log "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus"
"github.com/iron-io/go/common"
) )
type RunningApp struct { type RunningApp struct {
@@ -31,15 +31,15 @@ func init() {
func Run(w http.ResponseWriter, req *http.Request) { func Run(w http.ResponseWriter, req *http.Request) {
fmt.Println("RUN!!!!") fmt.Println("RUN!!!!")
golog.Infoln("HOST:", req.Host) log.Infoln("HOST:", req.Host)
appName := req.FormValue("app") appName := req.FormValue("app")
golog.Infoln("app_name", appName, "path:", req.URL.Path) log.Infoln("app_name", appName, "path:", req.URL.Path)
if appName != "" { if appName != "" {
// passed in the name // passed in the name
} else { } else {
host := strings.Split(req.Host, ":")[0] host := strings.Split(req.Host, ":")[0]
appName = strings.Split(host, ".")[0] appName = strings.Split(host, ".")[0]
golog.Infoln("app_name from host", appName) log.Infoln("app_name from host", appName)
} }
app, err := getApp(appName) app, err := getApp(appName)
@@ -47,7 +47,7 @@ func Run(w http.ResponseWriter, req *http.Request) {
common.SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err)) common.SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err))
return return
} }
golog.Infoln("app", app) log.Infoln("app", app)
// find route // find route
for _, el := range app.Routes { for _, el := range app.Routes {
@@ -60,11 +60,11 @@ func Run(w http.ResponseWriter, req *http.Request) {
return return
} }
if el.Type == "app" { if el.Type == "app" {
DockerHost(&el, w) DockerHost(el, w)
return return
} else { // "run" } else { // "run"
// TODO: timeout 59 seconds // TODO: timeout 59 seconds
DockerRun(el.Image, w) DockerRun(el.Image, w, req)
return return
} }
} }
@@ -73,8 +73,18 @@ func Run(w http.ResponseWriter, req *http.Request) {
} }
// TODO: use Docker utils from docker-job for this and a few others in here // TODO: use Docker utils from docker-job for this and a few others in here
func DockerRun(image string, w http.ResponseWriter) { func DockerRun(image string, w http.ResponseWriter, req *http.Request) {
cmd := exec.Command("docker", "run", "--rm", "-i", image)
payload, err := ioutil.ReadAll(req.Body)
if err != nil {
log.WithError(err).Errorln("Error reading request body")
return
}
log.WithField("payload", "---"+string(payload)+"---").Infoln("incoming request")
log.WithField("image", image).Infoln("About to run using this image")
// TODO: swap all this out with Titan's running via API
cmd := exec.Command("docker", "run", "--rm", "-i", "-e", fmt.Sprintf("PAYLOAD=%v", string(payload)), image)
stdout, err := cmd.StdoutPipe() stdout, err := cmd.StdoutPipe()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
@@ -94,14 +104,18 @@ func DockerRun(image string, w http.ResponseWriter) {
log.Printf("Waiting for command to finish...") log.Printf("Waiting for command to finish...")
if err = cmd.Wait(); err != nil { if err = cmd.Wait(); err != nil {
// this probably shouldn't be fatal? test with iron/error image // job failed
log.Fatal(err) log.Infoln("job finished with err:", err)
log.WithFields(log.Fields{"metric": "run.errors", "value": 1, "type": "count"}).Infoln("failed run")
// TODO: wrap error in json "error": buff
} else {
log.Infoln("Docker ran successfully:", b.String())
// print
log.WithFields(log.Fields{"metric": "run.success", "value": 1, "type": "count"}).Infoln("successful run")
} }
log.WithFields(log.Fields{"metric": "ran", "value": 1, "type": "count"}).Infoln("") log.WithFields(log.Fields{"metric": "run", "value": 1, "type": "count"}).Infoln("job ran")
log.Printf("Command finished with error: %v", err)
buff.Flush() buff.Flush()
golog.Infoln("Docker ran successfully:", b.String()) fmt.Fprintln(w, string(bytes.Trim(b.Bytes(), "\x00")))
fmt.Fprintln(w, b.String())
} }
func DockerHost(el *Route3, w http.ResponseWriter) { func DockerHost(el *Route3, w http.ResponseWriter) {
@@ -120,8 +134,8 @@ func DockerHost(el *Route3, w http.ResponseWriter) {
// TODO: Need to catch interrupt and stop all containers that are started, see devo/dj for how to do this // TODO: Need to catch interrupt and stop all containers that are started, see devo/dj for how to do this
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {
log.Fatal(err) log.Fatal(err)
}
// TODO: What if the app fails to start? Don't want to keep starting the container // TODO: What if the app fails to start? Don't want to keep starting the container
}
} else { } else {
// TODO: check if it's still running? // TODO: check if it's still running?
// TODO: if ra.terminated, then start new container? // TODO: if ra.terminated, then start new container?

5
ssh.sh Executable file
View File

@@ -0,0 +1,5 @@
# ssh into the box on staging that the prototype is running on
set -ex
ssh -i gateway.pem rancher@ec2-52-205-252-82.compute-1.amazonaws.com