Bunch of cleanup.

This commit is contained in:
Travis Reeder
2016-07-14 09:18:43 -07:00
parent ae07ed108f
commit c73dd4dd42
13 changed files with 357 additions and 216 deletions

1
.gitignore vendored
View File

@@ -11,5 +11,6 @@ app.zip
vendor/
/microgateway
/gateway
/functions
private.sh

View File

@@ -1,4 +1,35 @@
## Building/Testing
Build:
```sh
# one time:
glide install
# then every time
./build.sh
```
Test it, the iron token and project id are for cache.
```sh
docker run -e "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" -e "CLOUDFLARE_EMAIL=treeder@gmail.com" -e "CLOUDFLARE_API_KEY=X" --rm -it --privileged -p 8080:8080 iron/functions
```
Push it:
```sh
docker push iron/functions
```
Get it on a server and point router.iron.computer (on cloudflare) to the machine.
After deploying, run it with:
```sh
docker run -e --name functions -it --privileged -d -p 80:80 "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" -e PORT=80 iron/functions
```
## FOR INFLUX AND ANALYTICS

View File

@@ -1,7 +1,7 @@
FROM iron/dind
RUN mkdir /app
ADD gateway /app/
ADD functions /app/
WORKDIR /app
CMD ["./gateway"]
CMD ["./functions"]

View File

@@ -30,77 +30,13 @@ Test out the route:
Surf to: http://localhost:8080/hello?app=myapp
Now try mapping an app endpoint:
```sh
curl -H "Content-Type: application/json" -X POST -d '{"path":"/sinatra","image":"treeder/hello-sinatra", "type":"app", "cpath":"/"}' http://localhost:8080/api/v1/apps/myapp/routes
```
And test it out:
```sh
curl -i -X GET http://localhost:8080/sinatra?app=myapp
```
And another:
```sh
curl -H "Content-Type: application/json" -X POST -d '{"path":"/sinatra/ping","image":"treeder/hello-sinatra", "type":"app", "cpath":"/ping"}' http://localhost:8080/api/v1/apps/myapp/routes
```
And test it out:
```sh
curl -i -X GET http://localhost:8080/sinatra?app=myapp
```
You'all also get a custom URL like this when in production.
```
appname.iron.computer
myapp.ironfunctions.com/myroute
```
## Updating Your Images
Tag your images with a version, eg `treeder/guppy:0.0.5` then use that including the tag and update
the route.
## Building/Testing
Build:
```sh
# one time:
glide install
# then every time
./build.sh
```
Test it, the iron token and project id are for cache.
```sh
docker run -e "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" -e "CLOUDFLARE_EMAIL=treeder@gmail.com" -e "CLOUDFLARE_API_KEY=X" --rm -it --privileged -p 8080:8080 iron/gateway
```
Push it:
```sh
docker push iron/gateway
```
Get it on a server and point router.iron.computer (on cloudflare) to the machine.
After deploying, running it with:
```sh
docker run -e "IRON_TOKEN=GP8cqlKSrcpmqeR8x9WKD4qSAss" -e "IRON_PROJECT_ID=4fd2729368a0197d1102056b" --name irongateway -it --privileged --net=host -p 8080:8080 -d --name irongateway iron/gateway
```
## TODOS
* [ ] Check if image exists when registering the endpoint, not at run time
* [ ] Put stats into influxdb or something to show to user (requests, errors). Or maybe try Keen's new Native Analytics?? Could be faster and easier route: https://keen.io/native-analytics/
* [ ] Store recent logs. Get logs from STDERR, STDOUT is the response.
* [ ] Allow env vars for config on the app and routes (routes override apps).
* [ ] Provide a base url for each app, eg: appname.userid.iron.computer
* [ ] Allow setting content-type on a route, then use that when responding

View File

@@ -1,4 +1,4 @@
set -ex
docker run --rm -v "$PWD":/go/src/github.com/iron-io/microgateway -w /go/src/github.com/iron-io/microgateway iron/go:dev sh -c 'go build -o gateway'
docker build -t iron/gateway:latest .
docker run --rm -v "$PWD":/go/src/github.com/iron-io/functions -w /go/src/github.com/iron-io/functions iron/go:dev sh -c 'go build -o functions'
docker build -t iron/functions:latest .

View File

@@ -1,96 +0,0 @@
package main
import (
"github.com/iron-io/common"
"github.com/iron-io/iron_go/cache"
"encoding/json"
"log"
)
var config struct {
Iron struct {
Token string `json:"token"`
ProjectId string `json:"project_id"`
SuperToken string `json:"super_token"`
Host string `json:"host"`
} `json:"iron"`
MongoAuth common.MongoConfig `json:"mongo_auth"`
Logging struct {
To string `json:"to"`
Level string `json:"level"`
Prefix string `json:"prefix"`
}
}
type Route struct {
// TODO: Change destinations to a simple cache so it can expire entries after 55 minutes (the one we use in common?)
Host string `json:"host"`
Destinations []string `json:"destinations"`
ProjectId string `json:"project_id"`
Token string `json:"token"` // store this so we can queue up new workers on demand
CodeName string `json:"code_name"`
}
var icache = cache.New("routing-table")
func main() {
log.Println("STARTING")
var configFile string
var env string
env = "development"
configFile = "config_" + env + ".json"
common.LoadConfig("iron_mq", configFile, &config)
common.SetLogLevel(config.Logging.Level)
// common.SetLogLocation(config.Logging.To, config.Logging.Prefix)
icache.Settings.UseConfigMap(map[string]interface{}{
"token": config.Iron.Token,
"project_id": config.Iron.ProjectId,
"host": config.Iron.Host,
})
log.Println("icache settings:", icache.Settings)
host := "routertest5.irondns.info"
// r2 := Route{
// Host: host,
// }
// err2 := putRoute(&r2)
// log.Println("err2:", err2)
log.Println("CHECKING ROUTE")
route, err := getRoute(host)
log.Println("route:", route)
log.Println("err:", err)
// err = icache.Delete(host)
// log.Println("delete err:", err)
}
func getRoute(host string) (*Route, error) {
rx, err := icache.Get(host)
if err != nil {
return nil, err
}
rx2 := []byte(rx.(string))
route := Route{}
err = json.Unmarshal(rx2, &route)
if err != nil {
return nil, err
}
return &route, err
}
func putRoute(route *Route) error {
item := cache.Item{}
v, err := json.Marshal(route)
if err != nil {
return err
}
item.Value = string(v)
err = icache.Put(route.Host, &item)
return err
}

7
dns.go
View File

@@ -8,7 +8,6 @@ import (
"strings"
log "github.com/Sirupsen/logrus"
"github.com/iron-io/go/common"
)
type CloudFlareResult struct {
@@ -54,7 +53,7 @@ func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool {
resp, err := client.Do(req)
if err != nil {
log.Error("Could not register dns entry.", "err", err)
common.SendError(w, 500, fmt.Sprint("Could not register dns entry.", err))
SendError(w, 500, fmt.Sprint("Could not register dns entry.", err))
return false
}
defer resp.Body.Close()
@@ -62,14 +61,14 @@ func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool {
body, err := ioutil.ReadAll(resp.Body)
if resp.StatusCode != 200 {
log.Error("Could not register dns entry 2.", "code", resp.StatusCode, "body", string(body))
common.SendError(w, 500, fmt.Sprint("Could not register dns entry 2. ", resp.StatusCode))
SendError(w, 500, fmt.Sprint("Could not register dns entry 2. ", resp.StatusCode))
return false
}
cfResult := CloudFlareResponse{}
err = json.Unmarshal(body, &cfResult)
if err != nil {
log.Error("Could not parse DNS response.", "err", err, "code", resp.StatusCode, "body", string(body))
common.SendError(w, 500, fmt.Sprint("Could not parse DNS response. ", resp.StatusCode))
SendError(w, 500, fmt.Sprint("Could not parse DNS response. ", resp.StatusCode))
return false
}
fmt.Println("cfresult:", cfResult)

17
glide.lock generated
View File

@@ -1,20 +1,20 @@
hash: a104a522dc2ba982d25101330bbd5d44778565128f0cb09f34061c167921046c
updated: 2016-06-17T14:27:42.156301621-07:00
updated: 2016-07-14T08:40:37.798125603-07:00
imports:
- name: github.com/amir/raidman
version: 91c20f3f475cab75bb40ad7951d9bbdde357ade7
subpackages:
- proto
- name: github.com/BurntSushi/toml
version: f0aeabca5a127c4078abb8c8d64298b147264b55
version: bec2dacf4b590d26237cfebff4471e21ce543494
- name: github.com/cactus/go-statsd-client
version: 91c326c3f7bd20f0226d3d1c289dd9f8ce28d33d
subpackages:
- statsd
- name: github.com/dgrijalva/jwt-go
version: f0777076321ab64f6efc15a82d9d23b98539b943
version: 01aeca54ebda6e0fbfafd0a524d234159c05ec20
- name: github.com/golang/protobuf
version: 0c1f6d65b5a189c2250d10e71a5506f06f9fa0a0
version: 874264fbbb43f4d91e999fecb4b40143ed611400
subpackages:
- proto
- name: github.com/gorilla/context
@@ -29,14 +29,13 @@ imports:
- semaphore
- serverutil
- name: github.com/iron-io/go
version: 6b082f50f20cb0e0146bdf0ffbc6c07216eeea77
version: 1ed3de151aa27db91d6df5dc8c8ce164c833b57c
subpackages:
- common
- common/httpshutdown
- common/msgpack
- common/semaphore
- common/stats
- common/serverutil
- name: github.com/iron-io/golog
version: 5b80d97af5a2a5d386e7609efb82192ae99a7c67
- name: github.com/iron-io/iron_go
@@ -51,17 +50,17 @@ imports:
- name: github.com/mattn/go-isatty
version: 56b76bdf51f7708750eac80fa38b952bb9f32639
- name: github.com/Sirupsen/logrus
version: f3cfb454f4c209e6668c95216c4744b8fddb2356
version: 32055c351ea8b00b96d70f28db48d9840feaf0ec
- name: github.com/vmihailenco/bufio
version: 24e7e48f60fc2d9e99e43c07485d9fff42051e66
- name: github.com/vrischmann/envconfig
version: 9e6e1c4d3b73427d03118518603bb904d9c55236
- name: golang.org/x/net
version: d7bf3545bb0dacf009c535b3d3fbf53ac0a339ab
version: a728288923b47049b2ce791836767ffbe964a5bd
subpackages:
- proxy
- name: golang.org/x/sys
version: 62bee037599929a6e9146f29d10dd5208c43507d
version: b518c298ac9dc94b6ac0757394f50d10c5dfa25a
subpackages:
- unix
- name: gopkg.in/inconshreveable/log15.v2

View File

@@ -2,12 +2,8 @@ package: github.com/iron-io/microgateway
import:
- package: github.com/gorilla/mux
- package: github.com/iron-io/common
- package: github.com/iron-io/go
subpackages:
- common
- package: github.com/iron-io/golog
- package: github.com/iron-io/iron_go
subpackages:
- cache
- worker
- package: gopkg.in/inconshreveable/log15.v2

View File

@@ -1,21 +0,0 @@
package main
import (
"fmt"
"github.com/gorilla/mux"
"log"
"net/http"
)
func main() {
r := mux.NewRouter()
r.HandleFunc("/", Hello)
http.Handle("/", r)
log.Fatal(http.ListenAndServe(":8081", nil))
}
func Hello(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(w, "Hello world!")
}

266
helpers.go Normal file
View File

@@ -0,0 +1,266 @@
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"reflect"
"runtime/debug"
"strconv"
"strings"
"sync/atomic"
"time"
"gopkg.in/inconshreveable/log15.v2"
"github.com/Sirupsen/logrus"
)
func NotFound(w http.ResponseWriter, r *http.Request) {
SendError(w, http.StatusNotFound, "Not found")
}
func EndpointNotFound(w http.ResponseWriter, r *http.Request) {
SendError(w, http.StatusNotFound, "Endpoint not found")
}
func InternalError(w http.ResponseWriter, err error) {
logrus.Error("internal server error response", "err", err, "stack", string(debug.Stack()))
SendError(w, http.StatusInternalServerError, "internal error")
}
func InternalErrorDetailed(w http.ResponseWriter, r *http.Request, err error) {
logrus.Error("internal server error response", "err", err, "endpoint", r.URL.String(), "token", GetTokenString(r), "stack", string(debug.Stack()))
SendError(w, http.StatusInternalServerError, "internal error")
}
type HTTPError interface {
error
StatusCode() int
}
type response struct {
Msg string `json:"msg"`
}
func SendError(w http.ResponseWriter, code int, msg string) {
logrus.Debug("HTTP error", "status_code", code, "msg", msg)
resp := response{Msg: msg}
RespondCode(w, nil, code, &resp)
}
func SendSuccess(w http.ResponseWriter, msg string, params map[string]interface{}) {
var v interface{}
if params == nil {
v = &response{Msg: msg}
} else {
v = params
}
RespondCode(w, nil, http.StatusOK, v)
}
func Respond(w http.ResponseWriter, r *http.Request, v interface{}) {
RespondCode(w, r, http.StatusOK, v)
}
func RespondCode(w http.ResponseWriter, r *http.Request, code int, v interface{}) {
bytes, err := json.Marshal(v)
if err != nil {
logrus.Error("error marshalling HTTP response", "value", v, "type", reflect.TypeOf(v), "err", err)
InternalError(w, err)
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Length", strconv.Itoa(len(bytes)))
w.WriteHeader(code)
if _, err := w.Write(bytes); err != nil {
// older callers don't pass a request
if r != nil {
logrus.Error("unable to write HTTP response", "err", err)
} else {
logrus.Error("unable to write HTTP response", "err", err)
}
}
}
// GetTokenString returns the token string from either the Authorization header or
// the oauth query parameter.
func GetTokenString(r *http.Request) string {
tok, _ := GetTokenStringType(r)
return tok
}
func GetTokenStringType(r *http.Request) (tok string, jwt bool) {
tokenStr := r.URL.Query().Get("oauth")
if tokenStr != "" {
return tokenStr, false
}
tokenStr = r.URL.Query().Get("jwt")
jwt = tokenStr != ""
if tokenStr == "" {
authHeader := r.Header.Get("Authorization")
authFields := strings.Fields(authHeader)
if len(authFields) == 2 && (authFields[0] == "OAuth" || authFields[0] == "JWT") {
jwt = authFields[0] == "JWT"
tokenStr = authFields[1]
}
}
return tokenStr, jwt
}
func ReadJSONSize(w http.ResponseWriter, r *http.Request, v interface{}, n int64) (success bool) {
contentType := r.Header.Get("Content-Type")
i := strings.IndexByte(contentType, ';')
if i < 0 {
i = len(contentType)
}
if strings.TrimRight(contentType[:i], " ") != "application/json" {
SendError(w, http.StatusBadRequest, "Bad Content-Type.")
return false
}
if i < len(contentType) {
param := strings.Trim(contentType[i+1:], " ")
split := strings.SplitN(param, "=", 2)
if len(split) != 2 || strings.Trim(split[0], " ") != "charset" {
SendError(w, http.StatusBadRequest, "Invalid Content-Type parameter.")
return false
}
value := strings.Trim(split[1], " ")
if len(value) > 2 && value[0] == '"' && value[len(value)-1] == '"' {
// quoted string
value = value[1 : len(value)-1]
}
if !strings.EqualFold(value, "utf-8") {
SendError(w, http.StatusBadRequest, "Unsupported charset. JSON is always UTF-8 encoded.")
return false
}
}
if r.ContentLength > n {
SendError(w, http.StatusBadRequest, fmt.Sprint("Content-Length greater than", n, "bytes"))
return false
}
err := json.NewDecoder(&LimitedReader{r.Body, n}).Decode(v)
if err != nil {
jsonError(w, err)
return false
}
return true
}
func ReadJSON(w http.ResponseWriter, r *http.Request, v interface{}) bool {
return ReadJSONSize(w, r, v, 100*0xffff)
}
// Same as io.LimitedReader, but returns limitReached so we can
// distinguish between the limit being reached and actual EOF.
type LimitedReader struct {
R io.Reader
N int64
}
func (l *LimitedReader) Read(p []byte) (n int, err error) {
if l.N <= 0 {
return 0, LimitReached(l.N)
}
if int64(len(p)) > l.N {
p = p[:l.N]
}
n, err = l.R.Read(p)
l.N -= int64(n)
return
}
// LimitedWriter writes until n bytes are written, then writes
// an overage line and skips any further writes.
type LimitedWriter struct {
W io.Writer
N int64
}
func (l *LimitedWriter) Write(p []byte) (n int, err error) {
var overrage = []byte("maximum log file size exceeded")
// we expect there may be concurrent writers, so to be safe..
left := atomic.LoadInt64(&l.N)
if left <= 0 {
return 0, io.EOF // TODO EOF? really? does it matter?
}
n, err = l.W.Write(p)
left = atomic.AddInt64(&l.N, -int64(n))
if left <= 0 {
l.W.Write(overrage)
}
return n, err
}
type JSONError string
func (e JSONError) Error() string { return string(e) }
func jsonType(t reflect.Type) string {
switch t.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16,
reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return "integer"
case reflect.Float32, reflect.Float64:
return "number"
case reflect.Map, reflect.Struct:
return "object"
case reflect.Slice, reflect.Array:
return "array"
case reflect.Ptr:
return jsonType(t.Elem())
}
// bool, string, other cases not covered
return t.String()
}
func jsonError(w http.ResponseWriter, err error) {
var msg string
switch err := err.(type) {
case *json.InvalidUTF8Error:
msg = "Invalid UTF-8 in JSON: " + err.S
case *json.InvalidUnmarshalError, *json.UnmarshalFieldError,
*json.UnsupportedTypeError:
// should never happen
InternalError(w, err)
return
case *json.SyntaxError:
msg = fmt.Sprintf("In JSON, %v at position %v.", err, err.Offset)
case *json.UnmarshalTypeError:
msg = fmt.Sprintf("In JSON, cannot use %v as %v", err.Value, jsonType(err.Type))
case *time.ParseError:
msg = "Time strings must be in RFC 3339 format."
case LimitReached:
msg = err.Error()
case JSONError:
msg = string(err)
default:
if err != io.EOF {
log15.Error("unhandled json.Unmarshal error", "type", reflect.TypeOf(err), "err", err)
}
msg = "Failed to decode JSON."
}
SendError(w, http.StatusBadRequest, msg)
}
type sizer interface {
Size() int64
}
type LimitReached int64
func (e LimitReached) Error() string {
return fmt.Sprint("Request body greater than", int64(e), "bytes")
}

30
release.sh Executable file
View File

@@ -0,0 +1,30 @@
#!/bin/bash
set -ex
user="iron"
service="functions"
version_file="router.go"
tag="latest"
./build.sh
if [ -z $(grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" $version_file) ]; then
echo "did not find semantic version in $version_file"
exit 1
fi
perl -i -pe 's/\d+\.\d+\.\K(\d+)/$1+1/e' $version_file
version=$(grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" $version_file)
echo "Version: $version"
git add -u
git commit -m "$service: $version release"
git tag -a "$version" -m "version $version"
git push
git push --tags
# Finally tag and push docker images
docker tag $user/$service:$tag $user/$service:$version
docker push $user/$service:$version
docker push $user/$service:$tag

View File

@@ -44,7 +44,7 @@ var config struct {
}
}
var version = "0.0.19"
const Version = "0.0.19"
//var routingTable = map[string]*Route{}
var icache = cache.New("routing-table")
@@ -72,7 +72,7 @@ func main() {
configFile = "config_" + env + ".json"
}
common.LoadConfigFile(configFile, &config)
// common.LoadConfigFile(configFile, &config)
// common.SetLogging(common.LoggingConfig{To: config.Logging.To, Level: config.Logging.Level, Prefix: config.Logging.Prefix})
// TODO: validate inputs, iron tokens, cloudflare stuff, etc
@@ -80,7 +80,7 @@ func main() {
config.CloudFlare.AuthKey = os.Getenv("CLOUDFLARE_API_KEY")
log.Println("config:", config)
log.Infoln("Starting up router version", version)
log.Infoln("Starting up router version", Version)
r := mux.NewRouter()
@@ -92,7 +92,7 @@ func main() {
s.Handle("/v1/apps", &NewApp{})
s.HandleFunc("/v1/apps/{app_name}/routes", NewRoute)
s.HandleFunc("/ping", Ping)
s.HandleFunc("/version", Version)
s.HandleFunc("/version", VersionHandler)
// s.Handle("/addworker", &WorkerHandler{})
s.HandleFunc("/", Ping)
@@ -128,7 +128,7 @@ func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) {
log.Infoln("project_id:", projectId)
app := App{}
if !common.ReadJSON(w, req, &app) {
if !ReadJSON(w, req, &app) {
return
}
log.Infoln("body read into app:", app)
@@ -136,7 +136,7 @@ func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) {
_, err := getApp(app.Name)
if err == nil {
common.SendError(w, 400, fmt.Sprintln("An app with this name already exists.", err))
SendError(w, 400, fmt.Sprintln("An app with this name already exists.", err))
return
}
@@ -154,12 +154,12 @@ func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) {
err = putApp(&app)
if err != nil {
log.Infoln("couldn't create app:", err)
common.SendError(w, 400, fmt.Sprintln("Could not create app!", err))
SendError(w, 400, fmt.Sprintln("Could not create app!", err))
return
}
log.Infoln("registered app:", app)
v := map[string]interface{}{"app": app}
common.SendSuccess(w, "App created successfully.", v)
SendSuccess(w, "App created successfully.", v)
}
func NewRoute(w http.ResponseWriter, req *http.Request) {
@@ -170,7 +170,7 @@ func NewRoute(w http.ResponseWriter, req *http.Request) {
log.Infoln("project_id:", projectId, "app_name", appName)
route := &Route3{}
if !common.ReadJSON(w, req, &route) {
if !ReadJSON(w, req, &route) {
return
}
log.Infoln("body read into route:", route)
@@ -179,7 +179,7 @@ func NewRoute(w http.ResponseWriter, req *http.Request) {
app, err := getApp(appName)
if err != nil {
common.SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err))
SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err))
return
}
@@ -192,7 +192,7 @@ func NewRoute(w http.ResponseWriter, req *http.Request) {
err = putApp(app)
if err != nil {
log.Errorln("Couldn't create route!:", err)
common.SendError(w, 400, fmt.Sprintln("Could not create route!", err))
SendError(w, 400, fmt.Sprintln("Could not create route!", err))
return
}
log.Infoln("Route created:", route)
@@ -255,6 +255,6 @@ func Ping(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(w, "pong")
}
func Version(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(w, version)
func VersionHandler(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(w, Version)
}