diff --git a/.gitignore b/.gitignore index 0881c33b4..eafc00425 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ vendor/ /microgateway /gateway /functions +bolt.db private.sh .env diff --git a/api/api.go b/api/api.go deleted file mode 100644 index 8c737b76d..000000000 --- a/api/api.go +++ /dev/null @@ -1,205 +0,0 @@ -package api - -import ( - "encoding/json" - "fmt" - "net/http" - - log "github.com/Sirupsen/logrus" - "github.com/gorilla/mux" - "github.com/iron-io/iron_go/cache" -) - -var icache = cache.New("routing-table") -var config *Config - -func New(conf *Config) *Api { - config = conf - api := &Api{} - return api -} - -type Api struct { -} - -func (api *Api) Start() { - - log.Infoln("Starting up router version", Version) - - r := mux.NewRouter() - - // dev: - s := r.PathPrefix("/api").Subrouter() - // production: - // s := r.Host("router.irondns.info").Subrouter() - // s.Handle("/1/projects/{project_id}/register", &Register{}) - s.Handle("/v1/apps", &NewApp{}) - s.HandleFunc("/v1/apps/{app_name}/routes", NewRoute) - s.HandleFunc("/ping", Ping) - s.HandleFunc("/version", VersionHandler) - // s.Handle("/addworker", &WorkerHandler{}) - s.HandleFunc("/", Ping) - - r.HandleFunc("/elb-ping-router", Ping) // for ELB health check - - // for testing app responses, pass in app name, can use localhost - s4 := r.Queries("app", "").Subrouter() - // s4.HandleFunc("/appsr", Ping) - s4.HandleFunc("/{rest:.*}", Run) - s4.NotFoundHandler = http.HandlerFunc(Run) - - // s3 := r.Queries("rhost", "").Subrouter() - // s3.HandleFunc("/", ProxyFunc2) - - // This is where all the main incoming traffic goes - r.NotFoundHandler = http.HandlerFunc(Run) - - http.Handle("/", r) - port := 8080 - log.Infoln("Router started, listening and serving on port", port) - log.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%v", port), nil)) -} - -type NewApp struct{} - -// This registers a new host -func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) { - log.Println("NewApp called!") - - vars := mux.Vars(req) - projectId := vars["project_id"] - // token := common.GetToken(req) - log.Infoln("project_id:", projectId) - - app := App{} - if !ReadJSON(w, req, &app) { - return - } - log.Infoln("body read into app:", app) - app.ProjectId = projectId - - _, err := getApp(app.Name) - if err == nil { - SendError(w, 400, fmt.Sprintln("An app with this name already exists.", err)) - return - } - - app.Routes = make(map[string]*Route3) - - // create dns entry - // TODO: Add project id to this. eg: appname.projectid.iron.computer - log.Debug("Creating dns entry.") - regOk := registerHost(w, req, &app) - if !regOk { - return - } - - // todo: do we need to close body? - err = putApp(&app) - if err != nil { - log.Infoln("couldn't create app:", err) - SendError(w, 400, fmt.Sprintln("Could not create app!", err)) - return - } - log.Infoln("registered app:", app) - v := map[string]interface{}{"app": app, "msg": "App created successfully."} - SendSuccess(w, "App created successfully.", v) -} - -func NewRoute(w http.ResponseWriter, req *http.Request) { - fmt.Println("NewRoute") - vars := mux.Vars(req) - projectId := vars["project_id"] - appName := vars["app_name"] - log.Infoln("project_id: ", projectId, "app: ", appName) - - route := &Route3{} - if !ReadJSON(w, req, &route) { - return - } - log.Infoln("body read into route:", route) - - // TODO: validate route - - app, err := getApp(appName) - if err != nil { - SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err)) - return - } - - if route.Type == "" { - route.Type = "run" - } - - // app.Routes = append(app.Routes, route) - app.Routes[route.Path] = route - err = putApp(app) - if err != nil { - log.Errorln("Couldn't create route!:", err) - SendError(w, 400, fmt.Sprintln("Could not create route!", err)) - return - } - log.Infoln("Route created:", route) - v := map[string]interface{}{"url": fmt.Sprintf("http://%v%v", app.Dns, route.Path), "msg": "Route created successfully."} - SendSuccess(w, "Route created successfully.", v) -} - -func getRoute(host string) (*Route, error) { - log.Infoln("getRoute for host:", host) - rx, err := icache.Get(host) - if err != nil { - return nil, err - } - rx2 := []byte(rx.(string)) - route := Route{} - err = json.Unmarshal(rx2, &route) - if err != nil { - return nil, err - } - return &route, err -} - -func putRoute(route *Route) error { - item := cache.Item{} - v, err := json.Marshal(route) - if err != nil { - return err - } - item.Value = string(v) - err = icache.Put(route.Host, &item) - return err -} - -func getApp(name string) (*App, error) { - log.Infoln("getapp:", name) - rx, err := icache.Get(name) - if err != nil { - return nil, err - } - rx2 := []byte(rx.(string)) - app := App{} - err = json.Unmarshal(rx2, &app) - if err != nil { - return nil, err - } - return &app, err -} - -func putApp(app *App) error { - item := cache.Item{} - v, err := json.Marshal(app) - if err != nil { - return err - } - item.Value = string(v) - err = icache.Put(app.Name, &item) - return err -} - -func Ping(w http.ResponseWriter, req *http.Request) { - fmt.Fprintln(w, "pong") -} - -func VersionHandler(w http.ResponseWriter, req *http.Request) { - fmt.Fprintln(w, Version) -} diff --git a/api/config.go b/api/config.go deleted file mode 100644 index d03149200..000000000 --- a/api/config.go +++ /dev/null @@ -1,31 +0,0 @@ -package api - -type Config struct { - CloudFlare struct { - Email string `json:"email"` - ApiKey string `json:"api_key"` - ZoneId string `json:"zone_id"` - } `json:"cloudflare"` - Cache struct { - Host string `json:"host"` - Token string `json:"token"` - ProjectId string `json:"project_id"` - } - Iron struct { - Token string `json:"token"` - ProjectId string `json:"project_id"` - SuperToken string `json:"super_token"` - WorkerHost string `json:"worker_host"` - AuthHost string `json:"auth_host"` - } `json:"iron"` - Logging struct { - To string `json:"to"` - Level string `json:"level"` - Prefix string `json:"prefix"` - } -} - -func (c *Config) Validate() error { - // TODO: - return nil -} diff --git a/api/dns.go b/api/dns.go deleted file mode 100644 index 02b88b48c..000000000 --- a/api/dns.go +++ /dev/null @@ -1,80 +0,0 @@ -package api - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - - log "github.com/Sirupsen/logrus" -) - -type CloudFlareResult struct { - Id string `json:"id"` -} -type CloudFlareResponse struct { - Result CloudFlareResult `json:"result"` - Success bool -} - -/* -This function registers the host name provided (dns host) in IronCache which is used for -the routing table. Backend runners know this too. - -This will also create a dns entry for the worker in iron.computer. -*/ -func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool { - // Give it an iron.computer entry with format: APP_NAME.PROJECT_ID.ironfunctions.com - dnsHost := fmt.Sprintf("%v.%v.ironfunctions.com", app.Name, 123) - app.Dns = dnsHost - - if app.CloudFlareId == "" { - // Tad hacky, but their go lib is pretty confusing. - cfMethod := "POST" - cfUrl := fmt.Sprintf("https://api.cloudflare.com/client/v4/zones/%v/dns_records", config.CloudFlare.ZoneId) - if app.CloudFlareId != "" { - // Have this here in case we need to support updating the entry. If we do this, is how: - cfMethod = "PUT" - cfUrl = cfUrl + "/" + app.CloudFlareId - } - log.Info("registering dns: ", "dnsname: ", dnsHost, " url: ", cfUrl) - - cfbody := "{\"type\":\"CNAME\",\"name\":\"" + dnsHost + "\",\"content\":\"api.ironfunctions.com\",\"ttl\":120}" - client := &http.Client{} // todo: is default client fine? - req, err := http.NewRequest( - cfMethod, - cfUrl, - strings.NewReader(cfbody), - ) - req.Header.Set("X-Auth-Email", config.CloudFlare.Email) - req.Header.Set("X-Auth-Key", config.CloudFlare.ApiKey) - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - log.Error("Could not register dns entry.", "err", err) - SendError(w, 500, fmt.Sprint("Could not register dns entry.", err)) - return false - } - defer resp.Body.Close() - // todo: get error message from body for bad status code - body, err := ioutil.ReadAll(resp.Body) - if resp.StatusCode != 200 { - log.Error("Could not register dns entry 2.", "code", resp.StatusCode, "body", string(body)) - SendError(w, 500, fmt.Sprint("Could not register dns entry 2. ", resp.StatusCode)) - return false - } - cfResult := CloudFlareResponse{} - err = json.Unmarshal(body, &cfResult) - if err != nil { - log.Error("Could not parse DNS response.", "err", err, "code", resp.StatusCode, "body", string(body)) - SendError(w, 500, fmt.Sprint("Could not parse DNS response. ", resp.StatusCode)) - return false - } - fmt.Println("cfresult:", cfResult) - app.CloudFlareId = cfResult.Result.Id - } - - log.Info("host registered successfully with cloudflare", "app", app) - return true -} diff --git a/api/helpers.go b/api/helpers.go deleted file mode 100644 index 6ce907a1d..000000000 --- a/api/helpers.go +++ /dev/null @@ -1,266 +0,0 @@ -package api - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "reflect" - "runtime/debug" - "strconv" - "strings" - "sync/atomic" - "time" - - "gopkg.in/inconshreveable/log15.v2" - - "github.com/Sirupsen/logrus" -) - -func NotFound(w http.ResponseWriter, r *http.Request) { - SendError(w, http.StatusNotFound, "Not found") -} - -func EndpointNotFound(w http.ResponseWriter, r *http.Request) { - SendError(w, http.StatusNotFound, "Endpoint not found") -} - -func InternalError(w http.ResponseWriter, err error) { - logrus.Error("internal server error response", "err", err, "stack", string(debug.Stack())) - SendError(w, http.StatusInternalServerError, "internal error") -} - -func InternalErrorDetailed(w http.ResponseWriter, r *http.Request, err error) { - logrus.Error("internal server error response", "err", err, "endpoint", r.URL.String(), "token", GetTokenString(r), "stack", string(debug.Stack())) - SendError(w, http.StatusInternalServerError, "internal error") -} - -type HTTPError interface { - error - StatusCode() int -} - -type response struct { - Msg string `json:"msg"` -} - -func SendError(w http.ResponseWriter, code int, msg string) { - logrus.Debug("HTTP error", "status_code", code, "msg", msg) - resp := response{Msg: msg} - RespondCode(w, nil, code, &resp) -} - -func SendSuccess(w http.ResponseWriter, msg string, params map[string]interface{}) { - var v interface{} - if params == nil { - v = &response{Msg: msg} - } else { - v = params - } - RespondCode(w, nil, http.StatusOK, v) -} - -func Respond(w http.ResponseWriter, r *http.Request, v interface{}) { - RespondCode(w, r, http.StatusOK, v) -} - -func RespondCode(w http.ResponseWriter, r *http.Request, code int, v interface{}) { - bytes, err := json.Marshal(v) - if err != nil { - logrus.Error("error marshalling HTTP response", "value", v, "type", reflect.TypeOf(v), "err", err) - InternalError(w, err) - return - } - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Content-Length", strconv.Itoa(len(bytes))) - w.WriteHeader(code) - if _, err := w.Write(bytes); err != nil { - // older callers don't pass a request - if r != nil { - logrus.Error("unable to write HTTP response", "err", err) - } else { - logrus.Error("unable to write HTTP response", "err", err) - } - } -} - -// GetTokenString returns the token string from either the Authorization header or -// the oauth query parameter. -func GetTokenString(r *http.Request) string { - tok, _ := GetTokenStringType(r) - return tok -} - -func GetTokenStringType(r *http.Request) (tok string, jwt bool) { - tokenStr := r.URL.Query().Get("oauth") - if tokenStr != "" { - return tokenStr, false - } - tokenStr = r.URL.Query().Get("jwt") - jwt = tokenStr != "" - if tokenStr == "" { - authHeader := r.Header.Get("Authorization") - authFields := strings.Fields(authHeader) - if len(authFields) == 2 && (authFields[0] == "OAuth" || authFields[0] == "JWT") { - jwt = authFields[0] == "JWT" - tokenStr = authFields[1] - } - } - return tokenStr, jwt -} - -func ReadJSONSize(w http.ResponseWriter, r *http.Request, v interface{}, n int64) (success bool) { - contentType := r.Header.Get("Content-Type") - i := strings.IndexByte(contentType, ';') - if i < 0 { - i = len(contentType) - } - if strings.TrimRight(contentType[:i], " ") != "application/json" { - SendError(w, http.StatusBadRequest, "Bad Content-Type.") - return false - } - if i < len(contentType) { - param := strings.Trim(contentType[i+1:], " ") - split := strings.SplitN(param, "=", 2) - if len(split) != 2 || strings.Trim(split[0], " ") != "charset" { - SendError(w, http.StatusBadRequest, "Invalid Content-Type parameter.") - return false - } - value := strings.Trim(split[1], " ") - if len(value) > 2 && value[0] == '"' && value[len(value)-1] == '"' { - // quoted string - value = value[1 : len(value)-1] - } - if !strings.EqualFold(value, "utf-8") { - SendError(w, http.StatusBadRequest, "Unsupported charset. JSON is always UTF-8 encoded.") - return false - } - } - if r.ContentLength > n { - SendError(w, http.StatusBadRequest, fmt.Sprint("Content-Length greater than", n, "bytes")) - return false - } - - err := json.NewDecoder(&LimitedReader{r.Body, n}).Decode(v) - if err != nil { - jsonError(w, err) - return false - } - - return true -} - -func ReadJSON(w http.ResponseWriter, r *http.Request, v interface{}) bool { - return ReadJSONSize(w, r, v, 100*0xffff) -} - -// Same as io.LimitedReader, but returns limitReached so we can -// distinguish between the limit being reached and actual EOF. -type LimitedReader struct { - R io.Reader - N int64 -} - -func (l *LimitedReader) Read(p []byte) (n int, err error) { - if l.N <= 0 { - return 0, LimitReached(l.N) - } - if int64(len(p)) > l.N { - p = p[:l.N] - } - n, err = l.R.Read(p) - l.N -= int64(n) - return -} - -// LimitedWriter writes until n bytes are written, then writes -// an overage line and skips any further writes. -type LimitedWriter struct { - W io.Writer - N int64 -} - -func (l *LimitedWriter) Write(p []byte) (n int, err error) { - var overrage = []byte("maximum log file size exceeded") - - // we expect there may be concurrent writers, so to be safe.. - left := atomic.LoadInt64(&l.N) - if left <= 0 { - return 0, io.EOF // TODO EOF? really? does it matter? - } - n, err = l.W.Write(p) - left = atomic.AddInt64(&l.N, -int64(n)) - if left <= 0 { - l.W.Write(overrage) - } - return n, err -} - -type JSONError string - -func (e JSONError) Error() string { return string(e) } - -func jsonType(t reflect.Type) string { - switch t.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, - reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, - reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return "integer" - case reflect.Float32, reflect.Float64: - return "number" - case reflect.Map, reflect.Struct: - return "object" - case reflect.Slice, reflect.Array: - return "array" - case reflect.Ptr: - return jsonType(t.Elem()) - } - // bool, string, other cases not covered - return t.String() -} - -func jsonError(w http.ResponseWriter, err error) { - var msg string - switch err := err.(type) { - case *json.InvalidUTF8Error: - msg = "Invalid UTF-8 in JSON: " + err.S - - case *json.InvalidUnmarshalError, *json.UnmarshalFieldError, - *json.UnsupportedTypeError: - // should never happen - InternalError(w, err) - return - - case *json.SyntaxError: - msg = fmt.Sprintf("In JSON, %v at position %v.", err, err.Offset) - - case *json.UnmarshalTypeError: - msg = fmt.Sprintf("In JSON, cannot use %v as %v", err.Value, jsonType(err.Type)) - - case *time.ParseError: - msg = "Time strings must be in RFC 3339 format." - - case LimitReached: - msg = err.Error() - - case JSONError: - msg = string(err) - - default: - if err != io.EOF { - log15.Error("unhandled json.Unmarshal error", "type", reflect.TypeOf(err), "err", err) - } - msg = "Failed to decode JSON." - } - SendError(w, http.StatusBadRequest, msg) -} - -type sizer interface { - Size() int64 -} - -type LimitReached int64 - -func (e LimitReached) Error() string { - return fmt.Sprint("Request body greater than", int64(e), "bytes") -} diff --git a/api/models.go b/api/models.go deleted file mode 100644 index 2f65b56a5..000000000 --- a/api/models.go +++ /dev/null @@ -1,35 +0,0 @@ -package api - -type Route struct { - // TODO: Change destinations to a simple cache so it can expire entries after 55 minutes (the one we use in common?) - Host string `json:"host"` - Destinations []string `json:"destinations"` - ProjectId string `json:"project_id"` - Token string `json:"token"` // store this so we can queue up new workers on demand - CodeName string `json:"code_name"` -} - -// for adding new hosts -type Route2 struct { - Host string `json:"host"` - Dest string `json:"dest"` -} - -// An app is that base object for api gateway -type App struct { - Name string `json:"name"` - ProjectId string `json:"project_id"` - CloudFlareId string `json:"-"` - Dns string `json:"dns"` - Routes map[string]*Route3 `json:"routes"` -} - -// this is for the new api gateway -type Route3 struct { - Path string `json:"path"` // run/app - Image string `json:"image"` - Type string `json:"type"` - ContainerPath string `json:"cpath"` - // maybe user a header map for whatever user wants to send? - ContentType string `json:"content_type"` -} diff --git a/api/models/app.go b/api/models/app.go new file mode 100644 index 000000000..137182876 --- /dev/null +++ b/api/models/app.go @@ -0,0 +1,22 @@ +package models + +import "errors" + +type Apps []App + +var ( + ErrAppsCreate = errors.New("Could not create app") + ErrAppsUpdate = errors.New("Could not update app") + ErrAppsRemoving = errors.New("Could not remove app from datastore") + ErrAppsGet = errors.New("Could not get app from datastore") + ErrAppsList = errors.New("Could not list apps from datastore") + ErrAppsNotFound = errors.New("App not found") +) + +type App struct { + Name string `json:"name"` + Routes Routes `json:"routes"` +} + +type AppFilter struct { +} diff --git a/api/models/datastore.go b/api/models/datastore.go new file mode 100644 index 000000000..abc269e39 --- /dev/null +++ b/api/models/datastore.go @@ -0,0 +1,21 @@ +package models + +type Datastore interface { + GetApp(appName string) (*App, error) + GetApps(*AppFilter) ([]*App, error) + StoreApp(*App) (*App, error) + RemoveApp(appName string) error + + GetRoute(appName, routeName string) (*Route, error) + GetRoutes(*RouteFilter) (routes []*Route, err error) + StoreRoute(*Route) (*Route, error) + RemoveRoute(appName, routeName string) error +} + +func ApplyAppFilter(app *App, filter *AppFilter) bool { + return true +} + +func ApplyRouteFilter(route *Route, filter *RouteFilter) bool { + return true +} diff --git a/api/models/error.go b/api/models/error.go new file mode 100644 index 000000000..6996f8083 --- /dev/null +++ b/api/models/error.go @@ -0,0 +1,15 @@ +package models + +import "errors" + +type Error struct { + Error *ErrorBody `json:"error,omitempty"` +} + +func (m *Error) Validate() error { + return nil +} + +var ( + ErrInvalidJSON = errors.New("Could not create app") +) diff --git a/api/models/error_body.go b/api/models/error_body.go new file mode 100644 index 000000000..96e78ae2d --- /dev/null +++ b/api/models/error_body.go @@ -0,0 +1,11 @@ +package models + +type ErrorBody struct { + Fields string `json:"fields,omitempty"` + Message string `json:"message,omitempty"` +} + +// Validate validates this error body +func (m *ErrorBody) Validate() error { + return nil +} diff --git a/api/models/route.go b/api/models/route.go new file mode 100644 index 000000000..6dbe01323 --- /dev/null +++ b/api/models/route.go @@ -0,0 +1,31 @@ +package models + +import ( + "errors" + "net/http" +) + +var ( + ErrRoutesCreate = errors.New("Could not create route") + ErrRoutesUpdate = errors.New("Could not update route") + ErrRoutesRemoving = errors.New("Could not remove route from datastore") + ErrRoutesGet = errors.New("Could not get route from datastore") + ErrRoutesList = errors.New("Could not list routes from datastore") + ErrRoutesNotFound = errors.New("Route not found") +) + +type Routes []Route + +type Route struct { + Name string `json:"name"` + AppName string `json:"appname"` + Path string `json:"path"` + Image string `json:"image"` + Type string `json:"type"` + ContainerPath string `json:"container_path"` + Headers http.Header `json:"headers"` +} + +type RouteFilter struct { + AppName string +} diff --git a/api/runner.go b/api/runner/runner.go similarity index 58% rename from api/runner.go rename to api/runner/runner.go index 7baa1f2a5..d7faa263e 100644 --- a/api/runner.go +++ b/api/runner/runner.go @@ -1,27 +1,34 @@ -package api +package runner import ( "bufio" "bytes" + "errors" "fmt" "io" "io/ioutil" + "log" "math/rand" "net/http" "os" "os/exec" "strings" - log "github.com/Sirupsen/logrus" - "github.com/iron-io/go/common" + "github.com/Sirupsen/logrus" + "github.com/gin-gonic/gin" + "github.com/iron-io/functions/api/models" ) type RunningApp struct { - Route *Route3 + Route *models.Route Port int ContainerName string } +var ( + ErrRunnerRouteNotFound = errors.New("Route not found on that application") +) + var runningImages map[string]*RunningApp func init() { @@ -29,61 +36,56 @@ func init() { fmt.Println("ENV:", os.Environ()) } -func Run(w http.ResponseWriter, req *http.Request) { - fmt.Println("RUN!!!!") - log.Infoln("HOST:", req.Host) - rUrl := req.URL - appName := rUrl.Query().Get("app") - log.Infoln("app_name", appName, "path:", req.URL.Path) - if appName != "" { - // passed in the name - } else { - host := strings.Split(req.Host, ":")[0] +func Run(c *gin.Context) error { + log := c.MustGet("log").(logrus.FieldLogger) + store := c.MustGet("store").(models.Datastore) + + appName := c.Param("app") + + if appName == "" { + host := strings.Split(c.Request.Host, ":")[0] appName = strings.Split(host, ".")[0] - log.Infoln("app_name from host", appName) } - app, err := getApp(appName) + filter := &models.RouteFilter{ + AppName: appName, + } + + routes, err := store.GetRoutes(filter) if err != nil { - common.SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err)) - return + return err } - log.Infoln("app", app) - // find route - for _, el := range app.Routes { - // TODO: copy/use gorilla's pattern matching here - if el.Path == req.URL.Path { - // Boom, run it! + route := c.Param("route") + + log.WithFields(logrus.Fields{"app": appName}).Debug("Running app") + + for _, el := range routes { + if el.Path == route { err = checkAndPull(el.Image) if err != nil { - common.SendError(w, 404, fmt.Sprintln("The image could not be pulled:", err)) - return + return err } if el.Type == "app" { - DockerHost(el, w) - return - } else { // "run" - // TODO: timeout 59 seconds - DockerRun(el, w, req) - return + return DockerHost(el, c) + } else { + return DockerRun(el, c) } } } - common.SendError(w, 404, fmt.Sprintln("The requested endpoint does not exist.")) + + return ErrRunnerRouteNotFound } // TODO: use Docker utils from docker-job for this and a few others in here -func DockerRun(route *Route3, w http.ResponseWriter, req *http.Request) { - log.Infoln("route:", route) +func DockerRun(route *models.Route, c *gin.Context) error { image := route.Image - payload, err := ioutil.ReadAll(req.Body) + payload, err := ioutil.ReadAll(c.Request.Body) if err != nil { - log.WithError(err).Errorln("Error reading request body") - return + return err } - log.WithField("payload", "---"+string(payload)+"---").Infoln("incoming request") - log.WithField("image", image).Infoln("About to run using this image") + // log.WithField("payload", "---"+string(payload)+"---").Infoln("incoming request") + // log.WithField("image", image).Infoln("About to run using this image") // TODO: swap all this out with Titan's running via API cmd := exec.Command("docker", "run", "--rm", "-i", "-e", fmt.Sprintf("PAYLOAD=%v", string(payload)), image) @@ -107,23 +109,24 @@ func DockerRun(route *Route3, w http.ResponseWriter, req *http.Request) { log.Printf("Waiting for command to finish...") if err = cmd.Wait(); err != nil { // job failed - log.Infoln("job finished with err:", err) - log.WithFields(log.Fields{"metric": "run.errors", "value": 1, "type": "count"}).Infoln("failed run") + // log.Infoln("job finished with err:", err) + // log.WithFields(log.Fields{"metric": "run.errors", "value": 1, "type": "count"}).Infoln("failed run") + return err // TODO: wrap error in json "error": buff - } else { - log.Infoln("Docker ran successfully:", b.String()) - // print - log.WithFields(log.Fields{"metric": "run.success", "value": 1, "type": "count"}).Infoln("successful run") } - log.WithFields(log.Fields{"metric": "run", "value": 1, "type": "count"}).Infoln("job ran") + + // log.Infoln("Docker ran successfully:", b.String()) + // print + // log.WithFields(log.Fields{"metric": "run.success", "value": 1, "type": "count"}).Infoln("successful run") + // log.WithFields(log.Fields{"metric": "run", "value": 1, "type": "count"}).Infoln("job ran") buff.Flush() - if route.ContentType != "" { - w.Header().Set("Content-Type", route.ContentType) - } - fmt.Fprintln(w, string(bytes.Trim(b.Bytes(), "\x00"))) + + c.Data(http.StatusOK, "", bytes.Trim(b.Bytes(), "\x00")) + + return nil } -func DockerHost(el *Route3, w http.ResponseWriter) { +func DockerHost(el *models.Route, c *gin.Context) error { ra := runningImages[el.Image] if ra == nil { ra = &RunningApp{} @@ -134,11 +137,11 @@ func DockerHost(el *Route3, w http.ResponseWriter) { // TODO: timeout 59 minutes. Mark it in ra as terminated. cmd := exec.Command("docker", "run", "--name", ra.ContainerName, "--rm", "-i", "-p", fmt.Sprintf("%v:8080", ra.Port), el.Image) // TODO: What should we do with the output here? Store it? Send it to a log service? - cmd.Stdout = os.Stdout + // cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr // TODO: Need to catch interrupt and stop all containers that are started, see devo/dj for how to do this if err := cmd.Start(); err != nil { - log.Fatal(err) + return err // TODO: What if the app fails to start? Don't want to keep starting the container } } else { @@ -149,16 +152,16 @@ func DockerHost(el *Route3, w http.ResponseWriter) { // TODO: if connection fails, check if container still running? If not, start it again resp, err := http.Get(fmt.Sprintf("http://0.0.0.0:%v%v", ra.Port, el.ContainerPath)) if err != nil { - common.SendError(w, 404, fmt.Sprintln("The requested app endpoint does not exist.", err)) - return + return err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - common.SendError(w, 500, fmt.Sprintln("Error reading response body", err)) - return + return err } - fmt.Fprintln(w, string(body)) + + c.Data(http.StatusOK, "", body) + return nil } func checkAndPull(image string) error { @@ -193,7 +196,9 @@ func execAndPrint(cmdstr string, args []string) error { log.Printf("Waiting for cmd to finish...") err = cmd.Wait() - fmt.Println("stderr:", berr.String()) + if berr.Len() != 0 { + fmt.Println("stderr:", berr.String()) + } fmt.Println("stdout:", bout.String()) return err } diff --git a/api/server/config.go b/api/server/config.go new file mode 100644 index 000000000..d4ceb87c2 --- /dev/null +++ b/api/server/config.go @@ -0,0 +1,15 @@ +package server + +type Config struct { + DatabaseURL string `json:"db"` + Logging struct { + To string `json:"to"` + Level string `json:"level"` + Prefix string `json:"prefix"` + } +} + +func (c *Config) Validate() error { + // TODO: + return nil +} diff --git a/api/server/datastore/bolt/bolt.go b/api/server/datastore/bolt/bolt.go new file mode 100644 index 000000000..700157bb0 --- /dev/null +++ b/api/server/datastore/bolt/bolt.go @@ -0,0 +1,259 @@ +package bolt + +import ( + "encoding/json" + "net/url" + "os" + "path/filepath" + + "github.com/Sirupsen/logrus" + "github.com/boltdb/bolt" + "github.com/iron-io/functions/api/models" +) + +type BoltDatastore struct { + routesBucket []byte + appsBucket []byte + logsBucket []byte + db *bolt.DB + log logrus.FieldLogger +} + +func New(url *url.URL) (models.Datastore, error) { + dir := filepath.Dir(url.Path) + log := logrus.WithFields(logrus.Fields{"db": url.Scheme, "dir": dir}) + err := os.MkdirAll(dir, 0777) + if err != nil { + log.WithError(err).Errorln("Could not create data directory for db") + return nil, err + } + log.Infoln("Creating bolt db at ", url.Path) + db, err := bolt.Open(url.Path, 0600, nil) + if err != nil { + log.WithError(err).Errorln("Error on bolt.Open") + return nil, err + } + bucketPrefix := "fns-" + if url.Query()["bucket"] != nil { + bucketPrefix = url.Query()["bucket"][0] + } + routesBucketName := []byte(bucketPrefix + "routes") + appsBucketName := []byte(bucketPrefix + "apps") + logsBucketName := []byte(bucketPrefix + "logs") + err = db.Update(func(tx *bolt.Tx) error { + for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName} { + _, err := tx.CreateBucketIfNotExists(name) + if err != nil { + log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket") + return err + } + } + return nil + }) + if err != nil { + log.WithError(err).Errorln("Error creating bolt buckets") + return nil, err + } + + ds := &BoltDatastore{ + routesBucket: routesBucketName, + appsBucket: appsBucketName, + logsBucket: logsBucketName, + db: db, + log: log, + } + log.WithFields(logrus.Fields{"prefix": bucketPrefix, "file": url.Path}).Info("BoltDB initialized") + + return ds, nil +} + +func (ds *BoltDatastore) StoreApp(app *models.App) (*models.App, error) { + err := ds.db.Update(func(tx *bolt.Tx) error { + bIm := tx.Bucket(ds.appsBucket) + buf, err := json.Marshal(app) + if err != nil { + return err + } + err = bIm.Put([]byte(app.Name), buf) + if err != nil { + return err + } + bjParent := tx.Bucket(ds.routesBucket) + _, err = bjParent.CreateBucketIfNotExists([]byte(app.Name)) + if err != nil { + return err + } + return nil + }) + return app, err +} + +func (ds *BoltDatastore) RemoveApp(appName string) error { + err := ds.db.Update(func(tx *bolt.Tx) error { + bIm := tx.Bucket(ds.appsBucket) + err := bIm.Delete([]byte(appName)) + if err != nil { + return err + } + bjParent := tx.Bucket(ds.routesBucket) + err = bjParent.DeleteBucket([]byte(appName)) + if err != nil { + return err + } + return nil + }) + return err +} + +func (ds *BoltDatastore) GetApps(filter *models.AppFilter) ([]*models.App, error) { + res := []*models.App{} + err := ds.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket(ds.appsBucket) + err2 := b.ForEach(func(key, v []byte) error { + app := &models.App{} + err := json.Unmarshal(v, app) + if err != nil { + return err + } + res = append(res, app) + return nil + }) + if err2 != nil { + logrus.WithError(err2).Errorln("Couldn't get apps!") + } + return nil + }) + if err != nil { + return nil, err + } + return res, nil +} + +func (ds *BoltDatastore) GetApp(name string) (*models.App, error) { + var res *models.App + err := ds.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket(ds.appsBucket) + v := b.Get([]byte(name)) + if v != nil { + app := &models.App{} + err := json.Unmarshal(v, app) + if err != nil { + return err + } + res = app + } + return nil + }) + if err != nil { + return nil, err + } + return res, nil +} + +func (ds *BoltDatastore) getRouteBucketForApp(tx *bolt.Tx, appName string) (*bolt.Bucket, error) { + var err error + bp := tx.Bucket(ds.routesBucket) + b := bp.Bucket([]byte(appName)) + if b == nil { + b, err = bp.CreateBucket([]byte(appName)) + if err != nil { + return nil, err + } + } + return b, nil +} + +func (ds *BoltDatastore) StoreRoute(route *models.Route) (*models.Route, error) { + err := ds.db.Update(func(tx *bolt.Tx) error { + b, err := ds.getRouteBucketForApp(tx, route.AppName) + if err != nil { + return err + } + + buf, err := json.Marshal(route) + if err != nil { + return err + } + + err = b.Put([]byte(route.Name), buf) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + return route, nil +} + +func (ds *BoltDatastore) RemoveRoute(appName, routeName string) error { + err := ds.db.Update(func(tx *bolt.Tx) error { + b, err := ds.getRouteBucketForApp(tx, appName) + if err != nil { + return err + } + + err = b.Delete([]byte(routeName)) + if err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + return nil +} + +func (ds *BoltDatastore) GetRoute(appName, routeName string) (*models.Route, error) { + var route models.Route + err := ds.db.View(func(tx *bolt.Tx) error { + b, err := ds.getRouteBucketForApp(tx, appName) + if err != nil { + return err + } + + v := b.Get([]byte(routeName)) + if v == nil { + return models.ErrRoutesNotFound + } + err = json.Unmarshal(v, &route) + return err + }) + return &route, err +} + +func (ds *BoltDatastore) GetRoutes(filter *models.RouteFilter) ([]*models.Route, error) { + res := []*models.Route{} + err := ds.db.View(func(tx *bolt.Tx) error { + b, err := ds.getRouteBucketForApp(tx, filter.AppName) + if err != nil { + return err + } + + i := 0 + c := b.Cursor() + + var k, v []byte + k, v = c.Last() + + // Iterate backwards, newest first + for ; k != nil; k, v = c.Prev() { + var route models.Route + err := json.Unmarshal(v, &route) + if err != nil { + return err + } + if models.ApplyRouteFilter(&route, filter) { + i++ + res = append(res, &route) + } + } + return nil + }) + if err != nil { + return nil, err + } + return res, nil +} diff --git a/api/server/datastore/datastore.go b/api/server/datastore/datastore.go new file mode 100644 index 000000000..92085633e --- /dev/null +++ b/api/server/datastore/datastore.go @@ -0,0 +1,27 @@ +package datastore + +import ( + "fmt" + "net/url" + + "github.com/Sirupsen/logrus" + "github.com/iron-io/functions/api/models" + "github.com/iron-io/functions/api/server/datastore/bolt" + "github.com/iron-io/functions/api/server/datastore/postgres" +) + +func New(dbURL string) (models.Datastore, error) { + u, err := url.Parse(dbURL) + if err != nil { + logrus.WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL") + } + logrus.WithFields(logrus.Fields{"db": u.Scheme}).Info("creating new datastore") + switch u.Scheme { + case "bolt": + return bolt.New(u) + case "postgres": + return postgres.New(u) + default: + return nil, fmt.Errorf("db type not supported %v", u.Scheme) + } +} diff --git a/api/server/datastore/postgres/postgres.go b/api/server/datastore/postgres/postgres.go new file mode 100644 index 000000000..95bf6e384 --- /dev/null +++ b/api/server/datastore/postgres/postgres.go @@ -0,0 +1,262 @@ +package postgres + +import ( + "database/sql" + "encoding/json" + "fmt" + "net/url" + + "github.com/Sirupsen/logrus" + "github.com/iron-io/functions/api/models" + _ "github.com/lib/pq" +) + +const routesTableCreate = `CREATE TABLE IF NOT EXISTS routes ( + name character varying(256) NOT NULL PRIMARY KEY, + path text NOT NULL, + app_name character varying(256) NOT NULL, + image character varying(256) NOT NULL, + type character varying(256) NOT NULL, + container_path text NOT NULL, + headers text NOT NULL, +);` + +const appsTableCreate = `CREATE TABLE IF NOT EXISTS apps ( + name character varying(256) NOT NULL PRIMARY KEY, +);` + +const routeSelector = `SELECT path, app_name, image, type, container_path, headers FROM routes` + +// Tries to read in properties into `route` from `scanner`. Bubbles up errors. +// Capture sql.Row and sql.Rows +type rowScanner interface { + Scan(dest ...interface{}) error +} + +// Capture sql.DB and sql.Tx +type rowQuerier interface { + QueryRow(query string, args ...interface{}) *sql.Row +} + +func scanRoute(scanner rowScanner, route *models.Route) error { + err := scanner.Scan( + &route.Name, + &route.Path, + &route.AppName, + &route.Image, + &route.Type, + &route.ContainerPath, + &route.Headers, + ) + return err +} + +type PostgresDatastore struct { + db *sql.DB +} + +func New(url *url.URL) (models.Datastore, error) { + db, err := sql.Open("postgres", url.String()) + if err != nil { + return nil, err + } + + err = db.Ping() + if err != nil { + return nil, err + } + + maxIdleConns := 30 // c.MaxIdleConnections + db.SetMaxIdleConns(maxIdleConns) + logrus.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns}).Info("Postgres dialed") + + pg := &PostgresDatastore{ + db: db, + } + + for _, v := range []string{routesTableCreate, appsTableCreate} { + _, err = db.Exec(v) + if err != nil { + return nil, err + } + } + + return pg, nil +} + +func (ds *PostgresDatastore) StoreRoute(route *models.Route) (*models.Route, error) { + var headers string + + hbyte, err := json.Marshal(route.Headers) + if err != nil { + return nil, err + } + + headers = string(hbyte) + + err = ds.db.QueryRow(` + INSERT INTO routes ( + name, app_name, path, image, + type, container_path, headers + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + route.Name, + route.AppName, + route.Path, + route.Image, + route.Type, + route.ContainerPath, + headers, + ).Scan(nil) + if err != nil { + return nil, err + } + return route, nil +} + +func (ds *PostgresDatastore) RemoveRoute(appName, routeName string) error { + err := ds.db.QueryRow(` + DELETE FROM routes + WHERE name = $1`, + routeName, + ).Scan(nil) + if err != nil { + return err + } + return nil +} + +func getRoute(qr rowQuerier, routeName string) (*models.Route, error) { + var route models.Route + + row := qr.QueryRow(fmt.Sprintf("%s WHERE name=$1", routeSelector), routeName) + err := scanRoute(row, &route) + + if err == sql.ErrNoRows { + return nil, models.ErrRoutesNotFound + } else if err != nil { + return nil, err + } + return &route, nil +} + +func (ds *PostgresDatastore) GetRoute(appName, routeName string) (*models.Route, error) { + return getRoute(ds.db, routeName) +} + +// TODO: Add pagination +func (ds *PostgresDatastore) GetRoutes(filter *models.RouteFilter) ([]*models.Route, error) { + res := []*models.Route{} + filterQuery := buildFilterQuery(filter) + rows, err := ds.db.Query(fmt.Sprintf("%s %s", routeSelector, filterQuery)) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var route models.Route + err := scanRoute(rows, &route) + if err != nil { + return nil, err + } + res = append(res, &route) + + } + if err := rows.Err(); err != nil { + return nil, err + } + return res, nil +} + +func (ds *PostgresDatastore) GetApp(name string) (*models.App, error) { + row := ds.db.QueryRow("SELECT * FROM groups WHERE name=$1", name) + + var resName string + err := row.Scan(&resName) + + res := &models.App{ + Name: resName, + } + + if err != nil { + return nil, err + } + + return res, nil +} + +func (ds *PostgresDatastore) GetApps(filter *models.AppFilter) ([]*models.App, error) { + res := []*models.App{} + + rows, err := ds.db.Query(` + SELECT DISTINCT * + FROM apps + ORDER BY name`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var app models.App + err := rows.Scan(&app) + + if err != nil { + return nil, err + } + res = append(res, &app) + } + + if err := rows.Err(); err != nil { + return nil, err + } + return res, nil +} + +func (ds *PostgresDatastore) StoreApp(app *models.App) (*models.App, error) { + err := ds.db.QueryRow(` + INSERT INTO apps (name) + VALUES ($1) + `, app.Name).Scan(&app.Name) + // ON CONFLICT (name) DO UPDATE SET created_at = $2; + + if err != nil { + return nil, err + } + + return app, nil +} + +func (ds *PostgresDatastore) RemoveApp(appName string) error { + err := ds.db.QueryRow(` + DELETE FROM apps + WHERE name = $1 + `, appName).Scan(nil) + + if err != nil { + return err + } + + return nil +} + +func buildFilterQuery(filter *models.RouteFilter) string { + filterQuery := "" + + filterQueries := []string{} + if filter.AppName != "" { + filterQueries = append(filterQueries, fmt.Sprintf("app_name = '%s'", filter.AppName)) + } + + for i, field := range filterQueries { + if i == 0 { + filterQuery = fmt.Sprintf("WHERE %s ", field) + } else { + filterQuery = fmt.Sprintf("%s AND %s", filterQuery, field) + } + } + + return filterQuery +} diff --git a/api/server/server.go b/api/server/server.go new file mode 100644 index 000000000..7615a9458 --- /dev/null +++ b/api/server/server.go @@ -0,0 +1,58 @@ +package server + +import ( + "fmt" + "os" + "path" + + "github.com/Sirupsen/logrus" + "github.com/gin-gonic/gin" + "github.com/iron-io/functions/api/server/datastore" + "github.com/iron-io/functions/api/server/router" +) + +type Server struct { + router *gin.Engine + cfg *Config +} + +func New(config *Config) *Server { + return &Server{ + router: gin.Default(), + cfg: config, + } +} + +func extractFields(c *gin.Context) logrus.Fields { + fields := logrus.Fields{"action": path.Base(c.HandlerName())} + for _, param := range c.Params { + fields[param.Key] = param.Value + } + return fields +} + +func (s *Server) Start() { + if s.cfg.DatabaseURL == "" { + cwd, _ := os.Getwd() + s.cfg.DatabaseURL = fmt.Sprintf("bolt://%s/bolt.db?bucket=fns", cwd) + } + + ds, err := datastore.New(s.cfg.DatabaseURL) + if err != nil { + logrus.WithError(err).Fatalln("Invalid DB url.") + } + + logrus.SetOutput(os.Stdout) + logrus.SetLevel(logrus.DebugLevel) + + s.router.Use(func(c *gin.Context) { + c.Set("store", ds) + c.Set("log", logrus.WithFields(extractFields(c))) + c.Next() + }) + + router.Start(s.router) + + // Default to :8080 + s.router.Run() +} diff --git a/api/version.go b/api/version.go deleted file mode 100644 index 58b62165a..000000000 --- a/api/version.go +++ /dev/null @@ -1,3 +0,0 @@ -package api - -const Version = "0.0.28" diff --git a/main.go b/main.go index 9e4fc9957..8bcb920a1 100644 --- a/main.go +++ b/main.go @@ -11,23 +11,19 @@ import ( "os" log "github.com/Sirupsen/logrus" - "github.com/iron-io/functions/api" + "github.com/iron-io/functions/api/server" ) func main() { + config := &server.Config{} + config.DatabaseURL = os.Getenv("DB") - config := &api.Config{} - config.CloudFlare.Email = os.Getenv("CLOUDFLARE_EMAIL") - config.CloudFlare.ApiKey = os.Getenv("CLOUDFLARE_API_KEY") - config.CloudFlare.ZoneId = os.Getenv("CLOUDFLARE_ZONE_ID") - - // TODO: validate inputs, iron tokens, cloudflare stuff, etc err := config.Validate() if err != nil { log.WithError(err).Fatalln("Invalid config.") } log.Printf("config: %+v", config) - api := api.New(config) + api := server.New(config) api.Start() }