alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactoring API and added dbs: postgres, bolt

Browse Source
This commit is contained in:
Pedro Nasser
2016-07-21 16:04:58 -03:00
parent edc126eb81
commit 66fa3d4035
19 changed files with 791 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
api/api.go
View File

@@ -1,205 +0,0 @@
package api
import (
"encoding/json"
"fmt"
"net/http"
log "github.com/Sirupsen/logrus"
"github.com/gorilla/mux"
"github.com/iron-io/iron_go/cache"
)
var icache = cache.New("routing-table")
var config *Config
func New(conf *Config) *Api {
config = conf
api := &Api{}
return api
}
type Api struct {
}
func (api *Api) Start() {
log.Infoln("Starting up router version", Version)
r := mux.NewRouter()
// dev:
s := r.PathPrefix("/api").Subrouter()
// production:
// s := r.Host("router.irondns.info").Subrouter()
// s.Handle("/1/projects/{project_id}/register", &Register{})
s.Handle("/v1/apps", &NewApp{})
s.HandleFunc("/v1/apps/{app_name}/routes", NewRoute)
s.HandleFunc("/ping", Ping)
s.HandleFunc("/version", VersionHandler)
// s.Handle("/addworker", &WorkerHandler{})
s.HandleFunc("/", Ping)
r.HandleFunc("/elb-ping-router", Ping) // for ELB health check
// for testing app responses, pass in app name, can use localhost
s4 := r.Queries("app", "").Subrouter()
// s4.HandleFunc("/appsr", Ping)
s4.HandleFunc("/{rest:.*}", Run)
s4.NotFoundHandler = http.HandlerFunc(Run)
// s3 := r.Queries("rhost", "").Subrouter()
// s3.HandleFunc("/", ProxyFunc2)
// This is where all the main incoming traffic goes
r.NotFoundHandler = http.HandlerFunc(Run)
http.Handle("/", r)
port := 8080
log.Infoln("Router started, listening and serving on port", port)
log.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%v", port), nil))
}
type NewApp struct{}
// This registers a new host
func (r *NewApp) ServeHTTP(w http.ResponseWriter, req *http.Request) {
log.Println("NewApp called!")
vars := mux.Vars(req)
projectId := vars["project_id"]
// token := common.GetToken(req)
log.Infoln("project_id:", projectId)
app := App{}
if !ReadJSON(w, req, &app) {
return
}
log.Infoln("body read into app:", app)
app.ProjectId = projectId
_, err := getApp(app.Name)
if err == nil {
SendError(w, 400, fmt.Sprintln("An app with this name already exists.", err))
return
}
app.Routes = make(map[string]*Route3)
// create dns entry
// TODO: Add project id to this. eg: appname.projectid.iron.computer
log.Debug("Creating dns entry.")
regOk := registerHost(w, req, &app)
if !regOk {
return
}
// todo: do we need to close body?
err = putApp(&app)
if err != nil {
log.Infoln("couldn't create app:", err)
SendError(w, 400, fmt.Sprintln("Could not create app!", err))
return
}
log.Infoln("registered app:", app)
v := map[string]interface{}{"app": app, "msg": "App created successfully."}
SendSuccess(w, "App created successfully.", v)
}
func NewRoute(w http.ResponseWriter, req *http.Request) {
fmt.Println("NewRoute")
vars := mux.Vars(req)
projectId := vars["project_id"]
appName := vars["app_name"]
log.Infoln("project_id: ", projectId, "app: ", appName)
route := &Route3{}
if !ReadJSON(w, req, &route) {
return
}
log.Infoln("body read into route:", route)
// TODO: validate route
app, err := getApp(appName)
if err != nil {
SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err))
return
}
if route.Type == "" {
route.Type = "run"
}
// app.Routes = append(app.Routes, route)
app.Routes[route.Path] = route
err = putApp(app)
if err != nil {
log.Errorln("Couldn't create route!:", err)
SendError(w, 400, fmt.Sprintln("Could not create route!", err))
return
}
log.Infoln("Route created:", route)
v := map[string]interface{}{"url": fmt.Sprintf("http://%v%v", app.Dns, route.Path), "msg": "Route created successfully."}
SendSuccess(w, "Route created successfully.", v)
}
func getRoute(host string) (*Route, error) {
log.Infoln("getRoute for host:", host)
rx, err := icache.Get(host)
if err != nil {
return nil, err
}
rx2 := []byte(rx.(string))
route := Route{}
err = json.Unmarshal(rx2, &route)
if err != nil {
return nil, err
}
return &route, err
}
func putRoute(route *Route) error {
item := cache.Item{}
v, err := json.Marshal(route)
if err != nil {
return err
}
item.Value = string(v)
err = icache.Put(route.Host, &item)
return err
}
func getApp(name string) (*App, error) {
log.Infoln("getapp:", name)
rx, err := icache.Get(name)
if err != nil {
return nil, err
}
rx2 := []byte(rx.(string))
app := App{}
err = json.Unmarshal(rx2, &app)
if err != nil {
return nil, err
}
return &app, err
}
func putApp(app *App) error {
item := cache.Item{}
v, err := json.Marshal(app)
if err != nil {
return err
}
item.Value = string(v)
err = icache.Put(app.Name, &item)
return err
}
func Ping(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(w, "pong")
}
func VersionHandler(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(w, Version)
}

31
api/config.go
View File

@@ -1,31 +0,0 @@
package api
type Config struct {
CloudFlare struct {
Email string `json:"email"`
ApiKey string `json:"api_key"`
ZoneId string `json:"zone_id"`
} `json:"cloudflare"`
Cache struct {
Host string `json:"host"`
Token string `json:"token"`
ProjectId string `json:"project_id"`
}
Iron struct {
Token string `json:"token"`
ProjectId string `json:"project_id"`
SuperToken string `json:"super_token"`
WorkerHost string `json:"worker_host"`
AuthHost string `json:"auth_host"`
} `json:"iron"`
Logging struct {
To string `json:"to"`
Level string `json:"level"`
Prefix string `json:"prefix"`
}
}
func (c *Config) Validate() error {
// TODO:
return nil
}

80
api/dns.go
View File

@@ -1,80 +0,0 @@
package api
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
log "github.com/Sirupsen/logrus"
)
type CloudFlareResult struct {
Id string `json:"id"`
}
type CloudFlareResponse struct {
Result CloudFlareResult `json:"result"`
Success bool
}
/*
This function registers the host name provided (dns host) in IronCache which is used for
the routing table. Backend runners know this too.
This will also create a dns entry for the worker in iron.computer.
*/
func registerHost(w http.ResponseWriter, r *http.Request, app *App) bool {
// Give it an iron.computer entry with format: APP_NAME.PROJECT_ID.ironfunctions.com
dnsHost := fmt.Sprintf("%v.%v.ironfunctions.com", app.Name, 123)
app.Dns = dnsHost
if app.CloudFlareId == "" {
// Tad hacky, but their go lib is pretty confusing.
cfMethod := "POST"
cfUrl := fmt.Sprintf("https://api.cloudflare.com/client/v4/zones/%v/dns_records", config.CloudFlare.ZoneId)
if app.CloudFlareId != "" {
// Have this here in case we need to support updating the entry. If we do this, is how:
cfMethod = "PUT"
cfUrl = cfUrl + "/" + app.CloudFlareId
}
log.Info("registering dns: ", "dnsname: ", dnsHost, " url: ", cfUrl)
cfbody := "{\"type\":\"CNAME\",\"name\":\"" + dnsHost + "\",\"content\":\"api.ironfunctions.com\",\"ttl\":120}"
client := &http.Client{} // todo: is default client fine?
req, err := http.NewRequest(
cfMethod,
cfUrl,
strings.NewReader(cfbody),
)
req.Header.Set("X-Auth-Email", config.CloudFlare.Email)
req.Header.Set("X-Auth-Key", config.CloudFlare.ApiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Error("Could not register dns entry.", "err", err)
SendError(w, 500, fmt.Sprint("Could not register dns entry.", err))
return false
}
defer resp.Body.Close()
// todo: get error message from body for bad status code
body, err := ioutil.ReadAll(resp.Body)
if resp.StatusCode != 200 {
log.Error("Could not register dns entry 2.", "code", resp.StatusCode, "body", string(body))
SendError(w, 500, fmt.Sprint("Could not register dns entry 2. ", resp.StatusCode))
return false
}
cfResult := CloudFlareResponse{}
err = json.Unmarshal(body, &cfResult)
if err != nil {
log.Error("Could not parse DNS response.", "err", err, "code", resp.StatusCode, "body", string(body))
SendError(w, 500, fmt.Sprint("Could not parse DNS response. ", resp.StatusCode))
return false
}
fmt.Println("cfresult:", cfResult)
app.CloudFlareId = cfResult.Result.Id
}
log.Info("host registered successfully with cloudflare", "app", app)
return true
}

266
api/helpers.go
View File

@@ -1,266 +0,0 @@
package api
import (
"encoding/json"
"fmt"
"io"
"net/http"
"reflect"
"runtime/debug"
"strconv"
"strings"
"sync/atomic"
"time"
"gopkg.in/inconshreveable/log15.v2"
"github.com/Sirupsen/logrus"
)
func NotFound(w http.ResponseWriter, r *http.Request) {
SendError(w, http.StatusNotFound, "Not found")
}
func EndpointNotFound(w http.ResponseWriter, r *http.Request) {
SendError(w, http.StatusNotFound, "Endpoint not found")
}
func InternalError(w http.ResponseWriter, err error) {
logrus.Error("internal server error response", "err", err, "stack", string(debug.Stack()))
SendError(w, http.StatusInternalServerError, "internal error")
}
func InternalErrorDetailed(w http.ResponseWriter, r *http.Request, err error) {
logrus.Error("internal server error response", "err", err, "endpoint", r.URL.String(), "token", GetTokenString(r), "stack", string(debug.Stack()))
SendError(w, http.StatusInternalServerError, "internal error")
}
type HTTPError interface {
error
StatusCode() int
}
type response struct {
Msg string `json:"msg"`
}
func SendError(w http.ResponseWriter, code int, msg string) {
logrus.Debug("HTTP error", "status_code", code, "msg", msg)
resp := response{Msg: msg}
RespondCode(w, nil, code, &resp)
}
func SendSuccess(w http.ResponseWriter, msg string, params map[string]interface{}) {
var v interface{}
if params == nil {
v = &response{Msg: msg}
} else {
v = params
}
RespondCode(w, nil, http.StatusOK, v)
}
func Respond(w http.ResponseWriter, r *http.Request, v interface{}) {
RespondCode(w, r, http.StatusOK, v)
}
func RespondCode(w http.ResponseWriter, r *http.Request, code int, v interface{}) {
bytes, err := json.Marshal(v)
if err != nil {
logrus.Error("error marshalling HTTP response", "value", v, "type", reflect.TypeOf(v), "err", err)
InternalError(w, err)
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Length", strconv.Itoa(len(bytes)))
w.WriteHeader(code)
if _, err := w.Write(bytes); err != nil {
// older callers don't pass a request
if r != nil {
logrus.Error("unable to write HTTP response", "err", err)
} else {
logrus.Error("unable to write HTTP response", "err", err)
}
}
}
// GetTokenString returns the token string from either the Authorization header or
// the oauth query parameter.
func GetTokenString(r *http.Request) string {
tok, _ := GetTokenStringType(r)
return tok
}
func GetTokenStringType(r *http.Request) (tok string, jwt bool) {
tokenStr := r.URL.Query().Get("oauth")
if tokenStr != "" {
return tokenStr, false
}
tokenStr = r.URL.Query().Get("jwt")
jwt = tokenStr != ""
if tokenStr == "" {
authHeader := r.Header.Get("Authorization")
authFields := strings.Fields(authHeader)
if len(authFields) == 2 && (authFields[0] == "OAuth" || authFields[0] == "JWT") {
jwt = authFields[0] == "JWT"
tokenStr = authFields[1]
}
}
return tokenStr, jwt
}
func ReadJSONSize(w http.ResponseWriter, r *http.Request, v interface{}, n int64) (success bool) {
contentType := r.Header.Get("Content-Type")
i := strings.IndexByte(contentType, ';')
if i < 0 {
i = len(contentType)
}
if strings.TrimRight(contentType[:i], " ") != "application/json" {
SendError(w, http.StatusBadRequest, "Bad Content-Type.")
return false
}
if i < len(contentType) {
param := strings.Trim(contentType[i+1:], " ")
split := strings.SplitN(param, "=", 2)
if len(split) != 2 || strings.Trim(split[0], " ") != "charset" {
SendError(w, http.StatusBadRequest, "Invalid Content-Type parameter.")
return false
}
value := strings.Trim(split[1], " ")
if len(value) > 2 && value[0] == '"' && value[len(value)-1] == '"' {
// quoted string
value = value[1 : len(value)-1]
}
if !strings.EqualFold(value, "utf-8") {
SendError(w, http.StatusBadRequest, "Unsupported charset. JSON is always UTF-8 encoded.")
return false
}
}
if r.ContentLength > n {
SendError(w, http.StatusBadRequest, fmt.Sprint("Content-Length greater than", n, "bytes"))
return false
}
err := json.NewDecoder(&LimitedReader{r.Body, n}).Decode(v)
if err != nil {
jsonError(w, err)
return false
}
return true
}
func ReadJSON(w http.ResponseWriter, r *http.Request, v interface{}) bool {
return ReadJSONSize(w, r, v, 100*0xffff)
}
// Same as io.LimitedReader, but returns limitReached so we can
// distinguish between the limit being reached and actual EOF.
type LimitedReader struct {
R io.Reader
N int64
}
func (l *LimitedReader) Read(p []byte) (n int, err error) {
if l.N <= 0 {
return 0, LimitReached(l.N)
}
if int64(len(p)) > l.N {
p = p[:l.N]
}
n, err = l.R.Read(p)
l.N -= int64(n)
return
}
// LimitedWriter writes until n bytes are written, then writes
// an overage line and skips any further writes.
type LimitedWriter struct {
W io.Writer
N int64
}
func (l *LimitedWriter) Write(p []byte) (n int, err error) {
var overrage = []byte("maximum log file size exceeded")
// we expect there may be concurrent writers, so to be safe..
left := atomic.LoadInt64(&l.N)
if left <= 0 {
return 0, io.EOF // TODO EOF? really? does it matter?
}
n, err = l.W.Write(p)
left = atomic.AddInt64(&l.N, -int64(n))
if left <= 0 {
l.W.Write(overrage)
}
return n, err
}
type JSONError string
func (e JSONError) Error() string { return string(e) }
func jsonType(t reflect.Type) string {
switch t.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16,
reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return "integer"
case reflect.Float32, reflect.Float64:
return "number"
case reflect.Map, reflect.Struct:
return "object"
case reflect.Slice, reflect.Array:
return "array"
case reflect.Ptr:
return jsonType(t.Elem())
}
// bool, string, other cases not covered
return t.String()
}
func jsonError(w http.ResponseWriter, err error) {
var msg string
switch err := err.(type) {
case *json.InvalidUTF8Error:
msg = "Invalid UTF-8 in JSON: " + err.S
case *json.InvalidUnmarshalError, *json.UnmarshalFieldError,
*json.UnsupportedTypeError:
// should never happen
InternalError(w, err)
return
case *json.SyntaxError:
msg = fmt.Sprintf("In JSON, %v at position %v.", err, err.Offset)
case *json.UnmarshalTypeError:
msg = fmt.Sprintf("In JSON, cannot use %v as %v", err.Value, jsonType(err.Type))
case *time.ParseError:
msg = "Time strings must be in RFC 3339 format."
case LimitReached:
msg = err.Error()
case JSONError:
msg = string(err)
default:
if err != io.EOF {
log15.Error("unhandled json.Unmarshal error", "type", reflect.TypeOf(err), "err", err)
}
msg = "Failed to decode JSON."
}
SendError(w, http.StatusBadRequest, msg)
}
type sizer interface {
Size() int64
}
type LimitReached int64
func (e LimitReached) Error() string {
return fmt.Sprint("Request body greater than", int64(e), "bytes")
}

35
api/models.go
View File

@@ -1,35 +0,0 @@
package api
type Route struct {
// TODO: Change destinations to a simple cache so it can expire entries after 55 minutes (the one we use in common?)
Host string `json:"host"`
Destinations []string `json:"destinations"`
ProjectId string `json:"project_id"`
Token string `json:"token"` // store this so we can queue up new workers on demand
CodeName string `json:"code_name"`
}
// for adding new hosts
type Route2 struct {
Host string `json:"host"`
Dest string `json:"dest"`
}
// An app is that base object for api gateway
type App struct {
Name string `json:"name"`
ProjectId string `json:"project_id"`
CloudFlareId string `json:"-"`
Dns string `json:"dns"`
Routes map[string]*Route3 `json:"routes"`
}
// this is for the new api gateway
type Route3 struct {
Path string `json:"path"` // run/app
Image string `json:"image"`
Type string `json:"type"`
ContainerPath string `json:"cpath"`
// maybe user a header map for whatever user wants to send?
ContentType string `json:"content_type"`
}

22
api/models/app.go Normal file
View File

@@ -0,0 +1,22 @@
package models
import "errors"
type Apps []App
var (
ErrAppsCreate = errors.New("Could not create app")
ErrAppsUpdate = errors.New("Could not update app")
ErrAppsRemoving = errors.New("Could not remove app from datastore")
ErrAppsGet = errors.New("Could not get app from datastore")
ErrAppsList = errors.New("Could not list apps from datastore")
ErrAppsNotFound = errors.New("App not found")
)
type App struct {
Name string `json:"name"`
Routes Routes `json:"routes"`
}
type AppFilter struct {
}

21
api/models/datastore.go Normal file
View File

@@ -0,0 +1,21 @@
package models
type Datastore interface {
GetApp(appName string) (*App, error)
GetApps(*AppFilter) ([]*App, error)
StoreApp(*App) (*App, error)
RemoveApp(appName string) error
GetRoute(appName, routeName string) (*Route, error)
GetRoutes(*RouteFilter) (routes []*Route, err error)
StoreRoute(*Route) (*Route, error)
RemoveRoute(appName, routeName string) error
}
func ApplyAppFilter(app *App, filter *AppFilter) bool {
return true
}
func ApplyRouteFilter(route *Route, filter *RouteFilter) bool {
return true
}

15
api/models/error.go Normal file
View File

@@ -0,0 +1,15 @@
package models
import "errors"
type Error struct {
Error *ErrorBody `json:"error,omitempty"`
}
func (m *Error) Validate() error {
return nil
}
var (
ErrInvalidJSON = errors.New("Could not create app")
)

11
api/models/error_body.go Normal file
View File

@@ -0,0 +1,11 @@
package models
type ErrorBody struct {
Fields string `json:"fields,omitempty"`
Message string `json:"message,omitempty"`
}
// Validate validates this error body
func (m *ErrorBody) Validate() error {
return nil
}

31
api/models/route.go Normal file
View File

@@ -0,0 +1,31 @@
package models
import (
"errors"
"net/http"
)
var (
ErrRoutesCreate = errors.New("Could not create route")
ErrRoutesUpdate = errors.New("Could not update route")
ErrRoutesRemoving = errors.New("Could not remove route from datastore")
ErrRoutesGet = errors.New("Could not get route from datastore")
ErrRoutesList = errors.New("Could not list routes from datastore")
ErrRoutesNotFound = errors.New("Route not found")
)
type Routes []Route
type Route struct {
Name string `json:"name"`
AppName string `json:"appname"`
Path string `json:"path"`
Image string `json:"image"`
Type string `json:"type"`
ContainerPath string `json:"container_path"`
Headers http.Header `json:"headers"`
}
type RouteFilter struct {
AppName string
}

125
api/runner.go → api/runner/runner.go
View File

@@ -1,27 +1,34 @@
package api
package runner
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"os"
"os/exec"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/iron-io/go/common"
"github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin"
"github.com/iron-io/functions/api/models"
)
type RunningApp struct {
Route *Route3
Route *models.Route
Port int
ContainerName string
}
var (
ErrRunnerRouteNotFound = errors.New("Route not found on that application")
)
var runningImages map[string]*RunningApp
func init() {
@@ -29,61 +36,56 @@ func init() {
fmt.Println("ENV:", os.Environ())
}
func Run(w http.ResponseWriter, req *http.Request) {
fmt.Println("RUN!!!!")
log.Infoln("HOST:", req.Host)
rUrl := req.URL
appName := rUrl.Query().Get("app")
log.Infoln("app_name", appName, "path:", req.URL.Path)
if appName != "" {
// passed in the name
} else {
host := strings.Split(req.Host, ":")[0]
func Run(c *gin.Context) error {
log := c.MustGet("log").(logrus.FieldLogger)
store := c.MustGet("store").(models.Datastore)
appName := c.Param("app")
if appName == "" {
host := strings.Split(c.Request.Host, ":")[0]
appName = strings.Split(host, ".")[0]
log.Infoln("app_name from host", appName)
}
app, err := getApp(appName)
filter := &models.RouteFilter{
AppName: appName,
}
routes, err := store.GetRoutes(filter)
if err != nil {
common.SendError(w, 400, fmt.Sprintln("This app does not exist. Please create app first.", err))
return
return err
}
log.Infoln("app", app)
// find route
for _, el := range app.Routes {
// TODO: copy/use gorilla's pattern matching here
if el.Path == req.URL.Path {
// Boom, run it!
route := c.Param("route")
log.WithFields(logrus.Fields{"app": appName}).Debug("Running app")
for _, el := range routes {
if el.Path == route {
err = checkAndPull(el.Image)
if err != nil {
common.SendError(w, 404, fmt.Sprintln("The image could not be pulled:", err))
return
return err
}
if el.Type == "app" {
DockerHost(el, w)
return
} else { // "run"
// TODO: timeout 59 seconds
DockerRun(el, w, req)
return
return DockerHost(el, c)
} else {
return DockerRun(el, c)
}
}
}
common.SendError(w, 404, fmt.Sprintln("The requested endpoint does not exist."))
return ErrRunnerRouteNotFound
}
// TODO: use Docker utils from docker-job for this and a few others in here
func DockerRun(route *Route3, w http.ResponseWriter, req *http.Request) {
log.Infoln("route:", route)
func DockerRun(route *models.Route, c *gin.Context) error {
image := route.Image
payload, err := ioutil.ReadAll(req.Body)
payload, err := ioutil.ReadAll(c.Request.Body)
if err != nil {
log.WithError(err).Errorln("Error reading request body")
return
return err
}
log.WithField("payload", "---"+string(payload)+"---").Infoln("incoming request")
log.WithField("image", image).Infoln("About to run using this image")
// log.WithField("payload", "---"+string(payload)+"---").Infoln("incoming request")
// log.WithField("image", image).Infoln("About to run using this image")
// TODO: swap all this out with Titan's running via API
cmd := exec.Command("docker", "run", "--rm", "-i", "-e", fmt.Sprintf("PAYLOAD=%v", string(payload)), image)
@@ -107,23 +109,24 @@ func DockerRun(route *Route3, w http.ResponseWriter, req *http.Request) {
log.Printf("Waiting for command to finish...")
if err = cmd.Wait(); err != nil {
// job failed
log.Infoln("job finished with err:", err)
log.WithFields(log.Fields{"metric": "run.errors", "value": 1, "type": "count"}).Infoln("failed run")
// log.Infoln("job finished with err:", err)
// log.WithFields(log.Fields{"metric": "run.errors", "value": 1, "type": "count"}).Infoln("failed run")
return err
// TODO: wrap error in json "error": buff
} else {
log.Infoln("Docker ran successfully:", b.String())
// print
log.WithFields(log.Fields{"metric": "run.success", "value": 1, "type": "count"}).Infoln("successful run")
}
log.WithFields(log.Fields{"metric": "run", "value": 1, "type": "count"}).Infoln("job ran")
// log.Infoln("Docker ran successfully:", b.String())
// print
// log.WithFields(log.Fields{"metric": "run.success", "value": 1, "type": "count"}).Infoln("successful run")
// log.WithFields(log.Fields{"metric": "run", "value": 1, "type": "count"}).Infoln("job ran")
buff.Flush()
if route.ContentType != "" {
w.Header().Set("Content-Type", route.ContentType)
}
fmt.Fprintln(w, string(bytes.Trim(b.Bytes(), "\x00")))
c.Data(http.StatusOK, "", bytes.Trim(b.Bytes(), "\x00"))
return nil
}
func DockerHost(el *Route3, w http.ResponseWriter) {
func DockerHost(el *models.Route, c *gin.Context) error {
ra := runningImages[el.Image]
if ra == nil {
ra = &RunningApp{}
@@ -134,11 +137,11 @@ func DockerHost(el *Route3, w http.ResponseWriter) {
// TODO: timeout 59 minutes. Mark it in ra as terminated.
cmd := exec.Command("docker", "run", "--name", ra.ContainerName, "--rm", "-i", "-p", fmt.Sprintf("%v:8080", ra.Port), el.Image)
// TODO: What should we do with the output here? Store it? Send it to a log service?
cmd.Stdout = os.Stdout
// cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
// TODO: Need to catch interrupt and stop all containers that are started, see devo/dj for how to do this
if err := cmd.Start(); err != nil {
log.Fatal(err)
return err
// TODO: What if the app fails to start? Don't want to keep starting the container
}
} else {
@@ -149,16 +152,16 @@ func DockerHost(el *Route3, w http.ResponseWriter) {
// TODO: if connection fails, check if container still running? If not, start it again
resp, err := http.Get(fmt.Sprintf("http://0.0.0.0:%v%v", ra.Port, el.ContainerPath))
if err != nil {
common.SendError(w, 404, fmt.Sprintln("The requested app endpoint does not exist.", err))
return
return err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
common.SendError(w, 500, fmt.Sprintln("Error reading response body", err))
return
return err
}
fmt.Fprintln(w, string(body))
c.Data(http.StatusOK, "", body)
return nil
}
func checkAndPull(image string) error {
@@ -193,7 +196,9 @@ func execAndPrint(cmdstr string, args []string) error {
log.Printf("Waiting for cmd to finish...")
err = cmd.Wait()
fmt.Println("stderr:", berr.String())
if berr.Len() != 0 {
fmt.Println("stderr:", berr.String())
}
fmt.Println("stdout:", bout.String())
return err
}

15
api/server/config.go Normal file
View File

@@ -0,0 +1,15 @@
package server
type Config struct {
DatabaseURL string `json:"db"`
Logging struct {
To string `json:"to"`
Level string `json:"level"`
Prefix string `json:"prefix"`
}
}
func (c *Config) Validate() error {
// TODO:
return nil
}

259
api/server/datastore/bolt/bolt.go Normal file
View File

@@ -0,0 +1,259 @@
package bolt
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"github.com/Sirupsen/logrus"
"github.com/boltdb/bolt"
"github.com/iron-io/functions/api/models"
)
type BoltDatastore struct {
routesBucket []byte
appsBucket []byte
logsBucket []byte
db *bolt.DB
log logrus.FieldLogger
}
func New(url *url.URL) (models.Datastore, error) {
dir := filepath.Dir(url.Path)
log := logrus.WithFields(logrus.Fields{"db": url.Scheme, "dir": dir})
err := os.MkdirAll(dir, 0777)
if err != nil {
log.WithError(err).Errorln("Could not create data directory for db")
return nil, err
}
log.Infoln("Creating bolt db at ", url.Path)
db, err := bolt.Open(url.Path, 0600, nil)
if err != nil {
log.WithError(err).Errorln("Error on bolt.Open")
return nil, err
}
bucketPrefix := "fns-"
if url.Query()["bucket"] != nil {
bucketPrefix = url.Query()["bucket"][0]
}
routesBucketName := []byte(bucketPrefix + "routes")
appsBucketName := []byte(bucketPrefix + "apps")
logsBucketName := []byte(bucketPrefix + "logs")
err = db.Update(func(tx *bolt.Tx) error {
for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName} {
_, err := tx.CreateBucketIfNotExists(name)
if err != nil {
log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket")
return err
}
}
return nil
})
if err != nil {
log.WithError(err).Errorln("Error creating bolt buckets")
return nil, err
}
ds := &BoltDatastore{
routesBucket: routesBucketName,
appsBucket: appsBucketName,
logsBucket: logsBucketName,
db: db,
log: log,
}
log.WithFields(logrus.Fields{"prefix": bucketPrefix, "file": url.Path}).Info("BoltDB initialized")
return ds, nil
}
func (ds *BoltDatastore) StoreApp(app *models.App) (*models.App, error) {
err := ds.db.Update(func(tx *bolt.Tx) error {
bIm := tx.Bucket(ds.appsBucket)
buf, err := json.Marshal(app)
if err != nil {
return err
}
err = bIm.Put([]byte(app.Name), buf)
if err != nil {
return err
}
bjParent := tx.Bucket(ds.routesBucket)
_, err = bjParent.CreateBucketIfNotExists([]byte(app.Name))
if err != nil {
return err
}
return nil
})
return app, err
}
func (ds *BoltDatastore) RemoveApp(appName string) error {
err := ds.db.Update(func(tx *bolt.Tx) error {
bIm := tx.Bucket(ds.appsBucket)
err := bIm.Delete([]byte(appName))
if err != nil {
return err
}
bjParent := tx.Bucket(ds.routesBucket)
err = bjParent.DeleteBucket([]byte(appName))
if err != nil {
return err
}
return nil
})
return err
}
func (ds *BoltDatastore) GetApps(filter *models.AppFilter) ([]*models.App, error) {
res := []*models.App{}
err := ds.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(ds.appsBucket)
err2 := b.ForEach(func(key, v []byte) error {
app := &models.App{}
err := json.Unmarshal(v, app)
if err != nil {
return err
}
res = append(res, app)
return nil
})
if err2 != nil {
logrus.WithError(err2).Errorln("Couldn't get apps!")
}
return nil
})
if err != nil {
return nil, err
}
return res, nil
}
func (ds *BoltDatastore) GetApp(name string) (*models.App, error) {
var res *models.App
err := ds.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket(ds.appsBucket)
v := b.Get([]byte(name))
if v != nil {
app := &models.App{}
err := json.Unmarshal(v, app)
if err != nil {
return err
}
res = app
}
return nil
})
if err != nil {
return nil, err
}
return res, nil
}
func (ds *BoltDatastore) getRouteBucketForApp(tx *bolt.Tx, appName string) (*bolt.Bucket, error) {
var err error
bp := tx.Bucket(ds.routesBucket)
b := bp.Bucket([]byte(appName))
if b == nil {
b, err = bp.CreateBucket([]byte(appName))
if err != nil {
return nil, err
}
}
return b, nil
}
func (ds *BoltDatastore) StoreRoute(route *models.Route) (*models.Route, error) {
err := ds.db.Update(func(tx *bolt.Tx) error {
b, err := ds.getRouteBucketForApp(tx, route.AppName)
if err != nil {
return err
}
buf, err := json.Marshal(route)
if err != nil {
return err
}
err = b.Put([]byte(route.Name), buf)
if err != nil {
return err
}
return nil
})
if err != nil {
return nil, err
}
return route, nil
}
func (ds *BoltDatastore) RemoveRoute(appName, routeName string) error {
err := ds.db.Update(func(tx *bolt.Tx) error {
b, err := ds.getRouteBucketForApp(tx, appName)
if err != nil {
return err
}
err = b.Delete([]byte(routeName))
if err != nil {
return err
}
return nil
})
if err != nil {
return err
}
return nil
}
func (ds *BoltDatastore) GetRoute(appName, routeName string) (*models.Route, error) {
var route models.Route
err := ds.db.View(func(tx *bolt.Tx) error {
b, err := ds.getRouteBucketForApp(tx, appName)
if err != nil {
return err
}
v := b.Get([]byte(routeName))
if v == nil {
return models.ErrRoutesNotFound
}
err = json.Unmarshal(v, &route)
return err
})
return &route, err
}
func (ds *BoltDatastore) GetRoutes(filter *models.RouteFilter) ([]*models.Route, error) {
res := []*models.Route{}
err := ds.db.View(func(tx *bolt.Tx) error {
b, err := ds.getRouteBucketForApp(tx, filter.AppName)
if err != nil {
return err
}
i := 0
c := b.Cursor()
var k, v []byte
k, v = c.Last()
// Iterate backwards, newest first
for ; k != nil; k, v = c.Prev() {
var route models.Route
err := json.Unmarshal(v, &route)
if err != nil {
return err
}
if models.ApplyRouteFilter(&route, filter) {
i++
res = append(res, &route)
}
}
return nil
})
if err != nil {
return nil, err
}
return res, nil
}

27
api/server/datastore/datastore.go Normal file
View File

@@ -0,0 +1,27 @@
package datastore
import (
"fmt"
"net/url"
"github.com/Sirupsen/logrus"
"github.com/iron-io/functions/api/models"
"github.com/iron-io/functions/api/server/datastore/bolt"
"github.com/iron-io/functions/api/server/datastore/postgres"
)
func New(dbURL string) (models.Datastore, error) {
u, err := url.Parse(dbURL)
if err != nil {
logrus.WithFields(logrus.Fields{"url": dbURL}).Fatal("bad DB URL")
}
logrus.WithFields(logrus.Fields{"db": u.Scheme}).Info("creating new datastore")
switch u.Scheme {
case "bolt":
return bolt.New(u)
case "postgres":
return postgres.New(u)
default:
return nil, fmt.Errorf("db type not supported %v", u.Scheme)
}
}

262
api/server/datastore/postgres/postgres.go Normal file
View File

@@ -0,0 +1,262 @@
package postgres
import (
"database/sql"
"encoding/json"
"fmt"
"net/url"
"github.com/Sirupsen/logrus"
"github.com/iron-io/functions/api/models"
_ "github.com/lib/pq"
)
const routesTableCreate = `CREATE TABLE IF NOT EXISTS routes (
name character varying(256) NOT NULL PRIMARY KEY,
path text NOT NULL,
app_name character varying(256) NOT NULL,
image character varying(256) NOT NULL,
type character varying(256) NOT NULL,
container_path text NOT NULL,
headers text NOT NULL,
);`
const appsTableCreate = `CREATE TABLE IF NOT EXISTS apps (
name character varying(256) NOT NULL PRIMARY KEY,
);`
const routeSelector = `SELECT path, app_name, image, type, container_path, headers FROM routes`
// Tries to read in properties into `route` from `scanner`. Bubbles up errors.
// Capture sql.Row and sql.Rows
type rowScanner interface {
Scan(dest ...interface{}) error
}
// Capture sql.DB and sql.Tx
type rowQuerier interface {
QueryRow(query string, args ...interface{}) *sql.Row
}
func scanRoute(scanner rowScanner, route *models.Route) error {
err := scanner.Scan(
&route.Name,
&route.Path,
&route.AppName,
&route.Image,
&route.Type,
&route.ContainerPath,
&route.Headers,
)
return err
}
type PostgresDatastore struct {
db *sql.DB
}
func New(url *url.URL) (models.Datastore, error) {
db, err := sql.Open("postgres", url.String())
if err != nil {
return nil, err
}
err = db.Ping()
if err != nil {
return nil, err
}
maxIdleConns := 30 // c.MaxIdleConnections
db.SetMaxIdleConns(maxIdleConns)
logrus.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns}).Info("Postgres dialed")
pg := &PostgresDatastore{
db: db,
}
for _, v := range []string{routesTableCreate, appsTableCreate} {
_, err = db.Exec(v)
if err != nil {
return nil, err
}
}
return pg, nil
}
func (ds *PostgresDatastore) StoreRoute(route *models.Route) (*models.Route, error) {
var headers string
hbyte, err := json.Marshal(route.Headers)
if err != nil {
return nil, err
}
headers = string(hbyte)
err = ds.db.QueryRow(`
INSERT INTO routes (
name, app_name, path, image,
type, container_path, headers
)
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
route.Name,
route.AppName,
route.Path,
route.Image,
route.Type,
route.ContainerPath,
headers,
).Scan(nil)
if err != nil {
return nil, err
}
return route, nil
}
func (ds *PostgresDatastore) RemoveRoute(appName, routeName string) error {
err := ds.db.QueryRow(`
DELETE FROM routes
WHERE name = $1`,
routeName,
).Scan(nil)
if err != nil {
return err
}
return nil
}
func getRoute(qr rowQuerier, routeName string) (*models.Route, error) {
var route models.Route
row := qr.QueryRow(fmt.Sprintf("%s WHERE name=$1", routeSelector), routeName)
err := scanRoute(row, &route)
if err == sql.ErrNoRows {
return nil, models.ErrRoutesNotFound
} else if err != nil {
return nil, err
}
return &route, nil
}
func (ds *PostgresDatastore) GetRoute(appName, routeName string) (*models.Route, error) {
return getRoute(ds.db, routeName)
}
// TODO: Add pagination
func (ds *PostgresDatastore) GetRoutes(filter *models.RouteFilter) ([]*models.Route, error) {
res := []*models.Route{}
filterQuery := buildFilterQuery(filter)
rows, err := ds.db.Query(fmt.Sprintf("%s %s", routeSelector, filterQuery))
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var route models.Route
err := scanRoute(rows, &route)
if err != nil {
return nil, err
}
res = append(res, &route)
}
if err := rows.Err(); err != nil {
return nil, err
}
return res, nil
}
func (ds *PostgresDatastore) GetApp(name string) (*models.App, error) {
row := ds.db.QueryRow("SELECT * FROM groups WHERE name=$1", name)
var resName string
err := row.Scan(&resName)
res := &models.App{
Name: resName,
}
if err != nil {
return nil, err
}
return res, nil
}
func (ds *PostgresDatastore) GetApps(filter *models.AppFilter) ([]*models.App, error) {
res := []*models.App{}
rows, err := ds.db.Query(`
SELECT DISTINCT *
FROM apps
ORDER BY name`,
)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var app models.App
err := rows.Scan(&app)
if err != nil {
return nil, err
}
res = append(res, &app)
}
if err := rows.Err(); err != nil {
return nil, err
}
return res, nil
}
func (ds *PostgresDatastore) StoreApp(app *models.App) (*models.App, error) {
err := ds.db.QueryRow(`
INSERT INTO apps (name)
VALUES ($1)
`, app.Name).Scan(&app.Name)
// ON CONFLICT (name) DO UPDATE SET created_at = $2;
if err != nil {
return nil, err
}
return app, nil
}
func (ds *PostgresDatastore) RemoveApp(appName string) error {
err := ds.db.QueryRow(`
DELETE FROM apps
WHERE name = $1
`, appName).Scan(nil)
if err != nil {
return err
}
return nil
}
func buildFilterQuery(filter *models.RouteFilter) string {
filterQuery := ""
filterQueries := []string{}
if filter.AppName != "" {
filterQueries = append(filterQueries, fmt.Sprintf("app_name = '%s'", filter.AppName))
}
for i, field := range filterQueries {
if i == 0 {
filterQuery = fmt.Sprintf("WHERE %s ", field)
} else {
filterQuery = fmt.Sprintf("%s AND %s", filterQuery, field)
}
}
return filterQuery
}

58
api/server/server.go Normal file
View File

@@ -0,0 +1,58 @@
package server
import (
"fmt"
"os"
"path"
"github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin"
"github.com/iron-io/functions/api/server/datastore"
"github.com/iron-io/functions/api/server/router"
)
type Server struct {
router *gin.Engine
cfg *Config
}
func New(config *Config) *Server {
return &Server{
router: gin.Default(),
cfg: config,
}
}
func extractFields(c *gin.Context) logrus.Fields {
fields := logrus.Fields{"action": path.Base(c.HandlerName())}
for _, param := range c.Params {
fields[param.Key] = param.Value
}
return fields
}
func (s *Server) Start() {
if s.cfg.DatabaseURL == "" {
cwd, _ := os.Getwd()
s.cfg.DatabaseURL = fmt.Sprintf("bolt://%s/bolt.db?bucket=fns", cwd)
}
ds, err := datastore.New(s.cfg.DatabaseURL)
if err != nil {
logrus.WithError(err).Fatalln("Invalid DB url.")
}
logrus.SetOutput(os.Stdout)
logrus.SetLevel(logrus.DebugLevel)
s.router.Use(func(c *gin.Context) {
c.Set("store", ds)
c.Set("log", logrus.WithFields(extractFields(c)))
c.Next()
})
router.Start(s.router)
// Default to :8080
s.router.Run()
}

3
api/version.go
View File

@@ -1,3 +0,0 @@
package api
const Version = "0.0.28"