alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

mask models.Call blank fields in api, sqlx

Browse Source 
sqlx has nice facilities for using structs to do queries and using their
fields, so decided to move us all over to this. now when you take a look at
models.Call it's really obvious what's in db and what's not. added omitempty
to some json fields that were bleeding through api too.

deletes a lot of code in the sql package for scanning and made some queries
use struct based sqlx methods now which seem easier to read than what we
previously had. moves all json stuff into sql.Valuer and sql.Scanner methods
in models/config.go, these are the only 2 types that ever need this. sadly,
sqlx would have done this marshaling for us, but to keep compat, I added json.
we can do some migrations later maybe for a more efficient encoding, but did
not want to fuss with it today.

it seems like we should probably aim to keep models.Call as small as possible
in the db as there will be a lot of them. interestingly, most functions
platforms I looked at do not seem to expose this kind of information that I
could find. so, i think only having timestamps, status, id, app, path and
maybe docker stats is really all that should be in here (agree w/ Denys on
284 as these and logs will end up taking up most db space in prod. notably,
payload, headers, and env vars could be extremely large and in the general
case they are always a copy of the routes (this breaks apart when routes are
updated, which would be useful considering we don't have versioning --
versioning may be cheaper).

removed unused field in apps too

this is lined up behind #349 so that I could use the tests...

closes #345
closes #142
closes #284
This commit is contained in:
Reed Allman
2017-09-20 05:48:37 -07:00
parent 3083415611
commit 46dfbd362d
7 changed files with 198 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
api/models/app.go
View File

@@ -1,9 +1,8 @@
package models
type App struct {
Name string `json:"name"`
Routes Routes `json:"routes,omitempty"`
Config `json:"config"`
Name string `json:"name" db:"name"`
Config Config `json:"config" db:"config"`
}
func (a *App) Validate() error {
@@ -24,11 +23,6 @@ func (a *App) Validate() error {
func (a *App) Clone() *App {
var c App
c.Name = a.Name
if a.Routes != nil {
for i := range a.Routes {
c.Routes = append(c.Routes, a.Routes[i].Clone())
}
}
if a.Config != nil {
c.Config = make(Config)
for k, v := range a.Config {

46
api/models/call.go
View File

@@ -23,15 +23,15 @@ const (
var possibleStatuses = [...]string{"delayed", "queued", "running", "success", "error", "cancelled"}
type CallLog struct {
CallID string `json:"call_id"`
Log string `json:"log"`
AppName string `json:"app_name"`
CallID string `json:"call_id" db:"id"`
Log string `json:"log" db:"log"`
AppName string `json:"app_name" db:"app_name"`
}
// Call is a representation of a specific invocation of a route.
type Call struct {
// Unique identifier representing a specific call.
ID string `json:"id"`
ID string `json:"id" db:"id"`
// NOTE: this is stale, retries are not implemented atm, but this is nice, so leaving
// States and valid transitions.
@@ -68,65 +68,65 @@ type Call struct {
// - bad_exit - exited with non-zero status due to program termination/crash.
// * cancelled - cancelled via API. More information in the reason field.
// - client_request - Request was cancelled by a client.
Status string `json:"status"`
Status string `json:"status" db:"status"`
// App this call belongs to.
AppName string `json:"app_name"`
AppName string `json:"app_name" db:"app_name"`
// Path of the route that is responsible for this call
Path string `json:"path"`
Path string `json:"path" db:"path"`
// Name of Docker image to use.
Image string `json:"image"`
Image string `json:"image,omitempty" db:"-"`
// Number of seconds to wait before queueing the call for consumption for the
// first time. Must be a positive integer. Calls with a delay start in state
// "delayed" and transition to "running" after delay seconds.
Delay int32 `json:"delay,omitempty"`
Delay int32 `json:"delay,omitempty" db:"-"`
// Type indicates whether a task is to be run synchronously or asynchronously.
Type string `json:"type,omitempty"`
Type string `json:"type,omitempty" db:"-"`
// Format is the format to pass input into the function.
Format string `json:"format,omitempty"`
Format string `json:"format,omitempty" db:"-"`
// Payload for the call. This is only used by async calls, to store their input.
// TODO should we copy it into here too for debugging sync?
Payload string `json:"payload,omitempty"`
Payload string `json:"payload,omitempty" db:"-"`
// Full request url that spawned this invocation.
URL string `json:"url,omitempty"`
URL string `json:"url,omitempty" db:"-"`
// Method of the http request used to make this call.
Method string `json:"method,omitempty"`
Method string `json:"method,omitempty" db:"-"`
// Priority of the call. Higher has more priority. 3 levels from 0-2. Calls
// at same priority are processed in FIFO order.
Priority *int32 `json:"priority"`
Priority *int32 `json:"priority,omitempty" db:"-"`
// Maximum runtime in seconds.
Timeout int32 `json:"timeout,omitempty"`
Timeout int32 `json:"timeout,omitempty" db:"-"`
// Hot function idle timeout in seconds before termination.
IdleTimeout int32 `json:"idle_timeout,omitempty"`
IdleTimeout int32 `json:"idle_timeout,omitempty" db:"-"`
// Memory is the amount of RAM this call is allocated.
Memory uint64 `json:"memory,omitempty"`
Memory uint64 `json:"memory,omitempty" db:"-"`
// BaseEnv are the env vars for hot containers, not request specific.
BaseEnv map[string]string `json:"base_env,omitempty"`
BaseEnv map[string]string `json:"base_env,omitempty" db:"-"`
// Env vars for the call. Comes from the ones set on the Route.
EnvVars map[string]string `json:"env_vars,omitempty"`
EnvVars map[string]string `json:"env_vars,omitempty" db:"-"`
// Time when call completed, whether it was successul or failed. Always in UTC.
CompletedAt strfmt.DateTime `json:"completed_at,omitempty"`
CompletedAt strfmt.DateTime `json:"completed_at,omitempty" db:"completed_at"`
// Time when call was submitted. Always in UTC.
CreatedAt strfmt.DateTime `json:"created_at,omitempty"`
CreatedAt strfmt.DateTime `json:"created_at,omitempty" db:"created_at"`
// Time when call started execution. Always in UTC.
StartedAt strfmt.DateTime `json:"started_at,omitempty"`
StartedAt strfmt.DateTime `json:"started_at,omitempty" db:"started_at"`
}
type CallFilter struct {

89
api/models/config.go
View File

@@ -1,7 +1,96 @@
package models
import (
"bytes"
"database/sql/driver"
"encoding/json"
"fmt"
"net/http"
)
type Config map[string]string
func (c *Config) Validate() error {
return nil
}
// implements sql.Valuer, returning a string
func (c Config) Value() (driver.Value, error) {
if len(c) < 1 {
return driver.Value(string("")), nil
}
var b bytes.Buffer
err := json.NewEncoder(&b).Encode(c)
// return a string type
return driver.Value(b.String()), err
}
// implements sql.Scanner
func (c *Config) Scan(value interface{}) error {
if value == nil {
*c = nil
return nil
}
bv, err := driver.String.ConvertValue(value)
if err == nil {
var b []byte
switch x := bv.(type) {
case []byte:
b = x
case string:
b = []byte(x)
}
if len(b) > 0 {
return json.Unmarshal(b, c)
} else {
*c = nil
return nil
}
}
// otherwise, return an error
return fmt.Errorf("config invalid db format: %T %T value, err: %v", value, bv, err)
}
// Headers is an http.Header that implements additional methods.
type Headers http.Header
// implements sql.Valuer, returning a string
func (h Headers) Value() (driver.Value, error) {
if len(h) < 1 {
return driver.Value(string("")), nil
}
var b bytes.Buffer
err := json.NewEncoder(&b).Encode(h)
// return a string type
return driver.Value(b.String()), err
}
// implements sql.Scanner
func (h *Headers) Scan(value interface{}) error {
if value == nil {
*h = nil
return nil
}
bv, err := driver.String.ConvertValue(value)
if err == nil {
var b []byte
switch x := bv.(type) {
case []byte:
b = x
case string:
b = []byte(x)
}
if len(b) > 0 {
return json.Unmarshal(b, h)
} else {
*h = nil
return nil
}
}
// otherwise, return an error
return fmt.Errorf("headers invalid db format: %T %T value, err: %v", value, bv, err)
}

26
api/models/route.go
View File

@@ -15,16 +15,16 @@ const (
type Routes []*Route
type Route struct {
AppName string `json:"app_name"`
Path string `json:"path"`
Image string `json:"image"`
Memory uint64 `json:"memory"`
Headers http.Header `json:"headers"`
Type string `json:"type"`
Format string `json:"format"`
Timeout int32 `json:"timeout"`
IdleTimeout int32 `json:"idle_timeout"`
Config `json:"config"`
AppName string `json:"app_name" db:"app_name"`
Path string `json:"path" db:"path"`
Image string `json:"image" db:"image"`
Memory uint64 `json:"memory" db:"memory"`
Headers Headers `json:"headers" db:"headers"`
Type string `json:"type" db:"type"`
Format string `json:"format" db":format"`
Timeout int32 `json:"timeout" db:"timeout"`
IdleTimeout int32 `json:"idle_timeout" db:"idle_timeout"`
Config Config `json:"config" db:"config"`
}
// SetDefaults sets zeroed field to defaults.
@@ -42,7 +42,7 @@ func (r *Route) SetDefaults() {
}
if r.Headers == nil {
r.Headers = http.Header{}
r.Headers = Headers(http.Header{})
}
if r.Config == nil {
@@ -144,11 +144,11 @@ func (r *Route) Update(new *Route) {
}
if new.Headers != nil {
if r.Headers == nil {
r.Headers = make(http.Header)
r.Headers = Headers(make(http.Header))
}
for k, v := range new.Headers {
if len(v) == 0 {
r.Headers.Del(k)
http.Header(r.Headers).Del(k)
} else {
r.Headers[k] = v
}