clean up hotf(x) concurrency, rm max c

this patch gets rid of max concurrency for functions altogether, as discussed,
since it will be challenging to support across functions nodes. as a result of
doing so, the previous version of functions would fall over when offered 1000
functions, so there was some work needed in order to push this through.
further work is necessary as docker basically falls over when trying to start
enough containers at the same time, and with this patch essentially every
function can scale infinitely. it seems like we could add some kind of
adaptive restrictions based on task run length and configured wait time so
that fast running functions will line up to run in a hot container instead of
them all creating new hot containers.

this patch takes a first cut at whacking out some of the insanity that was the
previous concurrency model, which was problematic in that it limited
concurrency significantly across all functions since every task went through
the same unbuffered channel, which could create blocking issues for all
functions if the channel is not picked off fast enough (it's not apparent that
this was impossible in the previous implementation). in any event, each
request has a goroutine already, there's no reason not to use it. not too hard
to wrap a map in a lock, not sure what the benefits were (added insanity?) in effect
this is marginally easier to understand and less insane (marginally). after
getting rid of max c this adds a blocking mechanism for the first invocation
of any function so that all other hot functions will wait on the first one to
finish to avoid a herd issue (was making docker die...) -- this could be
slightly improved, but works in a pinch. reduced some memory usage by having
redundant maps of htfnsvr's and task.Requests (by a factor of 2!). cleaned up
some of the protocol stuff, need to clean this up further. anyway, it's a
first cut. have another patch that rewrites all of it but was getting into
rabbit hole territory, would be happy to oblige if anybody else has problems
understanding this rat's nest of channels. there is a good bit of work left to
make this prod ready (regardless of removing max c).

a warning that this will break the db schemas, didn't put the effort in to add
migration stuff since this isn't deployed anywhere in prod...

TODO need to clean out the htfnmgr bucket with LRU
TODO need to clean up runner interface
TODO need to unify the task running paths across protocols
TODO need to move the ram checking stuff into worker for noted reasons
TODO need better elasticity of hot f(x) containers

api/datastore/internal/datastoreutil/shared.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"database/sql"
 	"encoding/json"
-	"github.com/Sirupsen/logrus"
 	"fmt"
+	"github.com/Sirupsen/logrus"
 	"strings"
 
 	"gitlab-odx.oracle.com/odx/functions/api/models"
@@ -15,7 +15,6 @@ type RowScanner interface {
 	Scan(dest ...interface{}) error
 }
 
-
 func ScanRoute(scanner RowScanner, route *models.Route) error {
 	var headerStr string
 	var configStr string
@@ -25,7 +24,6 @@ func ScanRoute(scanner RowScanner, route *models.Route) error {
 		&route.Path,
 		&route.Image,
 		&route.Format,
-		&route.MaxConcurrency,
 		&route.Memory,
 		&route.Type,
 		&route.Timeout,
@@ -107,7 +105,6 @@ func BuildFilterRouteQuery(filter *models.RouteFilter, whereStm, andStm string)
 	return b.String(), args
 }
 
-
 func BuildFilterAppQuery(filter *models.AppFilter, whereStm string) (string, []interface{}) {
 	if filter == nil {
 		return "", nil
@@ -120,7 +117,6 @@ func BuildFilterAppQuery(filter *models.AppFilter, whereStm string) (string, []i
 	return "", nil
 }
 
-
 func BuildFilterCallQuery(filter *models.CallFilter, whereStm, andStm string) (string, []interface{}) {
 	if filter == nil {
 		return "", nil
@@ -269,7 +265,6 @@ func NewDatastore(dataSourceName, dialect string, tables []string) (*sql.DB, err
 	logrus.WithFields(logrus.Fields{"max_idle_connections": maxIdleConns}).Info(
 		fmt.Sprintf("%v datastore dialed", dialect))
 
-
 	for _, v := range tables {
 		_, err = db.Exec(v)
 		if err != nil {
@@ -372,4 +367,4 @@ func SQLRemoveRoute(db *sql.DB, appName, routePath, deleteStm string) error {
 
 	return nil
 
-}
+}

api/datastore/mysql/mysql.go

@@ -18,7 +18,6 @@ const routesTableCreate = `CREATE TABLE IF NOT EXISTS routes (
 	path varchar(256) NOT NULL,
 	image varchar(256) NOT NULL,
 	format varchar(16) NOT NULL,
-	maxc int NOT NULL,
 	memory int NOT NULL,
 	timeout int NOT NULL,
 	idle_timeout int NOT NULL,
@@ -38,7 +37,7 @@ const extrasTableCreate = `CREATE TABLE IF NOT EXISTS extras (
 	value varchar(256) NOT NULL
 );`
 
-const routeSelector = `SELECT app_name, path, image, format, maxc, memory, type, timeout, idle_timeout, headers, config FROM routes`
+const routeSelector = `SELECT app_name, path, image, format, memory, type, timeout, idle_timeout, headers, config FROM routes`
 
 const callTableCreate = `CREATE TABLE IF NOT EXISTS calls (
 	created_at varchar(256) NOT NULL,
@@ -53,7 +52,6 @@ const callTableCreate = `CREATE TABLE IF NOT EXISTS calls (
 
 const callSelector = `SELECT id, created_at, started_at, completed_at, status, app_name, path FROM calls`
 
-
 /*
 MySQLDatastore defines a basic MySQL Datastore struct.
 */
@@ -237,7 +235,6 @@ func (ds *MySQLDatastore) InsertRoute(ctx context.Context, route *models.Route)
 			path,
 			image,
 			format,
-			maxc,
 			memory,
 			type,
 			timeout,
@@ -250,7 +247,6 @@ func (ds *MySQLDatastore) InsertRoute(ctx context.Context, route *models.Route)
 			route.Path,
 			route.Image,
 			route.Format,
-			route.MaxConcurrency,
 			route.Memory,
 			route.Type,
 			route.Timeout,
@@ -296,7 +292,6 @@ func (ds *MySQLDatastore) UpdateRoute(ctx context.Context, newroute *models.Rout
 		UPDATE routes SET
 			image = ?,
 			format = ?,
-			maxc = ?,
 			memory = ?,
 			type = ?,
 			timeout = ?,
@@ -306,7 +301,6 @@ func (ds *MySQLDatastore) UpdateRoute(ctx context.Context, newroute *models.Rout
 		WHERE app_name = ? AND path = ?;`,
 			route.Image,
 			route.Format,
-			route.MaxConcurrency,
 			route.Memory,
 			route.Type,
 			route.Timeout,
@@ -341,7 +335,6 @@ RemoveRoute removes an existing route on MySQL.
 */
 func (ds *MySQLDatastore) RemoveRoute(ctx context.Context, appName, routePath string) error {
 	deleteStm := `DELETE FROM routes WHERE path = ? AND app_name = ?`
-
 	return datastoreutil.SQLRemoveRoute(ds.db, appName, routePath, deleteStm)
 }
 
@@ -429,7 +422,7 @@ func (ds *MySQLDatastore) Tx(f func(*sql.Tx) error) error {
 	return tx.Commit()
 }
 
-func (ds * MySQLDatastore) InsertTask(ctx context.Context, task *models.Task) error {
+func (ds *MySQLDatastore) InsertTask(ctx context.Context, task *models.Task) error {
 	stmt, err := ds.db.Prepare("INSERT calls SET id=?,created_at=?,started_at=?,completed_at=?,status=?,app_name=?,path=?")
 	if err != nil {
 		return err
@@ -445,13 +438,13 @@ func (ds * MySQLDatastore) InsertTask(ctx context.Context, task *models.Task) er
 	return nil
 }
 
-func (ds * MySQLDatastore) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
+func (ds *MySQLDatastore) GetTask(ctx context.Context, callID string) (*models.FnCall, error) {
 	whereStm := "%s WHERE id=?"
 
 	return datastoreutil.SQLGetCall(ds.db, callSelector, callID, whereStm)
 }
 
-func (ds * MySQLDatastore) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
+func (ds *MySQLDatastore) GetTasks(ctx context.Context, filter *models.CallFilter) (models.FnCalls, error) {
 	whereStm := "WHERE %s ?"
 	andStm := " AND %s ?"
 

api/datastore/postgres/postgres.go

@@ -1,17 +1,16 @@
 package postgres
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
 	"net/url"
 
-	"context"
-
-	"gitlab-odx.oracle.com/odx/functions/api/datastore/internal/datastoreutil"
-	"gitlab-odx.oracle.com/odx/functions/api/models"
 	"github.com/lib/pq"
 	_ "github.com/lib/pq"
+	"gitlab-odx.oracle.com/odx/functions/api/datastore/internal/datastoreutil"
+	"gitlab-odx.oracle.com/odx/functions/api/models"
 )
 
 const routesTableCreate = `
@@ -20,7 +19,6 @@ CREATE TABLE IF NOT EXISTS routes (
 	path text NOT NULL,
 	image character varying(256) NOT NULL,
 	format character varying(16) NOT NULL,
-	maxc integer NOT NULL,
 	memory integer NOT NULL,
 	timeout integer NOT NULL,
 	idle_timeout integer NOT NULL,
@@ -40,7 +38,7 @@ const extrasTableCreate = `CREATE TABLE IF NOT EXISTS extras (
 	value character varying(256) NOT NULL
 );`
 
-const routeSelector = `SELECT app_name, path, image, format, maxc, memory, type, timeout, idle_timeout, headers, config FROM routes`
+const routeSelector = `SELECT app_name, path, image, format, memory, type, timeout, idle_timeout, headers, config FROM routes`
 
 const callsTableCreate = `CREATE TABLE IF NOT EXISTS calls (
 	created_at character varying(256) NOT NULL,
@@ -203,7 +201,6 @@ func (ds *PostgresDatastore) InsertRoute(ctx context.Context, route *models.Rout
 			path,
 			image,
 			format,
-			maxc,
 			memory,
 			type,
 			timeout,
@@ -211,12 +208,11 @@ func (ds *PostgresDatastore) InsertRoute(ctx context.Context, route *models.Rout
 			headers,
 			config
 		)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11);`,
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);`,
 			route.AppName,
 			route.Path,
 			route.Image,
 			route.Format,
-			route.MaxConcurrency,
 			route.Memory,
 			route.Type,
 			route.Timeout,
@@ -259,19 +255,17 @@ func (ds *PostgresDatastore) UpdateRoute(ctx context.Context, newroute *models.R
 		UPDATE routes SET
 			image = $3,
 			format = $4,
-			maxc = $5,
-			memory = $6,
-			type = $7,
-			timeout = $8,
-			idle_timeout = $9,
-			headers = $10,
-			config = $11
+			memory = $5,
+			type = $6,
+			timeout = $7,
+			idle_timeout = $8,
+			headers = $9,
+			config = $10
 		WHERE app_name = $1 AND path = $2;`,
 			route.AppName,
 			route.Path,
 			route.Image,
 			route.Format,
-			route.MaxConcurrency,
 			route.Memory,
 			route.Type,
 			route.Timeout,
@@ -301,7 +295,6 @@ func (ds *PostgresDatastore) UpdateRoute(ctx context.Context, newroute *models.R
 
 func (ds *PostgresDatastore) RemoveRoute(ctx context.Context, appName, routePath string) error {
 	deleteStm := `DELETE FROM routes WHERE path = $1 AND app_name = $2`
-
 	return datastoreutil.SQLRemoveRoute(ds.db, appName, routePath, deleteStm)
 }