Initial work on async functions

api/config/config.go

@@ -13,6 +13,7 @@ func InitConfig() {
 	cwd, _ := os.Getwd()
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.SetDefault("log_level", "info")
+	viper.SetDefault("mq", fmt.Sprintf("bolt://%s/data/worker_mq.db", cwd))
 	viper.SetDefault("db", fmt.Sprintf("bolt://%s/data/bolt.db?bucket=funcs", cwd))
 	viper.SetConfigName("config")
 	viper.AddConfigPath(".")

api/models/complete.go

@@ -0,0 +1,30 @@
+package models
+
+import "github.com/go-openapi/strfmt"
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+/*Complete complete
+
+swagger:model Complete
+*/
+type Complete struct {
+
+	/* Time when task was completed. Always in UTC.
+	 */
+	CompletedAt strfmt.DateTime `json:"completed_at,omitempty"`
+
+	/* Error message, if status=error. Only used by the /error endpoint.
+	 */
+	Error string `json:"error,omitempty"`
+
+	/* Machine readable reason failure, if status=error. Only used by the /error endpoint.
+	 */
+	Reason string `json:"reason,omitempty"`
+}
+
+// Validate validates this complete
+func (m *Complete) Validate(formats strfmt.Registry) error {
+	return nil
+}

api/models/group.go

@@ -0,0 +1,71 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+	"github.com/go-openapi/swag"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*Group group
+
+swagger:model Group
+*/
+type Group struct {
+
+	/* Time when image first used/created.
+
+	Read Only: true
+	*/
+	CreatedAt strfmt.DateTime `json:"created_at,omitempty"`
+
+	/* User defined environment variables that will be passed in to each task in this group.
+	 */
+	EnvVars map[string]string `json:"env_vars,omitempty"`
+
+	/* Name of Docker image to use in this group. You should include the image tag, which should be a version number, to be more accurate. Can be overridden on a per task basis with task.image.
+	 */
+	Image string `json:"image,omitempty"`
+
+	/* The maximum number of tasks that will run at the exact same time in this group.
+	 */
+	MaxConcurrency int32 `json:"max_concurrency,omitempty"`
+
+	/* Name of this group. Must be different than the image name. Can ony contain alphanumeric, -, and _.
+
+	Read Only: true
+	*/
+	Name string `json:"name,omitempty"`
+}
+
+// Validate validates this group
+func (m *Group) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateEnvVars(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *Group) validateEnvVars(formats strfmt.Registry) error {
+
+	if swag.IsZero(m.EnvVars) { // not required
+		return nil
+	}
+
+	if err := validate.Required("env_vars", "body", m.EnvVars); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/group_wrapper.go

@@ -0,0 +1,50 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+)
+
+/*GroupWrapper group wrapper
+
+swagger:model GroupWrapper
+*/
+type GroupWrapper struct {
+
+	/* group
+
+	Required: true
+	*/
+	Group *Group `json:"group"`
+}
+
+// Validate validates this group wrapper
+func (m *GroupWrapper) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateGroup(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *GroupWrapper) validateGroup(formats strfmt.Registry) error {
+
+	if m.Group != nil {
+
+		if err := m.Group.Validate(formats); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/models/groups_wrapper.go

@@ -0,0 +1,48 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*GroupsWrapper groups wrapper
+
+swagger:model GroupsWrapper
+*/
+type GroupsWrapper struct {
+
+	/* groups
+
+	Required: true
+	*/
+	Groups []*Group `json:"groups"`
+}
+
+// Validate validates this groups wrapper
+func (m *GroupsWrapper) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateGroups(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *GroupsWrapper) validateGroups(formats strfmt.Registry) error {
+
+	if err := validate.Required("groups", "body", m.Groups); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/id_status.go

@@ -0,0 +1,115 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"encoding/json"
+
+	strfmt "github.com/go-openapi/strfmt"
+	"github.com/go-openapi/swag"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*IDStatus Id status
+
+swagger:model IdStatus
+*/
+type IDStatus struct {
+
+	/* Unique identifier representing a specific task.
+
+	Read Only: true
+	*/
+	ID string `json:"id,omitempty"`
+
+	/* States and valid transitions.
+
+	                 +---------+
+	       +---------> delayed <----------------+
+	                 +----+----+                |
+	                      |                     |
+	                      |                     |
+	                 +----v----+                |
+	       +---------> queued  <----------------+
+	                 +----+----+                *
+	                      |                     *
+	                      |               retry * creates new task
+	                 +----v----+                *
+	                 | running |                *
+	                 +--+-+-+--+                |
+	          +---------|-|-|-----+-------------+
+	      +---|---------+ | +-----|---------+   |
+	      |   |           |       |         |   |
+	+-----v---^-+      +--v-------^+     +--v---^-+
+	| success   |      | cancelled |     |  error |
+	+-----------+      +-----------+     +--------+
+
+	* delayed - has a delay.
+	* queued - Ready to be consumed when it's turn comes.
+	* running - Currently consumed by a runner which will attempt to process it.
+	* success - (or complete? success/error is common javascript terminology)
+	* error - Something went wrong. In this case more information can be obtained
+	  by inspecting the "reason" field.
+	  - timeout
+	  - killed - forcibly killed by worker due to resource restrictions or access
+	    violations.
+	  - bad_exit - exited with non-zero status due to program termination/crash.
+	* cancelled - cancelled via API. More information in the reason field.
+	  - client_request - Request was cancelled by a client.
+
+
+	Read Only: true
+	*/
+	Status string `json:"status,omitempty"`
+}
+
+// Validate validates this Id status
+func (m *IDStatus) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateStatus(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+var idStatusTypeStatusPropEnum []interface{}
+
+// prop value enum
+func (m *IDStatus) validateStatusEnum(path, location string, value string) error {
+	if idStatusTypeStatusPropEnum == nil {
+		var res []string
+		if err := json.Unmarshal([]byte(`["delayed","queued","running","success","error","cancelled"]`), &res); err != nil {
+			return err
+		}
+		for _, v := range res {
+			idStatusTypeStatusPropEnum = append(idStatusTypeStatusPropEnum, v)
+		}
+	}
+	if err := validate.Enum(path, location, value, idStatusTypeStatusPropEnum); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *IDStatus) validateStatus(formats strfmt.Registry) error {
+
+	if swag.IsZero(m.Status) { // not required
+		return nil
+	}
+
+	// value enum
+	if err := m.validateStatusEnum("status", "body", m.Status); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/mq.go

@@ -0,0 +1,52 @@
+package models
+
+// Titan uses a Message Queue to impose a total ordering on jobs that it will
+// execute in order. Tasks are added to the queue via the Push() interface. The
+// MQ must support a reserve-delete 2 step dequeue to allow Titan to implement
+// timeouts and retries.
+//
+// The Reserve() operation must return a job based on this total ordering
+// (described below). At this point, the MQ backend must start a timeout on the
+// job. If Delete() is not called on the Task within the timeout, the Task should
+// be restored to the queue.
+//
+// Total ordering: The queue should maintain an ordering based on priority and
+// logical time.  Priorities are currently 0-2 and available in the Task's
+// priority field.  Tasks with higher priority always get pulled off the queue
+// first.  Within the same priority, jobs should be available in FIFO order.
+
+// When a job is required to be restored to the queue, it should maintain it's
+// approximate order in the queue. That is, for jobs [A, B, C], with A being
+// the head of the queue:
+// Reserve() leads to A being passed to a consumer, and timeout started.
+// Next Reserve() leads to B being dequeued. This consumer finishes running the
+// task, leading to Delete() being called. B is now permanently erased from the
+// queue.
+// A's timeout occurs before the job is finished. At this point the ordering
+// should be [A, C] and not [C, A].
+type MessageQueue interface {
+	// Push a Task onto the queue. If any error is returned, the Task SHOULD not be
+	// queued. Note that this does not completely avoid double queueing, that is
+	// OK, Titan will perform a check against the datastore after a dequeue.
+	//
+	// If the job's Delay value is > 0, the job should NOT be enqueued. The job
+	// should only be available in the queue after at least Delay seconds have
+	// elapsed. No ordering is required among multiple jobs queued with similar
+	// delays. That is, if jobs {A, C} are queued at t seconds, both with Delay
+	// = 5 seconds, and the same priority, then they may be available on the
+	// queue as [C, A] or [A, C].
+	Push(*Task) (*Task, error)
+
+	// Remove a job from the front of the queue, reserve it for a timeout and
+	// return it. MQ implementations MUST NOT lose jobs in case of errors. That
+	// is, in case of reservation failure, it should be possible to retrieve the
+	// job on a future reservation.
+	Reserve() (*Task, error)
+
+	// If a reservation is pending, consider it acknowledged and delete it. If
+	// the job does not have an outstanding reservation, error. If a job did not
+	// exist, succeed.
+	Delete(*Task) error
+}
+
+type Enqueue func(*Task) (*Task, error)

api/models/new_job.go

@@ -0,0 +1,95 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*NewJob new job
+
+swagger:model NewJob
+*/
+type NewJob struct {
+
+	/* Number of seconds to wait before queueing the job for consumption for the first time. Must be a positive integer. Jobs with a delay start in state "delayed" and transition to "running" after delay seconds.
+	 */
+	Delay int32 `json:"delay,omitempty"`
+
+	/* Name of Docker image to use. This is optional and can be used to override the image defined at the group level.
+
+	Required: true
+	*/
+	Image *string `json:"image"`
+
+	/* "Number of automatic retries this job is allowed. A retry will be attempted if a task fails. Max 25. Automatic retries are performed by titan when a task reaches a failed state and has `max_retries` > 0. A retry is performed by queueing a new job with the same image id and payload. The new job's max_retries is one less than the original. The new job's `retry_of` field is set to the original Job ID.  Titan will delay the new job for retries_delay seconds before queueing it. Cancelled or successful tasks are never automatically retried."
+
+	 */
+	MaxRetries int32 `json:"max_retries,omitempty"`
+
+	/* Payload for the job. This is what you pass into each job to make it do something.
+	 */
+	Payload string `json:"payload,omitempty"`
+
+	/* Priority of the job. Higher has more priority. 3 levels from 0-2. Jobs at same priority are processed in FIFO order.
+
+	Required: true
+	*/
+	Priority *int32 `json:"priority"`
+
+	/* Time in seconds to wait before retrying the job. Must be a non-negative integer.
+	 */
+	RetriesDelay *int32 `json:"retries_delay,omitempty"`
+
+	/* Maximum runtime in seconds. If a consumer retrieves the
+	job, but does not change it's status within timeout seconds, the job
+	is considered failed, with reason timeout (Titan may allow a small
+	grace period). The consumer should also kill the job after timeout
+	seconds. If a consumer tries to change status after Titan has already
+	timed out the job, the consumer will be ignored.
+
+	*/
+	Timeout *int32 `json:"timeout,omitempty"`
+}
+
+// Validate validates this new job
+func (m *NewJob) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateImage(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if err := m.validatePriority(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *NewJob) validateImage(formats strfmt.Registry) error {
+
+	if err := validate.Required("image", "body", m.Image); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *NewJob) validatePriority(formats strfmt.Registry) error {
+
+	if err := validate.Required("priority", "body", m.Priority); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/new_jobs_wrapper.go

@@ -0,0 +1,48 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*NewJobsWrapper new jobs wrapper
+
+swagger:model NewJobsWrapper
+*/
+type NewJobsWrapper struct {
+
+	/* jobs
+
+	Required: true
+	*/
+	Jobs []*NewJob `json:"jobs"`
+}
+
+// Validate validates this new jobs wrapper
+func (m *NewJobsWrapper) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateJobs(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *NewJobsWrapper) validateJobs(formats strfmt.Registry) error {
+
+	if err := validate.Required("jobs", "body", m.Jobs); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/new_task.go

@@ -0,0 +1,95 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*NewTask new task
+
+swagger:model NewTask
+*/
+type NewTask struct {
+
+	/* Number of seconds to wait before queueing the task for consumption for the first time. Must be a positive integer. Tasks with a delay start in state "delayed" and transition to "running" after delay seconds.
+	 */
+	Delay int32 `json:"delay,omitempty"`
+
+	/* Name of Docker image to use. This is optional and can be used to override the image defined at the group level.
+
+	Required: true
+	*/
+	Image *string `json:"image"`
+
+	/* "Number of automatic retries this task is allowed.  A retry will be attempted if a task fails. Max 25. Automatic retries are performed by titan when a task reaches a failed state and has `max_retries` > 0. A retry is performed by queueing a new task with the same image id and payload. The new task's max_retries is one less than the original. The new task's `retry_of` field is set to the original Task ID. The old task's `retry_at` field is set to the new Task's ID.  Titan will delay the new task for retries_delay seconds before queueing it. Cancelled or successful tasks are never automatically retried."
+
+	 */
+	MaxRetries int32 `json:"max_retries,omitempty"`
+
+	/* Payload for the task. This is what you pass into each task to make it do something.
+	 */
+	Payload string `json:"payload,omitempty"`
+
+	/* Priority of the task. Higher has more priority. 3 levels from 0-2. Tasks at same priority are processed in FIFO order.
+
+	Required: true
+	*/
+	Priority *int32 `json:"priority"`
+
+	/* Time in seconds to wait before retrying the task. Must be a non-negative integer.
+	 */
+	RetriesDelay *int32 `json:"retries_delay,omitempty"`
+
+	/* Maximum runtime in seconds. If a consumer retrieves the
+	task, but does not change it's status within timeout seconds, the task
+	is considered failed, with reason timeout (Titan may allow a small
+	grace period). The consumer should also kill the task after timeout
+	seconds. If a consumer tries to change status after Titan has already
+	timed out the task, the consumer will be ignored.
+
+	*/
+	Timeout *int32 `json:"timeout,omitempty"`
+}
+
+// Validate validates this new task
+func (m *NewTask) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateImage(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if err := m.validatePriority(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *NewTask) validateImage(formats strfmt.Registry) error {
+
+	if err := validate.Required("image", "body", m.Image); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *NewTask) validatePriority(formats strfmt.Registry) error {
+
+	if err := validate.Required("priority", "body", m.Priority); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/new_tasks_wrapper.go

@@ -0,0 +1,48 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*NewTasksWrapper new tasks wrapper
+
+swagger:model NewTasksWrapper
+*/
+type NewTasksWrapper struct {
+
+	/* tasks
+
+	Required: true
+	*/
+	Tasks []*NewTask `json:"tasks"`
+}
+
+// Validate validates this new tasks wrapper
+func (m *NewTasksWrapper) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateTasks(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *NewTasksWrapper) validateTasks(formats strfmt.Registry) error {
+
+	if err := validate.Required("tasks", "body", m.Tasks); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/reason.go

@@ -0,0 +1,57 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"encoding/json"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*Reason Machine usable reason for job being in this state.
+Valid values for error status are `timeout | killed | bad_exit`.
+Valid values for cancelled status are `client_request`.
+For everything else, this is undefined.
+
+
+swagger:model Reason
+*/
+type Reason string
+
+// for schema
+var reasonEnum []interface{}
+
+func (m Reason) validateReasonEnum(path, location string, value Reason) error {
+	if reasonEnum == nil {
+		var res []Reason
+		if err := json.Unmarshal([]byte(`["timeout","killed","bad_exit","client_request"]`), &res); err != nil {
+			return err
+		}
+		for _, v := range res {
+			reasonEnum = append(reasonEnum, v)
+		}
+	}
+	if err := validate.Enum(path, location, value, reasonEnum); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Validate validates this reason
+func (m Reason) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	// value enum
+	if err := m.validateReasonEnum("", "body", m); err != nil {
+		return err
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

api/models/route.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"path"
 
@@ -26,6 +27,7 @@ type Route struct {
 	Image   string      `json:"image,omitempty"`
 	Memory  uint64      `json:"memory,omitempty"`
 	Headers http.Header `json:"headers,omitempty"`
+	Type    string      `json:"type,omitempty"`
 	Config  `json:"config"`
 }
 
@@ -35,6 +37,8 @@ var (
 	ErrRoutesValidationMissingAppName = errors.New("Missing route AppName")
 	ErrRoutesValidationMissingPath    = errors.New("Missing route Path")
 	ErrRoutesValidationInvalidPath    = errors.New("Invalid Path format")
+	ErrRoutesValidationMissingType    = errors.New("Missing route Type")
+	ErrRoutesValidationInvalidType    = errors.New("Invalid route Type")
 )
 
 func (r *Route) Validate() error {
@@ -60,6 +64,16 @@ func (r *Route) Validate() error {
 		res = append(res, ErrRoutesValidationInvalidPath)
 	}
 
+	if r.Type == "" {
+		r.Type = "sync"
+	}
+
+	if r.Type != "async" && r.Type != "sync" {
+		res = append(res, ErrRoutesValidationInvalidType)
+	}
+
+	fmt.Println(">>>", r.Type)
+
 	if len(res) > 0 {
 		return apiErrors.CompositeValidationError(res...)
 	}

api/models/start.go

@@ -0,0 +1,22 @@
+package models
+
+import "github.com/go-openapi/strfmt"
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+/*Start start
+
+swagger:model Start
+*/
+type Start struct {
+
+	/* Time when task started execution. Always in UTC.
+	 */
+	StartedAt strfmt.DateTime `json:"started_at,omitempty"`
+}
+
+// Validate validates this start
+func (m *Start) Validate(formats strfmt.Registry) error {
+	return nil
+}

api/models/task.go

@@ -0,0 +1,135 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"encoding/json"
+
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*Task task
+
+swagger:model Task
+*/
+type Task struct {
+	NewTask
+
+	IDStatus
+
+	/* Time when task completed, whether it was successul or failed. Always in UTC.
+	 */
+	CompletedAt strfmt.DateTime `json:"completed_at,omitempty"`
+
+	/* Time when task was submitted. Always in UTC.
+
+	Read Only: true
+	*/
+	CreatedAt strfmt.DateTime `json:"created_at,omitempty"`
+
+	/* Env vars for the task. Comes from the ones set on the Group.
+	 */
+	EnvVars map[string]string `json:"env_vars,omitempty"`
+
+	/* The error message, if status is 'error'. This is errors due to things outside the task itself. Errors from user code will be found in the log.
+	 */
+	Error string `json:"error,omitempty"`
+
+	/* Group this task belongs to.
+
+	Read Only: true
+	*/
+	GroupName string `json:"group_name,omitempty"`
+
+	/* Machine usable reason for task being in this state.
+	Valid values for error status are `timeout | killed | bad_exit`.
+	Valid values for cancelled status are `client_request`.
+	For everything else, this is undefined.
+
+	*/
+	Reason string `json:"reason,omitempty"`
+
+	/* If this field is set, then this task was retried by the task referenced in this field.
+
+	Read Only: true
+	*/
+	RetryAt string `json:"retry_at,omitempty"`
+
+	/* If this field is set, then this task is a retry of the ID in this field.
+
+	Read Only: true
+	*/
+	RetryOf string `json:"retry_of,omitempty"`
+
+	/* Time when task started execution. Always in UTC.
+	 */
+	StartedAt strfmt.DateTime `json:"started_at,omitempty"`
+}
+
+// Validate validates this task
+func (m *Task) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.NewTask.Validate(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if err := m.IDStatus.Validate(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if err := m.validateEnvVars(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if err := m.validateReason(formats); err != nil {
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *Task) validateEnvVars(formats strfmt.Registry) error {
+
+	if err := validate.Required("env_vars", "body", m.EnvVars); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+var taskTypeReasonPropEnum []interface{}
+
+// property enum
+func (m *Task) validateReasonEnum(path, location string, value string) error {
+	if taskTypeReasonPropEnum == nil {
+		var res []string
+		if err := json.Unmarshal([]byte(`["timeout","killed","bad_exit","client_request"]`), &res); err != nil {
+			return err
+		}
+		for _, v := range res {
+			taskTypeReasonPropEnum = append(taskTypeReasonPropEnum, v)
+		}
+	}
+	if err := validate.Enum(path, location, value, taskTypeReasonPropEnum); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *Task) validateReason(formats strfmt.Registry) error {
+
+	// value enum
+	if err := m.validateReasonEnum("reason", "body", m.Reason); err != nil {
+		return err
+	}
+
+	return nil
+}

api/models/task_wrapper.go

@@ -0,0 +1,50 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+)
+
+/*TaskWrapper task wrapper
+
+swagger:model TaskWrapper
+*/
+type TaskWrapper struct {
+
+	/* task
+
+	Required: true
+	*/
+	Task *Task `json:"task"`
+}
+
+// Validate validates this task wrapper
+func (m *TaskWrapper) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateTask(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *TaskWrapper) validateTask(formats strfmt.Registry) error {
+
+	if m.Task != nil {
+
+		if err := m.Task.Validate(formats); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/models/tasks_wrapper.go

@@ -0,0 +1,56 @@
+package models
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	strfmt "github.com/go-openapi/strfmt"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/validate"
+)
+
+/*TasksWrapper tasks wrapper
+
+swagger:model TasksWrapper
+*/
+type TasksWrapper struct {
+
+	/* Used to paginate results. If this is returned, pass it into the same query again to get more results.
+	 */
+	Cursor string `json:"cursor,omitempty"`
+
+	/* error
+	 */
+	Error *ErrorBody `json:"error,omitempty"`
+
+	/* tasks
+
+	Required: true
+	*/
+	Tasks []*Task `json:"tasks"`
+}
+
+// Validate validates this tasks wrapper
+func (m *TasksWrapper) Validate(formats strfmt.Registry) error {
+	var res []error
+
+	if err := m.validateTasks(formats); err != nil {
+		// prop
+		res = append(res, err)
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}
+
+func (m *TasksWrapper) validateTasks(formats strfmt.Registry) error {
+
+	if err := validate.Required("tasks", "body", m.Tasks); err != nil {
+		return err
+	}
+
+	return nil
+}

api/mqs/bolt.go

@@ -0,0 +1,336 @@
+package mqs
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/boltdb/bolt"
+	"github.com/iron-io/functions/api/models"
+)
+
+type BoltDbMQ struct {
+	db     *bolt.DB
+	ticker *time.Ticker
+}
+
+type BoltDbConfig struct {
+	FileName string `mapstructure:"filename"`
+}
+
+func jobKey(jobID string) []byte {
+	b := make([]byte, len(jobID)+1)
+	b[0] = 'j'
+	copy(b[1:], []byte(jobID))
+	return b
+}
+
+const timeoutToIDKeyPrefix = "id:"
+
+func timeoutToIDKey(timeout []byte) []byte {
+	b := make([]byte, len(timeout)+len(timeoutToIDKeyPrefix))
+	copy(b[:], []byte(timeoutToIDKeyPrefix))
+	copy(b[len(timeoutToIDKeyPrefix):], []byte(timeout))
+	return b
+}
+
+var delayQueueName = []byte("titan_delay")
+
+func queueName(i int) []byte {
+	return []byte(fmt.Sprintf("titan_%d_queue", i))
+}
+
+func timeoutName(i int) []byte {
+	return []byte(fmt.Sprintf("titan_%d_timeout", i))
+}
+
+func NewBoltMQ(url *url.URL) (*BoltDbMQ, error) {
+	dir := filepath.Dir(url.Path)
+	log := logrus.WithFields(logrus.Fields{"mq": url.Scheme, "dir": dir})
+	err := os.MkdirAll(dir, 0777)
+	if err != nil {
+		log.WithError(err).Errorln("Could not create data directory for mq")
+		return nil, err
+	}
+	db, err := bolt.Open(url.Path, 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		for i := 0; i < 3; i++ {
+			_, err := tx.CreateBucketIfNotExists(queueName(i))
+			if err != nil {
+				log.WithError(err).Errorln("Error creating bucket")
+				return err
+			}
+			_, err = tx.CreateBucketIfNotExists(timeoutName(i))
+			if err != nil {
+				log.WithError(err).Errorln("Error creating timeout bucket")
+				return err
+			}
+		}
+		_, err = tx.CreateBucketIfNotExists(delayQueueName)
+		if err != nil {
+			log.WithError(err).Errorln("Error creating delay bucket")
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		log.WithError(err).Errorln("Error creating timeout bucket")
+		return nil, err
+	}
+
+	ticker := time.NewTicker(time.Second)
+	mq := &BoltDbMQ{
+		ticker: ticker,
+		db:     db,
+	}
+	mq.Start()
+	log.WithFields(logrus.Fields{"file": url.Path}).Info("BoltDb initialized")
+	return mq, nil
+}
+
+func (mq *BoltDbMQ) Start() {
+	go func() {
+		// It would be nice to switch to a tick-less, next-event Timer based model.
+		for _ = range mq.ticker.C {
+			err := mq.db.Update(func(tx *bolt.Tx) error {
+				now := uint64(time.Now().UnixNano())
+				for i := 0; i < 3; i++ {
+					// Assume our timeouts bucket exists and has resKey encoded keys.
+					jobBucket := tx.Bucket(queueName(i))
+					timeoutBucket := tx.Bucket(timeoutName(i))
+					c := timeoutBucket.Cursor()
+
+					var err error
+					for k, v := c.Seek([]byte(resKeyPrefix)); k != nil; k, v = c.Next() {
+						reserved, id := resKeyToProperties(k)
+						if reserved > now {
+							break
+						}
+						err = jobBucket.Put(id, v)
+						if err != nil {
+							return err
+						}
+						timeoutBucket.Delete(k)
+						timeoutBucket.Delete(timeoutToIDKey(k))
+					}
+				}
+
+				return nil
+			})
+			if err != nil {
+				logrus.WithError(err).Error("boltdb reservation check error")
+			}
+
+			err = mq.db.Update(func(tx *bolt.Tx) error {
+				now := uint64(time.Now().UnixNano())
+				// Assume our timeouts bucket exists and has resKey encoded keys.
+				delayBucket := tx.Bucket(delayQueueName)
+				c := delayBucket.Cursor()
+
+				var err error
+				for k, v := c.Seek([]byte(resKeyPrefix)); k != nil; k, v = c.Next() {
+					reserved, id := resKeyToProperties(k)
+					if reserved > now {
+						break
+					}
+
+					priority := binary.BigEndian.Uint32(v)
+					job := delayBucket.Get(id)
+					if job == nil {
+						// oops
+						logrus.Warnf("Expected delayed job, none found with id %s", id)
+						continue
+					}
+
+					jobBucket := tx.Bucket(queueName(int(priority)))
+					err = jobBucket.Put(id, job)
+					if err != nil {
+						return err
+					}
+
+					err := delayBucket.Delete(k)
+					if err != nil {
+						return err
+					}
+
+					return delayBucket.Delete(id)
+				}
+				return nil
+			})
+			if err != nil {
+				logrus.WithError(err).Error("boltdb delay check error")
+			}
+		}
+	}()
+}
+
+// We insert a "reservation" at readyAt, and store the json blob at the msg
+// key. The timer loop plucks this out and puts it in the jobs bucket when the
+// time elapses. The value stored at the reservation key is the priority.
+func (mq *BoltDbMQ) delayTask(job *models.Task) (*models.Task, error) {
+	readyAt := time.Now().Add(time.Duration(job.Delay) * time.Second)
+	err := mq.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(delayQueueName)
+		id, _ := b.NextSequence()
+		buf, err := json.Marshal(job)
+		if err != nil {
+			return err
+		}
+
+		key := msgKey(id)
+		err = b.Put(key, buf)
+		if err != nil {
+			return err
+		}
+
+		pb := make([]byte, 4)
+		binary.BigEndian.PutUint32(pb[:], uint32(*job.Priority))
+		reservation := resKey(key, readyAt)
+		return b.Put(reservation, pb)
+	})
+	return job, err
+}
+
+func (mq *BoltDbMQ) Push(job *models.Task) (*models.Task, error) {
+	if job.Delay > 0 {
+		return mq.delayTask(job)
+	}
+
+	err := mq.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(queueName(int(*job.Priority)))
+
+		id, _ := b.NextSequence()
+
+		buf, err := json.Marshal(job)
+		if err != nil {
+			return err
+		}
+
+		return b.Put(msgKey(id), buf)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return job, nil
+
+}
+
+const msgKeyPrefix = "j:"
+const msgKeyLength = len(msgKeyPrefix) + 8
+const resKeyPrefix = "r:"
+
+// r:<timestamp>:msgKey
+// The msgKey is used to introduce uniqueness within the timestamp. It probably isn't required.
+const resKeyLength = len(resKeyPrefix) + msgKeyLength + 8
+
+func msgKey(v uint64) []byte {
+	b := make([]byte, msgKeyLength)
+	copy(b[:], []byte(msgKeyPrefix))
+	binary.BigEndian.PutUint64(b[len(msgKeyPrefix):], v)
+	return b
+}
+
+func resKey(jobKey []byte, reservedUntil time.Time) []byte {
+	b := make([]byte, resKeyLength)
+	copy(b[:], []byte(resKeyPrefix))
+	binary.BigEndian.PutUint64(b[len(resKeyPrefix):], uint64(reservedUntil.UnixNano()))
+	copy(b[len(resKeyPrefix)+8:], jobKey)
+	return b
+}
+
+func resKeyToProperties(key []byte) (uint64, []byte) {
+	if len(key) != resKeyLength {
+		return 0, nil
+	}
+
+	reservedUntil := binary.BigEndian.Uint64(key[len(resKeyPrefix):])
+	return reservedUntil, key[len(resKeyPrefix)+8:]
+}
+
+func (mq *BoltDbMQ) Reserve() (*models.Task, error) {
+	// Start a writable transaction.
+	tx, err := mq.db.Begin(true)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+
+	for i := 2; i >= 0; i-- {
+		// Use the transaction...
+		b := tx.Bucket(queueName(i))
+		c := b.Cursor()
+		key, value := c.Seek([]byte(msgKeyPrefix))
+		if key == nil {
+			// No jobs, try next bucket
+			continue
+		}
+
+		b.Delete(key)
+
+		var job models.Task
+		err = json.Unmarshal([]byte(value), &job)
+		if err != nil {
+			return nil, err
+		}
+
+		reservationKey := resKey(key, time.Now().Add(time.Minute))
+		b = tx.Bucket(timeoutName(i))
+		// Reserve introduces 3 keys in timeout bucket:
+		// Save reservationKey -> Task to allow release
+		// Save job.ID -> reservationKey to allow Deletes
+		// Save reservationKey -> job.ID to allow clearing job.ID -> reservationKey in recovery without unmarshaling the job.
+		// On Delete:
+		// We have job ID, we get the reservationKey
+		// Delete job.ID -> reservationKey
+		// Delete reservationKey -> job.ID
+		// Delete reservationKey -> Task
+		// On Release:
+		// We have reservationKey, we get the jobID
+		// Delete reservationKey -> job.ID
+		// Delete job.ID -> reservationKey
+		// Move reservationKey -> Task to job bucket.
+		b.Put(reservationKey, value)
+		b.Put(jobKey(job.ID), reservationKey)
+		b.Put(timeoutToIDKey(reservationKey), []byte(job.ID))
+
+		// Commit the transaction and check for error.
+		if err := tx.Commit(); err != nil {
+			return nil, err
+		}
+		return &job, nil
+	}
+
+	return nil, nil
+}
+
+func (mq *BoltDbMQ) Delete(job *models.Task) error {
+	return mq.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(timeoutName(int(*job.Priority)))
+		k := jobKey(job.ID)
+
+		reservationKey := b.Get(k)
+		if reservationKey == nil {
+			return errors.New("Not found")
+		}
+
+		for _, k := range [][]byte{k, timeoutToIDKey(reservationKey), reservationKey} {
+			err := b.Delete(k)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}

api/mqs/memory.go

@@ -0,0 +1,184 @@
+package mqs
+
+import (
+	"errors"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/google/btree"
+	"github.com/iron-io/functions/api/models"
+)
+
+type MemoryMQ struct {
+	// WorkQueue A buffered channel that we can send work requests on.
+	PriorityQueues []chan *models.Task
+	Ticker         *time.Ticker
+	BTree          *btree.BTree
+	Timeouts       map[string]*TaskItem
+	// Protects B-tree and Timeouts
+	// If this becomes a bottleneck, consider separating the two mutexes. The
+	// goroutine to clear up timed out messages could also become a bottleneck at
+	// some point. May need to switch to bucketing of some sort.
+	Mutex sync.Mutex
+}
+
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func randSeq(n int) string {
+	rand.Seed(time.Now().Unix())
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
+const NumPriorities = 3
+
+func NewMemoryMQ() *MemoryMQ {
+	var queues []chan *models.Task
+	for i := 0; i < NumPriorities; i++ {
+		queues = append(queues, make(chan *models.Task, 5000))
+	}
+	ticker := time.NewTicker(time.Second)
+	mq := &MemoryMQ{
+		PriorityQueues: queues,
+		Ticker:         ticker,
+		BTree:          btree.New(2),
+		Timeouts:       make(map[string]*TaskItem, 0),
+	}
+	mq.start()
+	logrus.Info("MemoryMQ initialized")
+	return mq
+}
+
+func (mq *MemoryMQ) start() {
+	// start goroutine to check for delayed jobs and put them onto regular queue when ready
+	go func() {
+		for _ = range mq.Ticker.C {
+			ji := &TaskItem{
+				StartAt: time.Now(),
+			}
+			mq.Mutex.Lock()
+			mq.BTree.AscendLessThan(ji, func(a btree.Item) bool {
+				logrus.WithFields(logrus.Fields{"queue": a}).Debug("delayed job move to queue")
+				ji2 := mq.BTree.Delete(a).(*TaskItem)
+				// put it onto the regular queue now
+				_, err := mq.pushForce(ji2.Task)
+				if err != nil {
+					logrus.WithError(err).Error("Couldn't push delayed message onto main queue")
+				}
+				return true
+			})
+			mq.Mutex.Unlock()
+		}
+	}()
+	// start goroutine to check for messages that have timed out and put them back onto regular queue
+	// TODO: this should be like the delayed messages above. Could even be the same thing as delayed messages, but remove them if job is completed.
+	go func() {
+		for _ = range mq.Ticker.C {
+			ji := &TaskItem{
+				StartAt: time.Now(),
+			}
+			mq.Mutex.Lock()
+			for _, jobItem := range mq.Timeouts {
+				if jobItem.Less(ji) {
+					delete(mq.Timeouts, jobItem.Task.ID)
+					_, err := mq.pushForce(jobItem.Task)
+					if err != nil {
+						logrus.WithError(err).Error("Couldn't push timed out message onto main queue")
+					}
+				}
+			}
+			mq.Mutex.Unlock()
+		}
+	}()
+}
+
+// TaskItem is for the Btree, implements btree.Item
+type TaskItem struct {
+	Task    *models.Task
+	StartAt time.Time
+}
+
+func (ji *TaskItem) Less(than btree.Item) bool {
+	// TODO: this could lose jobs: https://godoc.org/github.com/google/btree#Item
+	ji2 := than.(*TaskItem)
+	return ji.StartAt.Before(ji2.StartAt)
+}
+
+func (mq *MemoryMQ) Push(job *models.Task) (*models.Task, error) {
+
+	// It seems to me that using the job ID in the reservation is acceptable since each job can only have one outstanding reservation.
+	// job.MsgId = randSeq(20)
+	if job.Delay > 0 {
+		// then we'll put into short term storage until ready
+		ji := &TaskItem{
+			Task:    job,
+			StartAt: time.Now().Add(time.Second * time.Duration(job.Delay)),
+		}
+		mq.Mutex.Lock()
+		replaced := mq.BTree.ReplaceOrInsert(ji)
+		mq.Mutex.Unlock()
+		if replaced != nil {
+			logrus.Warn("Ooops! an item was replaced and therefore lost, not good.")
+		}
+		return job, nil
+	}
+
+	// Push the work onto the queue.
+	return mq.pushForce(job)
+}
+func (mq *MemoryMQ) pushTimeout(job *models.Task) error {
+
+	ji := &TaskItem{
+		Task:    job,
+		StartAt: time.Now().Add(time.Minute),
+	}
+	mq.Mutex.Lock()
+	mq.Timeouts[job.ID] = ji
+	mq.Mutex.Unlock()
+	return nil
+}
+
+func (mq *MemoryMQ) pushForce(job *models.Task) (*models.Task, error) {
+	mq.PriorityQueues[*job.Priority] <- job
+	return job, nil
+}
+
+// This is recursive, so be careful how many channels you pass in.
+func pickEarliestNonblocking(channels ...chan *models.Task) *models.Task {
+	if len(channels) == 0 {
+		return nil
+	}
+
+	select {
+	case job := <-channels[0]:
+		return job
+	default:
+		return pickEarliestNonblocking(channels[1:]...)
+	}
+}
+
+func (mq *MemoryMQ) Reserve() (*models.Task, error) {
+	job := pickEarliestNonblocking(mq.PriorityQueues[2], mq.PriorityQueues[1], mq.PriorityQueues[0])
+	if job == nil {
+		return nil, nil
+	}
+
+	return job, mq.pushTimeout(job)
+}
+
+func (mq *MemoryMQ) Delete(job *models.Task) error {
+	mq.Mutex.Lock()
+	defer mq.Mutex.Unlock()
+	_, exists := mq.Timeouts[job.ID]
+	if !exists {
+		return errors.New("Not reserved")
+	}
+
+	delete(mq.Timeouts, job.ID)
+	return nil
+}

api/mqs/new.go

@@ -0,0 +1,29 @@
+package mqs
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/iron-io/functions/api/models"
+)
+
+// New will parse the URL and return the correct MQ implementation.
+func New(mqURL string) (models.MessageQueue, error) {
+	// Play with URL schemes here: https://play.golang.org/p/xWAf9SpCBW
+	u, err := url.Parse(mqURL)
+	if err != nil {
+		logrus.WithError(err).WithFields(logrus.Fields{"url": mqURL}).Fatal("bad MQ URL")
+	}
+	logrus.WithFields(logrus.Fields{"mq": u.Scheme}).Info("selecting MQ")
+	switch u.Scheme {
+	case "memory":
+		return NewMemoryMQ(), nil
+	case "redis":
+		return NewRedisMQ(u)
+	case "bolt":
+		return NewBoltMQ(u)
+	}
+
+	return nil, fmt.Errorf("mq type not supported %v", u.Scheme)
+}

api/mqs/redis.go

@@ -0,0 +1,298 @@
+package mqs
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/garyburd/redigo/redis"
+	"github.com/iron-io/functions/api/models"
+)
+
+type RedisMQ struct {
+	pool      *redis.Pool
+	queueName string
+	ticker    *time.Ticker
+	prefix    string
+}
+
+func NewRedisMQ(url *url.URL) (*RedisMQ, error) {
+
+	pool := &redis.Pool{
+		MaxIdle: 4,
+		// I'm not sure if allowing the pool to block if more than 16 connections are required is a good idea.
+		MaxActive:   16,
+		Wait:        true,
+		IdleTimeout: 300 * time.Second,
+		Dial: func() (redis.Conn, error) {
+			return redis.DialURL(url.String())
+		},
+		TestOnBorrow: func(c redis.Conn, t time.Time) error {
+			_, err := c.Do("PING")
+			return err
+		},
+	}
+
+	// Force a connection so we can fail in case of error.
+	conn := pool.Get()
+	if err := conn.Err(); err != nil {
+		logrus.WithError(err).Fatal("Error connecting to redis")
+	}
+	conn.Close()
+
+	mq := &RedisMQ{
+		pool:   pool,
+		ticker: time.NewTicker(time.Second),
+		prefix: url.Path,
+	}
+	mq.queueName = mq.k("queue")
+	logrus.WithFields(logrus.Fields{"name": mq.queueName}).Info("Redis initialized with queue name")
+
+	mq.start()
+	return mq, nil
+}
+
+func (mq *RedisMQ) k(s string) string {
+	return mq.prefix + s
+}
+
+func getFirstKeyValue(resp map[string]string) (string, string, error) {
+
+	for key, value := range resp {
+		return key, value, nil
+	}
+	return "", "", errors.New("Blank map")
+}
+
+func (mq *RedisMQ) processPendingReservations(conn redis.Conn) {
+	resp, err := redis.StringMap(conn.Do("ZRANGE", mq.k("timeouts"), 0, 0, "WITHSCORES"))
+	if mq.checkNilResponse(err) || len(resp) == 0 {
+		return
+	}
+	if err != nil {
+		logrus.WithError(err).Error("Redis command error")
+	}
+
+	reservationId, timeoutString, err := getFirstKeyValue(resp)
+	if err != nil {
+		logrus.WithError(err).Error("error getting kv")
+		return
+	}
+
+	timeout, err := strconv.ParseInt(timeoutString, 10, 64)
+	if err != nil || timeout > time.Now().Unix() {
+		return
+	}
+	response, err := redis.Bytes(conn.Do("HGET", mq.k("timeout_jobs"), reservationId))
+	if mq.checkNilResponse(err) {
+		return
+	}
+	if err != nil {
+		logrus.WithError(err).Error("redis get timeout_jobs error")
+		return
+	}
+
+	var job models.Task
+	err = json.Unmarshal(response, &job)
+	if err != nil {
+		logrus.WithError(err).Error("error unmarshaling job json")
+		return
+	}
+
+	conn.Do("ZREM", mq.k("timeouts"), reservationId)
+	conn.Do("HDEL", mq.k("timeout_jobs"), reservationId)
+	conn.Do("HDEL", mq.k("reservations"), job.ID)
+	redisPush(conn, mq.queueName, &job)
+}
+
+func (mq *RedisMQ) processDelayedTasks(conn redis.Conn) {
+	// List of reservation ids between -inf time and the current time will get us
+	// everything that is now ready to be queued.
+	now := time.Now().UTC().Unix()
+	resIds, err := redis.Strings(conn.Do("ZRANGEBYSCORE", mq.k("delays"), "-inf", now))
+	if err != nil {
+		logrus.WithError(err).Error("Error getting delayed jobs")
+		return
+	}
+
+	for _, resId := range resIds {
+		// Might be a good idea to do this transactionally so we do not have left over reservationIds if the delete fails.
+		buf, err := redis.Bytes(conn.Do("HGET", mq.k("delayed_jobs"), resId))
+		// If:
+		// a) A HSET in Push() failed, or
+		// b) A previous zremrangebyscore failed,
+		// we can get ids that we never associated with a job, or already placed in the queue, just skip these.
+		if err == redis.ErrNil {
+			continue
+		} else if err != nil {
+			logrus.WithError(err).WithFields(logrus.Fields{"reservationId": resId}).Error("Error HGET delayed_jobs")
+			continue
+		}
+
+		var job models.Task
+		err = json.Unmarshal(buf, &job)
+		if err != nil {
+			logrus.WithError(err).WithFields(logrus.Fields{"buf": buf, "reservationId": resId}).Error("Error unmarshaling job")
+			return
+		}
+
+		_, err = redisPush(conn, mq.queueName, &job)
+		if err != nil {
+			logrus.WithError(err).WithFields(logrus.Fields{"reservationId": resId}).Error("Pushing delayed job")
+			return
+		}
+		conn.Do("HDEL", mq.k("delayed_jobs"), resId)
+	}
+
+	// Remove everything we processed.
+	conn.Do("ZREMRANGEBYSCORE", mq.k("delays"), "-inf", now)
+}
+
+func (mq *RedisMQ) start() {
+	go func() {
+		conn := mq.pool.Get()
+		defer conn.Close()
+		if err := conn.Err(); err != nil {
+			logrus.WithError(err).Fatal("Could not start redis MQ reservation system")
+		}
+
+		for _ = range mq.ticker.C {
+			mq.processPendingReservations(conn)
+			mq.processDelayedTasks(conn)
+		}
+	}()
+}
+
+func redisPush(conn redis.Conn, queue string, job *models.Task) (*models.Task, error) {
+	buf, err := json.Marshal(job)
+	if err != nil {
+		return nil, err
+	}
+	_, err = conn.Do("LPUSH", fmt.Sprintf("%s%d", queue, *job.Priority), buf)
+	if err != nil {
+		return nil, err
+	}
+	return job, nil
+}
+
+func (mq *RedisMQ) delayTask(conn redis.Conn, job *models.Task) (*models.Task, error) {
+	buf, err := json.Marshal(job)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := redis.Int64(conn.Do("INCR", mq.k("delays_counter")))
+	if err != nil {
+		return nil, err
+	}
+
+	reservationId := strconv.FormatInt(resp, 10)
+
+	// Timestamp -> resID
+	_, err = conn.Do("ZADD", mq.k("delays"), time.Now().UTC().Add(time.Duration(job.Delay)*time.Second).Unix(), reservationId)
+	if err != nil {
+		return nil, err
+	}
+
+	// resID -> Task
+	_, err = conn.Do("HSET", mq.k("delayed_jobs"), reservationId, buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return job, nil
+}
+
+func (mq *RedisMQ) Push(job *models.Task) (*models.Task, error) {
+	conn := mq.pool.Get()
+	defer conn.Close()
+
+	if job.Delay > 0 {
+		return mq.delayTask(conn, job)
+	}
+	return redisPush(conn, mq.queueName, job)
+}
+func (mq *RedisMQ) checkNilResponse(err error) bool {
+	return err != nil && err.Error() == redis.ErrNil.Error()
+}
+
+// Would be nice to switch to this model http://redis.io/commands/rpoplpush#pattern-reliable-queue
+func (mq *RedisMQ) Reserve() (*models.Task, error) {
+
+	conn := mq.pool.Get()
+	defer conn.Close()
+	var job models.Task
+	var resp []byte
+	var err error
+	for i := 2; i >= 0; i-- {
+		resp, err = redis.Bytes(conn.Do("RPOP", fmt.Sprintf("%s%d", mq.queueName, i)))
+		if mq.checkNilResponse(err) {
+			if i == 0 {
+				// Out of queues!
+				return nil, nil
+			}
+
+			// No valid job on this queue, try lower priority queue.
+			continue
+		} else if err != nil {
+			// Some other error!
+			return nil, err
+		}
+
+		// We got a valid high priority job.
+		break
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(resp, &job)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := redis.Int64(conn.Do("INCR", mq.queueName+"_incr"))
+	if err != nil {
+		return nil, err
+	}
+	reservationId := strconv.FormatInt(response, 10)
+	_, err = conn.Do("ZADD", "timeout:", time.Now().Add(time.Minute).Unix(), reservationId)
+	if err != nil {
+		return nil, err
+	}
+	_, err = conn.Do("HSET", "timeout", reservationId, resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Map from job.ID -> reservation ID
+	_, err = conn.Do("HSET", "reservations", job.ID, reservationId)
+	if err != nil {
+		return nil, err
+	}
+
+	return &job, nil
+}
+
+func (mq *RedisMQ) Delete(job *models.Task) error {
+	conn := mq.pool.Get()
+	defer conn.Close()
+	resId, err := conn.Do("HGET", "reservations", job.ID)
+	if err != nil {
+		return err
+	}
+	_, err = conn.Do("HDEL", "reservations", job.ID)
+	if err != nil {
+		return err
+	}
+	_, err = conn.Do("ZREM", "timeout:", resId)
+	if err != nil {
+		return err
+	}
+	_, err = conn.Do("HDEL", "timeout", resId)
+	return err
+}

api/server/helpers.go

@@ -44,7 +44,9 @@ func testRouter() *gin.Engine {
 		c.Set("ctx", ctx)
 		c.Next()
 	})
-	bindHandlers(r)
+	bindHandlers(r, func(ctx *gin.Context) {
+		handleRequest(ctx, nil)
+	})
 	return r
 }
 

api/server/routes_create.go

@@ -52,6 +52,7 @@ func handleRouteCreate(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, simpleError(models.ErrAppsGet))
 		return
 	}
+
 	if app == nil {
 		newapp := &models.App{Name: wroute.Route.AppName}
 		if err := newapp.Validate(); err != nil {

api/server/runner.go

@@ -17,6 +17,7 @@ import (
 	"github.com/iron-io/functions/api/models"
 	"github.com/iron-io/functions/api/runner"
 	titancommon "github.com/iron-io/worker/common"
+	"github.com/iron-io/worker/runner/drivers"
 	"github.com/satori/go.uuid"
 )
 
@@ -31,7 +32,7 @@ func handleSpecial(c *gin.Context) {
 	}
 }
 
-func handleRunner(c *gin.Context) {
+func handleRequest(c *gin.Context, enqueue models.Enqueue) {
 	if strings.HasPrefix(c.Request.URL.Path, "/v1") {
 		c.Status(http.StatusNotFound)
 		return
@@ -151,10 +152,29 @@ func handleRunner(c *gin.Context) {
 				Memory:  el.Memory,
 			}
 
-			if result, err := Api.Runner.Run(c, cfg); err != nil {
-				log.WithError(err).Error(models.ErrRunnerRunRoute)
-				c.JSON(http.StatusInternalServerError, simpleError(models.ErrRunnerRunRoute))
-			} else {
+			// Request count metric
+			metricBaseName := "server.handleRequest." + appName + "."
+			runner.LogMetricCount(ctx, (metricBaseName + "requests"), 1)
+
+			metricStart := time.Now()
+
+			var err error
+			var result drivers.RunResult
+			switch el.Type {
+			case "async":
+				// TODO: Create Task
+				priority := int32(0)
+				task := &models.Task{}
+				task.Image = &cfg.Image
+				task.ID = cfg.ID
+				task.GroupName = cfg.AppName
+				task.Priority = &priority
+				// TODO: Push to queue
+				enqueue(task)
+			default:
+				if result, err = Api.Runner.Run(c, cfg); err != nil {
+					break
+				}
 				for k, v := range el.Headers {
 					c.Header(k, v[0])
 				}
@@ -166,6 +186,16 @@ func handleRunner(c *gin.Context) {
 					c.AbortWithStatus(http.StatusInternalServerError)
 				}
 			}
+
+			if err != nil {
+				log.WithError(err).Error(models.ErrRunnerRunRoute)
+				c.JSON(http.StatusInternalServerError, simpleError(models.ErrRunnerRunRoute))
+			}
+
+			// Execution time metric
+			metricElapsed := time.Since(metricStart)
+			runner.LogMetricTime(ctx, (metricBaseName + "time"), metricElapsed)
+			runner.LogMetricTime(ctx, "server.handleRunner.exec_time", metricElapsed)
 			return
 		}
 	}

api/server/server.go

@@ -21,14 +21,16 @@ type Server struct {
 	Runner          *runner.Runner
 	Router          *gin.Engine
 	Datastore       models.Datastore
+	MQ              models.MessageQueue
 	AppListeners    []ifaces.AppListener
 	SpecialHandlers []ifaces.SpecialHandler
 }
 
-func New(ds models.Datastore, r *runner.Runner) *Server {
+func New(ds models.Datastore, mq models.MessageQueue, r *runner.Runner) *Server {
 	Api = &Server{
 		Router:    gin.Default(),
 		Datastore: ds,
+		MQ:        mq,
 		Runner:    r,
 	}
 	return Api
@@ -75,10 +77,17 @@ func (s *Server) UseSpecialHandlers(ginC *gin.Context) error {
 		}
 	}
 	// now call the normal runner call
-	handleRunner(ginC)
+	handleRequest(ginC, nil)
 	return nil
 }
 
+func (s *Server) handleRequest(ginC *gin.Context) {
+	enqueue := func(task *models.Task) (*models.Task, error) {
+		return s.MQ.Push(task)
+	}
+	handleRequest(ginC, enqueue)
+}
+
 func extractFields(c *gin.Context) logrus.Fields {
 	fields := logrus.Fields{"action": path.Base(c.HandlerName())}
 	for _, param := range c.Params {
@@ -95,14 +104,14 @@ func (s *Server) Run(ctx context.Context) {
 		c.Next()
 	})
 
-	bindHandlers(s.Router)
+	bindHandlers(s.Router, s.handleRequest)
 
 	// By default it serves on :8080 unless a
 	// PORT environment variable was defined.
 	s.Router.Run()
 }
 
-func bindHandlers(engine *gin.Engine) {
+func bindHandlers(engine *gin.Engine, reqHandler func(ginC *gin.Context)) {
 	engine.GET("/", handlePing)
 	engine.GET("/version", handleVersion)
 
@@ -127,7 +136,7 @@ func bindHandlers(engine *gin.Engine) {
 		}
 	}
 
-	engine.Any("/r/:app/*route", handleRunner)
+	engine.Any("/r/:app/*route", reqHandler)
 
 	// This final route is used for extensions, see Server.Add
 	engine.NoRoute(handleSpecial)