mirror of
https://github.com/fnproject/fn.git
synced 2022-10-28 21:29:17 +03:00
this patch gets rid of max concurrency for functions altogether, as discussed, since it will be challenging to support across functions nodes. as a result of doing so, the previous version of functions would fall over when offered 1000 functions, so there was some work needed in order to push this through. further work is necessary as docker basically falls over when trying to start enough containers at the same time, and with this patch essentially every function can scale infinitely. it seems like we could add some kind of adaptive restrictions based on task run length and configured wait time so that fast running functions will line up to run in a hot container instead of them all creating new hot containers. this patch takes a first cut at whacking out some of the insanity that was the previous concurrency model, which was problematic in that it limited concurrency significantly across all functions since every task went through the same unbuffered channel, which could create blocking issues for all functions if the channel is not picked off fast enough (it's not apparent that this was impossible in the previous implementation). in any event, each request has a goroutine already, there's no reason not to use it. not too hard to wrap a map in a lock, not sure what the benefits were (added insanity?) in effect this is marginally easier to understand and less insane (marginally). after getting rid of max c this adds a blocking mechanism for the first invocation of any function so that all other hot functions will wait on the first one to finish to avoid a herd issue (was making docker die...) -- this could be slightly improved, but works in a pinch. reduced some memory usage by having redundant maps of htfnsvr's and task.Requests (by a factor of 2!). cleaned up some of the protocol stuff, need to clean this up further. anyway, it's a first cut. have another patch that rewrites all of it but was getting into rabbit hole territory, would be happy to oblige if anybody else has problems understanding this rat's nest of channels. there is a good bit of work left to make this prod ready (regardless of removing max c). a warning that this will break the db schemas, didn't put the effort in to add migration stuff since this isn't deployed anywhere in prod... TODO need to clean out the htfnmgr bucket with LRU TODO need to clean up runner interface TODO need to unify the task running paths across protocols TODO need to move the ram checking stuff into worker for noted reasons TODO need better elasticity of hot f(x) containers
218 lines
4.9 KiB
Go
218 lines
4.9 KiB
Go
package main
|
|
|
|
/*
|
|
usage: fn init <name>
|
|
|
|
If there's a Dockerfile found, this will generate the basic file with just the image name. exit
|
|
It will then try to decipher the runtime based on the files in the current directory, if it can't figure it out, it will ask.
|
|
It will then take a best guess for what the entrypoint will be based on the language, it it can't guess, it will ask.
|
|
|
|
*/
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"strings"
|
|
|
|
"github.com/urfave/cli"
|
|
"gitlab-odx.oracle.com/odx/functions/fn/langs"
|
|
)
|
|
|
|
var (
|
|
fileExtToRuntime = map[string]string{
|
|
".go": "go",
|
|
".js": "node",
|
|
".rb": "ruby",
|
|
".py": "python",
|
|
".php": "php",
|
|
".rs": "rust",
|
|
".cs": "dotnet",
|
|
".fs": "dotnet",
|
|
".java": "java",
|
|
}
|
|
|
|
fnInitRuntimes []string
|
|
)
|
|
|
|
func init() {
|
|
for rt := range fileExtToRuntime {
|
|
fnInitRuntimes = append(fnInitRuntimes, rt)
|
|
}
|
|
}
|
|
|
|
type initFnCmd struct {
|
|
name string
|
|
force bool
|
|
runtime string
|
|
entrypoint string
|
|
cmd string
|
|
format string
|
|
}
|
|
|
|
func initFn() cli.Command {
|
|
a := initFnCmd{}
|
|
|
|
return cli.Command{
|
|
Name: "init",
|
|
Usage: "create a local func.yaml file",
|
|
Description: "Creates a func.yaml file in the current directory. ",
|
|
ArgsUsage: "<DOCKERHUB_USERNAME/FUNCTION_NAME>",
|
|
Action: a.init,
|
|
Flags: []cli.Flag{
|
|
cli.BoolFlag{
|
|
Name: "force, f",
|
|
Usage: "overwrite existing func.yaml",
|
|
Destination: &a.force,
|
|
},
|
|
cli.StringFlag{
|
|
Name: "runtime",
|
|
Usage: "choose an existing runtime - " + strings.Join(fnInitRuntimes, ", "),
|
|
Destination: &a.runtime,
|
|
},
|
|
cli.StringFlag{
|
|
Name: "entrypoint",
|
|
Usage: "entrypoint is the command to run to start this function - equivalent to Dockerfile ENTRYPOINT.",
|
|
Destination: &a.entrypoint,
|
|
},
|
|
cli.StringFlag{
|
|
Name: "format",
|
|
Usage: "hot function IO format - json or http",
|
|
Destination: &a.format,
|
|
Value: "",
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (a *initFnCmd) init(c *cli.Context) error {
|
|
if !a.force {
|
|
ff, err := loadFuncfile()
|
|
if _, ok := err.(*notFoundError); !ok && err != nil {
|
|
return err
|
|
}
|
|
if ff != nil {
|
|
return errors.New("Function file already exists")
|
|
}
|
|
}
|
|
|
|
runtimeSpecified := a.runtime != ""
|
|
|
|
err := a.buildFuncFile(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if runtimeSpecified {
|
|
err := a.generateBoilerplate()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
var ffmt *string
|
|
if a.format != "" {
|
|
ffmt = &a.format
|
|
}
|
|
|
|
ff := &funcfile{
|
|
Name: a.name,
|
|
Runtime: &a.runtime,
|
|
Version: initialVersion,
|
|
Entrypoint: a.entrypoint,
|
|
Cmd: a.cmd,
|
|
Format: ffmt,
|
|
}
|
|
|
|
_, path := appNamePath(ff.FullName())
|
|
ff.Path = &path
|
|
|
|
if err := encodeFuncfileYAML("func.yaml", ff); err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Println("func.yaml created")
|
|
return nil
|
|
}
|
|
|
|
func (a *initFnCmd) generateBoilerplate() error {
|
|
helper := langs.GetLangHelper(a.runtime)
|
|
if helper != nil && helper.HasBoilerplate() {
|
|
if err := helper.GenerateBoilerplate(); err != nil {
|
|
if err == langs.ErrBoilerplateExists {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
fmt.Println("function boilerplate generated.")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *initFnCmd) buildFuncFile(c *cli.Context) error {
|
|
pwd, err := os.Getwd()
|
|
if err != nil {
|
|
return fmt.Errorf("error detecting current working directory: %s", err)
|
|
}
|
|
|
|
a.name = c.Args().First()
|
|
if a.name == "" || strings.Contains(a.name, ":") {
|
|
return errors.New("please specify a name for your function in the following format <DOCKERHUB_USERNAME>/<FUNCTION_NAME>.\nTry: fn init <DOCKERHUB_USERNAME>/<FUNCTION_NAME>")
|
|
}
|
|
|
|
if exists("Dockerfile") {
|
|
fmt.Println("Dockerfile found. Let's use that to build...")
|
|
return nil
|
|
}
|
|
|
|
var rt string
|
|
if a.runtime == "" {
|
|
rt, err = detectRuntime(pwd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
a.runtime = rt
|
|
fmt.Printf("Found %v, assuming %v runtime.\n", rt, rt)
|
|
} else {
|
|
fmt.Println("Runtime:", a.runtime)
|
|
}
|
|
helper := langs.GetLangHelper(a.runtime)
|
|
if helper == nil {
|
|
fmt.Printf("init does not support the %s runtime, you'll have to create your own Dockerfile for this function", a.runtime)
|
|
}
|
|
|
|
if a.entrypoint == "" {
|
|
if helper != nil {
|
|
a.entrypoint = helper.Entrypoint()
|
|
}
|
|
}
|
|
if a.cmd == "" {
|
|
if helper != nil {
|
|
a.cmd = helper.Cmd()
|
|
}
|
|
}
|
|
if a.entrypoint == "" && a.cmd == "" {
|
|
return fmt.Errorf("could not detect entrypoint or cmd for %v, use --entrypoint and/or --cmd to set them explicitly", a.runtime)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func detectRuntime(path string) (runtime string, err error) {
|
|
for ext, runtime := range fileExtToRuntime {
|
|
filenames := []string{
|
|
filepath.Join(path, fmt.Sprintf("func%s", ext)),
|
|
filepath.Join(path, fmt.Sprintf("Func%s", ext)),
|
|
filepath.Join(path, fmt.Sprintf("src/main%s", ext)), // rust
|
|
}
|
|
for _, filename := range filenames {
|
|
if exists(filename) {
|
|
return runtime, nil
|
|
}
|
|
}
|
|
}
|
|
return "", fmt.Errorf("no supported files found to guess runtime, please set runtime explicitly with --runtime flag.")
|
|
}
|