From 9ac2539aebedbf8a77851ef897da2b1855617020 Mon Sep 17 00:00:00 2001 From: C Cirello Date: Thu, 1 Dec 2016 18:11:13 +0100 Subject: [PATCH] fn: improve UX and publish/deploy command (#359) * fn: improve UX and publish/deploy command * fn: remove wrong use cases for deploy * fn: fix regression introduced by merge --- docs/function-file.md | 9 +- docs/lambda/getting-started.md | 12 +- docs/lambda/import.md | 4 +- docs/packaging.md | 2 +- examples/blog/func.yaml | 3 + examples/blog/func.yml | 3 - examples/caddy-lb/func.yaml | 3 + examples/caddy-lb/func.yml | 3 - examples/checker/func.yaml | 3 + examples/checker/func.yml | 3 - examples/echo/func.yaml | 3 + examples/echo/func.yml | 3 - examples/error/func.yaml | 3 + examples/error/func.yml | 3 - examples/hello/node/README.md | 2 +- examples/hello/php/README.md | 17 ++- examples/hello/php/func.yaml | 8 +- examples/hello/python/README.md | 16 +- examples/hello/python/func.yaml | 5 - examples/hello/ruby/README.md | 4 +- examples/redis/func.yaml | 3 + examples/redis/func.yml | 3 - examples/slackbot/guppy/README.md | 4 +- examples/sleeper/func.yaml | 2 +- examples/twitter/func.yaml | 3 + examples/twitter/func.yml | 3 - fn/README.md | 61 ++------ fn/build.go | 38 +++-- fn/bump.go | 41 +++-- fn/common.go | 165 +++++---------------- fn/deploy.go | 239 ++++++++++++++++++++++++++++++ fn/funcfile.go | 20 ++- fn/init.go | 14 +- fn/lambda.go | 5 +- fn/main.go | 2 +- fn/publish.go | 153 ------------------- fn/push.go | 47 +++--- fn/routes.go | 8 +- fn/run.go | 2 +- 39 files changed, 453 insertions(+), 469 deletions(-) create mode 100644 examples/blog/func.yaml delete mode 100644 examples/blog/func.yml create mode 100644 examples/caddy-lb/func.yaml delete mode 100644 examples/caddy-lb/func.yml create mode 100644 examples/checker/func.yaml delete mode 100644 examples/checker/func.yml create mode 100644 examples/echo/func.yaml delete mode 100644 examples/echo/func.yml create mode 100644 examples/error/func.yaml delete mode 100644 examples/error/func.yml delete mode 100644 examples/hello/python/func.yaml create mode 100644 examples/redis/func.yaml delete mode 100644 examples/redis/func.yml create mode 100644 examples/twitter/func.yaml delete mode 100644 examples/twitter/func.yml create mode 100644 fn/deploy.go delete mode 100644 fn/publish.go diff --git a/docs/function-file.md b/docs/function-file.md index 4f947d8dd..a47943cba 100644 --- a/docs/function-file.md +++ b/docs/function-file.md @@ -23,16 +23,13 @@ build: - make test ``` -`app` (optional) is the application name to which this function will be pushed -to. - -`image` is the name and tag to which this function will be pushed to and the +`name` is the name and tag to which this function will be pushed to and the route updated to use it. -`route` (optional) allows you to overwrite the calculated route from the path +`path` (optional) allows you to overwrite the calculated route from the path position. You may use it to override the calculated route. -`version` represents current version of the function. When publishing, it is +`version` represents current version of the function. When deploying, it is appended to the image as a tag. `type` (optional) allows you to set the type of the route. `sync`, for functions diff --git a/docs/lambda/getting-started.md b/docs/lambda/getting-started.md index 156f0a151..11dfb2093 100644 --- a/docs/lambda/getting-started.md +++ b/docs/lambda/getting-started.md @@ -42,12 +42,12 @@ the name of the function to run, in the form that python expects (`module.function`). Where you would package the files into a `.zip` to upload to Lambda, we just pass the list of files to `fn`. -## Publishing the function to IronFunctions +## Deploying the function to IronFunctions -Next we want to publish the function to our IronFunctions +Next we want to deploy the function to our IronFunctions ```sh - $ fn publish -v -f -d ./irontest - publishing irontest/hello_world:1/function.yaml + $ fn deploy -v -d ./irontest irontest + deploying irontest/hello_world:1/function.yaml Sending build context to Docker daemon 4.096 kB Step 1 : FROM iron/lambda-python2.7 latest: Pulling from iron/lambda-python2.7 @@ -82,7 +82,7 @@ Next we want to publish the function to our IronFunctions irontest/hello_world:1/function.yaml done ``` -This will publish the generated function under the app `irontest` with `hello_world` as a route, e.g: +This will deploy the generated function under the app `irontest` with `hello_world` as a route, e.g: `http:///r/irontest/hello_world:1`, You should also now see the generated Docker image. @@ -108,7 +108,7 @@ You should see the output. ## Calling the function from IronFunctions -The `fn call` command can call the published version with a given payload. +The `fn call` command can call the deployed version with a given payload. ```sh $ echo '{ "first_name": "Jon", "last_name": "Snow" }' | ./fn call irontest /hello_world:1 diff --git a/docs/lambda/import.md b/docs/lambda/import.md index fed82fabf..249336cb4 100644 --- a/docs/lambda/import.md +++ b/docs/lambda/import.md @@ -48,8 +48,8 @@ If you only want to download the code, pass the `--download-only` flag. The you tweak the settings on a command level. Finally, you can import a different version of your lambda function than the latest one by passing `--version .` -You can then publish the imported lambda as follows: +You can then deploy the imported lambda as follows: ``` -./fn publish -d ./user/my-function +./fn deploy -d ./user/my-function user ```` Now the function can be reached via ```http://$HOSTNAME/r/user/my-function``` \ No newline at end of file diff --git a/docs/packaging.md b/docs/packaging.md index 2dda554c8..9f0a3189c 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -9,7 +9,7 @@ Once it's pushed to a registry, you can use it by referencing it when adding a r ## Using fn -This is the easiest way to build, package and publish your functions. +This is the easiest way to build, package and deploy your functions. diff --git a/examples/blog/func.yaml b/examples/blog/func.yaml new file mode 100644 index 000000000..541608545 --- /dev/null +++ b/examples/blog/func.yaml @@ -0,0 +1,3 @@ +name: iron/func-blog +build: + - ./build.sh \ No newline at end of file diff --git a/examples/blog/func.yml b/examples/blog/func.yml deleted file mode 100644 index 5dd678f12..000000000 --- a/examples/blog/func.yml +++ /dev/null @@ -1,3 +0,0 @@ -image: iron/func-blog -build: - - ./build.sh \ No newline at end of file diff --git a/examples/caddy-lb/func.yaml b/examples/caddy-lb/func.yaml new file mode 100644 index 000000000..6be246270 --- /dev/null +++ b/examples/caddy-lb/func.yaml @@ -0,0 +1,3 @@ +name: iron/func-caddy-lb +build: + - ./build.sh \ No newline at end of file diff --git a/examples/caddy-lb/func.yml b/examples/caddy-lb/func.yml deleted file mode 100644 index 953a3b564..000000000 --- a/examples/caddy-lb/func.yml +++ /dev/null @@ -1,3 +0,0 @@ -image: iron/func-caddy-lb -build: - - ./build.sh \ No newline at end of file diff --git a/examples/checker/func.yaml b/examples/checker/func.yaml new file mode 100644 index 000000000..645f6ed01 --- /dev/null +++ b/examples/checker/func.yaml @@ -0,0 +1,3 @@ +name: iron/func-checker +build: + - ./build.sh \ No newline at end of file diff --git a/examples/checker/func.yml b/examples/checker/func.yml deleted file mode 100644 index 6ba395003..000000000 --- a/examples/checker/func.yml +++ /dev/null @@ -1,3 +0,0 @@ -image: iron/func-checker -build: - - ./build.sh \ No newline at end of file diff --git a/examples/echo/func.yaml b/examples/echo/func.yaml new file mode 100644 index 000000000..7266fe8b3 --- /dev/null +++ b/examples/echo/func.yaml @@ -0,0 +1,3 @@ +name: iron/func-echo +build: + - ./build.sh \ No newline at end of file diff --git a/examples/echo/func.yml b/examples/echo/func.yml deleted file mode 100644 index de5fcfac4..000000000 --- a/examples/echo/func.yml +++ /dev/null @@ -1,3 +0,0 @@ -image: iron/func-echo -build: - - ./build.sh \ No newline at end of file diff --git a/examples/error/func.yaml b/examples/error/func.yaml new file mode 100644 index 000000000..37345b04f --- /dev/null +++ b/examples/error/func.yaml @@ -0,0 +1,3 @@ +name: iron/func-error +build: + - ./build.sh \ No newline at end of file diff --git a/examples/error/func.yml b/examples/error/func.yml deleted file mode 100644 index 34fa569f8..000000000 --- a/examples/error/func.yml +++ /dev/null @@ -1,3 +0,0 @@ -image: iron/func-error -build: - - ./build.sh \ No newline at end of file diff --git a/examples/hello/node/README.md b/examples/hello/node/README.md index dbddfca5f..cc8ca5040 100644 --- a/examples/hello/node/README.md +++ b/examples/hello/node/README.md @@ -9,7 +9,7 @@ fn init /hello fn build # test it cat hello.payload.json | fn run -# push it to Docker Hub for use with IronFunctions +# push it to Docker Hub fn push # Create a route to this function on IronFunctions fn routes create myapp /hello diff --git a/examples/hello/php/README.md b/examples/hello/php/README.md index 266c248ff..88b01afcf 100644 --- a/examples/hello/php/README.md +++ b/examples/hello/php/README.md @@ -5,11 +5,11 @@ This example will show you how to test and deploy Go (Golang) code to IronFuncti ### 1. Prepare the `func.yaml` file: At func.yaml you will find: + ```yml -app: phpapp -route: /hello -image: USERNAME/hello +name: USERNAME/hello version: 0.0.1 +path: /hello build: - docker run --rm -v "$PWD":/worker -w /worker iron/php:dev composer install ``` @@ -21,7 +21,14 @@ the moment you try to test this function. ### 2. Build: ```sh -fn publish +# build the function +fn build +# test it +cat hello.payload.json | fn run +# push it to Docker Hub +fn push +# Create a route to this function on IronFunctions +fn routes create phpapp /hello ``` `-v` is optional, but it allows you to see how this function is being built. @@ -31,7 +38,7 @@ fn publish Now you can start jobs on your function. Let's quickly queue up a job to try it out. ```sh -cat hello.payload.json | fn run phpapp /hello +cat hello.payload.json | fn call phpapp /hello ``` Here's a curl example to show how easy it is to do in any language: diff --git a/examples/hello/php/func.yaml b/examples/hello/php/func.yaml index 4900091fc..4a09ae8b5 100644 --- a/examples/hello/php/func.yaml +++ b/examples/hello/php/func.yaml @@ -1,5 +1,5 @@ -app: phpapp -route: /hello -image: USERNAME/hello:0.0.1 +name: USERNAME/hello +version: 0.0.1 +path: /hello build: -- docker run --rm -v "$PWD":/worker -w /worker iron/php:dev composer install \ No newline at end of file +- docker run --rm -v "$PWD":/worker -w /worker iron/php:dev composer install diff --git a/examples/hello/python/README.md b/examples/hello/python/README.md index 20361699b..3a05e76a7 100644 --- a/examples/hello/python/README.md +++ b/examples/hello/python/README.md @@ -7,10 +7,9 @@ This example will show you how to test and deploy Go (Golang) code to IronFuncti At func.yaml you will find: ```yml -app: pythonapp -route: /hello -image: USERNAME/hello +name: USERNAME/hello version: 0.0.1 +path: /hello build: - docker run --rm -v "$PWD":/worker -w /worker iron/python:2-dev pip install -t packages -r requirements.txt ``` @@ -22,7 +21,14 @@ the moment you try to test this function. ### 2. Build: ```sh -fn publish +# build the function +fn build +# test it +cat hello.payload.json | fn run +# push it to Docker Hub +fn push +# Create a route to this function on IronFunctions +fn routes create pythonapp /hello ``` `-v` is optional, but it allows you to see how this function is being built. @@ -32,7 +38,7 @@ fn publish Now you can start jobs on your function. Let's quickly queue up a job to try it out. ```sh -cat hello.payload.json | fn run pythonapp /hello +cat hello.payload.json | fn call pythonapp /hello ``` Here's a curl example to show how easy it is to do in any language: diff --git a/examples/hello/python/func.yaml b/examples/hello/python/func.yaml deleted file mode 100644 index 023c4f75c..000000000 --- a/examples/hello/python/func.yaml +++ /dev/null @@ -1,5 +0,0 @@ -app: pythonapp -route: /hello -image: USERNAME/hello:0.0.1 -build: -- docker run --rm -v "$PWD":/worker -w /worker iron/python:2-dev pip install -t packages -r requirements.txt \ No newline at end of file diff --git a/examples/hello/ruby/README.md b/examples/hello/ruby/README.md index 288bd9607..102b5c607 100644 --- a/examples/hello/ruby/README.md +++ b/examples/hello/ruby/README.md @@ -11,10 +11,8 @@ docker run --rm -v "$PWD":/worker -w /worker iron/ruby:dev bundle install --stan fn build # test it cat hello.payload.json | fn run -# push it to Docker Hub for use with IronFunctions -fn push # Create a route to this function on IronFunctions -fn routes create myapp /hello +fn deploy myapp ``` Now surf to: http://localhost:8080/r/myapp/hello diff --git a/examples/redis/func.yaml b/examples/redis/func.yaml new file mode 100644 index 000000000..c3762d507 --- /dev/null +++ b/examples/redis/func.yaml @@ -0,0 +1,3 @@ +name: iron/func-redis +build: + - ./build.sh \ No newline at end of file diff --git a/examples/redis/func.yml b/examples/redis/func.yml deleted file mode 100644 index 7a63e0777..000000000 --- a/examples/redis/func.yml +++ /dev/null @@ -1,3 +0,0 @@ -image: iron/func-redis -build: - - ./build.sh \ No newline at end of file diff --git a/examples/slackbot/guppy/README.md b/examples/slackbot/guppy/README.md index b2773354f..220ac9d91 100644 --- a/examples/slackbot/guppy/README.md +++ b/examples/slackbot/guppy/README.md @@ -11,10 +11,8 @@ docker run --rm -v "$PWD":/worker -w /worker iron/ruby:dev bundle install --stan fn build # test it cat slack.payload | fn run -# push it to Docker Hub for use with IronFunctions -fn push # Create a route to this function on IronFunctions -fn routes create slackbot /guppy +fn deploy slackbot # Change the route response header content-type to application/json curl -X PUT http://127.0.0.1:8080/v1/apps/slackbot/routes/guppy -d '{ "route": { "headers": { "Content-type": ["application/json"] } } }' ``` diff --git a/examples/sleeper/func.yaml b/examples/sleeper/func.yaml index c6c42b691..953ec06a1 100644 --- a/examples/sleeper/func.yaml +++ b/examples/sleeper/func.yaml @@ -1,2 +1,2 @@ name: iron/sleeper -version: 0.0.2 +version: 0.0.2 \ No newline at end of file diff --git a/examples/twitter/func.yaml b/examples/twitter/func.yaml new file mode 100644 index 000000000..5ede21210 --- /dev/null +++ b/examples/twitter/func.yaml @@ -0,0 +1,3 @@ +name: iron/func-twitter +build: + - ./build.sh \ No newline at end of file diff --git a/examples/twitter/func.yml b/examples/twitter/func.yml deleted file mode 100644 index c041ecb15..000000000 --- a/examples/twitter/func.yml +++ /dev/null @@ -1,3 +0,0 @@ -image: iron/func-twitter -build: - - ./build.sh \ No newline at end of file diff --git a/fn/README.md b/fn/README.md index 453a05896..b15e9bd67 100644 --- a/fn/README.md +++ b/fn/README.md @@ -62,9 +62,8 @@ myapp $ fn apps create otherapp # create new app otherapp created -$ fn apps describe otherapp # describe an app -app: otherapp -no specific configuration +$ fn apps config otherapp # show app-specific configuration +this application has no configurations $ fn apps myapp @@ -156,60 +155,20 @@ $ export API_URL="http://myfunctions.example.org/" $ fn ... ``` -## Publish +## Bulk deploy -Also there is the publish command that is going to scan all local directory for +Also there is the `deploy` command that is going to scan all local directory for functions, rebuild them and push them to Docker Hub and update them in -IronFunction. +IronFunction. It will use the `route` entry in the existing function file to +see the update in the daemon. + ```sh -$ fn publish -path result -/app/hello done -/app/hello-sync error: no Dockerfile found for this function -/app/test done +$ fn deploy APP ``` -It works by scanning all children directories of the current working directory, -following this convention: - -
┌───────┐
-│  ./   │
-└───┬───┘
-    │     ┌───────┐
-    ├────▶│ myapp │
-    │     └───┬───┘
-    │         │     ┌───────┐
-    │         ├────▶│route1 │
-    │         │     └───────┘
-    │         │         │     ┌─────────┐
-    │         │         ├────▶│subroute1│
-    │         │         │     └─────────┘
-    │
-    │     ┌───────┐
-    ├────▶│ other │
-    │     └───┬───┘
-    │         │     ┌───────┐
-    │         ├────▶│route1 │
-    │         │     └───────┘
- - -It will render this pattern of updates: - -```sh -$ fn publish -path result -/myapp/route1/subroute1 done -/other/route1 done -``` - -It means that first subdirectory are always considered app names (e.g. `myapp` -and `other`), each subdirectory of these firsts are considered part of the route -(e.g. `route1/subroute1`). - -`fn publish` expects that each directory to contain a file `func.yaml` -which instructs `fn` on how to act with that particular update, and a -Dockerfile which it is going to use to build the image and push to Docker Hub. +`fn deploy` expects that each directory to contain a file `func.yaml` +which instructs `fn` on how to act with that particular update. ## Contributing diff --git a/fn/build.go b/fn/build.go index de68b5dd6..84b44eff4 100644 --- a/fn/build.go +++ b/fn/build.go @@ -8,35 +8,45 @@ import ( ) func build() cli.Command { - cmd := buildcmd{commoncmd: &commoncmd{}} + cmd := buildcmd{} flags := append([]cli.Flag{}, cmd.flags()...) return cli.Command{ Name: "build", Usage: "build function version", Flags: flags, - Action: cmd.scan, + Action: cmd.build, } } type buildcmd struct { - *commoncmd + verbose bool } -func (b *buildcmd) scan(c *cli.Context) error { - b.commoncmd.scan(b.walker) - return nil -} - -func (b *buildcmd) walker(path string, info os.FileInfo, err error) error { - walker(path, info, err, b.build) - return nil +func (b *buildcmd) flags() []cli.Flag { + return []cli.Flag{ + cli.BoolFlag{ + Name: "v", + Usage: "verbose mode", + Destination: &b.verbose, + }, + } } // build will take the found valid function and build it -func (b *buildcmd) build(path string) error { - fmt.Fprintln(b.verbwriter, "building", path) +func (b *buildcmd) build(c *cli.Context) error { + verbwriter := verbwriter(b.verbose) - ff, err := b.buildfunc(path) + path, err := os.Getwd() + if err != nil { + return err + } + fn, err := findFuncfile(path) + if err != nil { + return err + } + + fmt.Fprintln(verbwriter, "building", fn) + ff, err := buildfunc(verbwriter, fn) if err != nil { return err } diff --git a/fn/bump.go b/fn/bump.go index 19f3d7ded..e7c8e7270 100644 --- a/fn/bump.go +++ b/fn/bump.go @@ -15,35 +15,46 @@ var ( ) func bump() cli.Command { - cmd := bumpcmd{commoncmd: &commoncmd{}} + cmd := bumpcmd{} flags := append([]cli.Flag{}, cmd.flags()...) return cli.Command{ Name: "bump", Usage: "bump function version", Flags: flags, - Action: cmd.scan, + Action: cmd.bump, } } type bumpcmd struct { - *commoncmd + verbose bool } -func (b *bumpcmd) scan(c *cli.Context) error { - b.commoncmd.scan(b.walker) - return nil -} - -func (b *bumpcmd) walker(path string, info os.FileInfo, err error) error { - walker(path, info, err, b.bump) - return nil +func (b *bumpcmd) flags() []cli.Flag { + return []cli.Flag{ + cli.BoolFlag{ + Name: "v", + Usage: "verbose mode", + Destination: &b.verbose, + }, + } } // bump will take the found valid function and bump its version -func (b *bumpcmd) bump(path string) error { - fmt.Fprintln(b.verbwriter, "bumping version for", path) +func (b *bumpcmd) bump(c *cli.Context) error { + verbwriter := verbwriter(b.verbose) - funcfile, err := parsefuncfile(path) + path, err := os.Getwd() + if err != nil { + return err + } + fn, err := findFuncfile(path) + if err != nil { + return err + } + + fmt.Fprintln(verbwriter, "bumping version for", fn) + + funcfile, err := parsefuncfile(fn) if err != nil { return err } @@ -66,7 +77,7 @@ func (b *bumpcmd) bump(path string) error { funcfile.Version = newver.String() - if err := storefuncfile(path, funcfile); err != nil { + if err := storefuncfile(fn, funcfile); err != nil { return err } diff --git a/fn/common.go b/fn/common.go index 3aba3b160..c86a4e03f 100644 --- a/fn/common.go +++ b/fn/common.go @@ -11,159 +11,41 @@ import ( "path/filepath" "strings" "text/template" - "time" "github.com/iron-io/functions/fn/langs" - "github.com/urfave/cli" ) -func isFuncfile(path string, info os.FileInfo) bool { - if info.IsDir() { - return false +func verbwriter(verbose bool) io.Writer { + verbwriter := ioutil.Discard + if verbose { + verbwriter = os.Stderr } - - basefn := filepath.Base(path) - for _, fn := range validfn { - if basefn == fn { - return true - } - } - - return false + return verbwriter } -func walker(path string, info os.FileInfo, err error, f func(path string) error) { - if err := f(path); err != nil { - fmt.Fprintln(os.Stderr, path, err) - } -} - -type commoncmd struct { - wd string - verbose bool - force bool - recursively bool - - verbwriter io.Writer -} - -func (c *commoncmd) flags() []cli.Flag { - return []cli.Flag{ - cli.StringFlag{ - Name: "d", - Usage: "working directory", - Destination: &c.wd, - EnvVar: "WORK_DIR", - Value: "./", - }, - cli.BoolFlag{ - Name: "v", - Usage: "verbose mode", - Destination: &c.verbose, - }, - cli.BoolFlag{ - Name: "f", - Usage: "force updating of all functions that are already up-to-date", - Destination: &c.force, - }, - cli.BoolFlag{ - Name: "r", - Usage: "recursively scan all functions", - Destination: &c.recursively, - }, - } -} - -func (c *commoncmd) scan(walker func(path string, info os.FileInfo, err error) error) { - c.verbwriter = ioutil.Discard - if c.verbose { - c.verbwriter = os.Stderr - } - - var walked bool - - err := filepath.Walk(c.wd, func(path string, info os.FileInfo, err error) error { - if !c.recursively && path != c.wd && info.IsDir() { - return filepath.SkipDir - } - - if !isFuncfile(path, info) { - return nil - } - - if c.recursively && !c.force && !isstale(path) { - return nil - } - - e := walker(path, info, err) - now := time.Now() - os.Chtimes(path, now, now) - walked = true - return e - }) - if err != nil { - fmt.Fprintf(c.verbwriter, "file walk error: %s\n", err) - } - - if !walked { - fmt.Println("No function file found.") - return - } -} - -// Theory of operation: this takes an optimistic approach to detect whether a -// package must be rebuild/bump/published. It loads for all files mtime's and -// compare with functions.json own mtime. If any file is younger than -// functions.json, it triggers a rebuild. -// The problem with this approach is that depending on the OS running it, the -// time granularity of these timestamps might lead to false negatives - that is -// a package that is stale but it is not recompiled. A more elegant solution -// could be applied here, like https://golang.org/src/cmd/go/pkg.go#L1111 -func isstale(path string) bool { - fi, err := os.Stat(path) - if err != nil { - return true - } - - fnmtime := fi.ModTime() - dir := filepath.Dir(path) - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - return nil - } - if info.ModTime().After(fnmtime) { - return errors.New("found stale package") - } - return nil - }) - - return err != nil -} - -func (c commoncmd) buildfunc(path string) (*funcfile, error) { +func buildfunc(verbwriter io.Writer, path string) (*funcfile, error) { funcfile, err := parsefuncfile(path) if err != nil { return nil, err } - if err := c.localbuild(path, funcfile.Build); err != nil { + if err := localbuild(verbwriter, path, funcfile.Build); err != nil { return nil, err } - if err := c.dockerbuild(path, funcfile); err != nil { + if err := dockerbuild(verbwriter, path, funcfile); err != nil { return nil, err } return funcfile, nil } -func (c commoncmd) localbuild(path string, steps []string) error { +func localbuild(verbwriter io.Writer, path string, steps []string) error { for _, cmd := range steps { exe := exec.Command("/bin/sh", "-c", cmd) exe.Dir = filepath.Dir(path) - exe.Stderr = c.verbwriter - exe.Stdout = c.verbwriter - fmt.Fprintf(c.verbwriter, "- %s:\n", cmd) + exe.Stderr = verbwriter + exe.Stdout = verbwriter if err := exe.Run(); err != nil { return fmt.Errorf("error running command %v (%v)", cmd, err) } @@ -172,7 +54,7 @@ func (c commoncmd) localbuild(path string, steps []string) error { return nil } -func (c commoncmd) dockerbuild(path string, ff *funcfile) error { +func dockerbuild(verbwriter io.Writer, path string, ff *funcfile) error { dir := filepath.Dir(path) var helper langs.LangHelper @@ -275,7 +157,6 @@ func writeTmpDockerfile(dir string, ff *funcfile) error { buffer.WriteString(s) buffer.WriteString("\"") } - fmt.Println(buffer.String()) t := template.Must(template.New("Dockerfile").Parse(tplDockerfile)) err = t.Execute(fd, struct { @@ -293,3 +174,25 @@ func extractEnvConfig(configs []string) map[string]string { } return c } + +func dockerpush(ff *funcfile) error { + cmd := exec.Command("docker", "push", ff.FullName()) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + if err := cmd.Run(); err != nil { + return fmt.Errorf("error running docker push: %v", err) + } + return nil +} + +func appNamePath(img string) (string, string) { + sep := strings.Index(img, "/") + if sep < 0 { + return "", "" + } + tag := strings.Index(img[sep:], ":") + if tag < 0 { + tag = len(img[sep:]) + } + return img[:sep], img[sep : sep+tag] +} diff --git a/fn/deploy.go b/fn/deploy.go new file mode 100644 index 000000000..4390077f1 --- /dev/null +++ b/fn/deploy.go @@ -0,0 +1,239 @@ +package main + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + functions "github.com/iron-io/functions_go" + "github.com/urfave/cli" +) + +func deploy() cli.Command { + cmd := deploycmd{ + RoutesApi: functions.NewRoutesApi(), + } + var flags []cli.Flag + flags = append(flags, cmd.flags()...) + return cli.Command{ + Name: "deploy", + ArgsUsage: "`APPNAME`", + Usage: "scan local directory for functions, build and push all of them to `APPNAME`.", + Flags: flags, + Action: cmd.scan, + } +} + +type deploycmd struct { + appName string + *functions.RoutesApi + + wd string + verbose bool + incremental bool + skippush bool + + verbwriter io.Writer +} + +func (p *deploycmd) flags() []cli.Flag { + return []cli.Flag{ + cli.BoolFlag{ + Name: "v", + Usage: "verbose mode", + Destination: &p.verbose, + }, + cli.StringFlag{ + Name: "d", + Usage: "working directory", + Destination: &p.wd, + EnvVar: "WORK_DIR", + Value: "./", + }, + cli.BoolFlag{ + Name: "i", + Usage: "uses incremental building", + Destination: &p.incremental, + }, + cli.BoolFlag{ + Name: "skip-push", + Usage: "does not push Docker built images onto Docker Hub - useful for local development.", + Destination: &p.skippush, + }, + } +} + +func (p *deploycmd) scan(c *cli.Context) error { + if c.Args().First() == "" { + return errors.New("application name is missing") + } + p.appName = c.Args().First() + p.verbwriter = verbwriter(p.verbose) + + var walked bool + + err := filepath.Walk(p.wd, func(path string, info os.FileInfo, err error) error { + if path != p.wd && info.IsDir() { + return filepath.SkipDir + } + + if !isFuncfile(path, info) { + return nil + } + + if p.incremental && !isstale(path) { + return nil + } + + e := p.deploy(path) + if err != nil { + fmt.Fprintln(p.verbwriter, path, e) + } + + now := time.Now() + os.Chtimes(path, now, now) + walked = true + return e + }) + if err != nil { + fmt.Fprintf(p.verbwriter, "file walk error: %s\n", err) + } + + if !walked { + return errors.New("No function file found.") + } + + return nil +} + +// deploy will take the found function and check for the presence of a +// Dockerfile, and run a three step process: parse functions file, build and +// push the container, and finally it will update function's route. Optionally, +// the route can be overriden inside the functions file. +func (p *deploycmd) deploy(path string) error { + fmt.Fprintln(p.verbwriter, "deploying", path) + + funcfile, err := buildfunc(p.verbwriter, path) + if err != nil { + return err + } + + if p.skippush { + return nil + } + + if err := dockerpush(funcfile); err != nil { + return err + } + + return p.route(path, funcfile) +} + +func (p *deploycmd) route(path string, ff *funcfile) error { + if err := resetBasePath(p.Configuration); err != nil { + return fmt.Errorf("error setting endpoint: %v", err) + } + + if ff.Path == nil { + _, path := appNamePath(ff.FullName()) + ff.Path = &path + } + + if ff.Memory == nil { + ff.Memory = new(int64) + } + if ff.Type == nil { + ff.Type = new(string) + } + if ff.Format == nil { + ff.Format = new(string) + } + if ff.MaxConcurrency == nil { + ff.MaxConcurrency = new(int) + } + if ff.Timeout == nil { + dur := time.Duration(0) + ff.Timeout = &dur + } + + body := functions.RouteWrapper{ + Route: functions.Route{ + Path: *ff.Path, + Image: ff.FullName(), + Memory: *ff.Memory, + Type_: *ff.Type, + Config: expandEnvConfig(ff.Config), + Headers: ff.Headers, + Format: *ff.Format, + MaxConcurrency: int32(*ff.MaxConcurrency), + Timeout: int32(ff.Timeout.Seconds()), + }, + } + + fmt.Fprintf(p.verbwriter, "updating API with app: %s route: %s name: %s \n", p.appName, *ff.Path, ff.Name) + + wrapper, resp, err := p.AppsAppRoutesPost(p.appName, body) + if err != nil { + return fmt.Errorf("error getting routes: %v", err) + } + if resp.StatusCode == http.StatusBadRequest { + return fmt.Errorf("error storing this route: %s", wrapper.Error_.Message) + } + + return nil +} + +func expandEnvConfig(configs map[string]string) map[string]string { + for k, v := range configs { + configs[k] = os.ExpandEnv(v) + } + return configs +} + +func isFuncfile(path string, info os.FileInfo) bool { + if info.IsDir() { + return false + } + + basefn := filepath.Base(path) + for _, fn := range validfn { + if basefn == fn { + return true + } + } + + return false +} + +// Theory of operation: this takes an optimistic approach to detect whether a +// package must be rebuild/bump/deployed. It loads for all files mtime's and +// compare with functions.json own mtime. If any file is younger than +// functions.json, it triggers a rebuild. +// The problem with this approach is that depending on the OS running it, the +// time granularity of these timestamps might lead to false negatives - that is +// a package that is stale but it is not recompiled. A more elegant solution +// could be applied here, like https://golang.org/src/cmd/go/pkg.go#L1111 +func isstale(path string) bool { + fi, err := os.Stat(path) + if err != nil { + return true + } + + fnmtime := fi.ModTime() + dir := filepath.Dir(path) + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + if info.ModTime().After(fnmtime) { + return errors.New("found stale package") + } + return nil + }) + + return err != nil +} diff --git a/fn/funcfile.go b/fn/funcfile.go index 9b349a9cd..198744c13 100644 --- a/fn/funcfile.go +++ b/fn/funcfile.go @@ -24,12 +24,11 @@ var ( ) type funcfile struct { - App *string `yaml:"app,omitempty",json:"app,omitempty"` Name string `yaml:"name,omitempty",json:"name,omitempty"` Version string `yaml:"version,omitempty",json:"version,omitempty"` Runtime *string `yaml:"runtime,omitempty",json:"runtime,omitempty"` Entrypoint *string `yaml:"entrypoint,omitempty",json:"entrypoint,omitempty"` - Route *string `yaml:"route,omitempty",json:"route,omitempty"` + Path *string `yaml:"path,omitempty",json:"path,omitempty"` Type *string `yaml:"type,omitempty",json:"type,omitempty"` Memory *int64 `yaml:"memory,omitempty",json:"memory,omitempty"` Format *string `yaml:"format,omitempty",json:"format,omitempty"` @@ -62,13 +61,22 @@ func (ff *funcfile) RuntimeTag() (runtime, tag string) { return rt[:tagpos], rt[tagpos+1:] } -func findFuncfile() (*funcfile, error) { +func findFuncfile(path string) (string, error) { for _, fn := range validfn { - if exists(fn) { - return parsefuncfile(fn) + fullfn := filepath.Join(path, fn) + if exists(fullfn) { + return fullfn, nil } } - return nil, newNotFoundError("could not find function file") + return "", newNotFoundError("could not find function file") +} + +func loadFuncfile() (*funcfile, error) { + fn, err := findFuncfile(".") + if err != nil { + return nil, err + } + return parsefuncfile(fn) } func parsefuncfile(path string) (*funcfile, error) { diff --git a/fn/init.go b/fn/init.go index 1c99c65ec..7b831f59a 100644 --- a/fn/init.go +++ b/fn/init.go @@ -91,7 +91,7 @@ func initFn() cli.Command { func (a *initFnCmd) init(c *cli.Context) error { if !a.force { - ff, err := findFuncfile() + ff, err := loadFuncfile() if _, ok := err.(*notFoundError); !ok && err != nil { return err } @@ -105,15 +105,23 @@ func (a *initFnCmd) init(c *cli.Context) error { return err } + var ffmt *string + if a.format != "" { + ffmt = &a.format + } + ff := &funcfile{ Name: a.name, Runtime: &a.runtime, Version: initialVersion, Entrypoint: &a.entrypoint, - Format: &a.format, + Format: ffmt, MaxConcurrency: &a.maxConcurrency, } + _, path := appNamePath(ff.FullName()) + ff.Path = &path + if err := encodeFuncfileYAML("func.yaml", ff); err != nil { return err } @@ -130,7 +138,7 @@ func (a *initFnCmd) buildFuncFile(c *cli.Context) error { a.name = c.Args().First() if a.name == "" || strings.Contains(a.name, ":") { - return errors.New("Please specify a name for your function in the following format /") + return errors.New("Please specify a name for your function in the following format /.\nTry: fn init /") } if exists("Dockerfile") { diff --git a/fn/lambda.go b/fn/lambda.go index 6fed71928..4629d1903 100644 --- a/fn/lambda.go +++ b/fn/lambda.go @@ -276,11 +276,10 @@ func basicImportHandler(functionName, tmpFileName string, opts *createImageOptio func createFunctionYaml(opts createImageOptions) error { strs := strings.Split(opts.Name, "/") - route := fmt.Sprintf("/%s", strs[1]) + path := fmt.Sprintf("/%s", strs[1]) funcDesc := &funcfile{ - App: &strs[0], Name: opts.Name, - Route: &route, + Path: &path, Config: opts.Config, } diff --git a/fn/main.go b/fn/main.go index b83113a90..2c521c8d7 100644 --- a/fn/main.go +++ b/fn/main.go @@ -27,9 +27,9 @@ ENVIRONMENT VARIABLES: build(), bump(), call(), + deploy(), initFn(), lambda(), - publish(), push(), routes(), run(), diff --git a/fn/publish.go b/fn/publish.go deleted file mode 100644 index b93976b77..000000000 --- a/fn/publish.go +++ /dev/null @@ -1,153 +0,0 @@ -package main - -import ( - "fmt" - "net/http" - "os" - "os/exec" - "strings" - - functions "github.com/iron-io/functions_go" - "github.com/urfave/cli" -) - -func publish() cli.Command { - cmd := publishcmd{ - commoncmd: &commoncmd{}, - RoutesApi: functions.NewRoutesApi(), - } - var flags []cli.Flag - flags = append(flags, cmd.flags()...) - flags = append(flags, cmd.commoncmd.flags()...) - return cli.Command{ - Name: "publish", - Usage: "scan local directory for functions, build and publish them.", - Flags: flags, - Action: cmd.scan, - } -} - -type publishcmd struct { - *commoncmd - *functions.RoutesApi - - skippush bool -} - -func (p *publishcmd) flags() []cli.Flag { - return []cli.Flag{ - cli.BoolFlag{ - Name: "skip-push", - Usage: "does not push Docker built images onto Docker Hub - useful for local development.", - Destination: &p.skippush, - }, - } -} - -func (p *publishcmd) scan(c *cli.Context) error { - p.commoncmd.scan(p.walker) - return nil -} - -func (p *publishcmd) walker(path string, info os.FileInfo, err error) error { - walker(path, info, err, p.publish) - return nil -} - -// publish will take the found function and check for the presence of a -// Dockerfile, and run a three step process: parse functions file, build and -// push the container, and finally it will update function's route. Optionally, -// the route can be overriden inside the functions file. -func (p *publishcmd) publish(path string) error { - fmt.Fprintln(p.verbwriter, "publishing", path) - - funcfile, err := p.buildfunc(path) - if err != nil { - return err - } - - if p.skippush { - return nil - } - - if err := p.dockerpush(funcfile); err != nil { - return err - } - - return p.route(path, funcfile) -} - -func (p publishcmd) dockerpush(ff *funcfile) error { - cmd := exec.Command("docker", "push", ff.FullName()) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - if err := cmd.Run(); err != nil { - return fmt.Errorf("error running docker push: %v", err) - } - return nil -} - -func (p *publishcmd) route(path string, ff *funcfile) error { - if err := resetBasePath(p.Configuration); err != nil { - return fmt.Errorf("error setting endpoint: %v", err) - } - - // TODO: This is just a nasty hack and should be cleaned up all the way - pathsSplit := strings.Split(ff.FullName(), "/") - - if ff.App == nil { - ff.App = &pathsSplit[0] - } - if ff.Route == nil { - path := "/" + strings.Split(pathsSplit[1], ":")[0] - ff.Route = &path - } - - if ff.Memory == nil { - ff.Memory = new(int64) - } - if ff.Type == nil { - ff.Type = new(string) - } - - body := functions.RouteWrapper{ - Route: functions.Route{ - Path: *ff.Route, - Image: ff.FullName(), - AppName: *ff.App, - Memory: *ff.Memory, - Type_: *ff.Type, - Config: expandEnvConfig(ff.Config), - Headers: ff.Headers, - Timeout: int32(ff.Timeout.Seconds()), - MaxConcurrency: int32(*ff.MaxConcurrency), - }, - } - - fmt.Fprintf(p.verbwriter, "updating API with app: %s route: %s name: %s \n", *ff.App, *ff.Route, ff.Name) - - wrapper, resp, err := p.AppsAppRoutesPost(*ff.App, body) - if err != nil { - return fmt.Errorf("error getting routes: %v", err) - } - if resp.StatusCode == http.StatusBadRequest { - return fmt.Errorf("error storing this route: %s", wrapper.Error_.Message) - } - - return nil -} - -func expandEnvConfig(configs map[string]string) map[string]string { - for k, v := range configs { - configs[k] = os.ExpandEnv(v) - } - return configs -} - -func reverse(s string) string { - r := []rune(s) - for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 { - r[i], r[j] = r[j], r[i] - } - return string(r) -} diff --git a/fn/push.go b/fn/push.go index 9cabba5dd..92b33d98d 100644 --- a/fn/push.go +++ b/fn/push.go @@ -1,60 +1,59 @@ package main import ( + "errors" "fmt" - "os" - functions "github.com/iron-io/functions_go" "github.com/urfave/cli" ) func push() cli.Command { - cmd := pushcmd{ - publishcmd: &publishcmd{ - commoncmd: &commoncmd{}, - RoutesApi: functions.NewRoutesApi(), - }, - } + cmd := pushcmd{} var flags []cli.Flag - flags = append(flags, cmd.commoncmd.flags()...) + flags = append(flags, cmd.flags()...) return cli.Command{ Name: "push", Usage: "push function to Docker Hub", Flags: flags, - Action: cmd.scan, + Action: cmd.push, } } type pushcmd struct { - *publishcmd + verbose bool } -func (p *pushcmd) scan(c *cli.Context) error { - p.commoncmd.scan(p.walker) - return nil -} - -func (p *pushcmd) walker(path string, info os.FileInfo, err error) error { - walker(path, info, err, p.push) - return nil +func (p *pushcmd) flags() []cli.Flag { + return []cli.Flag{ + cli.BoolFlag{ + Name: "v", + Usage: "verbose mode", + Destination: &p.verbose, + }, + } } // push will take the found function and check for the presence of a // Dockerfile, and run a three step process: parse functions file, // push the container, and finally it will update function's route. Optionally, // the route can be overriden inside the functions file. -func (p *pushcmd) push(path string) error { - fmt.Fprintln(p.verbwriter, "pushing", path) +func (p *pushcmd) push(c *cli.Context) error { + verbwriter := verbwriter(p.verbose) - funcfile, err := parsefuncfile(path) + ff, err := loadFuncfile() if err != nil { + if _, ok := err.(*notFoundError); ok { + return errors.New("error: image name is missing or no function file found") + } return err } - if err := p.dockerpush(funcfile); err != nil { + fmt.Fprintln(verbwriter, "pushing", ff.FullName()) + + if err := dockerpush(ff); err != nil { return err } - fmt.Printf("Function %v pushed successfully to Docker Hub.\n", funcfile.FullName()) + fmt.Printf("Function %v pushed successfully to Docker Hub.\n", ff.FullName()) return nil } diff --git a/fn/routes.go b/fn/routes.go index f3af6b57f..9600952c9 100644 --- a/fn/routes.go +++ b/fn/routes.go @@ -279,7 +279,7 @@ func (a *routesCmd) create(c *cli.Context) error { timeout time.Duration ) if image == "" { - ff, err := findFuncfile() + ff, err := loadFuncfile() if err != nil { if _, ok := err.(*notFoundError); ok { return errors.New("error: image name is missing or no function file found") @@ -512,7 +512,7 @@ func (a *routesCmd) headersList(c *cli.Context) error { return errors.New("this route has no headers") } - fmt.Println(wrapper.Route.AppName, wrapper.Route.Path, "headers:") + fmt.Println(appName, wrapper.Route.Path, "headers:") w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, ' ', 0) for k, v := range headers { fmt.Fprint(w, k, ":\t", v, "\n") @@ -557,7 +557,7 @@ func (a *routesCmd) headersSet(c *cli.Context) error { return fmt.Errorf("error updating route configuration: %v", err) } - fmt.Println(wrapper.Route.AppName, wrapper.Route.Path, "headers updated", key, "with", value) + fmt.Println(appName, wrapper.Route.Path, "headers updated", key, "with", value) return nil } @@ -600,6 +600,6 @@ func (a *routesCmd) headersUnset(c *cli.Context) error { return fmt.Errorf("error updating route configuration: %v", err) } - fmt.Println(wrapper.Route.AppName, wrapper.Route.Path, "removed header", key) + fmt.Println(appName, wrapper.Route.Path, "removed header", key) return nil } diff --git a/fn/run.go b/fn/run.go index db61f5be7..2cfc64a04 100644 --- a/fn/run.go +++ b/fn/run.go @@ -36,7 +36,7 @@ func runflags() []cli.Flag { func (r *runCmd) run(c *cli.Context) error { image := c.Args().First() if image == "" { - ff, err := findFuncfile() + ff, err := loadFuncfile() if err != nil { if _, ok := err.(*notFoundError); ok { return errors.New("error: image name is missing or no function file found")