fn: support for functions testing (#379)

* fn: add test framework

* fn: make routes creation smarter

* fn: add testframework examples

* fn: remove unnecessary dependency

* fn: update doc

* fn: fix consistenty between runff, runlocaltest and runremotetest

docs/function-file.md

@@ -27,7 +27,8 @@ build:
 route updated to use it.
 
 `path` (optional) allows you to overwrite the calculated route from the path
-position. You may use it to override the calculated route.
+position. You may use it to override the calculated route. If you plan to use
+`fn test --remote=""`, this is mandatory.
 
 `version` represents current version of the function. When deploying, it is
 appended to the image as a tag.
@@ -47,4 +48,32 @@ during functions execution.
 
 `build` (optional) is an array of shell calls which are used to helping building
 the image. These calls are executed before `fn` calls `docker build` and
-`docker push`.
+`docker push`.
+
+## Testing functions
+
+`tests` (optional) is an array of tests that can be used to valid functions both
+locally and remotely. It has the following structure
+
+```yaml
+tests:
+- name: envvar
+  in: "inserted stdin"
+  out: "expected stdout"
+  err: "expected stderr"
+  env:
+    envvar: trololo
+```
+
+`in` (optional) is a string that is going to be sent to the file's declared
+function.
+
+`out` (optional) is the expected output for this function test. It is present
+both in local and remote executions.
+
+`err` (optional) similar to `out`, however it read from `stderr`. It is only
+available for local machine tests.
+
+`env` (optional) is a map of environment variables that are injected during
+tests.
+

examples/testframework/local/README.md

@@ -0,0 +1,16 @@
+# Example of IronFunctions test framework - running functions locally
+
+This example will show you how to run a test suite on a function.
+
+```sh
+# build the test image (testframework:0.0.1)
+fn build
+# test it
+fn test
+```
+
+Alternatively, you can force a rebuild before the test suite with:
+```sh
+# build and test it
+fn test -b
+```

examples/testframework/local/func.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	envvar := os.Getenv("HEADER_ENVVAR")
+	if envvar != "" {
+		fmt.Println("HEADER_ENVVAR:", envvar)
+	}
+	fmt.Println("hw")
+}

examples/testframework/local/func.yaml

@@ -0,0 +1,15 @@
+name: testframework
+version: 0.0.1
+runtime: go
+entrypoint: ./func
+path: /tests
+tests:
+- name: simple
+  out: |
+    hw
+- name: envvar
+  out: |
+    HEADER_ENVVAR: trololo
+    hw
+  env:
+    envvar: trololo

examples/testframework/remote/README.md

@@ -0,0 +1,14 @@
+# Example of IronFunctions test framework - running functions remotely
+
+This example will show you how to run a test suite on a function.
+
+```sh
+# build the test image (iron/functions-testframework:0.0.1)
+fn build
+# push it
+fn push
+# create a route for the testframework
+fn routes create testframework
+# test it
+fn test --remote testframework
+```

examples/testframework/remote/func.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	envvar := os.Getenv("HEADER_ENVVAR")
+	if envvar != "" {
+		fmt.Println("HEADER_ENVVAR:", envvar)
+	}
+	fmt.Println("hw")
+}

examples/testframework/remote/func.yaml

@@ -0,0 +1,15 @@
+name: iron/functions-testframework
+version: 0.0.1
+runtime: go
+entrypoint: ./func
+path: /tests
+tests:
+- name: simple
+  out: |
+    hw
+- name: envvar
+  out: |
+    HEADER_ENVVAR: trololo
+    hw
+  env:
+    envvar: trololo

fn/README.md

@@ -170,6 +170,42 @@ $ fn deploy APP
 `fn deploy` expects that each directory to contain a file `func.yaml`
 which instructs `fn` on how to act with that particular update.
 
+## Testing functions
+
+If you added `tests` to the `func.yaml` file, you can have them tested using
+`fn test`.
+
+```sh
+$ fn test
+```
+
+During local development cycles, you can easily force a build before test:
+```sh
+$ fn test -b
+```
+
+When preparing to deploy you application, remember adding `path` to `func.yaml`,
+it will simplify both the creation of the route, and the execution of remote
+tests:
+```yaml
+name: me/myapp
+version: 1.0.0
+path: /myfunc
+```
+
+Once you application is done and deployed, you can run tests remotely:
+```
+# test the function locally first
+$ fn test -b
+
+# push it to Docker Hub and IronFunctions
+$ fn push
+$ fn routes create myapp
+
+# test it remotely
+$ fn test --remote myapp
+```
+
 ## Contributing
 
 Ensure you have Go configured and installed in your environment. Once it is

fn/funcfile.go

@@ -23,6 +23,14 @@ var (
 	errUnexpectedFileFormat = errors.New("unexpected file format for function file")
 )
 
+type fftest struct {
+	Name string            `yaml:"name,omitempty",json:"name,omitempty"`
+	In   *string           `yaml:"in,omitempty",json:"in,omitempty"`
+	Out  *string           `yaml:"out,omitempty",json:"out,omitempty"`
+	Err  *string           `yaml:"err,omitempty",json:"err,omitempty"`
+	Env  map[string]string `yaml:"env,omitempty",json:"env,omitempty"`
+}
+
 type funcfile struct {
 	Name           string              `yaml:"name,omitempty",json:"name,omitempty"`
 	Version        string              `yaml:"version,omitempty",json:"version,omitempty"`
@@ -37,6 +45,7 @@ type funcfile struct {
 	Headers        map[string][]string `yaml:"headers,omitempty",json:"headers,omitempty"`
 	Config         map[string]string   `yaml:"config,omitempty",json:"config,omitempty"`
 	Build          []string            `yaml:"build,omitempty",json:"build,omitempty"`
+	Tests          []fftest            `yaml:"tests,omitempty",json:"tests,omitempty"`
 }
 
 func (ff *funcfile) FullName() string {

fn/glide.lock

@@ -1,5 +1,5 @@
-hash: 7c5768e12a63862a0bea40ee5d6c51234c2e4a8bae3d7c944721d5a29dfe99a2
-updated: 2016-12-01T17:52:59.626069479+01:00
+hash: f3c0e4634313b824f30782a3431b6fb8ad2feaf382c765e6f6930bfd50f53750
+updated: 2016-12-01T21:51:57.931016569+01:00
 imports:
 - name: github.com/aws/aws-sdk-go
   version: 90dec2183a5f5458ee79cbaf4b8e9ab910bc81a6
@@ -107,10 +107,6 @@ imports:
   version: d26492970760ca5d33129d2d799e34be5c4782eb
 - name: github.com/urfave/cli
   version: d86a009f5e13f83df65d0d6cee9a2e3f1445f0da
-- name: golang.org/x/crypto
-  version: c10c31b5e94b6f7a0283272dc2bb27163dcea24b
-  subpackages:
-  - ssh/terminal
 - name: golang.org/x/net
   version: f315505cf3349909cdf013ea56690da34e96a451
   subpackages:

fn/glide.yaml

@@ -1,5 +1,11 @@
 package: github.com/iron-io/functions/fn
 import:
+- package: github.com/aws/aws-sdk-go
+  subpackages:
+  - aws
+  - aws/credentials
+  - aws/session
+  - service/lambda
 - package: github.com/docker/docker
   subpackages:
   - pkg/jsonmessage
@@ -7,16 +13,10 @@ import:
   subpackages:
   - bump
   - storage
-- package: github.com/iron-io/iron_go3
-  subpackages:
-  - config
+- package: github.com/iron-io/functions_go
+  version: 429df8920abd7c47dfcd6777dba278d6122ab93d
 - package: github.com/iron-io/lambda
   subpackages:
   - lambda
 - package: github.com/urfave/cli
-- package: golang.org/x/crypto
-  subpackages:
-  - ssh/terminal
 - package: gopkg.in/yaml.v2
-- package: github.com/iron-io/functions_go
-  version: 429df8920abd7c47dfcd6777dba278d6122ab93d

fn/main.go

@@ -33,6 +33,7 @@ ENVIRONMENT VARIABLES:
 		push(),
 		routes(),
 		run(),
+		testfn(),
 		version(),
 	}
 	app.Run(os.Args)

fn/routes.go

@@ -15,7 +15,6 @@ import (
 
 	functions "github.com/iron-io/functions_go"
 	"github.com/urfave/cli"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 type routesCmd struct {
@@ -215,36 +214,37 @@ func (a *routesCmd) call(c *cli.Context) error {
 		return fmt.Errorf("error setting endpoint: %v", err)
 	}
 
+	appName := c.Args().Get(0)
+	route := c.Args().Get(1)
+
 	baseURL, err := url.Parse(a.Configuration.BasePath)
 	if err != nil {
 		return fmt.Errorf("error parsing base path: %v", err)
 	}
 
-	appName := c.Args().Get(0)
-	route := c.Args().Get(1)
-
 	u, err := url.Parse("../")
 	u.Path = path.Join(u.Path, "r", appName, route)
+	content := stdin()
 
-	var content io.Reader
-	if !terminal.IsTerminal(int(os.Stdin.Fd())) {
-		content = os.Stdin
-	}
+	return callfn(baseURL.ResolveReference(u).String(), content, os.Stdout, c.StringSlice("e"))
+}
 
-	req, err := http.NewRequest("POST", baseURL.ResolveReference(u).String(), content)
+func callfn(u string, content io.Reader, output io.Writer, env []string) error {
+	req, err := http.NewRequest("POST", u, content)
 	if err != nil {
 		return fmt.Errorf("error running route: %v", err)
 	}
+
 	req.Header.Set("Content-Type", "application/json")
 
-	envAsHeader(req, c.StringSlice("e"))
+	envAsHeader(req, env)
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return fmt.Errorf("error running route: %v", err)
 	}
 
-	io.Copy(os.Stdout, resp.Body)
+	io.Copy(output, resp.Body)
 	return nil
 }
 
@@ -262,8 +262,8 @@ func envAsHeader(req *http.Request, selectedEnv []string) {
 }
 
 func (a *routesCmd) create(c *cli.Context) error {
-	if c.Args().Get(0) == "" || c.Args().Get(1) == "" {
-		return errors.New("error: routes creation takes three arguments: an app name, a route path and an image")
+	if c.Args().Get(0) == "" {
+		return errors.New("error: routes creation takes at least one argument: an app name")
 	}
 
 	if err := resetBasePath(a.Configuration); err != nil {
@@ -297,6 +297,16 @@ func (a *routesCmd) create(c *cli.Context) error {
 		if ff.Timeout != nil {
 			timeout = *ff.Timeout
 		}
+		if ff.Path != nil {
+			route = *ff.Path
+		}
+	}
+
+	if route == "" {
+		return errors.New("error: route path is missing")
+	}
+	if image == "" {
+		return errors.New("error: function image name is missing")
 	}
 
 	if f := c.String("format"); f != "" {

fn/run.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"strings"
@@ -46,12 +47,16 @@ func (r *runCmd) run(c *cli.Context) error {
 		image = ff.FullName()
 	}
 
+	return runff(image, stdin(), os.Stdout, os.Stderr, c.StringSlice("e"))
+}
+
+func runff(image string, stdin io.Reader, stdout, stderr io.Writer, restrictedEnv []string) error {
 	sh := []string{"docker", "run", "--rm", "-i"}
 
 	var env []string
 	detectedEnv := os.Environ()
-	if se := c.StringSlice("e"); len(se) > 0 {
-		detectedEnv = se
+	if len(restrictedEnv) > 0 {
+		detectedEnv = restrictedEnv
 	}
 
 	for _, e := range detectedEnv {
@@ -67,26 +72,9 @@ func (r *runCmd) run(c *cli.Context) error {
 
 	sh = append(sh, image)
 	cmd := exec.Command(sh[0], sh[1:]...)
-	// Check if stdin is being piped, and if not, create our own pipe with nothing in it
-	// http://stackoverflow.com/questions/22744443/check-if-there-is-something-to-read-on-stdin-in-golang
-	stat, err := os.Stdin.Stat()
-	if err != nil {
-		// On Windows, this gets an error if nothing is piped in.
-		// If something is piped in, it works fine.
-		// Turns out, this works just fine in our case as the piped stuff works properly and the non-piped doesn't hang either.
-		// See: https://github.com/golang/go/issues/14853#issuecomment-260170423
-		// log.Println("Warning: couldn't stat stdin, you are probably on Windows. Be sure to pipe something into this command, eg: 'echo \"hello\" | fn run'")
-	} else {
-		if (stat.Mode() & os.ModeCharDevice) == 0 {
-			// log.Println("data is being piped to stdin")
-			cmd.Stdin = os.Stdin
-		} else {
-			// log.Println("stdin is from a terminal")
-			cmd.Stdin = strings.NewReader("")
-		}
-	}
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
+	cmd.Stdin = stdin
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
 	cmd.Env = env
 	return cmd.Run()
 }

fn/run_others.go

@@ -0,0 +1,18 @@
+// +build !windows
+
+package main
+
+import (
+	"io"
+	"os"
+	"strings"
+)
+
+func stdin() io.Reader {
+	var stdin io.Reader = os.Stdin
+	stat, err := os.Stdin.Stat()
+	if err != nil || (stat.Mode()&os.ModeCharDevice) != 0 {
+		stdin = strings.NewReader("")
+	}
+	return stdin
+}

fn/run_windows.go

@@ -0,0 +1,25 @@
+// +build windows
+
+package main
+
+import (
+	"io"
+	"os"
+	"strings"
+	"syscall"
+	"unsafe"
+)
+
+func getStdin() io.Reader {
+	var stdin io.Reader = os.Stdin
+	if isTerminal(int(os.Stdin.Fd())) {
+		stdin = strings.NewReader("")
+	}
+	return stdin
+}
+
+func isTerminal(fd int) bool {
+	var st uint32
+	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
+	return r != 0 && e == 0
+}

fn/testfn.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	functions "github.com/iron-io/functions_go"
+	"github.com/urfave/cli"
+)
+
+func testfn() cli.Command {
+	cmd := testcmd{RoutesApi: functions.NewRoutesApi()}
+	return cli.Command{
+		Name:   "test",
+		Usage:  "run functions test if present",
+		Flags:  cmd.flags(),
+		Action: cmd.test,
+	}
+}
+
+type testcmd struct {
+	*functions.RoutesApi
+
+	build  bool
+	remote string
+}
+
+func (t *testcmd) flags() []cli.Flag {
+	return []cli.Flag{
+		cli.BoolFlag{
+			Name:        "b",
+			Usage:       "build before test",
+			Destination: &t.build,
+		},
+		cli.StringFlag{
+			Name:        "remote",
+			Usage:       "run tests by calling the function on IronFunctions daemon on `appname`",
+			Destination: &t.remote,
+		},
+	}
+}
+
+func (t *testcmd) test(c *cli.Context) error {
+	if t.build {
+		b := &buildcmd{verbose: true}
+		if err := b.build(c); err != nil {
+			return err
+		}
+		fmt.Println()
+	}
+
+	ff, err := loadFuncfile()
+	if err != nil {
+		return err
+	}
+
+	if len(ff.Tests) == 0 {
+		return errors.New("no tests found for this function")
+	}
+
+	target := ff.FullName()
+	runtest := runlocaltest
+	if t.remote != "" {
+		if ff.Path == nil || *ff.Path == "" {
+			return errors.New("execution of tests on remote server demand that this function to have a `path`.")
+		}
+		if err := resetBasePath(t.Configuration); err != nil {
+			return fmt.Errorf("error setting endpoint: %v", err)
+		}
+		baseURL, err := url.Parse(t.Configuration.BasePath)
+		if err != nil {
+			return fmt.Errorf("error parsing base path: %v", err)
+		}
+
+		u, err := url.Parse("../")
+		u.Path = path.Join(u.Path, "r", t.remote, *ff.Path)
+		target = baseURL.ResolveReference(u).String()
+		runtest = runremotetest
+	}
+
+	var foundErr bool
+	fmt.Println("running tests on", ff.FullName(), ":")
+	for _, tt := range ff.Tests {
+		start := time.Now()
+		var err error
+		err = runtest(target, tt.In, tt.Out, tt.Err, tt.Env)
+
+		fmt.Print("\t - ", tt.Name, " (", time.Since(start), "): ")
+
+		if err != nil {
+			fmt.Println()
+			foundErr = true
+			scanner := bufio.NewScanner(strings.NewReader(err.Error()))
+			for scanner.Scan() {
+				fmt.Println("\t\t", scanner.Text())
+			}
+			if err := scanner.Err(); err != nil {
+				fmt.Fprintln(os.Stderr, "reading test result:", err)
+				break
+			}
+			continue
+		}
+
+		fmt.Println("OK")
+	}
+
+	if foundErr {
+		return errors.New("errors found")
+	}
+	return nil
+}
+
+func runlocaltest(target string, in, expectedOut, expectedErr *string, env map[string]string) error {
+	stdin := &bytes.Buffer{}
+	if in != nil {
+		stdin = bytes.NewBufferString(*in)
+	}
+
+	var stdout, stderr bytes.Buffer
+	var restrictedEnv []string
+	for k, v := range env {
+		oldv := os.Getenv(k)
+		defer func(oldk, oldv string) {
+			os.Setenv(oldk, oldv)
+		}(k, oldv)
+		os.Setenv(k, v)
+		restrictedEnv = append(restrictedEnv, k)
+	}
+
+	if err := runff(target, stdin, &stdout, &stderr, restrictedEnv); err != nil {
+		return fmt.Errorf("%v\nstdout:%s\nstderr:%s\n", err, stdout.String(), stderr.String())
+	}
+
+	out := stdout.String()
+	if expectedOut == nil && out != "" {
+		return fmt.Errorf("unexpected output found: %s", out)
+	} else if expectedOut != nil && *expectedOut != out {
+		return fmt.Errorf("mismatched output found.\nexpected (%d bytes):\n%s\ngot (%d bytes):\n%s\n", len(*expectedOut), *expectedOut, len(out), out)
+	}
+
+	err := stderr.String()
+	if expectedErr == nil && err != "" {
+		return fmt.Errorf("unexpected error output found: %s", err)
+	} else if expectedErr != nil && *expectedErr != err {
+		return fmt.Errorf("mismatched error output found.\nexpected (%d bytes):\n%s\ngot (%d bytes):\n%s\n", len(*expectedErr), *expectedErr, len(err), err)
+	}
+
+	return nil
+}
+
+func runremotetest(target string, in, expectedOut, expectedErr *string, env map[string]string) error {
+	stdin := &bytes.Buffer{}
+	if in != nil {
+		stdin = bytes.NewBufferString(*in)
+	}
+
+	var stdout bytes.Buffer
+
+	var restrictedEnv []string
+	for k, v := range env {
+		oldv := os.Getenv(k)
+		defer func(oldk, oldv string) {
+			os.Setenv(oldk, oldv)
+		}(k, oldv)
+		os.Setenv(k, v)
+		restrictedEnv = append(restrictedEnv, k)
+	}
+	if err := callfn(target, stdin, &stdout, restrictedEnv); err != nil {
+		return fmt.Errorf("%v\nstdout:%s\n", err, stdout.String())
+	}
+
+	out := stdout.String()
+	if expectedOut == nil && out != "" {
+		return fmt.Errorf("unexpected output found: %s", out)
+	} else if expectedOut != nil && *expectedOut != out {
+		return fmt.Errorf("mismatched output found.\nexpected (%d bytes):\n%s\ngot (%d bytes):\n%s\n", len(*expectedOut), *expectedOut, len(out), out)
+	}
+
+	if expectedErr != nil {
+		return fmt.Errorf("cannot process stderr in remote calls")
+	}
+
+	return nil
+}