diff --git a/test/fnlb-test-harness/README.md b/test/fnlb-test-harness/README.md new file mode 100644 index 000000000..07959946f --- /dev/null +++ b/test/fnlb-test-harness/README.md @@ -0,0 +1,88 @@ +# fnlb-test-harness +Test harness that exercises the fnlb load balancer in order to verify that it works properly. +## How it works +This is a test harness that makes calls to an IronFunctions route through the fnlb load balancer, which routes traffic to multiple IronFunctions nodes. +The test harness keeps track of which node each request was routed to so we can assess how the requests are being distributed across the nodes. The functionality +of fnlb is to normally route traffic to the same small number of nodes so that efficiences can be achieved and to support reuse of hot functions. +### Primes function +The test harness utilizes the "primes" function, which calculates prime numbers as an excuse for consuming CPU resources. The function is invoked as follows: +``` +curl http://host:8080/r/primesapp/primes?max=1000000&loops=1 +``` +where: +- *max*: calculate all primes <= max (increasing max will increase memory usage, due to the Sieve of Eratosthenes algorithm) +- *loops*: number of times to calculate the primes (repeating the count consumes additional CPU without consuming additional memory) + +## How to use it +The test harness requires running one or more IronFunctions nodes and one instance of fnlb. The list of nodes must be provided both to fnlb and to the test harness +because the test harness must call each node directly one time in order to discover the node's container id. + +After it has run, examine the results to see how the requests were distributed across the nodes. +### How to run it locally +Each of the IronFunctions nodes needs to connect to the same database. + +STEP 1: Create a route for the primes function. Example: +``` +fn apps create primesapp +fn routes create primesapp /primes jconning/primes:0.0.1 +``` +STEP 2: Run five IronFunctions nodes locally. Example (runs five nodes in the background using Docker): +``` +sudo docker run -d -it --name functions-8082 --privileged -v ${HOME}/data-8082:/app/data -p 8082:8080 -e "DB_URL=postgres://dbUser:dbPassword@dbHost:5432/dbName" iron/functions +sudo docker run -d -it --name functions-8083 --privileged -v ${HOME}/data-8083:/app/data -p 8083:8080 -e "DB_URL=postgres://dbUser:dbPassword@dbHost:5432/dbName" iron/functions +sudo docker run -d -it --name functions-8084 --privileged -v ${HOME}/data-8084:/app/data -p 8084:8080 -e "DB_URL=postgres://dbUser:dbPassword@dbHost:5432/dbName" iron/functions +sudo docker run -d -it --name functions-8085 --privileged -v ${HOME}/data-8085:/app/data -p 8085:8080 -e "DB_URL=postgres://dbUser:dbPassword@dbHost:5432/dbName" iron/functions +sudo docker run -d -it --name functions-8086 --privileged -v ${HOME}/data-8086:/app/data -p 8086:8080 -e "DB_URL=postgres://dbUser:dbPassword@dbHost:5432/dbName" iron/functions +``` +STEP 3: Run fnlb locally. Example (runs fnlb on the default port 8081): +``` +fnlb -nodes localhost:8082,localhost:8083,localhost:8084,localhost:8085,localhost:8086 +``` +STEP 4: Run the test harness. Note that the 'nodes' parameter should be the same that was used with fnlb. Example: +``` +cd functions/test/fnlb-test-harness +go run main.go -nodes localhost:8082,localhost:8083,localhost:8084,localhost:8085,localhost:8086 -calls 10 -v +``` +STEP 5: Examine the output to determine how many times fnlb called each node. Assess whether it is working properly. + +### Usage +go run main.go -help + +Command line parameters: +- *-calls*: number of times to call the route (default 100) +- *-lb*: host and port of load balancer (default "localhost:8081") +- *-loops*: number of times to execute the primes calculation (ex: '-loops 2' means run the primes calculation twice) (default 1) +- *-max*: maximum number to search for primes (higher number consumes more memory) (default 1000000) +- *-nodes*: comma-delimited list of nodes (host:port) balanced by the load balancer (needed to discover container id of each) (default "localhost:8080") +- *-route*: path representing the route to the primes function (default "/r/primesapp/primes") +- *-v*: flag indicating verbose output + +### Examples: quick vs long running + +**Quick function:**: calculate primes up to 1000 +``` +go run main.go -nodes localhost:8082,localhost:8083,localhost:8084,localhost:8085,localhost:8086 -max 1000 -v +``` +where *-max* is default of 1M, *-calls* is default of 100, *-route* is default of "/r/primesapp/primes", *-lb* is default localhost:8081 + +**Normal function**: calculate primes up to 1M +``` +go run main.go -nodes localhost:8082,localhost:8083,localhost:8084,localhost:8085,localhost:8086 -v +``` +where *-max* is default of 1M, *-calls* is default of 100, *-route* is default of "/r/primesapp/primes", *-lb* is default localhost:8081 + +**Longer running function**: calculate primes up to 1M and perform the calculation ten times +``` +go run main.go -nodes localhost:8082,localhost:8083,localhost:8084,localhost:8085,localhost:8086 -loops 10 -v +``` +where *-max* is default of 1M, *-calls* is default of 100, *-route* is default of "/r/primesapp/primes", *-lb* is default localhost:8081 + +**1000 calls to the route**: send 1000 requests through the load balancer +``` +go run main.go -nodes localhost:8082,localhost:8083,localhost:8084,localhost:8085,localhost:8086 -calls 1000 -v +``` +where *-max* is default of 1M, *-calls* is default of 100, *-route* is default of "/r/primesapp/primes", *-lb* is default localhost:8081 + +## Planned Enhancements +- Create 1000 routes and distribute calls amongst them. +- Use concurrent programming to enable the test harness to call multiple routes at the same time. diff --git a/test/fnlb-test-harness/main.go b/test/fnlb-test-harness/main.go new file mode 100644 index 000000000..6b02de166 --- /dev/null +++ b/test/fnlb-test-harness/main.go @@ -0,0 +1,127 @@ +package main + +import ( + "fmt" + "net/http" + "io/ioutil" + "encoding/json" + "time" + "flag" + "log" + "strings" +) + +type execution struct { + DurationSeconds float64 + Hostname string + node string + body string + responseSeconds float64 +} + +var ( + lbHostPort, nodesStr, route string + numExecutions, maxPrime, numLoops int + nodes []string + nodesByContainerId map[string]string = make(map[string]string) + verbose bool +) + +func init() { + flag.StringVar(&lbHostPort, "lb", "localhost:8081", "host and port of load balancer") + flag.StringVar(&nodesStr, "nodes", "localhost:8080", "comma-delimited list of nodes (host:port) balanced by the load balancer (needed to discover container id of each)") + flag.StringVar(&route, "route", "/r/primesapp/primes", "path representing the route to the primes function") + flag.IntVar(&numExecutions, "calls", 100, "number of times to call the route") + flag.IntVar(&maxPrime, "max", 1000000, "maximum number to search for primes (higher number consumes more memory)") + flag.IntVar(&numLoops, "loops", 1, "number of times to execute the primes calculation (ex: 'loops=2' means run the primes calculation twice)") + flag.BoolVar(&verbose, "v", false, "true for more verbose output") + flag.Parse() + + if maxPrime < 3 { + log.Fatal("-max must be 3 or greater") + } + if numLoops < 1 { + log.Fatal("-loops must be 1 or greater") + } + + nodes = strings.Split(nodesStr, ",") +} + +func executeFunction(hostPort, path string, max, loops int) (execution, error) { + var e execution + + start := time.Now() + resp, err := http.Get(fmt.Sprintf("http://%s%s?max=%d&loops=%d", hostPort, path, max, loops)) + e.responseSeconds = time.Since(start).Seconds() + if err != nil { + return e, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return e, fmt.Errorf("function returned status code: %d", resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return e, err + } + + err = json.Unmarshal(body, &e) + if err != nil { + e.body = string(body) // set the body in the execution so that it is available for logging + return e, err + } + e.node = nodesByContainerId[e.Hostname] + + return e, nil +} + +func invokeLoadBalancer(hostPort, path string, numExecutions, max, loops int) { + executionsByNode := make(map[string][]execution) + fmt.Printf("All primes will be calculated up to %d, a total of %d time(s)\n", maxPrime, numLoops) + fmt.Printf("Calling route %s %d times (through the load balancer)...\n", route, numExecutions) + + for i := 0; i < numExecutions; i++ { + e, err := executeFunction(hostPort, path, max, loops) + if err == nil { + if ex, ok := executionsByNode[e.node]; ok { + executionsByNode[e.node] = append(ex, e) + } else { + // Create a slice to contain the list of executions for this host + executionsByNode[e.node] = []execution{e} + } + if verbose { + fmt.Printf(" %s in-function duration: %fsec, response time: %fsec\n", e.node, e.DurationSeconds, e.responseSeconds) + } + } else { + fmt.Printf(" Ignoring failed execution on node %s: %v\n", e.node, err) + fmt.Printf(" JSON: %s\n", e.body) + } + } + + fmt.Println("Results (executions per node):") + for node, ex := range executionsByNode { + fmt.Printf(" %s %d\n", node, len(ex)) + } +} + +func discoverContainerIds() { + // Discover the Docker hostname of each node; create a mapping of hostnames to host/port. + // This is needed because IronFunctions doesn't make the host/port available to the function (as of Mar 2017). + fmt.Println("Discovering container ids for every node (use Docker's HOSTNAME env var as a container id)...") + for _, s := range nodes { + if e, err := executeFunction(s, route, 100, 1); err == nil { + nodesByContainerId[e.Hostname] = s + fmt.Printf(" %s %s\n", s, e.Hostname) + } else { + fmt.Printf(" Ignoring host %s which returned error: %v\n", s, err) + fmt.Printf(" JSON: %s\n", e.body) + } + } +} + +func main() { + discoverContainerIds() + invokeLoadBalancer(lbHostPort, route, numExecutions, maxPrime, numLoops) +} + diff --git a/test/fnlb-test-harness/primes-func/func.go b/test/fnlb-test-harness/primes-func/func.go new file mode 100644 index 000000000..eddecb72f --- /dev/null +++ b/test/fnlb-test-harness/primes-func/func.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + "strings" + "strconv" + "time" +) + +// return list of primes less than N +// source: http://stackoverflow.com/a/21923233 +func sieveOfEratosthenes(N int) (primes []int) { + b := make([]bool, N) + for i := 2; i < N; i++ { + if b[i] == true { + continue + } + primes = append(primes, i) + for k := i * i; k < N; k += i { + b[k] = true + } + } + return +} + +func main() { + start := time.Now() + maxPrime := 1000000 + numLoops := 1 + + // Parse the query string + s := strings.Split(os.Getenv("REQUEST_URL"), "?") + if len(s) > 1 { + for _, pair := range strings.Split(s[1], "&") { + kv := strings.Split(pair, "=") + if len(kv) > 1 { + key, value := kv[0], kv[1] + if key == "max" { + maxPrime, _ = strconv.Atoi(value) + } + if key == "loops" { + numLoops, _ = strconv.Atoi(value) + } + } + } + } + + // Repeat the calculation of primes simply to give the CPU more work to do without consuming additional memory + for i := 0; i < numLoops; i++ { + primes := sieveOfEratosthenes(maxPrime) + _ = primes + if i == numLoops - 1 { + //fmt.Printf("Highest three primes: %d %d %d\n", primes[len(primes) - 1], primes[len(primes) - 2], primes[len(primes) - 3]) + } + } + fmt.Printf("{\"durationSeconds\": %f, \"hostname\": \"%s\", \"max\": %d, \"loops\": %d}", time.Since(start).Seconds(), os.Getenv("HOSTNAME"), maxPrime, numLoops) +} + diff --git a/test/fnlb-test-harness/primes-func/func.yaml b/test/fnlb-test-harness/primes-func/func.yaml new file mode 100644 index 000000000..71aa5f03f --- /dev/null +++ b/test/fnlb-test-harness/primes-func/func.yaml @@ -0,0 +1,6 @@ +name: jconning/primes +version: 0.0.1 +runtime: go +entrypoint: ./func +path: /primes +max_concurrency: 1