From 2cced6f2c315c29e52527169d575e20253d58761 Mon Sep 17 00:00:00 2001 From: Travis Reeder Date: Thu, 13 Oct 2016 20:27:39 -0700 Subject: [PATCH] Copied a bunch of examples from Titan. --- examples/checker/.gitignore | 2 + examples/checker/Dockerfile | 9 + examples/checker/Gemfile | 3 + examples/checker/Gemfile.lock | 13 + examples/checker/README.md | 22 ++ examples/checker/checker.rb | 13 + examples/echo/.gitignore | 2 + examples/echo/Dockerfile | 9 + examples/echo/Gemfile | 4 + examples/echo/Gemfile.lock | 25 ++ examples/echo/README.md | 24 ++ examples/echo/echo.rb | 7 + examples/error/.gitignore | 2 + examples/error/Dockerfile | 9 + examples/error/Gemfile | 4 + examples/error/Gemfile.lock | 25 ++ examples/error/README.md | 9 + examples/error/error.rb | 8 + examples/sleeper/.gitignore | 2 + examples/sleeper/Dockerfile | 9 + examples/sleeper/Gemfile | 3 + examples/sleeper/Gemfile.lock | 13 + examples/sleeper/README.md | 23 ++ examples/sleeper/sleeper.rb | 8 + test/Dockerfile | 9 + test/Gemfile | 4 + test/Gemfile.lock | 30 ++ test/README.md | 21 ++ test/build.sh | 3 + test/run.sh | 4 + test/test.rb | 511 ++++++++++++++++++++++++++++++++++ test/utils.rb | 38 +++ 32 files changed, 868 insertions(+) create mode 100644 examples/checker/.gitignore create mode 100644 examples/checker/Dockerfile create mode 100644 examples/checker/Gemfile create mode 100644 examples/checker/Gemfile.lock create mode 100644 examples/checker/README.md create mode 100644 examples/checker/checker.rb create mode 100644 examples/echo/.gitignore create mode 100644 examples/echo/Dockerfile create mode 100644 examples/echo/Gemfile create mode 100644 examples/echo/Gemfile.lock create mode 100644 examples/echo/README.md create mode 100644 examples/echo/echo.rb create mode 100644 examples/error/.gitignore create mode 100644 examples/error/Dockerfile create mode 100644 examples/error/Gemfile create mode 100644 examples/error/Gemfile.lock create mode 100644 examples/error/README.md create mode 100644 examples/error/error.rb create mode 100644 examples/sleeper/.gitignore create mode 100644 examples/sleeper/Dockerfile create mode 100644 examples/sleeper/Gemfile create mode 100644 examples/sleeper/Gemfile.lock create mode 100644 examples/sleeper/README.md create mode 100644 examples/sleeper/sleeper.rb create mode 100644 test/Dockerfile create mode 100644 test/Gemfile create mode 100644 test/Gemfile.lock create mode 100644 test/README.md create mode 100755 test/build.sh create mode 100755 test/run.sh create mode 100644 test/test.rb create mode 100644 test/utils.rb diff --git a/examples/checker/.gitignore b/examples/checker/.gitignore new file mode 100644 index 000000000..e715a8a80 --- /dev/null +++ b/examples/checker/.gitignore @@ -0,0 +1,2 @@ +bundle/ +.bundle/ diff --git a/examples/checker/Dockerfile b/examples/checker/Dockerfile new file mode 100644 index 000000000..a5d9a5a87 --- /dev/null +++ b/examples/checker/Dockerfile @@ -0,0 +1,9 @@ +FROM iron/ruby:dev + +WORKDIR /worker +ADD Gemfile* /worker/ +RUN bundle install + +ADD . /worker/ + +ENTRYPOINT ["ruby", "checker.rb"] diff --git a/examples/checker/Gemfile b/examples/checker/Gemfile new file mode 100644 index 000000000..e05a90012 --- /dev/null +++ b/examples/checker/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' + +gem 'json', '> 1.8.2' diff --git a/examples/checker/Gemfile.lock b/examples/checker/Gemfile.lock new file mode 100644 index 000000000..28ab22fe5 --- /dev/null +++ b/examples/checker/Gemfile.lock @@ -0,0 +1,13 @@ +GEM + remote: https://rubygems.org/ + specs: + json (1.8.3) + +PLATFORMS + ruby + +DEPENDENCIES + json (> 1.8.2) + +BUNDLED WITH + 1.10.5 diff --git a/examples/checker/README.md b/examples/checker/README.md new file mode 100644 index 000000000..dd1672638 --- /dev/null +++ b/examples/checker/README.md @@ -0,0 +1,22 @@ +This is a worker that we can use to check inputs to the job, such as env vars. + +Pass in checks via the payload: + +```json +{ + "env_vars": { + "foo": "bar" + } +} +``` + +That will check that there is an env var called foo with the value bar passed to the task. + +## Building Image + +Install [dj](https://github.com/treeder/dj/), then run: + +``` +docker build -t iron/checker . +docker run -e 'PAYLOAD={"env_vars": {"FOO": "bar"}}' -e "FOO=bar" iron/checker +``` diff --git a/examples/checker/checker.rb b/examples/checker/checker.rb new file mode 100644 index 000000000..73c2c61b5 --- /dev/null +++ b/examples/checker/checker.rb @@ -0,0 +1,13 @@ +require 'json' + +payload = JSON.parse(ENV['PAYLOAD']) + +# payload contains checks +if payload["env_vars"] + payload["env_vars"].each do |k,v| + if ENV[k] != v + raise "Env var #{k} does not match" + end + end +end +puts "all good" diff --git a/examples/echo/.gitignore b/examples/echo/.gitignore new file mode 100644 index 000000000..e715a8a80 --- /dev/null +++ b/examples/echo/.gitignore @@ -0,0 +1,2 @@ +bundle/ +.bundle/ diff --git a/examples/echo/Dockerfile b/examples/echo/Dockerfile new file mode 100644 index 000000000..75dcd174f --- /dev/null +++ b/examples/echo/Dockerfile @@ -0,0 +1,9 @@ +FROM iron/ruby:dev + +WORKDIR /worker +ADD Gemfile* /worker/ +RUN bundle install + +ADD . /worker/ + +ENTRYPOINT ["ruby", "echo.rb"] diff --git a/examples/echo/Gemfile b/examples/echo/Gemfile new file mode 100644 index 000000000..651ef62b4 --- /dev/null +++ b/examples/echo/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +gem 'json', '> 1.8.2' +gem 'iron_worker', '>= 3.0.0' diff --git a/examples/echo/Gemfile.lock b/examples/echo/Gemfile.lock new file mode 100644 index 000000000..b3c8c257d --- /dev/null +++ b/examples/echo/Gemfile.lock @@ -0,0 +1,25 @@ +GEM + remote: https://rubygems.org/ + specs: + iron_core (1.0.9) + rest (>= 3.0.4) + iron_worker (3.2.2) + iron_core (>= 0.5.1) + json (> 1.8.1) + rest (>= 3.0.6) + json (1.8.3) + net-http-persistent (2.9.4) + netrc (0.11.0) + rest (3.0.6) + net-http-persistent (>= 2.9.1) + netrc + +PLATFORMS + ruby + +DEPENDENCIES + iron_worker (>= 3.0.0) + json (> 1.8.2) + +BUNDLED WITH + 1.10.5 diff --git a/examples/echo/README.md b/examples/echo/README.md new file mode 100644 index 000000000..5b1c0a265 --- /dev/null +++ b/examples/echo/README.md @@ -0,0 +1,24 @@ +This is a worker that just echoes the "input" param in the payload. + +eg: + +This input: + +```json +{ + "input": "Yo dawg" +} +``` + +Will output: + +``` +Yo dawg +``` + +## Building Image + +``` +docker build -t iron/echo . +docker run -e 'PAYLOAD={"input": "yoooo"}' iron/echo +``` diff --git a/examples/echo/echo.rb b/examples/echo/echo.rb new file mode 100644 index 000000000..c221ebf0e --- /dev/null +++ b/examples/echo/echo.rb @@ -0,0 +1,7 @@ +require 'iron_worker' + +# p IronWorker.payload +# puts "#{IronWorker.payload["input"]}" + +payload = JSON.parse(ENV['PAYLOAD']) +puts payload['input'] diff --git a/examples/error/.gitignore b/examples/error/.gitignore new file mode 100644 index 000000000..e715a8a80 --- /dev/null +++ b/examples/error/.gitignore @@ -0,0 +1,2 @@ +bundle/ +.bundle/ diff --git a/examples/error/Dockerfile b/examples/error/Dockerfile new file mode 100644 index 000000000..f810193e9 --- /dev/null +++ b/examples/error/Dockerfile @@ -0,0 +1,9 @@ +FROM iron/ruby:dev + +WORKDIR /worker +ADD Gemfile* /worker/ +RUN bundle install + +ADD . /worker/ + +ENTRYPOINT ["ruby", "error.rb"] diff --git a/examples/error/Gemfile b/examples/error/Gemfile new file mode 100644 index 000000000..651ef62b4 --- /dev/null +++ b/examples/error/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +gem 'json', '> 1.8.2' +gem 'iron_worker', '>= 3.0.0' diff --git a/examples/error/Gemfile.lock b/examples/error/Gemfile.lock new file mode 100644 index 000000000..b3c8c257d --- /dev/null +++ b/examples/error/Gemfile.lock @@ -0,0 +1,25 @@ +GEM + remote: https://rubygems.org/ + specs: + iron_core (1.0.9) + rest (>= 3.0.4) + iron_worker (3.2.2) + iron_core (>= 0.5.1) + json (> 1.8.1) + rest (>= 3.0.6) + json (1.8.3) + net-http-persistent (2.9.4) + netrc (0.11.0) + rest (3.0.6) + net-http-persistent (>= 2.9.1) + netrc + +PLATFORMS + ruby + +DEPENDENCIES + iron_worker (>= 3.0.0) + json (> 1.8.2) + +BUNDLED WITH + 1.10.5 diff --git a/examples/error/README.md b/examples/error/README.md new file mode 100644 index 000000000..f954cbcf7 --- /dev/null +++ b/examples/error/README.md @@ -0,0 +1,9 @@ +This is a worker that errors out (ie: exits with non-zero exit code). + + +## Building Image + +``` +docker build -t iron/error . +docker run -e 'PAYLOAD={"input": "yoooo"}' iron/error +``` diff --git a/examples/error/error.rb b/examples/error/error.rb new file mode 100644 index 000000000..1ffd9c4b6 --- /dev/null +++ b/examples/error/error.rb @@ -0,0 +1,8 @@ +require 'iron_worker' + +if ENV['PAYLOAD'] && ENV['PAYLOAD'] != "" + payload = JSON.parse(ENV['PAYLOAD']) + puts payload['input'] +end + +raise "Something went terribly wrong!" diff --git a/examples/sleeper/.gitignore b/examples/sleeper/.gitignore new file mode 100644 index 000000000..e715a8a80 --- /dev/null +++ b/examples/sleeper/.gitignore @@ -0,0 +1,2 @@ +bundle/ +.bundle/ diff --git a/examples/sleeper/Dockerfile b/examples/sleeper/Dockerfile new file mode 100644 index 000000000..a0cba87d5 --- /dev/null +++ b/examples/sleeper/Dockerfile @@ -0,0 +1,9 @@ +FROM iron/ruby:dev + +WORKDIR /worker +ADD Gemfile* /worker/ +RUN bundle install + +ADD . /worker/ + +ENTRYPOINT ["ruby", "sleeper.rb"] diff --git a/examples/sleeper/Gemfile b/examples/sleeper/Gemfile new file mode 100644 index 000000000..e05a90012 --- /dev/null +++ b/examples/sleeper/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' + +gem 'json', '> 1.8.2' diff --git a/examples/sleeper/Gemfile.lock b/examples/sleeper/Gemfile.lock new file mode 100644 index 000000000..28ab22fe5 --- /dev/null +++ b/examples/sleeper/Gemfile.lock @@ -0,0 +1,13 @@ +GEM + remote: https://rubygems.org/ + specs: + json (1.8.3) + +PLATFORMS + ruby + +DEPENDENCIES + json (> 1.8.2) + +BUNDLED WITH + 1.10.5 diff --git a/examples/sleeper/README.md b/examples/sleeper/README.md new file mode 100644 index 000000000..a493acc58 --- /dev/null +++ b/examples/sleeper/README.md @@ -0,0 +1,23 @@ +This is a worker that just echoes the "input" param in the payload. + +eg: + +This input: + +```json +{ + "sleep": 5 +} +``` + +Will make this container sleep for 5 seconds. + + +## Building Image + +Install [dj](https://github.com/treeder/dj/), then run: + +``` +docker build -t iron/sleeper . +docker run -e 'PAYLOAD={"sleep": 5}' iron/sleeper +``` diff --git a/examples/sleeper/sleeper.rb b/examples/sleeper/sleeper.rb new file mode 100644 index 000000000..490d027cb --- /dev/null +++ b/examples/sleeper/sleeper.rb @@ -0,0 +1,8 @@ +require 'json' + +payload = JSON.parse(ENV['PAYLOAD']) + +i = payload['sleep'].to_i +puts "Sleeping for #{i} seconds..." +sleep i +puts "I'm awake!" diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 000000000..594f5e9d7 --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,9 @@ +FROM iron/ruby:dev + +WORKDIR /app +ADD Gemfile* /app/ +RUN bundle install + +ADD . /app/ + +# ENTRYPOINT ["ruby", "test.rb"] diff --git a/test/Gemfile b/test/Gemfile new file mode 100644 index 000000000..f251cb286 --- /dev/null +++ b/test/Gemfile @@ -0,0 +1,4 @@ +source "http://rubygems.org" + +gem 'test-unit', '>3' +gem 'worker_ruby', '>=0.3.5', :git => 'https://github.com/iron-io/worker_ruby' diff --git a/test/Gemfile.lock b/test/Gemfile.lock new file mode 100644 index 000000000..8f57330ee --- /dev/null +++ b/test/Gemfile.lock @@ -0,0 +1,30 @@ +GIT + remote: https://github.com/iron-io/worker_ruby + revision: cc72095f82eb9530d1e161b2dda9092f4068e4eb + specs: + worker_ruby (0.5.7) + json (~> 1.8, >= 1.8.3) + typhoeus (~> 1.0, >= 1.0.1) + +GEM + remote: http://rubygems.org/ + specs: + ethon (0.9.0) + ffi (>= 1.3.0) + ffi (1.9.14) + json (1.8.3) + power_assert (0.3.0) + test-unit (3.2.1) + power_assert + typhoeus (1.1.0) + ethon (>= 0.9.0) + +PLATFORMS + ruby + +DEPENDENCIES + test-unit (> 3) + worker_ruby (>= 0.3.5)! + +BUNDLED WITH + 1.12.5 diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..64cb39f30 --- /dev/null +++ b/test/README.md @@ -0,0 +1,21 @@ + +The tests in these directory will test all the API endpoints and options. + +One time: + +```sh +bundle install +``` + +Start `iron/worker-api` and `iron/worker-runner` + +Then: + +```sh +HOST=localhost:8080 bundle exec ruby test.rb +``` + +To run single test, add `-n testname` + +To test private images use env variables `TEST_AUTH` and `TEST_PRIVATE_IMAGE` +`TEST_AUTH` is encoded to base64 `user:pass` string. diff --git a/test/build.sh b/test/build.sh new file mode 100755 index 000000000..dc8c021fe --- /dev/null +++ b/test/build.sh @@ -0,0 +1,3 @@ +set -ex + +docker build -t iron/worker-test . diff --git a/test/run.sh b/test/run.sh new file mode 100755 index 000000000..a9db9c69a --- /dev/null +++ b/test/run.sh @@ -0,0 +1,4 @@ +set -ex + +./build.sh +docker run --rm --link worker-api iron/worker-test bundle exec ruby test.rb diff --git a/test/test.rb b/test/test.rb new file mode 100644 index 000000000..df4940b9f --- /dev/null +++ b/test/test.rb @@ -0,0 +1,511 @@ +require 'test/unit' +require 'worker_ruby' +require 'net/http' + +class TestTitan < Test::Unit::TestCase + + class << self + def startup + puts 'runs only once at start' + + end + def shutdown + puts 'runs only once at end' + end + + end + + + def setup + # puts "TEST SETUP" + # puts ENV['HOST'] + host = ENV['HOST'] || "worker-api:8080" + IronWorker.configure do |config| + config.host = "#{host}" + config.scheme = "http" + # config.debugging = true + end + + @titan = IronWorker::TasksApi.new + @titan_groups = IronWorker::GroupsApi.new + end + + def wait_for_completion(j) + max = 120 + i = 0 + while true do + i += 1 + if i >= max + raise "Task never seemed to finish!" + end + task = @titan.groups_name_tasks_id_get(j.group_name, j.id).task + # puts "#{task.id} task status: #{task.status}" + if task.status != "delayed" && task.status != "queued" && task.status != "running" + return task + end + sleep 1 + end + end + + def wait_for_running(j) + max = 120 + i = 0 + while true do + i += 1 + if i >= max + raise "Task never seemed to start!" + end + task = @titan.groups_name_tasks_id_get(j.group_name, j.id).task + # puts "#{task.id} task status: #{task.status}" + if task.status == "delayed" || task.status == "queued" + sleep 1 + elsif task.status == "running" + return task + else + raise "Task already finished" + end + end + end + + def post_simple_task(group_name) + return @titan.groups_name_tasks_post(group_name, tasks: [{image: 'iron/hello', payload: {name: "Johnny Utah"}.to_json}]) + end + + def test_basics + puts 'test_basics' + group_name = 'basics' + r = post_simple_task(group_name) + assert_equal 1, r.tasks.length + task = r.tasks[0] + # puts "Task after put:" + # p task + assert task.id.length > 0 + r = @titan.groups_name_tasks_id_get(group_name, task.id) + task = r.task + # puts "Task after get:" + # p task + assert task.id.length > 0 + assert ["queued","running"].include? task.status + puts "task created at: #{task.created_at.to_time} vs #{Time.now-5}" + # The clock is getting off here. Wth? + assert task.created_at.to_time > (Time.now-5) + + task = wait_for_completion(task) + assert_equal "success", task.status + assert_equal nil, task.reason + assert task.started_at >= task.created_at + assert task.completed_at >= task.started_at + + task = post_simple_task(group_name).tasks[0] + task = wait_for_completion(task) + assert_equal "success", task.status + + end + + def test_private_images + group_name = 'private_images' + if ENV['TEST_AUTH'] != nil + assert_not_nil ENV['TEST_AUTH'] + assert_not_nil ENV['TEST_PRIVATE_IMAGE'] + r = @titan.groups_name_tasks_post(group_name, tasks: [{image: ENV['TEST_PRIVATE_IMAGE'], payload: {name: "Johnny Utah"}.to_json}]) + assert_equal 1, r.tasks.length + task = r.tasks[0] + puts "Task after put:" + # p task + assert task.id.length > 0 + r = @titan.groups_name_tasks_id_get(group_name, task.id) + task = r.task + puts "Task after get:" + # p task + assert task.id.length > 0 + assert ["queued","running"].include? task.status + puts "task created at: #{task.created_at.to_time} vs #{Time.now-5}" + # The clock is getting off here. Wth? + assert task.created_at.to_time > (Time.now-5) + + task = wait_for_completion(task) + # p task + assert_equal "success", task.status + assert task.started_at > task.created_at + assert task.completed_at > task.started_at + end + end + + def test_logs + group_name = 'logs' + input_string = "Test Input" + r_payload = @titan.groups_name_tasks_post(group_name, tasks: [{image: 'iron/echo:latest', payload: {input: input_string}.to_json }]) + p r_payload + task = wait_for_completion(r_payload.tasks[0]) + # p task + log = @titan.groups_name_tasks_id_log_get(group_name, r_payload.tasks[0].id) + # echo image log should be exactly the input string + assert_equal input_string, log.chomp + end + + def test_get_tasks_by_group + now = Time.now() + group_name = "fortasklist" + image = 'iron/hello' + posted = [] + 10.times do |i| + posted << post_simple_task(group_name).tasks[0] + end + jarray = @titan.groups_name_tasks_get(group_name, created_after: now.to_datetime.rfc3339) + assert_equal 10, jarray.tasks.length + jarray.tasks.each do |task| + p task + assert_equal group_name, task.group_name + assert_equal image, task.image + assert task.created_at.to_time > now + end + wait_for_completion(posted.last) + end + + def test_priorities + # Wait a while for earlier tasks from other tests to fizzle out so that + # their priorities do not affect things. + # Would be nice to have some way to clean the queues without having to + # sleep. + sleep 5 + group_name = 'priorities' + # Test priorities from 0-2... but how to ensure this is reproducible? + # Post a bunch at once, then ensure start date on higher priority wins + tasks = [] + 10.times do |i| + priority = i < 5 ? 0 : i < 8 ? 1 : 2 + j = {image: 'iron/echo:latest', priority: priority, payload: {input: "task-#{i}"}.to_json} + p j + tasks << j + end + tasks = @titan.groups_name_tasks_post(group_name, tasks: tasks).tasks + sleep 2 + j4 = wait_for_completion(tasks[4]) # should in theory be the last one + puts "j4: #{j4.inspect}" + puts "j4 started_at: #{j4.started_at}" + j9 = wait_for_completion(tasks[9]) + puts "j9 started_at: #{j9.started_at}" + assert j9.started_at <= j4.started_at + # Should add more comparisons here + end + + # todo: Enable after fixing #176[] + #def test_delay + # group_name = 'delay' + # delayed_tasks = [] + # 5.times do |i| + # delayed_tasks << @titan.groups_name_tasks_post(group_name, tasks: [{image: 'treeder/echo:latest', delay: 5, payload: {input: 'Test Input'}.to_json }]).tasks[0] + # end + # p "Finished queueing tasks at #{Time.now}" + # delayed_tasks.each do |task| + # task_delay = @titan.groups_name_tasks_id_get(group_name, task.id).task + # assert_equal "delayed", task_delay.status + # end + + # sleep 2 + # delayed_tasks.each do |task| + # task_delay = @titan.groups_name_tasks_id_get(group_name, task.id).task + # assert_equal "delayed", task_delay.status + # end + + # sleep 8 + # p "Starting to check queued/running/success status at #{Time.now}" + # delayed_tasks.each do |task| + # task_delay = @titan.groups_name_tasks_id_get(group_name, task.id).task + # p "status is", task_delay.status + # assert ["queued", "running", "success"].include?(task_delay.status) + # if task_delay.status == "success" + # assert task_delay.created_at.to_time < task_delay.started_at.to_time - 5 + # end + # end + #end + + def test_error + group_name = 'error' + r = @titan.groups_name_tasks_post(group_name, tasks: [{image: 'iron/error'}]) + p r + task = wait_for_completion(r.tasks[0]) + assert_equal "error", task.status + assert_equal "bad_exit", task.reason + end + + def test_timeout + group_name = 'timeout' + # this one should be fine + r1 = @titan.groups_name_tasks_post(group_name, tasks: [{ + image: 'iron/sleeper:latest', + payload: {sleep: 10}.to_json, + timeout: 30 + }]) + # this one should timeout + r2 = @titan.groups_name_tasks_post(group_name, tasks: [{ + image: 'iron/sleeper:latest', + payload: {sleep: 10}.to_json, + timeout: 5 + }]) + task1 = wait_for_completion(r1.tasks[0]) + assert_equal "success", task1.status + task2 = wait_for_completion(r2.tasks[0]) + assert_equal "error", task2.status + # Fix swagger reason enum thing! + assert_equal "timeout", task2.reason + end + + # need to set retry_id field before this test can run. + # todo: + def test_autoretries + group_name = 'retries' + r = @titan.groups_name_tasks_post(group_name, tasks: [{ + image: 'treeder/error:0.0.2', + retries_delay: 5, + max_retries: 2 + }]) + task = wait_for_completion(r.tasks[0]) + assert_equal "error", task.status + p task + # Fix this once retries set things correctly. + assert task.retry_at && task.retry_at != "" + assert_equal 2, task.max_retries + # should retry in 5 seconds. + task2 = @titan.groups_name_tasks_id_get(group_name, task.retry_at).task + assert "delayed", task2.status # shouldn't start for 5 seconds + assert task2.id == task.retry_at + assert task2.retry_of == task.id + task2 = wait_for_completion(task2) + assert task.completed_at.to_time < task2.started_at.to_time - 5 + assert_equal 1, task2.max_retries # decremented + # last retry + task3 = @titan.groups_name_tasks_id_get(group_name, task2.retry_at).task + assert "delayed", task3.status # shouldn't start for 5 seconds + task3 = wait_for_completion(task3) + assert task2.completed_at.to_time < task3.started_at.to_time - 5 + assert task3.retry_at == nil || task3.retry_at == 0 + assert task3.max_retries == nil || task3.max_retries == 0 + end + + # todo: why is this commented out? + # todo: + #def test_retries + # group_name = 'retries' + + # # non-existent task + # exc = assert_raise IronWorker::ApiError do + # @titan.groups_name_tasks_id_retry_post(group_name, '-1') + # end + # assert_equal 404, exc.code + + # r = @titan.groups_name_tasks_post(group_name, tasks: [{ + # image: 'treeder/error:0.0.2', + # retries_delay: 5, + # }]) + + # # task should not be retry-able until it is finished. + # exc = assert_raise IronWorker::ApiError do + # @titan.groups_name_tasks_id_retry_post(group_name, r.tasks[0].id) + # end + # assert_equal 409, exc.code + + # task = wait_for_completion(r.tasks[0]) + # assert_equal "error", task.status + # assert_equal 0, task.max_retries + # task2 = @titan.groups_name_tasks_id_retry_post(group_name, task.id).task + # # should retry in 5 seconds. + # task = @titan.groups_name_tasks_id_get(group_name, task.id).task + # assert task.id, task2.retry_of + # task2 = wait_for_completion(task2) + # assert_equal task.max_retries, task2.max_retries + #end + + def test_auto_groups + group_name = 'test_groups' + # todo: need to delete image before this. + r = post_simple_task(group_name) + r = post_simple_task(group_name) + task = r.tasks[0] + iw = @titan_groups.groups_name_get(group_name) + assert_not_nil iw.group + assert iw.group.name == group_name + r = @titan_groups.groups_name_get(group_name) + assert_equal group_name, r.group.name + wait_for_completion(task) + end + + # Try cancelling completed task + def test_cancel_succeeded + group_name = 'cancellation' + + r = post_simple_task(group_name) + wait_for_completion(r.tasks[0]) + exc = assert_raise IronWorker::ApiError do + @titan.groups_name_tasks_id_cancel_post(group_name, r.tasks[0].id) + end + assert_equal 409, exc.code + end + + # Try cancelling failed task + def test_cancel_error + group_name = 'cancellation' + r = @titan.groups_name_tasks_post(group_name, tasks: [{ + image: 'iron/error', + }]) + task = wait_for_completion(r.tasks[0]) + assert_equal "error", task.status + exc = assert_raise IronWorker::ApiError do + @titan.groups_name_tasks_id_cancel_post(group_name, r.tasks[0].id) + end + assert_equal 409, exc.code + end + + # Try cancelling running task + def test_cancel_running + group_name = 'cancellation' + r = @titan.groups_name_tasks_post(group_name, tasks: [{ + image: 'iron/sleeper:latest', + payload: {sleep: 10}.to_json, + }]) + task = wait_for_running(r.tasks[0]) + r = @titan.groups_name_tasks_id_cancel_post(group_name, r.tasks[0].id) + assert_equal r.task.id, task.id + assert_equal r.task.status, "cancelled" + + sleep 15 + r = @titan.groups_name_tasks_id_get(group_name, task.id) + # The task should not transition to success or error + assert_equal r.task.status, "cancelled" + end + + # Try cancelling cancelled task + def test_cancel_cancel + group_name = 'cancellation' + r = @titan.groups_name_tasks_post(group_name, tasks: [{ + image: 'iron/sleeper:latest', + payload: {sleep: 10}.to_json, + }]) + task = wait_for_running(r.tasks[0]) + r = @titan.groups_name_tasks_id_cancel_post(group_name, task.id) + assert_equal r.task.id, task.id + assert_equal r.task.status, "cancelled" + + sleep 15 + r = @titan.groups_name_tasks_id_get(group_name, task.id) + # The task should not transition to success or error + assert_equal r.task.status, "cancelled" + + exc = assert_raise IronWorker::ApiError do + @titan.groups_name_tasks_id_cancel_post(group_name, task.id) + end + assert_equal 409, exc.code + end + + # Try cancelling queued task. + def test_cancel_queued + group_name = 'cancellation' + last_running_task = nil + last_task = nil + 5.times do |i| + last_running_task = last_task + last_task = post_simple_task(group_name).tasks[0] + end + + r = @titan.groups_name_tasks_id_get(group_name, last_task.id) + assert_equal r.task.status, "queued" + r = @titan.groups_name_tasks_id_cancel_post(group_name, last_task.id) + assert_equal r.task.id, last_task.id + assert_equal r.task.status, "cancelled" + + task = wait_for_completion(last_running_task) + assert_equal task.status, "success" + sleep 2 + + r = @titan.groups_name_tasks_id_get(group_name, last_task.id) + # The task should not transition to success or error, nor should it have + # a log. + assert_equal last_task.id, r.task.id + assert_equal r.task.status, "cancelled" + puts "cancelled task id: #{r.task.id}" + exc = assert_raise IronWorker::ApiError do + @titan.groups_name_tasks_id_log_get(group_name, last_task.id) + end + assert_equal 404, exc.code + end + + # Try cancelling non-existent task + def test_cancel_non_existent + group_name = 'cancellation' + + exc = assert_raise IronWorker::ApiError do + @titan.groups_name_tasks_id_cancel_post(group_name, "-1") + end + assert_equal 404, exc.code + end + + def test_cancel_delayed + group_name = 'cancellation' + task = @titan.groups_name_tasks_post(group_name, tasks: [{image: 'iron/hello', payload: {name: "Johnny Utah"}.to_json, delay: 5}]).tasks[0] + + r = @titan.groups_name_tasks_id_get(group_name, task.id) + assert_equal r.task.status, "delayed" + r = @titan.groups_name_tasks_id_cancel_post(group_name, task.id) + assert_equal r.task.id, task.id + assert_equal r.task.status, "cancelled" + + sleep 10 + r = @titan.groups_name_tasks_id_get(group_name, task.id) + # The task should not transition to success or error, nor should it have + # a log. + assert_equal r.task.status, "cancelled" + exc = assert_raise IronWorker::ApiError do + @titan.groups_name_tasks_id_log_get(group_name, task.id) + end + assert_equal 404, exc.code + end + + def test_groups + group_name = 'groups_test' + g = @titan_groups.groups_name_put(group_name, group: {image: "iron/checker", env_vars: {"FOO"=>"bar", "DB_URL" => "postgres://abc.com/"}, max_concurrency: 10}) + p g + g2 = @titan_groups.groups_name_get(group_name).group + assert_equal "bar", g2.env_vars["FOO"] + task = @titan.groups_name_tasks_post(group_name, tasks: [{payload: {env_vars: {"FOO"=>"bar"}}.to_json}]).tasks[0] + task = wait_for_completion(task) + assert_equal task.status, "success" + + end + + def test_pagination + # post 50 tasks, get them in groups of 10 and compare ids/timestamps or something + group_name = 'paging_test' + posted_tasks = [] + num_tasks = 25 + num_tasks.times do |i| + task = @titan.groups_name_tasks_post(group_name, tasks: [{image: 'iron/hello', payload: {name: "Johnny Utah"}.to_json}]).tasks[0] + puts "posted task id #{task.id}" + posted_tasks << task + end + + ji = num_tasks + cursor = nil + n = 5 + (num_tasks/n).times do |i| + tasksw = @titan.groups_name_tasks_get(group_name, n: n, cursor: cursor) + tasksw.tasks.each do |task| + ji -= 1 + puts "got task #{task.id}" + assert_equal posted_tasks[ji].id, task.id + end + cursor = tasksw.cursor + puts "cursor #{cursor}" + break if cursor == nil + end + + wait_for_completion(posted_tasks.last) + end + + def cancel_tasks(tasks) + tasks.each do |j| + @titan.groups_name_tasks_id_cancel_post(j.group_name, j.id) + end + + end + +end diff --git a/test/utils.rb b/test/utils.rb new file mode 100644 index 000000000..d93d1c299 --- /dev/null +++ b/test/utils.rb @@ -0,0 +1,38 @@ +require 'open3' + +class ExecError < StandardError + attr_accessor :exit_status, :last_line + + def initialize(exit_status, last_line) + super("Error on cmd. #{exit_status}") + self.exit_status = exit_status + self.last_line = last_line + end +end + +def stream_exec(cmd) + puts "Executing cmd: #{cmd}" + exit_status = nil + last_line = "" + Open3.popen2e(cmd) do |stdin, stdout_stderr, wait_thread| + Thread.new do + stdout_stderr.each {|l| + puts l + # Save last line for error checking + last_line = l + } + end + + # stdin.puts 'ls' + # stdin.close + + exit_status = wait_thread.value + raise ExecError.new(exit_status, last_line) if exit_status.exitstatus != 0 + end + return exit_status +end + +def exec(cmd) + puts "Executing: #{cmd}" + puts `#{cmd}` +end