From 7c91b98a728c39bc5448dc5865148257e95cbdb2 Mon Sep 17 00:00:00 2001 From: Tolga Ceylan Date: Wed, 10 Jan 2018 14:14:19 -0800 Subject: [PATCH] fn: hot container launcher adjustment (#673) Latency stats are not always read-time updated and if calls are stuck in waiting state, isNewContainerNeeded() needs to be a bit more aggresive if the wait queue grows. --- api/agent/slots.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/api/agent/slots.go b/api/agent/slots.go index f6c5ab259..511d666b9 100644 --- a/api/agent/slots.go +++ b/api/agent/slots.go @@ -204,12 +204,24 @@ func (a *slotQueue) isNewContainerNeeded() (bool, slotQueueStats) { return false, stats } - // no executors? Start a container now. + // this is a bit aggresive and assumes that we only + // want to queue as much as num of containers. executors := starters + stats.states[SlotQueueRunner] - if executors == 0 { + if executors < waiters { return true, stats } + // WARNING: Below is a few heuristics that are + // speculative, which may (and will) likely need + // adjustments. + + // WARNING: latencies below are updated after a call + // switches to/from different states. Do not assume + // the metrics below are always up-to-date. For example, + // a sudden burst of incoming requests will increase + // waiter count but not necessarily wait latency until + // those requests switch from waiter state. + runLat := stats.latencies[SlotQueueRunner] waitLat := stats.latencies[SlotQueueWaiter] startLat := stats.latencies[SlotQueueStarter]