Extend stats to report Failed calls

api/agent/agent.go

@@ -173,6 +173,7 @@ func (a *agent) Submit(callI Call) error {
 	default:
 	}
 
+	// increment queued count
 	a.stats.Enqueue(callI.Model().Path)
 
 	call := callI.(*call)
@@ -202,13 +203,20 @@ func (a *agent) Submit(callI Call) error {
 		return err
 	}
 
-	a.stats.Start(callI.Model().Path)
+	// decrement queued count, increment running count
+	a.stats.DequeueAndStart(callI.Model().Path)
 
 	err = slot.exec(ctx, call)
 	// pass this error (nil or otherwise) to end directly, to store status, etc
 	// End may rewrite the error or elect to return it
 
-	a.stats.Complete(callI.Model().Path)
+	if err == nil {
+		// decrement running count, increment completed count
+		a.stats.Complete(callI.Model().Path)
+	} else {
+		// decrement running count, increment failed count
+		a.stats.Failed(callI.Model().Path)
+	}
 
 	// TODO: we need to allocate more time to store the call + logs in case the call timed out,
 	// but this could put us over the timeout if the call did not reply yet (need better policy).

api/agent/stats.go

@@ -13,6 +13,7 @@ type stats struct {
 	queue    uint64
 	running  uint64
 	complete uint64
+	failed   uint64
 	// statistics for individual functions, keyed by function path
 	functionStatsMap map[string]*functionStats
 }
@@ -22,6 +23,7 @@ type functionStats struct {
 	queue    uint64
 	running  uint64
 	complete uint64
+	failed   uint64
 }
 
 type Stats struct {
@@ -29,6 +31,7 @@ type Stats struct {
 	Queue    uint64
 	Running  uint64
 	Complete uint64
+	Failed   uint64
 	// statistics for individual functions, keyed by function path
 	FunctionStatsMap map[string]*FunctionStats
 }
@@ -38,6 +41,7 @@ type FunctionStats struct {
 	Queue    uint64
 	Running  uint64
 	Complete uint64
+	Failed   uint64
 }
 
 func (s *stats) getStatsForFunction(path string) *functionStats {
@@ -68,7 +72,7 @@ func (s *stats) Dequeue(path string) {
 	s.mu.Unlock()
 }
 
-func (s *stats) Start(path string) {
+func (s *stats) DequeueAndStart(path string) {
 	s.mu.Lock()
 	s.queue--
 	s.getStatsForFunction(path).queue--
@@ -86,15 +90,34 @@ func (s *stats) Complete(path string) {
 	s.mu.Unlock()
 }
 
+func (s *stats) Failed(path string) {
+	s.mu.Lock()
+	s.running--
+	s.getStatsForFunction(path).running--
+	s.failed++
+	s.getStatsForFunction(path).failed++
+	s.mu.Unlock()
+}
+
+func (s *stats) DequeueAndFail(path string) {
+	s.mu.Lock()
+	s.queue--
+	s.getStatsForFunction(path).queue--
+	s.failed++
+	s.getStatsForFunction(path).failed++
+	s.mu.Unlock()
+}
+
 func (s *stats) Stats() Stats {
 	var stats Stats
 	s.mu.Lock()
 	stats.Running = s.running
 	stats.Complete = s.complete
 	stats.Queue = s.queue
+	stats.Failed = s.failed
 	stats.FunctionStatsMap = make(map[string]*FunctionStats)
 	for key, value := range s.functionStatsMap {
-		thisFunctionStats := &FunctionStats{Queue: value.queue, Running: value.running, Complete: value.complete}
+		thisFunctionStats := &FunctionStats{Queue: value.queue, Running: value.running, Complete: value.complete, Failed: value.failed}
 		stats.FunctionStatsMap[key] = thisFunctionStats
 	}
 	s.mu.Unlock()