fn: handle go-lang time.Now() skew (#1572)

Even for CLOCK_MONOTONIC, NTP adjustments can be made for advancing
the clock forward. When reporting metrics, let's handle this as
callLatency zero (in other words, execution latency is almost
same as overall latency.)

api/agent/lb_agent.go

@@ -385,15 +385,12 @@ func recordCallLatency(ctx context.Context, call *call, status string) {
 	// We want to exclude time spent in user-code. Today, this is container
 	// request processing latency as observed by runner agent.
 	execLatency := call.GetUserExecutionTime()
-
-	// some sanity check before. If sanity checks flags something, then
-	// this is likely that runners are sending malicious/suspicious data.
 	if execLatency != nil {
 		if *execLatency >= callLatency {
-			common.Logger(ctx).Errorf("invalid latency callLatency=%v execLatency=%v", callLatency, execLatency)
-			return
+			callLatency = 0 // NTP may skew, even for CLOCK_MONOTONIC in time.Now()
+		} else {
+			callLatency -= *execLatency
 		}
-		callLatency -= *execLatency
 	}
 
 	statsCallLatency(ctx, callLatency, status)