fn: error handling updates (#1337)

* docker-pull timeout is now a 504 which classifies it as a
service error. Avoid using 503 to make sure LB does not retry.

* Only applicable to detached mode, a timeout on LB is
now a ErrServiceReservationFailure (500). In detached mode,
this is unlikely to make it back to a client and it is mostly
for documentation/metrics purposes.
  
* For Triggers, avoid scrubbing service code.

api/agent/lb_agent.go

@@ -304,6 +304,18 @@ func (a *lbAgent) handleCallEnd(ctx context.Context, call *call, err error, isFo
 		if err == nil {
 			statsComplete(ctx)
 			recordCallLatency(ctx, call, completedMetricName)
+		} else if err == context.DeadlineExceeded {
+			// We are here because we were unable to service this request for the given
+			// reservation. In detached case, the reservation is calculated based on estimated
+			// total time to run a request. (See: spawnPlaceCall) Otherwise, there's no set
+			// deadline in the request context. This is also a bit more robust going forward
+			// if we start enforcing a maximum overall deadline for clients. For detached case, the
+			// error is unlikely to be delivered to the client since this is essentially an async
+			// operation.
+			statsTimedout(ctx)
+			recordCallLatency(ctx, call, timedoutMetricName)
+			// We have failed: http 500 Internal Server Error
+			return models.ErrServiceReservationFailure
 		}
 	} else {
 		statsDequeue(ctx)
@@ -318,10 +330,6 @@ func (a *lbAgent) handleCallEnd(ctx context.Context, call *call, err error, isFo
 		statsTooBusy(ctx)
 		recordCallLatency(ctx, call, serverBusyMetricName)
 		return models.ErrCallTimeoutServerBusy
-	} else if err == context.DeadlineExceeded {
-		statsTimedout(ctx)
-		recordCallLatency(ctx, call, timedoutMetricName)
-		return models.ErrCallTimeout
 	} else if err == context.Canceled {
 		statsCanceled(ctx)
 		recordCallLatency(ctx, call, canceledMetricName)