alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

fn: container initialization monitoring (#1288)

Browse Source 
Container initialization phase consumes resource tracker
resources (token), during lengthy operations.
In order for agent stability/liveness, this phase has
to be evictable/cancelable and time bounded.

With this change, introducing a new system wide environment setting
to bound the time spent in container initialization phase. This phase
includes docker-pull, docker-create, docker-attach, docker-start
and UDS wait operations. This initialization period is also now
considered evictable.
This commit is contained in:
Tolga Ceylan
2018-11-15 13:37:43 -08:00
committed by GitHub
parent fe2b9fb53d
commit 6eaf1578e6
8 changed files with 700 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
api/agent/agent_test.go
View File

@@ -7,9 +7,13 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
@@ -21,11 +25,8 @@ import (
"github.com/fnproject/fn/api/logs"
"github.com/fnproject/fn/api/models"
"github.com/fnproject/fn/api/mqs"
"github.com/sirupsen/logrus"
"io/ioutil"
"net"
"os"
"path/filepath"
)
func init() {
@@ -436,6 +437,48 @@ type dummyReader struct {
io.Reader
}
func TestHungFDK(t *testing.T) {
app := &models.App{ID: "app_id"}
fn := &models.Fn{
ID: "fn_id",
Image: "fnproject/fn-test-utils",
Config: models.Config{"ENABLE_INIT_DELAY_MSEC": "5000"},
ResourceConfig: models.ResourceConfig{
Timeout: 5,
IdleTimeout: 10,
Memory: 128,
},
}
url := "http://127.0.0.1:8080/invoke/" + fn.ID
ls := logs.NewMock()
cfg, err := NewConfig()
cfg.MaxDockerRetries = 1
cfg.HotStartTimeout = time.Duration(3) * time.Second
a := New(NewDirectCallDataAccess(ls, new(mqs.Mock)), WithConfig(cfg))
defer checkClose(t, a)
req, err := http.NewRequest("GET", url, &dummyReader{Reader: strings.NewReader(`{}`)})
if err != nil {
t.Fatal("unexpected error building request", err)
}
var out bytes.Buffer
callI, err := a.GetCall(FromHTTPFnRequest(app, fn, req), WithWriter(&out))
if err != nil {
t.Fatal(err)
}
err = a.Submit(callI)
if err == nil {
t.Fatal("submit should error!")
}
if err != models.ErrContainerInitTimeout {
t.Fatalf("unexpected error %v", err)
}
}
func TestDockerPullHungRepo(t *testing.T) {
hung, cancel := context.WithCancel(context.Background())
garbageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -467,7 +510,7 @@ func TestDockerPullHungRepo(t *testing.T) {
ls := logs.NewMock()
cfg, err := NewConfig()
cfg.MaxDockerRetries = 1
cfg.HotStartTimeout = time.Duration(5) * time.Second
cfg.HotPullTimeout = time.Duration(5) * time.Second
a := New(NewDirectCallDataAccess(ls, new(mqs.Mock)), WithConfig(cfg))
defer checkClose(t, a)
@@ -486,8 +529,7 @@ func TestDockerPullHungRepo(t *testing.T) {
if err == nil {
t.Fatal("submit should error!")
}
errS := err.Error()
if !strings.HasPrefix(errS, "Failed to pull image ") || !strings.Contains(errS, "context deadline exceeded") {
if err != models.ErrDockerPullTimeout {
t.Fatalf("unexpected error %v", err)
}
}
@@ -535,7 +577,7 @@ func TestDockerPullBadRepo(t *testing.T) {
if err == nil {
t.Fatal("submit should error!")
}
if !strings.HasPrefix(err.Error(), "Failed to pull image ") {
if !models.IsAPIError(err) || !strings.HasPrefix(err.Error(), "Failed to pull image ") {
t.Fatalf("unexpected error %v", err)
}
}
@@ -1219,8 +1261,11 @@ func TestDockerAuthExtn(t *testing.T) {
ctx := context.TODO()
c, err := newHotContainer(ctx, call, cfg)
if err != nil {
errC := make(chan error, 10)
c := newHotContainer(ctx, call, cfg, id.New().String(), errC)
if c == nil {
err := <-errC
t.Fatal("got unexpected err: ", err)
}
da, err := c.DockerAuth()
@@ -1236,8 +1281,9 @@ func TestDockerAuthExtn(t *testing.T) {
extn["FN_REGISTRY_TOKEN"] = "TestRegistryToken"
call.extensions = extn
c, err = newHotContainer(ctx, call, cfg)
if err != nil {
c = newHotContainer(ctx, call, cfg, id.New().String(), errC)
if c == nil {
err := <-errC
t.Fatal("got unexpected err: ", err)
}
da, err = c.DockerAuth()