Cpu resources (#642)

* fn: cpu quota implementation
2022-10-28 21:29:17 +03:00 · 2018-01-12 11:38:28 -08:00
parent fa59400a97
commit 39b2cb2d9b
29 changed files with 856 additions and 91 deletions
--- a/api/agent/resource_test.go
+++ b/api/agent/resource_test.go
@@ -1,6 +1,8 @@
 package agent

 import (
+	"context"
+	"errors"
 	"testing"
 	"time"
 )
@@ -8,11 +10,17 @@ import (
 func setTrackerTestVals(tr *resourceTracker, vals *trackerVals) {
 	tr.cond.L.Lock()

-	tr.ramSyncTotal = vals.st
-	tr.ramSyncUsed = vals.su
-	tr.ramAsyncTotal = vals.at
-	tr.ramAsyncUsed = vals.au
-	tr.ramAsyncHWMark = vals.am
+	tr.ramSyncTotal = vals.mst
+	tr.ramSyncUsed = vals.msu
+	tr.ramAsyncTotal = vals.mat
+	tr.ramAsyncUsed = vals.mau
+	tr.ramAsyncHWMark = vals.mam
+
+	tr.cpuSyncTotal = vals.cst
+	tr.cpuSyncUsed = vals.csu
+	tr.cpuAsyncTotal = vals.cat
+	tr.cpuAsyncUsed = vals.cau
+	tr.cpuAsyncHWMark = vals.cam

 	tr.cond.L.Unlock()
 	tr.cond.Broadcast()
@@ -22,24 +30,72 @@ func getTrackerTestVals(tr *resourceTracker, vals *trackerVals) {

 	tr.cond.L.Lock()

-	vals.st = tr.ramSyncTotal
-	vals.su = tr.ramSyncUsed
-	vals.at = tr.ramAsyncTotal
-	vals.au = tr.ramAsyncUsed
-	vals.am = tr.ramAsyncHWMark
+	vals.mst = tr.ramSyncTotal
+	vals.msu = tr.ramSyncUsed
+	vals.mat = tr.ramAsyncTotal
+	vals.mau = tr.ramAsyncUsed
+	vals.mam = tr.ramAsyncHWMark
+
+	vals.cst = tr.cpuSyncTotal
+	vals.csu = tr.cpuSyncUsed
+	vals.cat = tr.cpuAsyncTotal
+	vals.cau = tr.cpuAsyncUsed
+	vals.cam = tr.cpuAsyncHWMark

 	tr.cond.L.Unlock()
 }

+// helper to debug print (fields correspond to resourceTracker CPU/MEM fields)
 type trackerVals struct {
-	st uint64
-	su uint64
-	at uint64
-	au uint64
-	am uint64
+	mst uint64
+	msu uint64
+	mat uint64
+	mau uint64
+	mam uint64
+	cst uint64
+	csu uint64
+	cat uint64
+	cau uint64
+	cam uint64
 }

-func TestResourceAsyncMem(t *testing.T) {
+func (vals *trackerVals) setDefaults() {
+	// set set these to known vals (4GB total: 1GB sync, 3 async)
+	vals.mst = 1 * Mem1GB
+	vals.msu = 0
+	vals.mat = 3 * Mem1GB
+	vals.mau = 0
+	vals.mam = 1 * Mem1GB
+
+	// let's assume 10 CPUs (2 CPU sync, 8 CPU async)
+	vals.cst = 2000
+	vals.csu = 0
+	vals.cat = 8000
+	vals.cau = 0
+	vals.cam = 6000
+}
+
+func fetchToken(ch <-chan ResourceToken) (ResourceToken, error) {
+	select {
+	case tok := <-ch:
+		return tok, nil
+	case <-time.After(time.Duration(500) * time.Millisecond):
+		return nil, errors.New("expected token")
+	}
+}
+
+func isClosed(ch <-chan ResourceToken) bool {
+	select {
+	case _, ok := <-ch:
+		if !ok {
+			return true
+		}
+	default:
+	}
+	return false
+}
+
+func TestResourceAsyncWait(t *testing.T) {

 	var vals trackerVals

@@ -48,36 +104,242 @@ func TestResourceAsyncMem(t *testing.T) {
 	tr := trI.(*resourceTracker)

 	getTrackerTestVals(tr, &vals)
-	if vals.st <= 0 || vals.su != 0 || vals.at <= 0 || vals.au != 0 || vals.am <= 0 {
-		t.Fatalf("faulty init %#v", vals)
+	if vals.mst <= 0 || vals.msu != 0 || vals.mat <= 0 || vals.mau != 0 || vals.mam <= 0 {
+		t.Fatalf("faulty init MEM %#v", vals)
+	}
+	if vals.cst <= 0 || vals.csu != 0 || vals.cat <= 0 || vals.cau != 0 || vals.cam <= 0 {
+		t.Fatalf("faulty init CPU %#v", vals)
 	}

-	// set set these to known vals
-	vals.st = 1 * 1024 * 1024
-	vals.su = 0
-	vals.at = 2 * 1024 * 1024
-	vals.au = 0
-	vals.am = 1 * 1024 * 1024
+	vals.setDefaults()

 	// should block & wait
-	vals.au = vals.am
+	vals.mau = vals.mam
 	setTrackerTestVals(tr, &vals)
 	ch := tr.WaitAsyncResource()

 	select {
 	case <-ch:
-		t.Fatal("high water mark over, should not trigger")
+		t.Fatal("high water mark MEM over, should not trigger")
 	case <-time.After(time.Duration(500) * time.Millisecond):
 	}

 	// should not block & wait
-	vals.au = 0
+	vals.mau = 0
 	setTrackerTestVals(tr, &vals)

 	select {
 	case <-ch:
 	case <-time.After(time.Duration(500) * time.Millisecond):
-		t.Fatal("high water mark not over, should trigger")
+		t.Fatal("high water mark MEM not over, should trigger")
+	}
+
+	// get a new channel to prevent previous test interference
+	ch = tr.WaitAsyncResource()
+
+	// should block & wait
+	vals.cau = vals.cam
+	setTrackerTestVals(tr, &vals)
+
+	select {
+	case <-ch:
+		t.Fatal("high water mark CPU over, should not trigger")
+	case <-time.After(time.Duration(500) * time.Millisecond):
+	}
+
+	// should not block & wait
+	vals.cau = 0
+	setTrackerTestVals(tr, &vals)
+
+	select {
+	case <-ch:
+	case <-time.After(time.Duration(500) * time.Millisecond):
+		t.Fatal("high water mark CPU not over, should trigger")
+	}
+
+}
+
+func TestResourceGetSimple(t *testing.T) {
+
+	var vals trackerVals
+	trI := NewResourceTracker()
+	tr := trI.(*resourceTracker)
+
+	vals.setDefaults()
+
+	// let's make it like CPU and MEM are 100% full
+	vals.mau = vals.mat
+	vals.cau = vals.cat
+
+	setTrackerTestVals(tr, &vals)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// ask for 4GB and 10 CPU
+	ch := trI.GetResourceToken(ctx, 4*1024, 1000, false)
+
+	_, err := fetchToken(ch)
+	if err == nil {
+		t.Fatalf("full system should not hand out token")
+	}
+
+	// reset back
+	vals.setDefaults()
+	setTrackerTestVals(tr, &vals)
+
+	tok, err := fetchToken(ch)
+	if err != nil {
+		t.Fatalf("empty system should hand out token")
+	}
+
+	// ask for another 4GB and 10 CPU
+	ch = trI.GetResourceToken(ctx, 4*1024, 1000, false)
+
+	_, err = fetchToken(ch)
+	if err == nil {
+		t.Fatalf("full system should not hand out token")
+	}
+
+	// close means, giant token resources released
+	tok.Close()
+
+	tok, err = fetchToken(ch)
+	if err != nil {
+		t.Fatalf("empty system should hand out token")
+	}
+
+	tok.Close()
+
+	// POOLS should all be empty now
+	getTrackerTestVals(tr, &vals)
+	if vals.msu != 0 || vals.mau != 0 {
+		t.Fatalf("faulty state MEM %#v", vals)
+	}
+	if vals.csu != 0 || vals.cau != 0 {
+		t.Fatalf("faulty state CPU %#v", vals)
+	}
+}
+
+func TestResourceGetCombo(t *testing.T) {
+
+	var vals trackerVals
+	trI := NewResourceTracker()
+	tr := trI.(*resourceTracker)
+
+	vals.setDefaults()
+	setTrackerTestVals(tr, &vals)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// impossible request
+	ch := trI.GetResourceToken(ctx, 20*1024, 20000, false)
+	if !isClosed(ch) {
+		t.Fatalf("impossible request should return closed channel")
+	}
+
+	cancel()
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// let's use up 2 GB of 3GB async pool
+	ch = trI.GetResourceToken(ctx, 2*1024, 10, true)
+	tok1, err := fetchToken(ch)
+	if err != nil {
+		t.Fatalf("empty async system should hand out token1")
+	}
+
+	cancel()
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// remaining 1 GB async
+	ch = trI.GetResourceToken(ctx, 1*1024, 11, true)
+	tok2, err := fetchToken(ch)
+	if err != nil {
+		t.Fatalf("empty async system should hand out token2")
+	}
+
+	cancel()
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// NOW ASYNC POOL IS FULL
+	// SYNC POOL HAS 1GB
+
+	// we no longer can get async token
+	ch = trI.GetResourceToken(ctx, 1*1024, 12, true)
+	_, err = fetchToken(ch)
+	if err == nil {
+		t.Fatalf("full async system should not hand out a token")
+	}
+
+	cancel()
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// but we should get 1GB sync token
+	ch = trI.GetResourceToken(ctx, 1*1024, 13, false)
+	tok3, err := fetchToken(ch)
+	if err != nil {
+		t.Fatalf("empty sync system should hand out token3")
+	}
+	cancel()
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// NOW ASYNC AND SYNC POOLS ARE FULL
+
+	// this should fail
+	ch = trI.GetResourceToken(ctx, 1*1024, 14, false)
+	_, err = fetchToken(ch)
+	if err == nil {
+		t.Fatalf("full system should not hand out a token")
+	}
+	cancel()
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// now let's free up some async pool, release tok2 (1GB)
+	tok2.Close()
+
+	// NOW ASYNC POOL HAS 1GB FREE
+	// SYNC POOL IS FULL
+
+	// async pool should provide this
+	ch = trI.GetResourceToken(ctx, 1*1024, 15, false)
+	tok4, err := fetchToken(ch)
+	if err != nil {
+		t.Fatalf("async system should hand out token4")
+	}
+	cancel()
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// NOW ASYNC AND SYNC POOLS ARE FULL
+
+	tok4.Close()
+	tok3.Close()
+
+	// NOW ASYNC POOL HAS 1GB FREE
+	// SYNC POOL HAS 1GB FREE
+
+	// now, we ask for 2GB sync token, it should be provided from both async+sync pools
+	ch = trI.GetResourceToken(ctx, 2*1024, 16, false)
+	tok5, err := fetchToken(ch)
+	if err != nil {
+		t.Fatalf("async+sync system should hand out token5")
+	}
+	cancel()
+
+	// NOW ASYNC AND SYNC POOLS ARE FULL
+
+	tok1.Close()
+	tok5.Close()
+
+	// attempt to close tok2 twice.. This should be OK.
+	tok2.Close()
+
+	// POOLS should all be empty now
+	getTrackerTestVals(tr, &vals)
+	if vals.msu != 0 || vals.mau != 0 {
+		t.Fatalf("faulty state MEM %#v", vals)
+	}
+	if vals.csu != 0 || vals.cau != 0 {
+		t.Fatalf("faulty state CPU %#v", vals)
 	}

 }