Add annotations to routes and apps (#866)

Adds 'annotations' attribute to Routes and Apps
This commit is contained in:
Owen Cliffe
2018-03-20 18:02:49 +00:00
committed by GitHub
parent 845f40ee86
commit d25b5af59d
31 changed files with 1838 additions and 669 deletions

238
api/models/annotations.go Normal file
View File

@@ -0,0 +1,238 @@
package models
import (
"bytes"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"regexp"
)
// Annotations encapsulates key-value metadata associated with resource. The structure is immutable via its public API and nil-safe for its contract
// permissive nilability is here to simplify updates and reduce the need for nil handling in extensions - annotations should be updated by over-writing the original object:
// target.Annotations = target.Annotations.With("fooKey",1)
// old MD remains empty
// Annotations is lenable
type Annotations map[string]*annotationValue
// annotationValue encapsulates a value in the annotations map,
// This is stored in its compacted, un-parsed JSON format for later (re-) parsing into specific structs or values
// annotationValue objects are immutable after JSON load
type annotationValue []byte
const (
maxAnnotationValueBytes = 512
maxAnnotationKeyBytes = 128
maxAnnotationsKeys = 100
)
// Equals is defined based on un-ordered k/v comparison at of the annotation keys and (compacted) values of annotations, JSON object-value equality for values is property-order dependent
func (m Annotations) Equals(other Annotations) bool {
if len(m) != len(other) {
return false
}
for k1, v1 := range m {
v2, _ := other[k1]
if v2 == nil {
return false
}
if !bytes.Equal(*v1, *v2) {
return false
}
}
return true
}
func EmptyAnnotations() Annotations {
return nil
}
func (mv *annotationValue) String() string {
return string(*mv)
}
func (v *annotationValue) MarshalJSON() ([]byte, error) {
return *v, nil
}
func (mv *annotationValue) isEmptyValue() bool {
sval := string(*mv)
return sval == "\"\"" || sval == "null"
}
// UnmarshalJSON compacts annotation values but does not alter key-ordering for keys
func (mv *annotationValue) UnmarshalJSON(val []byte) error {
buf := bytes.Buffer{}
err := json.Compact(&buf, val)
if err != nil {
return err
}
if err != nil {
return err
}
*mv = buf.Bytes()
return nil
}
var validKeyRegex = regexp.MustCompile("^[!-~]+$")
func validateField(key string, value annotationValue) APIError {
if !validKeyRegex.Match([]byte(key)) {
return ErrInvalidAnnotationKey
}
keyLen := len([]byte(key))
if keyLen > maxAnnotationKeyBytes {
return ErrInvalidAnnotationKeyLength
}
if value.isEmptyValue() {
return ErrInvalidAnnotationValue
}
if len(value) > maxAnnotationValueBytes {
return ErrInvalidAnnotationValueLength
}
return nil
}
// With Creates a new annotations object containing the specified value - this does not perform size checks on the total number of keys
// this validates the correctness of the key and value. this returns a new the annotations object with the key set.
func (m Annotations) With(key string, data interface{}) (Annotations, error) {
if data == nil || data == "" {
return nil, errors.New("empty annotation value")
}
jsonBytes, err := json.Marshal(data)
if err != nil {
return nil, err
}
newVal := jsonBytes
err = validateField(key, newVal)
if err != nil {
return nil, err
}
var newMd Annotations
if m == nil {
newMd = make(Annotations, 1)
} else {
newMd = m.clone()
}
mv := annotationValue(newVal)
newMd[key] = &mv
return newMd, nil
}
// Validate validates a final annotations object prior to store,
// This will reject partial/patch changes with empty values (containing deletes)
func (m Annotations) Validate() APIError {
for k, v := range m {
err := validateField(k, *v)
if err != nil {
return err
}
}
if len(m) > maxAnnotationsKeys {
return ErrTooManyAnnotationKeys
}
return nil
}
// Get returns a raw JSON value of a annotation key
func (m Annotations) Get(key string) ([]byte, bool) {
if v, ok := m[key]; ok {
return *v, ok
}
return nil, false
}
// Without returns a new annotations object with a value excluded
func (m Annotations) Without(key string) Annotations {
nuVal := m.clone()
delete(nuVal, key)
return nuVal
}
// MergeChange merges a delta (possibly including deletes) with an existing annotations object and returns a new (copy) annotations object or an error.
// This assumes that both old and new annotations objects contain only valid keys and only newVs may contain deletes
func (m Annotations) MergeChange(newVs Annotations) Annotations {
newMd := m.clone()
for k, v := range newVs {
if v.isEmptyValue() {
delete(newMd, k)
} else {
if newMd == nil {
newMd = make(Annotations)
}
newMd[k] = v
}
}
if len(newMd) == 0 {
return EmptyAnnotations()
}
return newMd
}
// clone produces a key-wise copy of the underlying annotations
// publically MD can be copied by reference as it's (by contract) immutable
func (m Annotations) clone() Annotations {
if m == nil {
return nil
}
newMd := make(Annotations, len(m))
for ok, ov := range m {
newMd[ok] = ov
}
return newMd
}
// Value implements sql.Valuer, returning a string
func (m Annotations) Value() (driver.Value, error) {
if len(m) < 1 {
return driver.Value(string("")), nil
}
var b bytes.Buffer
err := json.NewEncoder(&b).Encode(m)
return driver.Value(b.String()), err
}
// Scan implements sql.Scanner
func (m *Annotations) Scan(value interface{}) error {
if value == nil || value == "" {
*m = nil
return nil
}
bv, err := driver.String.ConvertValue(value)
if err == nil {
var b []byte
switch x := bv.(type) {
case []byte:
b = x
case string:
b = []byte(x)
}
if len(b) > 0 {
return json.Unmarshal(b, m)
}
*m = nil
return nil
}
// otherwise, return an error
return fmt.Errorf("annotations invalid db format: %T %T value, err: %v", value, bv, err)
}

View File

@@ -0,0 +1,261 @@
package models
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
)
type testObj struct {
Md Annotations `json:"annotations,omitempty"`
}
type myJson struct {
Foo string `json:"foo,omitempty"`
Bar string `json:"bar,omitempty"`
}
func (m Annotations) withRawKey(key string, val string) Annotations {
newMd := make(Annotations)
for k, v := range m {
newMd[k] = v
}
v := annotationValue([]byte(val))
newMd[key] = &v
return newMd
}
func mustParseMd(t *testing.T, md string) Annotations {
mdObj := make(Annotations)
err := json.Unmarshal([]byte(md), &mdObj)
if err != nil {
t.Fatalf("Failed to parse must-parse value %s %v", md, err)
}
return mdObj
}
func TestAnnotationsEqual(t *testing.T) {
annWithVal, _ := EmptyAnnotations().With("foo", "Bar")
tcs := []struct {
a Annotations
b Annotations
equals bool
}{
{EmptyAnnotations(), EmptyAnnotations(), true},
{annWithVal, EmptyAnnotations(), false},
{annWithVal, annWithVal, true},
{EmptyAnnotations().withRawKey("v1", `"a"`), EmptyAnnotations().withRawKey("v1", `"b"`), false},
{EmptyAnnotations().withRawKey("v1", `"a"`), EmptyAnnotations().withRawKey("v2", `"a"`), false},
{annWithVal.Without("foo"), EmptyAnnotations(), true},
{mustParseMd(t,
"{ \r\n\t"+`"md.1":{ `+"\r\n\t"+`
"subkey1": "value\n with\n newlines",
"subkey2": true
}
}`), mustParseMd(t, `{"md.1":{"subkey1":"value\n with\n newlines", "subkey2":true}}`), true},
}
for _, tc := range tcs {
if tc.a.Equals(tc.b) != tc.equals {
t.Errorf("Annotations equality mismatch - expecting (%v == %v) = %v", tc.b, tc.a, tc.equals)
}
if tc.b.Equals(tc.a) != tc.equals {
t.Errorf("Annotations reflexive equality mismatch - expecting (%v == %v) = %v", tc.b, tc.a, tc.equals)
}
}
}
var annCases = []struct {
val *testObj
valString string
}{
{val: &testObj{Md: EmptyAnnotations()}, valString: "{}"},
{val: &testObj{Md: EmptyAnnotations().withRawKey("stringval", `"bar"`)}, valString: `{"annotations":{"stringval":"bar"}}`},
{val: &testObj{Md: EmptyAnnotations().withRawKey("intval", `1001`)}, valString: `{"annotations":{"intval":1001}}`},
{val: &testObj{Md: EmptyAnnotations().withRawKey("floatval", "3.141")}, valString: `{"annotations":{"floatval":3.141}}`},
{val: &testObj{Md: EmptyAnnotations().withRawKey("objval", `{"foo":"fooval","bar":"barval"}`)}, valString: `{"annotations":{"objval":{"foo":"fooval","bar":"barval"}}}`},
{val: &testObj{Md: EmptyAnnotations().withRawKey("objval", `{"foo":"fooval","bar":{"barbar":"barbarval"}}`)}, valString: `{"annotations":{"objval":{"foo":"fooval","bar":{"barbar":"barbarval"}}}}`},
{val: &testObj{Md: EmptyAnnotations().withRawKey("objval", `{"foo":"JSON newline \\n string"}`)}, valString: `{"annotations":{"objval":{"foo":"JSON newline \\n string"}}}`},
}
func TestAnnotationsJSONMarshalling(t *testing.T) {
for _, tc := range annCases {
v, err := json.Marshal(tc.val)
if err != nil {
t.Fatalf("Failed to marshal json into %s: %v", tc.valString, err)
}
if string(v) != tc.valString {
t.Errorf("Invalid annotations value, expected %s, got %s", tc.valString, string(v))
}
}
}
func TestAnnotationsJSONUnMarshalling(t *testing.T) {
for _, tc := range annCases {
tv := testObj{}
err := json.Unmarshal([]byte(tc.valString), &tv)
if err != nil {
t.Fatalf("Failed to unmarshal json into %s: %v", tc.valString, err)
}
if !reflect.DeepEqual(&tv, tc.val) {
t.Errorf("Invalid annotations value, expected %v, got %v", tc.val, tv)
}
}
}
func TestAnnotationsWithHonorsKeyLimits(t *testing.T) {
var validKeys = []string{
"ok",
strings.Repeat("a", maxAnnotationKeyBytes),
"fnproject/internal/foo",
"foo.bar.com.baz",
"foo$bar!_+-()[]:@/<>$",
}
for _, k := range validKeys {
m, err := EmptyAnnotations().With(k, "value")
if err != nil {
t.Errorf("Should have accepted valid key %s,%v", k, err)
}
err = m.Validate()
if err != nil {
t.Errorf("Should have validate valid key %s,%v", k, err)
}
}
var invalidKeys = []struct {
key string
err APIError
}{
{"", ErrInvalidAnnotationKey},
{" ", ErrInvalidAnnotationKey},
{"\u00e9", ErrInvalidAnnotationKey},
{"foo bar", ErrInvalidAnnotationKey},
{strings.Repeat("a", maxAnnotationKeyBytes+1), ErrInvalidAnnotationKeyLength},
}
for _, kc := range invalidKeys {
_, err := EmptyAnnotations().With(kc.key, "value")
if err == nil {
t.Errorf("Should have rejected invalid key %s", kc.key)
}
m := EmptyAnnotations().withRawKey(kc.key, "\"data\"")
err = m.Validate()
if err != kc.err {
t.Errorf("Should have returned validation error %v, for key %s got %v", kc.err, kc.key, err)
}
}
}
func TestAnnotationsHonorsValueLimits(t *testing.T) {
validValues := []interface{}{
"ok",
&myJson{Foo: "foo"},
strings.Repeat("a", maxAnnotationValueBytes-2),
[]string{strings.Repeat("a", maxAnnotationValueBytes-4)},
1,
[]string{"a", "b", "c"},
true,
}
for _, v := range validValues {
_, err := EmptyAnnotations().With("key", v)
if err != nil {
t.Errorf("Should have accepted valid value %s,%v", v, err)
}
rawJson, err := json.Marshal(v)
if err != nil {
panic(err)
}
md := EmptyAnnotations().withRawKey("key", string(rawJson))
err = md.Validate()
if err != nil {
t.Errorf("Should have validated valid value successfully %s, got error %v", string(rawJson), err)
}
}
invalidValues := []struct {
val interface{}
err APIError
}{
{"", ErrInvalidAnnotationValue},
{nil, ErrInvalidAnnotationValue},
{strings.Repeat("a", maxAnnotationValueBytes-1), ErrInvalidAnnotationValueLength},
{[]string{strings.Repeat("a", maxAnnotationValueBytes-3)}, ErrInvalidAnnotationValueLength},
}
for _, v := range invalidValues {
_, err := EmptyAnnotations().With("key", v.val)
if err == nil {
t.Errorf("Should have rejected invalid value \"%v\"", v)
}
rawJson, err := json.Marshal(v.val)
if err != nil {
panic(err)
}
md := EmptyAnnotations().withRawKey("key", string(rawJson))
err = md.Validate()
if err != v.err {
t.Errorf("Expected validation error %v for '%s', got %v", v.err, string(rawJson), err)
}
}
}
func TestMergeAnnotations(t *testing.T) {
mdWithNKeys := func(n int) Annotations {
md := EmptyAnnotations()
for i := 0; i < n; i++ {
md = md.withRawKey(fmt.Sprintf("key-%d", i), "val")
}
return md
}
validCases := []struct {
first Annotations
second Annotations
result Annotations
}{
{first: EmptyAnnotations(), second: EmptyAnnotations(), result: EmptyAnnotations()},
{first: EmptyAnnotations().withRawKey("key1", "\"val\""), second: EmptyAnnotations(), result: EmptyAnnotations().withRawKey("key1", "\"val\"")},
{first: EmptyAnnotations(), second: EmptyAnnotations().withRawKey("key1", "\"val\""), result: EmptyAnnotations().withRawKey("key1", "\"val\"")},
{first: EmptyAnnotations().withRawKey("key1", "\"val\""), second: EmptyAnnotations().withRawKey("key1", "\"val\""), result: EmptyAnnotations().withRawKey("key1", "\"val\"")},
{first: EmptyAnnotations().withRawKey("key1", "\"val1\""), second: EmptyAnnotations().withRawKey("key2", "\"val2\""), result: EmptyAnnotations().withRawKey("key1", "\"val1\"").withRawKey("key2", "\"val2\"")},
{first: EmptyAnnotations().withRawKey("key1", "\"val1\""), second: EmptyAnnotations().withRawKey("key1", "\"\""), result: EmptyAnnotations()},
{first: EmptyAnnotations().withRawKey("key1", "\"val1\""), second: EmptyAnnotations().withRawKey("key2", "\"\""), result: EmptyAnnotations().withRawKey("key1", "\"val1\"")},
{first: mdWithNKeys(maxAnnotationsKeys - 1), second: EmptyAnnotations().withRawKey("newkey", "\"val\""), result: mdWithNKeys(maxAnnotationsKeys-1).withRawKey("newkey", "\"val\"")},
}
for _, v := range validCases {
newMd := v.first.MergeChange(v.second)
if !reflect.DeepEqual(newMd, v.result) {
t.Errorf("Change %v + %v : expected %v got %v", v.first, v.second, v.result, newMd)
}
}
}

View File

@@ -8,10 +8,11 @@ import (
)
type App struct {
Name string `json:"name" db:"name"`
Config Config `json:"config,omitempty" db:"config"`
CreatedAt strfmt.DateTime `json:"created_at,omitempty" db:"created_at"`
UpdatedAt strfmt.DateTime `json:"updated_at,omitempty" db:"updated_at"`
Name string `json:"name" db:"name"`
Config Config `json:"config,omitempty" db:"config"`
Annotations Annotations `json:"annotations,omitempty" db:"annotations"`
CreatedAt strfmt.DateTime `json:"created_at,omitempty" db:"created_at"`
UpdatedAt strfmt.DateTime `json:"updated_at,omitempty" db:"updated_at"`
}
func (a *App) SetDefaults() {
@@ -39,6 +40,10 @@ func (a *App) Validate() error {
return ErrAppsInvalidName
}
}
err := a.Annotations.Validate()
if err != nil {
return err
}
return nil
}
@@ -53,6 +58,7 @@ func (a *App) Clone() *App {
clone.Config[k] = v
}
}
return clone
}
@@ -63,6 +69,7 @@ func (a1 *App) Equals(a2 *App) bool {
eq := true
eq = eq && a1.Name == a2.Name
eq = eq && a1.Config.Equals(a2.Config)
eq = eq && a1.Annotations.Equals(a2.Annotations)
// NOTE: datastore tests are not very fun to write with timestamp checks,
// and these are not values the user may set so we kind of don't care.
//eq = eq && time.Time(a1.CreatedAt).Equal(time.Time(a2.CreatedAt))
@@ -70,15 +77,15 @@ func (a1 *App) Equals(a2 *App) bool {
return eq
}
// Update adds entries from patch to a.Config, and removes entries with empty values.
func (a *App) Update(src *App) {
// Update adds entries from patch to a.Config and a.Annotations, and removes entries with empty values.
func (a *App) Update(patch *App) {
original := a.Clone()
if src.Config != nil {
if patch.Config != nil {
if a.Config == nil {
a.Config = make(Config)
}
for k, v := range src.Config {
for k, v := range patch.Config {
if v == "" {
delete(a.Config, k)
} else {
@@ -87,6 +94,8 @@ func (a *App) Update(src *App) {
}
}
a.Annotations = a.Annotations.MergeChange(patch.Annotations)
if !a.Equals(original) {
a.UpdatedAt = strfmt.DateTime(time.Now())
}

View File

@@ -185,6 +185,26 @@ var (
code: http.StatusBadGateway,
error: fmt.Errorf("function response too large"),
}
ErrInvalidAnnotationKey = err{
code: http.StatusBadRequest,
error: errors.New("Invalid annotation key, annotation keys must be non-empty ascii strings excluding whitespace"),
}
ErrInvalidAnnotationKeyLength = err{
code: http.StatusBadRequest,
error: fmt.Errorf("Invalid annotation key length, annotation keys may not be larger than %d bytes", maxAnnotationKeyBytes),
}
ErrInvalidAnnotationValue = err{
code: http.StatusBadRequest,
error: errors.New("Invalid annotation value, annotation values may only be non-empty strings, numbers, objects, or arrays"),
}
ErrInvalidAnnotationValueLength = err{
code: http.StatusBadRequest,
error: fmt.Errorf("Invalid annotation value length, annotation values may not be larger than %d bytes when serialized as JSON", maxAnnotationValueBytes),
}
ErrTooManyAnnotationKeys = err{
code: http.StatusBadRequest,
error: fmt.Errorf("Invalid annotation change, new key(s) exceed maximum permitted number of annotations keys (%d)", maxAnnotationsKeys),
}
)
// APIError any error that implements this interface will return an API response

View File

@@ -36,6 +36,7 @@ type Route struct {
Timeout int32 `json:"timeout" db:"timeout"`
IdleTimeout int32 `json:"idle_timeout" db:"idle_timeout"`
Config Config `json:"config,omitempty" db:"config"`
Annotations Annotations `json:"annotations,omitempty" db:"annotations"`
CreatedAt strfmt.DateTime `json:"created_at,omitempty" db:"created_at"`
UpdatedAt strfmt.DateTime `json:"updated_at,omitempty" db:"updated_at"`
}
@@ -129,6 +130,11 @@ func (r *Route) Validate() error {
return ErrRoutesInvalidMemory
}
err = r.Annotations.Validate()
if err != nil {
return err
}
return nil
}
@@ -169,6 +175,7 @@ func (r1 *Route) Equals(r2 *Route) bool {
eq = eq && r1.Timeout == r2.Timeout
eq = eq && r1.IdleTimeout == r2.IdleTimeout
eq = eq && r1.Config.Equals(r2.Config)
eq = eq && r1.Annotations.Equals(r2.Annotations)
// NOTE: datastore tests are not very fun to write with timestamp checks,
// and these are not values the user may set so we kind of don't care.
//eq = eq && time.Time(r1.CreatedAt).Equal(time.Time(r2.CreatedAt))
@@ -179,35 +186,35 @@ func (r1 *Route) Equals(r2 *Route) bool {
// Update updates fields in r with non-zero field values from new, and sets
// updated_at if any of the fields change. 0-length slice Header values, and
// empty-string Config values trigger removal of map entry.
func (r *Route) Update(new *Route) {
func (r *Route) Update(patch *Route) {
original := r.Clone()
if new.Image != "" {
r.Image = new.Image
if patch.Image != "" {
r.Image = patch.Image
}
if new.Memory != 0 {
r.Memory = new.Memory
if patch.Memory != 0 {
r.Memory = patch.Memory
}
if new.CPUs != 0 {
r.CPUs = new.CPUs
if patch.CPUs != 0 {
r.CPUs = patch.CPUs
}
if new.Type != "" {
r.Type = new.Type
if patch.Type != "" {
r.Type = patch.Type
}
if new.Timeout != 0 {
r.Timeout = new.Timeout
if patch.Timeout != 0 {
r.Timeout = patch.Timeout
}
if new.IdleTimeout != 0 {
r.IdleTimeout = new.IdleTimeout
if patch.IdleTimeout != 0 {
r.IdleTimeout = patch.IdleTimeout
}
if new.Format != "" {
r.Format = new.Format
if patch.Format != "" {
r.Format = patch.Format
}
if new.Headers != nil {
if patch.Headers != nil {
if r.Headers == nil {
r.Headers = Headers(make(http.Header))
}
for k, v := range new.Headers {
for k, v := range patch.Headers {
if len(v) == 0 {
http.Header(r.Headers).Del(k)
} else {
@@ -215,11 +222,11 @@ func (r *Route) Update(new *Route) {
}
}
}
if new.Config != nil {
if patch.Config != nil {
if r.Config == nil {
r.Config = make(Config)
}
for k, v := range new.Config {
for k, v := range patch.Config {
if v == "" {
delete(r.Config, k)
} else {
@@ -228,6 +235,8 @@ func (r *Route) Update(new *Route) {
}
}
r.Annotations = r.Annotations.MergeChange(patch.Annotations)
if !r.Equals(original) {
r.UpdatedAt = strfmt.DateTime(time.Now())
}