1
0
mirror of https://github.com/TomWright/dasel.git synced 2022-05-22 02:32:45 +03:00

Add JSON support, more tests and comments

This commit is contained in:
Tom Wright
2020-09-22 15:06:37 +01:00
parent 1529dcad15
commit 29f5e3a310
13 changed files with 343 additions and 128 deletions

1
.gitignore vendored
View File

@@ -1 +1,2 @@
.idea/
dasel

View File

@@ -1,8 +1,45 @@
# dasel
Read and modify data structures using selectors.
[![Go Report Card](https://goreportcard.com/badge/github.com/TomWright/dasel)](https://goreportcard.com/report/github.com/TomWright/dasel)
[![Documentation](https://godoc.org/github.com/TomWright/dasel?status.svg)](https://godoc.org/github.com/TomWright/dasel)
![Test](https://github.com/TomWright/dasel/workflows/Test/badge.svg)
![Build](https://github.com/TomWright/dasel/workflows/Build/badge.svg)
Dasel (short for data-selector) allows you to query and modify data structures using selector strings.
## Usage
You can import dasel as a package and use it in your applications, or you can use a pre-built binary to modify files from the command line.
### Import
As with any other go package, just use `go get`.
```
go get github.com/tomwright/dasel/cmd/dasel
```
### Command line
You can `go get` the `main` package and go should automatically build and install dasel for you.
```
go get github.com/tomwright/dasel/cmd/dasel
```
Alternatively you can download a compiled executable from the [latest release](https://github.com/TomWright/dasel/releases/latest):
This one liner should work for you - be sure to change the targeted release executable if needed. It currently targets `dasel_linux_amd64`.
```
curl -s https://api.github.com/repos/tomwright/dasel/releases/latest | grep browser_download_url | grep linux_amd64 | cut -d '"' -f 4 | wget -qi - && mv dasel_linux_amd64 dasel && chmod +x dasel
```
## Support data types
Dasel attempts to find the correct parser for the given file type, but if that fails you can choose which parser to use with the `-p` or `--parser` flag.
- YAML - `-p yaml`
- JSON - `-p json`
## Selectors
Selectors are used to define a path through a set of data. This path is usually defined as a chain of nodes.
A selector is made up of different parts separated by a dot `.`, each part being used to identify the next node in the chain.
The following YAML data structure will be used as a reference in the following examples.
```
name: Tom
@@ -21,26 +58,49 @@ colourCodes:
rgb: 0000ff
```
### Root Element
Just use the root element name as a string.
### Property
Property selectors are used to reference a single property of an object.
Just use the property name as a string.
```
$ dasel select -f ./tests/assets/example.yaml -s "name"
Tom
```
- `name` == `Tom`
### Child Element
Just separate the parent element from the parent element using a `.`:
### Child Elements
Just separate the child element from the parent element using a `.`:
```
$ dasel select -f ./tests/assets/example.yaml -s "preferences.favouriteColour"
red
```
- `preferences.favouriteColour` == `red`
#### Index
When you have a list, you can use square brackets to access or modify a specific item.
When you have a list, you can use square brackets to access a specific item in the list by its index.
```
$ dasel select -f ./tests/assets/example.yaml -s "colours.[1]"
green
```
- `colours.[0]` == `red`
- `colours.[1]` == `green`
- `colours.[2]` == `blue`
#### Next Available Index
Next available index selector is used when adding to a list of items.
#### Next Available Index - WIP
Next available index selector is used when adding to a list of items. It allows you to append to a list.
- `colours.[]`
#### Look up
Look ups are defined in brackets and allow you to dynamically select an object to use.
#### Dynamic
Dynamic selectors are used with lists when you don't know the index of the item, but instead know the value of a property of an object within the list.
Look ups are defined in brackets. You can use multiple dynamic selectors within the same part to perform multiple checks.
```
$ dasel select -f ./tests/assets/example.yaml -s ".colourCodes.(name=red).rgb"
ff0000
$ dasel select -f ./tests/assets/example.yaml -s ".colourCodes.(name=blue)(rgb=0000ff)"
map[name:blue rgb:0000ff]
```
- `.colourCodes.(name=red).rgb` == `ff0000`
- `.colourCodes.(name=green).rgb` == `00ff00`
- `.colourCodes.(name=blue).rgb` == `0000ff`

View File

@@ -2,11 +2,15 @@ package dasel
import "fmt"
// EqualCondition lets you check for an exact match.
type EqualCondition struct {
// Key is the key of the value to check against.
Key string
// Value is the value we are looking for.
Value string
}
// Check checks to see if other contains the required key value pair.
func (c EqualCondition) Check(other interface{}) (bool, error) {
switch o := other.(type) {
case map[string]string:
@@ -20,6 +24,7 @@ func (c EqualCondition) Check(other interface{}) (bool, error) {
}
}
// Condition defines a Check we can use within dynamic selectors.
type Condition interface {
Check(other interface{}) (bool, error)
}

View File

@@ -5,54 +5,71 @@ import (
"fmt"
)
// ErrMissingPreviousNode is returned when FindValue doesn't have access to the previous node.
var ErrMissingPreviousNode = errors.New("missing previous node")
// UnknownComparisonOperatorErr is returned when
type UnknownComparisonOperatorErr struct {
Operator string
}
// Error returns the error message.
func (e UnknownComparisonOperatorErr) Error() string {
return fmt.Sprintf("unknown comparison operator: %s", e.Operator)
}
// InvalidIndexErr is returned when a selector targets an index that does not exist.
type InvalidIndexErr struct {
Index string
}
// Error returns the error message.
func (e InvalidIndexErr) Error() string {
return fmt.Sprintf("invalid index: %s", e.Index)
}
// UnsupportedSelector is returned when a specific selector type is used in the wrong context.
type UnsupportedSelector struct {
Selector string
}
// Error returns the error message.
func (e UnsupportedSelector) Error() string {
return fmt.Sprintf("selector is not supported here: %s", e.Selector)
}
// UnsupportedTypeForSelector is returned when a selector attempts to handle a data type it can't handle.
type UnsupportedTypeForSelector struct {
Selector Selector
Value interface{}
}
// Error returns the error message.
func (e UnsupportedTypeForSelector) Error() string {
return fmt.Sprintf("selector [%s] does not support value: %T: %v", e.Selector.Type, e.Value, e.Value)
}
type NotFound struct {
// ValueNotFound is returned when a selector string cannot be fully resolved.
type ValueNotFound struct {
Selector string
Node *Node
}
func (e NotFound) Error() string {
return fmt.Sprintf("nothing found for selector: %s", e.Selector)
// Error returns the error message.
func (e ValueNotFound) Error() string {
var previousValue interface{}
if e.Node != nil && e.Node.Previous != nil {
previousValue = e.Node.Previous.Value
}
return fmt.Sprintf("no value found for selector: %s: %v", e.Selector, previousValue)
}
// UnexpectedPreviousNilValue is returned when the previous node contains a nil value.
type UnexpectedPreviousNilValue struct {
Selector string
}
// Error returns the error message.
func (e UnexpectedPreviousNilValue) Error() string {
return fmt.Sprintf("previous value is nil: %s", e.Selector)
}

View File

@@ -2,18 +2,18 @@ package command
import (
"github.com/spf13/cobra"
"github.com/tomwright/dasel/internal"
)
// RootCMD is the root command for use with cobra.
var RootCMD = &cobra.Command{
Use: "dasel",
Aliases: nil,
SuggestFor: nil,
Short: "A small helper to manage kubernetes configurations.",
Short: "Query and modify data structures using selector strings.",
}
func init() {
RootCMD.Version = internal.Version
RootCMD.AddCommand(
selectCommand(),
versionCommand(),
)
}

View File

@@ -8,23 +8,32 @@ import (
)
func selectCommand() *cobra.Command {
var file, selector, parser string
var fileFlag, selectorFlag, parserFlag string
cmd := &cobra.Command{
Use: "select -f <file> -s <selector>",
Short: "Select properties from the given files.",
Short: "Select properties from the given file.",
Args: cobra.ExactArgs(0),
RunE: func(cmd *cobra.Command, args []string) error {
parser, err := storage.FromString(parser)
var parser storage.Parser
var err error
if parserFlag == "" {
parser, err = storage.NewParserFromFilename(fileFlag)
if err != nil {
return err
return fmt.Errorf("could not get parser from filename: %w", err)
}
value, err := parser.LoadFromFile(file)
} else {
parser, err = storage.NewParserFromString(parserFlag)
if err != nil {
return fmt.Errorf("could not get parser: %w", err)
}
}
value, err := storage.LoadFromFile(fileFlag, parser)
if err != nil {
return fmt.Errorf("could not load file: %w", err)
}
rootNode := dasel.New(value)
res, err := rootNode.Query(selector)
res, err := rootNode.Query(selectorFlag)
if err != nil {
return fmt.Errorf("could not query node: %w", err)
}
@@ -35,9 +44,15 @@ func selectCommand() *cobra.Command {
},
}
cmd.Flags().StringVarP(&file, "file", "f", "", "The file to query.")
cmd.Flags().StringVarP(&selector, "selector", "s", "", "The selector to use when querying.")
cmd.Flags().StringVarP(&parser, "parser", "p", "yaml", "The parser to use with the given file.")
cmd.Flags().StringVarP(&fileFlag, "file", "f", "", "The file to query.")
cmd.Flags().StringVarP(&selectorFlag, "selector", "s", "", "The selector to use when querying the data structure.")
cmd.Flags().StringVarP(&parserFlag, "parser", "p", "", "The parser to use with the given file.")
for _, f := range []string{"file", "selector"} {
if err := cmd.MarkFlagRequired(f); err != nil {
panic("could not mark flag as required: " + f)
}
}
return cmd
}

View File

@@ -1,20 +0,0 @@
package command
import (
"fmt"
"github.com/spf13/cobra"
"github.com/tomwright/dasel/internal"
)
func versionCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "version",
Short: "Prints the dasel version.",
Args: cobra.ExactArgs(0),
Run: func(cmd *cobra.Command, args []string) {
fmt.Println(internal.Version)
},
}
return cmd
}

19
internal/storage/json.go Normal file
View File

@@ -0,0 +1,19 @@
package storage
import (
"encoding/json"
"fmt"
)
// JSONParser is a Parser implementation to handle yaml files.
type JSONParser struct {
}
// FromBytes returns some Data that is represented by the given bytes.
func (p *JSONParser) FromBytes(byteData []byte) (interface{}, error) {
var data interface{}
if err := json.Unmarshal(byteData, &data); err != nil {
return data, fmt.Errorf("could not unmarshal config data: %w", err)
}
return data, nil
}

View File

@@ -2,46 +2,53 @@ package storage
import (
"fmt"
"gopkg.in/yaml.v2"
"io/ioutil"
"path/filepath"
"strings"
)
type unknownParserErr struct {
// UnknownParserErr is returned when an invalid parser name is given.
type UnknownParserErr struct {
parser string
}
func (e unknownParserErr) Error() string {
// Error returns the error message.
func (e UnknownParserErr) Error() string {
return fmt.Sprintf("unknown parser: %s", e.parser)
}
// Parser can be used to load and save files from/to disk.
type Parser interface {
FromBytes(byteData []byte) (interface{}, error)
LoadFromFile(filename string) (interface{}, error)
}
func FromString(parser string) (Parser, error) {
// NewParserFromFilename returns a Parser from the given filename.
func NewParserFromFilename(filename string) (Parser, error) {
ext := strings.ToLower(filepath.Ext(filename))
switch ext {
case ".yaml", ".yml":
return &YAMLParser{}, nil
case ".json":
return &JSONParser{}, nil
default:
return nil, &UnknownParserErr{parser: ext}
}
}
// NewParserFromString returns a Parser from the given parser name.
func NewParserFromString(parser string) (Parser, error) {
switch parser {
case "yaml":
return &YAMLParser{}, nil
case "json":
return &JSONParser{}, nil
default:
return nil, &unknownParserErr{parser: parser}
return nil, &UnknownParserErr{parser: parser}
}
}
type YAMLParser struct {
}
// FromBytes returns some Data that is represented by the given bytes.
func (p *YAMLParser) FromBytes(byteData []byte) (interface{}, error) {
var data interface{}
if err := yaml.Unmarshal(byteData, &data); err != nil {
return data, fmt.Errorf("could not unmarshal config data: %w", err)
}
return data, nil
}
// LoadFromFile loads Data from the given file.
func (p *YAMLParser) LoadFromFile(filename string) (interface{}, error) {
// LoadFromFile loads data from the given file.
func LoadFromFile(filename string, p Parser) (interface{}, error) {
byteData, err := ioutil.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("could not read config file: %w", err)

19
internal/storage/yaml.go Normal file
View File

@@ -0,0 +1,19 @@
package storage
import (
"fmt"
"gopkg.in/yaml.v2"
)
// YAMLParser is a Parser implementation to handle yaml files.
type YAMLParser struct {
}
// FromBytes returns some Data that is represented by the given bytes.
func (p *YAMLParser) FromBytes(byteData []byte) (interface{}, error) {
var data interface{}
if err := yaml.Unmarshal(byteData, &data); err != nil {
return data, fmt.Errorf("could not unmarshal config data: %w", err)
}
return data, nil
}

108
node.go
View File

@@ -39,6 +39,7 @@ type Node struct {
Selector Selector `json:"selector"`
}
// String returns a string representation of the node. It does this by marshaling it.
func (n *Node) String() string {
b, err := json.MarshalIndent(n, "", " ")
if err != nil {
@@ -113,6 +114,7 @@ func ParseSelector(selector string) (Selector, error) {
return sel, nil
}
// New returns a new root note with the given value.
func New(value interface{}) *Node {
rootNode := &Node{
Previous: nil,
@@ -129,6 +131,7 @@ func New(value interface{}) *Node {
return rootNode
}
// Query uses the given selector to query the current node and return the result.
func (n Node) Query(selector string) (*Node, error) {
n.Selector.Remaining = selector
rootNode := &n
@@ -159,7 +162,7 @@ func (n Node) Query(selector string) (*Node, error) {
nextNode.Value, err = FindValue(nextNode)
// Populate the value for the new node.
if err != nil {
return nil, &NotFound{Selector: nextNode.Selector.Current, Node: nextNode}
return nil, err
}
previousNode = nextNode
@@ -179,14 +182,14 @@ func findValueProperty(n *Node) (interface{}, error) {
if ok {
return v, nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
case map[interface{}]interface{}:
v, ok := p[n.Selector.Property]
if ok {
return v, nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
default:
return nil, &UnsupportedTypeForSelector{Selector: n.Selector, Value: n.Previous.Value}
@@ -204,55 +207,76 @@ func findValueIndex(n *Node) (interface{}, error) {
if n.Selector.Index >= 0 && n.Selector.Index < l {
return p[n.Selector.Index], nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
case []map[string]interface{}:
l := int64(len(p))
if n.Selector.Index >= 0 && n.Selector.Index < l {
return p[n.Selector.Index], nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
case map[interface{}]interface{}:
l := int64(len(p))
if n.Selector.Index >= 0 && n.Selector.Index < l {
return p[n.Selector.Index], nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
case map[int]interface{}:
l := int64(len(p))
if n.Selector.Index >= 0 && n.Selector.Index < l {
return p[int(n.Selector.Index)], nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
case []interface{}:
l := int64(len(p))
if n.Selector.Index >= 0 && n.Selector.Index < l {
return p[n.Selector.Index], nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
case []string:
l := int64(len(p))
if n.Selector.Index >= 0 && n.Selector.Index < l {
return p[n.Selector.Index], nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
case []int:
l := int64(len(p))
if n.Selector.Index >= 0 && n.Selector.Index < l {
return p[n.Selector.Index], nil
} else {
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
}
default:
return nil, &UnsupportedTypeForSelector{Selector: n.Selector, Value: n.Previous.Value}
}
}
// processFindDynamicItem is used by findValueDynamic.
func processFindDynamicItem(n *Node, object interface{}) (interface{}, bool, error) {
// Loop through each condition.
allConditionsMatched := true
for _, c := range n.Selector.Conditions {
// If the object doesn't match any checks, return a ValueNotFound.
found, err := c.Check(object)
if err != nil {
return nil, false, err
}
if !found {
allConditionsMatched = false
break
}
}
if allConditionsMatched {
return object, true, nil
}
return nil, false, nil
}
// findValueDynamic finds the value for the given node using the dynamic selector
// information.
func findValueDynamic(n *Node) (interface{}, error) {
@@ -260,65 +284,49 @@ func findValueDynamic(n *Node) (interface{}, error) {
case nil:
return nil, &UnexpectedPreviousNilValue{Selector: n.Previous.Selector.Current}
case []map[interface{}]interface{}:
for _, v := range p {
// Loop through each condition.
allConditionsMatched := true
for _, c := range n.Selector.Conditions {
// If the object doesn't match any checks, return a NotFound.
found, err := c.Check(v)
for _, object := range p {
value, found, err := processFindDynamicItem(n, object)
if err != nil {
return nil, err
}
if !found {
allConditionsMatched = false
break
if found {
return value, nil
}
}
if allConditionsMatched {
return v, nil
}
}
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
case []map[string]interface{}:
for _, v := range p {
// Loop through each condition.
allConditionsMatched := true
for _, c := range n.Selector.Conditions {
// If the object doesn't match any checks, return a NotFound.
found, err := c.Check(v)
for _, object := range p {
value, found, err := processFindDynamicItem(n, object)
if err != nil {
return nil, err
}
if !found {
allConditionsMatched = false
break
if found {
return value, nil
}
}
if allConditionsMatched {
return v, nil
}
}
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
case []map[string]string:
for _, v := range p {
// Loop through each condition.
allConditionsMatched := true
for _, c := range n.Selector.Conditions {
// If the object doesn't match any checks, return a NotFound.
found, err := c.Check(v)
for _, object := range p {
value, found, err := processFindDynamicItem(n, object)
if err != nil {
return nil, err
}
if !found {
allConditionsMatched = false
break
if found {
return value, nil
}
}
if allConditionsMatched {
return v, nil
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
case []interface{}:
for _, object := range p {
value, found, err := processFindDynamicItem(n, object)
if err != nil {
return nil, err
}
if found {
return value, nil
}
}
return nil, &NotFound{Selector: n.Selector.Current, Node: n}
return nil, &ValueNotFound{Selector: n.Selector.Current, Node: n}
default:
return nil, &UnsupportedTypeForSelector{Selector: n.Selector, Value: n.Previous.Value}
}

View File

@@ -2,7 +2,9 @@ package dasel_test
import (
"errors"
"fmt"
"github.com/tomwright/dasel"
"github.com/tomwright/dasel/internal/storage"
"reflect"
"testing"
)
@@ -114,6 +116,63 @@ func TestNode_Query(t *testing.T) {
}
})
t.Run("File", func(t *testing.T) {
tests := []struct {
Name string
Selector string
Exp string
}{
{Name: "Property", Selector: "name", Exp: "Tom"},
{Name: "ChildProperty", Selector: "preferences.favouriteColour", Exp: "red"},
{Name: "Index", Selector: "colours.[0]", Exp: "red"},
{Name: "Index", Selector: "colours.[1]", Exp: "green"},
{Name: "Index", Selector: "colours.[2]", Exp: "blue"},
{Name: "IndexProperty", Selector: "colourCodes.[0].name", Exp: "red"},
{Name: "IndexProperty", Selector: "colourCodes.[1].name", Exp: "green"},
{Name: "IndexProperty", Selector: "colourCodes.[2].name", Exp: "blue"},
{Name: "DynamicProperty", Selector: "colourCodes.(name=red).rgb", Exp: "ff0000"},
{Name: "DynamicProperty", Selector: "colourCodes.(name=green).rgb", Exp: "00ff00"},
{Name: "DynamicProperty", Selector: "colourCodes.(name=blue).rgb", Exp: "0000ff"},
{Name: "MultipleDynamicProperty", Selector: "colourCodes.(name=red)(rgb=ff0000).name", Exp: "red"},
{Name: "MultipleDynamicProperty", Selector: "colourCodes.(name=green)(rgb=00ff00).name", Exp: "green"},
{Name: "MultipleDynamicProperty", Selector: "colourCodes.(name=blue)(rgb=0000ff).name", Exp: "blue"},
}
fileTest := func(filename string) func(t *testing.T) {
return func(t *testing.T) {
parser, err := storage.NewParserFromFilename(filename)
if err != nil {
t.Errorf("could not get parser: %s", err)
return
}
value, err := storage.LoadFromFile(filename, parser)
if err != nil {
t.Errorf("could not load value from file: %s", err)
return
}
for _, testCase := range tests {
tc := testCase
t.Run(tc.Name, func(t *testing.T) {
node, err := dasel.New(value).Query(tc.Selector)
if err != nil {
t.Errorf("unexpected error: %s", err)
return
}
if exp, got := tc.Exp, fmt.Sprint(node.Value); exp != got {
t.Errorf("expected value `%s`, got `%s`", exp, got)
}
})
}
}
}
t.Run("JSON", fileTest("./tests/assets/example.json"))
t.Run("YAML", fileTest("./tests/assets/example.yaml"))
})
t.Run("Traversal", func(t *testing.T) {
tests := parseTest{
Selector: ".a.b.c.thing",

25
tests/assets/example.json Normal file
View File

@@ -0,0 +1,25 @@
{
"name": "Tom",
"preferences": {
"favouriteColour": "red"
},
"colours": [
"red",
"green",
"blue"
],
"colourCodes": [
{
"name": "red",
"rgb": "ff0000"
},
{
"name": "green",
"rgb": "00ff00"
},
{
"name": "blue",
"rgb": "0000ff"
}
]
}