Merge pull request #151 from TomWright/output-formatting

Output formatting

CHANGELOG.md

@@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-- Nothing yet.
+### Added
+
+- `--format` flag to `select` command.
 
 ## [v1.17.0] - 2021-08-08
 

internal/command/delete.go

@@ -73,7 +73,7 @@ func runDeleteCommand(opts deleteOptions, cmd *cobra.Command) error {
 		})
 	}
 
-	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File)
+	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File, "")
 	if err != nil {
 		return err
 	}

internal/command/put.go

@@ -70,7 +70,12 @@ func getReadParser(fileFlag string, readParserFlag string, parserFlag string) (s
 	return parser, nil
 }
 
-func getWriteParser(readParser storage.ReadParser, writeParserFlag string, parserFlag string, outFlag string, fileFlag string) (storage.WriteParser, error) {
+func getWriteParser(readParser storage.ReadParser, writeParserFlag string, parserFlag string,
+	outFlag string, fileFlag string, formatTemplateFlag string) (storage.WriteParser, error) {
+	if formatTemplateFlag != "" {
+		writeParserFlag = "plain"
+	}
+
 	if writeParserFlag == "" {
 		writeParserFlag = parserFlag
 	}
@@ -136,11 +141,12 @@ func getRootNode(opts getRootNodeOpts, cmd *cobra.Command) (*dasel.Node, error)
 }
 
 type writeNodeToOutputOpts struct {
-	Node   *dasel.Node
-	Parser storage.WriteParser
-	File   string
-	Out    string
-	Writer io.Writer
+	Node           *dasel.Node
+	Parser         storage.WriteParser
+	File           string
+	Out            string
+	Writer         io.Writer
+	FormatTemplate string
 }
 
 type customErrorHandlingOpts struct {
@@ -195,7 +201,20 @@ func writeNodeToOutput(opts writeNodeToOutputOpts, cmd *cobra.Command, options .
 	opts.Writer = writer
 	defer writerCleanUp()
 
-	if err := storage.Write(opts.Parser, opts.Node.InterfaceValue(), opts.Node.OriginalValue, opts.Writer, options...); err != nil {
+	var value, originalValue interface{}
+	if opts.FormatTemplate == "" {
+		value = opts.Node.InterfaceValue()
+		originalValue = opts.Node.OriginalValue
+	} else {
+		result, err := dasel.FormatNode(opts.Node, opts.FormatTemplate)
+		if err != nil {
+			return fmt.Errorf("could not format node: %w", err)
+		}
+		value = result.String()
+		originalValue = value
+	}
+
+	if err := storage.Write(opts.Parser, value, originalValue, opts.Writer, options...); err != nil {
 		return fmt.Errorf("could not write to output file: %w", err)
 	}
 
@@ -203,11 +222,12 @@ func writeNodeToOutput(opts writeNodeToOutputOpts, cmd *cobra.Command, options .
 }
 
 type writeNodesToOutputOpts struct {
-	Nodes  []*dasel.Node
-	Parser storage.WriteParser
-	File   string
-	Out    string
-	Writer io.Writer
+	Nodes          []*dasel.Node
+	Parser         storage.WriteParser
+	File           string
+	Out            string
+	Writer         io.Writer
+	FormatTemplate string
 }
 
 func writeNodesToOutput(opts writeNodesToOutputOpts, cmd *cobra.Command, options ...storage.ReadWriteOption) error {
@@ -222,9 +242,10 @@ func writeNodesToOutput(opts writeNodesToOutputOpts, cmd *cobra.Command, options
 
 	for i, n := range opts.Nodes {
 		subOpts := writeNodeToOutputOpts{
-			Node:   n,
-			Parser: opts.Parser,
-			Writer: buf,
+			Node:           n,
+			Parser:         opts.Parser,
+			Writer:         buf,
+			FormatTemplate: opts.FormatTemplate,
 		}
 		if err := writeNodeToOutput(subOpts, cmd, options...); err != nil {
 			return fmt.Errorf("could not write node %d to output: %w", i, err)
@@ -375,7 +396,7 @@ func runGenericPutCommand(opts genericPutOptions, cmd *cobra.Command) error {
 		}
 	}
 
-	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File)
+	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File, "")
 	if err != nil {
 		return err
 	}

internal/command/put_document_internal.go

@@ -58,7 +58,7 @@ func runPutDocumentCommand(opts putDocumentOpts, cmd *cobra.Command) error {
 		}
 	}
 
-	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File)
+	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File, "")
 	if err != nil {
 		return err
 	}

internal/command/put_object_internal.go

@@ -76,7 +76,7 @@ func runPutObjectCommand(opts putObjectOpts, cmd *cobra.Command) error {
 		}
 	}
 
-	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File)
+	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, opts.Out, opts.File, "")
 	if err != nil {
 		return err
 	}

internal/command/root_select_test.go

@@ -649,3 +649,48 @@ ESTAT,Eurostat
 ILO,International Labor Organization
 `, nil, "-w", "csv"))
 }
+
+func TestRootCmd_Select_JSON_Format(t *testing.T) {
+	t.Run("RootElementFormattedToProperty", selectTest(jsonData, "json", ".", newline(`1111`), nil,
+		"--format", `{{ query ".id" }}`))
+	t.Run("SelectorFormatted", selectTest(jsonData, "json", ".id", newline(`1111`), nil,
+		"--format", `{{ . }}`))
+	t.Run("SelectorFormattedMultiple", selectTest(jsonData, "json", ".details.addresses.[*]",
+		newline(`101 Some Street
+34 Another Street`), nil,
+		"-m", "--format", `{{ query ".street" }}`))
+	t.Run("SelectorFormattedToMultiple", selectTest(jsonData, "json", ".",
+		newline(`101 Some Street
+34 Another Street`), nil,
+		"-m", "--format", `{{ queryMultiple ".details.addresses.[*]" | format "{{ .street }}{{ if not isLast }}{{ newline }}{{end}}" }}`))
+
+	// https://github.com/TomWright/dasel/discussions/146
+	t.Run("Discussion146", selectTest(
+		`[{"name": "click", "version": "7.1.2", "latest_version": "8.0.1", "latest_filetype": "wheel"}, {"name": "decorator", "version": "4.4.2", "latest_version": "5.0.9", "latest_filetype": "wheel"}, {"name": "ipython", "version": "7.20.0", "latest_version": "7.25.0", "latest_filetype": "wheel"}, {"name": "pandas", "version": "1.3.0", "latest_version": "1.3.1", "latest_filetype": "wheel"}, {"name": "parso", "version": "0.8.1", "latest_version": "0.8.2", "latest_filetype": "wheel"}, {"name": "pip", "version": "21.1.3", "latest_version": "21.2.1", "latest_filetype": "wheel"}, {"name": "prompt-toolkit", "version": "3.0.14", "latest_version": "3.0.19", "latest_filetype": "wheel"}, {"name": "Pygments", "version": "2.7.4", "latest_version": "2.9.0", "latest_filetype": "wheel"}, {"name": "setuptools", "version": "49.2.1", "latest_version": "57.4.0", "latest_filetype": "wheel"}, {"name": "tomli", "version": "1.0.4", "latest_version": "1.1.0", "latest_filetype": "wheel"}]`,
+		"json", ".(name!=setuptools)(name!=six)(name!=pip)(name!=pip-tools)",
+		newline(`click
+7.1.2
+8.0.1
+decorator
+4.4.2
+5.0.9
+ipython
+7.20.0
+7.25.0
+pandas
+1.3.0
+1.3.1
+parso
+0.8.1
+0.8.2
+prompt-toolkit
+3.0.14
+3.0.19
+Pygments
+2.7.4
+2.9.0
+tomli
+1.0.4
+1.1.0`), nil,
+		"-m", "--format", `{{ query ".name" }}{{ newline }}{{ query ".version" }}{{ newline }}{{ query ".latest_version" }}`))
+}

internal/command/select.go

@@ -22,6 +22,7 @@ type selectOptions struct {
 	Compact             bool
 	DisplayLength       bool
 	MergeInputDocuments bool
+	FormatTemplate      string
 }
 
 func outputNodeLength(writer io.Writer, nodes ...*dasel.Node) error {
@@ -72,9 +73,10 @@ func runSelectMultiCommand(cmd *cobra.Command, rootNode *dasel.Node, opts select
 	}
 
 	if err := writeNodesToOutput(writeNodesToOutputOpts{
-		Nodes:  results,
-		Parser: writeParser,
-		Writer: opts.Writer,
+		Nodes:          results,
+		Parser:         writeParser,
+		Writer:         opts.Writer,
+		FormatTemplate: opts.FormatTemplate,
 	}, cmd, writeOptions...); err != nil {
 		return fmt.Errorf("could not write output: %w", err)
 	}
@@ -106,7 +108,7 @@ func runSelectCommand(opts selectOptions, cmd *cobra.Command) error {
 		opts.Writer = cmd.OutOrStdout()
 	}
 
-	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, "-", opts.File)
+	writeParser, err := getWriteParser(readParser, opts.WriteParser, opts.Parser, "-", opts.File, opts.FormatTemplate)
 	if err != nil {
 		return err
 	}
@@ -151,9 +153,10 @@ func runSelectCommand(opts selectOptions, cmd *cobra.Command) error {
 	}
 
 	if err := writeNodeToOutput(writeNodeToOutputOpts{
-		Node:   res,
-		Parser: writeParser,
-		Writer: opts.Writer,
+		Node:           res,
+		Parser:         writeParser,
+		Writer:         opts.Writer,
+		FormatTemplate: opts.FormatTemplate,
 	}, cmd, writeOptions...); err != nil {
 		return fmt.Errorf("could not write output: %w", err)
 	}
@@ -162,7 +165,7 @@ func runSelectCommand(opts selectOptions, cmd *cobra.Command) error {
 }
 
 func selectCommand() *cobra.Command {
-	var fileFlag, selectorFlag, parserFlag, readParserFlag, writeParserFlag string
+	var fileFlag, selectorFlag, parserFlag, readParserFlag, writeParserFlag, formatTemplateFlag string
 	var plainFlag, multiFlag, nullValueNotFoundFlag, compactFlag, lengthFlag, mergeInputDocumentsFlag bool
 
 	cmd := &cobra.Command{
@@ -187,6 +190,7 @@ func selectCommand() *cobra.Command {
 				Compact:             compactFlag,
 				DisplayLength:       lengthFlag,
 				MergeInputDocuments: mergeInputDocumentsFlag,
+				FormatTemplate:      formatTemplateFlag,
 			}, cmd)
 		},
 	}
@@ -202,6 +206,7 @@ func selectCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&lengthFlag, "length", false, "Output the length of the selected value.")
 	cmd.Flags().BoolVar(&mergeInputDocumentsFlag, "merge-input-documents", false, "Merge multiple input documents into an array.")
 	cmd.Flags().BoolVarP(&compactFlag, "compact", "c", false, "Compact the output by removing all pretty-printing where possible.")
+	cmd.Flags().StringVar(&formatTemplateFlag, "format", "", "Formatting template to use when writing results.")
 
 	_ = cmd.MarkFlagFilename("file")
 

node.go

@@ -60,6 +60,12 @@ type Node struct {
 	wasInitialised bool
 }
 
+// String returns the value of the node as a string.
+// No formatting is done here, you get the raw value.
+func (n *Node) String() string {
+	return fmt.Sprint(n.InterfaceValue())
+}
+
 // InterfaceValue returns the value stored within the node as an interface{}.
 func (n *Node) InterfaceValue() interface{} {
 	// We shouldn't be able to get here but this will stop a panic if we do.

output_formatter.go

@@ -0,0 +1,129 @@
+package dasel
+
+import (
+	"bytes"
+	"text/template"
+)
+
+// FormatNode formats a node with the format template and returns the result.
+func FormatNode(node *Node, format string) (*bytes.Buffer, error) {
+	tpl, err := formatNodeTemplate(
+		&templateNode{
+			Node:    node,
+			isFirst: true,
+			isLast:  true,
+		},
+	).Parse(format)
+	if err != nil {
+		return nil, err
+	}
+
+	buf := new(bytes.Buffer)
+
+	err = tpl.Execute(buf, node.InterfaceValue())
+	return buf, err
+}
+
+type templateNode struct {
+	*Node
+
+	isFirst bool
+	isLast  bool
+}
+
+// FormatNodes formats a slice of nodes with the format template and returns the result.
+func FormatNodes(nodes []*Node, format string) (*bytes.Buffer, error) {
+	buf := new(bytes.Buffer)
+	nodesLen := len(nodes)
+	for k, node := range nodes {
+		tpl, err := formatNodeTemplate(
+			&templateNode{
+				Node:    node,
+				isFirst: k == 0,
+				isLast:  k == (nodesLen - 1),
+			},
+		).Parse(format)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := tpl.Execute(buf, node.InterfaceValue()); err != nil {
+			return nil, err
+		}
+	}
+	return buf, nil
+}
+
+type formatTemplateFuncs struct {
+	node *templateNode
+}
+
+func (funcs *formatTemplateFuncs) funcMap() template.FuncMap {
+	return template.FuncMap{
+		"query":          funcs.query,
+		"queryMultiple":  funcs.queryMultiple,
+		"select":         funcs.query,
+		"selectMultiple": funcs.queryMultiple,
+		"format":         funcs.format,
+		"isFirst":        funcs.isFirst,
+		"isLast":         funcs.isLast,
+		"newline":        funcs.newline,
+	}
+}
+
+func (funcs *formatTemplateFuncs) newline() string {
+	return "\n"
+}
+
+func (funcs *formatTemplateFuncs) isFirst() bool {
+	return funcs.node.isFirst
+}
+
+func (funcs *formatTemplateFuncs) isLast() bool {
+	return funcs.node.isLast
+}
+
+func (funcs *formatTemplateFuncs) query(selector string) *Node {
+	res, err := funcs.node.Query(selector)
+	if err != nil {
+		return nil
+	}
+	return res
+}
+
+func (funcs *formatTemplateFuncs) queryMultiple(selector string) []*Node {
+	res, err := funcs.node.QueryMultiple(selector)
+	if err != nil {
+		return nil
+	}
+	return res
+}
+
+func (funcs *formatTemplateFuncs) format(format string, target interface{}) string {
+	switch t := target.(type) {
+	case []*Node:
+		buf, err := FormatNodes(t, format)
+		if err != nil {
+			return err.Error()
+		}
+		res := buf.String()
+		return res
+	case *Node:
+		buf, err := FormatNode(t, format)
+		if err != nil {
+			return err.Error()
+		}
+		return buf.String()
+	}
+
+	return "<nil>"
+}
+
+func formatNodeTemplate(node *templateNode) *template.Template {
+	funcs := &formatTemplateFuncs{
+		node: node,
+	}
+	tpl := template.New("nodeFormat")
+	tpl.Funcs(funcs.funcMap())
+	return tpl
+}

output_formatter_test.go

@@ -0,0 +1,228 @@
+package dasel_test
+
+import (
+	"github.com/tomwright/dasel"
+	"testing"
+)
+
+func testFormatNode(value interface{}, format string, exp string) func(t *testing.T) {
+	return func(t *testing.T) {
+		node := dasel.New(value)
+		buf, err := dasel.FormatNode(node, format)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		got := buf.String()
+		if exp != got {
+			t.Errorf("expected %s, got %s", exp, got)
+		}
+	}
+}
+
+func testFormatNodes(values []interface{}, format string, exp string) func(t *testing.T) {
+	return func(t *testing.T) {
+		nodes := make([]*dasel.Node, len(values))
+		for k, v := range values {
+			nodes[k] = dasel.New(v)
+		}
+		buf, err := dasel.FormatNodes(nodes, format)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		got := buf.String()
+		if exp != got {
+			t.Errorf("expected %s, got %s", exp, got)
+		}
+	}
+}
+
+func TestFormatNode(t *testing.T) {
+	t.Run("InvalidFormatTemplate", func(t *testing.T) {
+		_, err := dasel.FormatNode(nil, "{{")
+		if err == nil {
+			t.Errorf("expected error but got none")
+		}
+	})
+	t.Run("PropertyAccess", testFormatNode(
+		map[string]interface{}{
+			"name":  "Tom",
+			"email": "contact@tomwright.me",
+		},
+		`{{ .name }}, {{ .email }}`,
+		`Tom, contact@tomwright.me`,
+	))
+	t.Run("QueryAccess", testFormatNode(
+		map[string]interface{}{
+			"name":  "Tom",
+			"email": "contact@tomwright.me",
+		},
+		`{{ query ".name" }}, {{ query ".email" }}`,
+		`Tom, contact@tomwright.me`,
+	))
+	t.Run("SelectAccess", testFormatNode(
+		map[string]interface{}{
+			"name":  "Tom",
+			"email": "contact@tomwright.me",
+		},
+		`{{ select ".name" }}, {{ select ".email" }}`,
+		`Tom, contact@tomwright.me`,
+	))
+	t.Run("Format", testFormatNode(
+		map[string]interface{}{
+			"name":  "Tom",
+			"email": "contact@tomwright.me",
+		},
+		`{{ query ".name" | format "{{ . }}" }}, {{ query ".email" | format "{{ . }}" }}`,
+		`Tom, contact@tomwright.me`,
+	))
+	t.Run("QueryAccessInvalidSelector", testFormatNode(
+		map[string]interface{}{
+			"name":  "Tom",
+			"email": "contact@tomwright.me",
+		},
+		`{{ query ".bad" }}`,
+		`<nil>`,
+	))
+	t.Run("QueryMultipleCommaSeparated", testFormatNode(
+		map[string]interface{}{
+			"users": []map[string]interface{}{
+				{
+					"name": "Tom",
+				},
+				{
+					"name": "Jim",
+				},
+				{
+					"name": "Frank",
+				},
+			},
+		},
+		`{{ queryMultiple ".users.[*]" | format "{{ .name }}{{ if not isLast }},{{ end }}" }}`,
+		`Tom,Jim,Frank`,
+	))
+	t.Run("QueryMultipleLineSeparated", testFormatNode(
+		map[string]interface{}{
+			"users": []map[string]interface{}{
+				{
+					"name": "Tom",
+				},
+				{
+					"name": "Jim",
+				},
+				{
+					"name": "Frank",
+				},
+			},
+		},
+		`{{ queryMultiple ".users.[*]" | format "{{ .name }}{{ if not isLast }}{{ newline }}{{ end }}" }}`,
+		`Tom
+Jim
+Frank`,
+	))
+	t.Run("QueryMultipleDashSeparated", testFormatNode(
+		map[string]interface{}{
+			"users": []map[string]interface{}{
+				{
+					"name": "Tom",
+				},
+				{
+					"name": "Jim",
+				},
+				{
+					"name": "Frank",
+				},
+			},
+		},
+		`{{ queryMultiple ".users.[*]" | format "{{ if not isFirst }}---{{ newline }}{{ end }}{{ .name }}{{ if not isLast }}{{ newline }}{{ end }}" }}`,
+		`Tom
+---
+Jim
+---
+Frank`,
+	))
+	t.Run("QueryMultipleBadSelector", testFormatNode(
+		map[string]interface{}{
+			"users": []map[string]interface{}{
+				{
+					"name": "Tom",
+				},
+				{
+					"name": "Jim",
+				},
+				{
+					"name": "Frank",
+				},
+			},
+		},
+		`{{ queryMultiple ".users.[*].names" | format "{{ . }}{{ if not isLast }}{{ newline }}{{ end }}" }}`,
+		``,
+	))
+}
+
+func TestFormatNodes(t *testing.T) {
+	t.Run("InvalidFormatTemplate", func(t *testing.T) {
+		_, err := dasel.FormatNodes([]*dasel.Node{dasel.New("")}, "{{")
+		if err == nil {
+			t.Errorf("expected error but got none")
+		}
+	})
+	t.Run("PropertyAccess", testFormatNodes(
+		[]interface{}{
+			map[string]interface{}{
+				"name":  "Tom",
+				"email": "contact@tomwright.me",
+			},
+			map[string]interface{}{
+				"name":  "Jim",
+				"email": "jim@gmail.com",
+			},
+		},
+		"{{ .name }}, {{ .email }}{{ if not isLast }}{{ newline }}{{ end }}",
+		`Tom, contact@tomwright.me
+Jim, jim@gmail.com`,
+	))
+	t.Run("QueryAccess", testFormatNodes(
+		[]interface{}{
+			map[string]interface{}{
+				"name":  "Tom",
+				"email": "contact@tomwright.me",
+			},
+			map[string]interface{}{
+				"name":  "Jim",
+				"email": "jim@gmail.com",
+			},
+		},
+		`{{ query ".name" }}, {{ query ".email" }}{{ if not isLast }}{{ newline }}{{ end }}`,
+		`Tom, contact@tomwright.me
+Jim, jim@gmail.com`))
+	t.Run("SelectAccess", testFormatNodes(
+		[]interface{}{
+			map[string]interface{}{
+				"name":  "Tom",
+				"email": "contact@tomwright.me",
+			},
+			map[string]interface{}{
+				"name":  "Jim",
+				"email": "jim@gmail.com",
+			},
+		},
+		`{{ select ".name" }}, {{ select ".email" }}{{ if not isLast }}{{ newline }}{{ end }}`,
+		`Tom, contact@tomwright.me
+Jim, jim@gmail.com`))
+	t.Run("QueryAccessInvalidSelector", testFormatNodes(
+		[]interface{}{
+			map[string]interface{}{
+				"name":  "Tom",
+				"email": "contact@tomwright.me",
+			},
+			map[string]interface{}{
+				"name":  "Jim",
+				"email": "jim@gmail.com",
+			},
+		},
+		`{{ query ".bad" }}{{ newline }}`,
+		`<nil>
+<nil>
+`,
+	))
+}