Merge pull request #165 from brenol/issue-164

xml: add support for other charsets

.github/workflows/build-test.yaml

@@ -3,6 +3,10 @@ on:
     branches-ignore:
       - master
       - main
+  pull_request:
+    branches:
+      - master
+      - main
 name: Build Test
 jobs:
   publish:

go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/clbanning/mxj/v2 v2.3.3-0.20201214204241-e937bdee5a3e
 	github.com/pelletier/go-toml v1.9.3
 	github.com/spf13/cobra v1.2.1
+	golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167
+	golang.org/x/text v0.3.7
 	gopkg.in/yaml.v2 v2.4.0
 )
 

go.sum

@@ -336,6 +336,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167 h1:eDd+TJqbgfXruGQ5sJRU7tEtp/58OAx4+Ayjxg4SM+4=
+golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -401,6 +403,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -411,6 +414,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

internal/storage/xml.go

@@ -3,14 +3,19 @@ package storage
 import (
 	"bytes"
 	"fmt"
-	"github.com/clbanning/mxj/v2"
 	"strings"
+
+	"github.com/clbanning/mxj/v2"
+	"golang.org/x/net/html/charset"
 )
 
 func init() {
 	// Required for https://github.com/TomWright/dasel/issues/61
 	mxj.XMLEscapeCharsDecoder(true)
 
+	// Required for https://github.com/TomWright/dasel/issues/164
+	mxj.XmlCharsetReader = charset.NewReaderLabel
+
 	registerReadParser([]string{"xml"}, []string{".xml"}, &XMLParser{})
 	registerWriteParser([]string{"xml"}, []string{".xml"}, &XMLParser{})
 }

internal/storage/xml_test.go

@@ -1,9 +1,15 @@
 package storage_test
 
 import (
-	"github.com/tomwright/dasel/internal/storage"
+	"bytes"
+	"fmt"
+	"io"
 	"reflect"
 	"testing"
+
+	"github.com/tomwright/dasel/internal/storage"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/unicode"
 )
 
 var xmlBytes = []byte(`<user>
@@ -15,6 +21,11 @@ var xmlMap = map[string]interface{}{
 		"name": "Tom",
 	},
 }
+var encodedXmlMap = map[string]interface{}{
+	"user": map[string]interface{}{
+		"name": "Tõm",
+	},
+}
 
 func TestXMLParser_FromBytes(t *testing.T) {
 	got, err := (&storage.XMLParser{}).FromBytes(xmlBytes)
@@ -177,3 +188,56 @@ func TestXMLParser_ToBytes_Entities(t *testing.T) {
 		}
 	})
 }
+
+func TestXMLParser_DifferentEncodings(t *testing.T) {
+	newXmlBytes := func(newWriter func(io.Writer) io.Writer, encoding, text string) []byte {
+		const encodedXmlBytesFmt = `<?xml version='1.0' encoding='%s'?>`
+		const xmlBody = `<user><name>%s</name></user>`
+
+		var buf bytes.Buffer
+
+		w := newWriter(&buf)
+		fmt.Fprintf(w, xmlBody, text)
+
+		return []byte(fmt.Sprintf(encodedXmlBytesFmt, encoding) + buf.String())
+	}
+
+	testCases := []struct {
+		name string
+		xml  []byte
+	}{
+		{
+			name: "supports ISO-8859-1",
+			xml:  newXmlBytes(charmap.ISO8859_1.NewEncoder().Writer, "ISO-8859-1", "Tõm"),
+		},
+		{
+			name: "supports UTF-8",
+			xml:  newXmlBytes(unicode.UTF8.NewEncoder().Writer, "UTF-8", "Tõm"),
+		},
+		{
+			name: "supports latin1",
+			xml:  newXmlBytes(charmap.Windows1252.NewEncoder().Writer, "latin1", "Tõm"),
+		},
+		{
+			name: "supports UTF-16",
+			xml:  newXmlBytes(unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Writer, "UTF-16", "Tõm"),
+		},
+		{
+			name: "supports UTF-16 (big endian)",
+			xml:  newXmlBytes(unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder().Writer, "UTF-16BE", "Tõm"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := (&storage.XMLParser{}).FromBytes(tc.xml)
+			if err != nil {
+				t.Errorf("unexpected error: %s", err)
+				return
+			}
+			if !reflect.DeepEqual(&storage.BasicSingleDocument{Value: encodedXmlMap}, got) {
+				t.Errorf("expected %v, got %v", encodedXmlMap, got)
+			}
+		})
+	}
+}