doing experiments with a server side decryption utility, still very WIP

arc/js/aes.js

@@ -54,13 +54,33 @@ function checkPrerequisites() {
     return errors;
 }
 
+function buf2hex(buf) {
+    var hexStr = '';
+    for (var i = 0; i < buf.length; i++) {
+        var hex = '';
+        if( typeof(buf) == 'string' ) {
+            hex = ( buf[i].charCodeAt(i) & 0xff ).toString(16); 
+        } else {
+            hex = (buf[i] & 0xff).toString(16);
+        }
+
+        hex = (hex.length === 1) ? '0' + hex : hex;
+        hexStr += hex;
+    }
+
+    return hexStr.toUpperCase();
+}
+
 function merge(salt, iv, ciphertext) {
+    console.log( "SALT: " + buf2hex(salt) );
     var buff = new Uint8Array( PBKDF_SALT_SIZE + AES_IV_SIZE + ciphertext.length );
 
     buff.set( salt );
     buff.set( iv, PBKDF_SALT_SIZE );
     buff.set( ciphertext, PBKDF_SALT_SIZE + AES_IV_SIZE );
 
+    console.log( buf2hex(buff) );
+
     return buf2a(buff);
 }
 
@@ -128,6 +148,10 @@ function encrypt(message, passphrase) {
 function decrypt(data, passphrase) {
     const [ salt, iv, ciphertext ] = unmerge(data);
 
+    console.log( "SALT   : " + buf2hex(salt));
+    console.log( "IV     : " + buf2hex(iv));
+    console.log( "CIPHER : " + buf2hex(ciphertext));
+
     var doDeriveKey = PBKDF2( passphrase, salt );
 
     return doDeriveKey.then( derivedKey  => 

arc/js/api.js

@@ -38,6 +38,7 @@ Arc.prototype.Api = function( method, path, data, success, error, raw ) {
         contentType: "application/json",
         dataType: raw ? undefined : 'json',
         cache: false,
+        processData: false,
         timeout: 60 * 60 * 1000
     });
 }

arc/js/app.js

@@ -677,6 +677,9 @@ app.controller('PMController', ['$scope', function (scope) {
             scope.progressAt = new Date();
             scope.uploading = false;
             scope.arc.GetRecordBuffer( secret.id, function(data){
+                console.log( "DATA.TYP = " + typeof(data) );
+                console.log( "DATA.LEN = " + data.length );
+                console.log( buf2hex(a2buf(data)) );
                 // start decrypting data when message is updated
                 scope.showLoader( "Decrypting data ...", function() {
                     var record = new Record(secret.title);

arcd/Makefile

@@ -1,7 +1,5 @@
 .PHONY: build fmt lint run test vet deps install
 
-SRC_PATH=.
-TARGET=arcd
 PREFIX_DIR=/usr/local
 BIN_DIR=$(PREFIX_DIR)/bin
 CONFIG_DIR=$(PREFIX_DIR)/etc
@@ -11,23 +9,26 @@ SERVICE_LN_DIR=/etc/systemd/system
 
 default: build
 
-build: deps fmt vet lint 
-	@go build $(FLAGS) -o $(TARGET) $(SRC_PATH)
+build: deps fmt lint build_arcd build_arc_decrypt
 
-vet:
-	@go vet $(SRC_PATH)
+
+build_arcd:
+	@go build $(FLAGS) -o arcd ./cmd/arcd/
+
+build_arc_decrypt:
+	@go build $(FLAGS) -o arc_decrypt ./cmd/arc_decrypt/
 
 fmt:
-	@go fmt $(SRC_PATH)/...
+	@go fmt ./...
 
 lint:
-	@golint $(SRC_PATH)
+	@golint .
 
 test:
-	@go test $(SRC_PATH)/...
+	@go test ./...
 
 clean:
-	@rm -rf $(TARGET)
+	@rm -rf arcd arc_decrypt
 
 deps:
 	@go get github.com/gin-gonic/gin
@@ -40,11 +41,13 @@ deps:
 
 # runs on previlege
 install: build
-	@echo "Installing $(TARGET) in $(PREFIX_DIR)"
-	@install -D -m 744 (SRC_PATH)/$(TARGET) $(BIN_DIR)/$(TARGET)
-	@setcap 'cap_net_bind_service=+ep' $(BIN_DIR)/$(TARGET)
+	@echo "Installing arcd in $(PREFIX_DIR)" 
+	@install -D -m 744 (SRC_PATH)/arcd $(BIN_DIR)/arcd
+	@setcap 'cap_net_bind_service=+ep' $(BIN_DIR)/arcd
+	@echo "Installing arc_decrypt in $(PREFIX_DIR)" 
+	@install -D -m 744 (SRC_PATH)/arc_decrypt $(BIN_DIR)/arc_decrypt
 	@cp -r ../arc $(WEBAPP_DIR)/arc
-	@install -D -m 644 $(SRC_PATH)/sample_config.json $(CONFIG_DIR)/$(TARGET)/config.json
-	@install -D -m 644 $(SRC_PATH)/arcd@.service $(SERVICE_DIR)/arcd@.service
+	@install -D -m 644 ./sample_config.json $(CONFIG_DIR)/arcd/config.json
+	@install -D -m 644 ./arcd@.service $(SERVICE_DIR)/arcd@.service
 	@ln -s $(SERVICE_DIR)/arcd@.service $(SERVICE_LN_DIR)/arcd@.service || echo "symlink already exists...skipping"
 	@echo "Done."

arcd/cmd/arc_decrypt/main.go

@@ -0,0 +1,179 @@
+/*
+ * Arc - Copyleft of Simone 'evilsocket' Margaritelli.
+ * evilsocket at protonmail dot com
+ * https://www.evilsocket.net/
+ *
+ * See LICENSE.
+ */
+package main
+
+import (
+	"compress/gzip"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"github.com/evilsocket/arc/arcd/db"
+	"github.com/evilsocket/arc/arcd/log"
+	"github.com/evilsocket/arc/arcd/utils"
+	"golang.org/x/crypto/pbkdf2"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"unicode/utf8"
+)
+
+var (
+	basePath        = ""
+	key             = ""
+	authMessage     = "Thanks to JP Aumasson > https://twitter.com/veorq/status/943506635317825536/////////////////////////////////////////////////////"
+	saltSize        = int64(16)
+	ivSize          = int64(16)
+	pbkdfIterations = int(10000)
+)
+
+func init() {
+	flag.StringVar(&basePath, "record", "", "Path containing the data and meta.json files of the basePath to decrypt.")
+	flag.StringVar(&key, "key", "", "Decryption key.")
+
+	flag.Int64Var(&saltSize, "salt-size", saltSize, "Salt size.")
+	flag.Int64Var(&ivSize, "iv-size", ivSize, "IV size.")
+	flag.IntVar(&pbkdfIterations, "iterations", pbkdfIterations, "PBKDF2 iterations.")
+}
+
+func main() {
+	var err error
+	flag.Parse()
+
+	basePath, err := filepath.Abs(basePath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	metaFile := path.Join(basePath, "meta.json")
+	dataFile := path.Join(basePath, "data")
+
+	if utils.Exists(metaFile) == false {
+		log.Fatal(fmt.Errorf("File %s not found.", metaFile))
+	} else if utils.Exists(dataFile) == false {
+		log.Fatal(fmt.Errorf("File %s not found.", dataFile))
+	}
+
+	log.Infof("Decrypting record %s ...", log.Bold(basePath))
+
+	meta, err := db.OpenMeta(metaFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if meta.Encryption != "aes" {
+		log.Fatal(fmt.Errorf("This tool only supports AES256 encrypted records."))
+	}
+
+	log.Infof("")
+
+	log.Infof("(%d) %s", meta.Id, log.Bold(meta.Title))
+	log.Infof("Created: %s", meta.CreatedAt)
+	if meta.Compressed {
+		log.Infof("Compression: %s", log.Bold("on"))
+	} else {
+		log.Infof("Compression: %s", "off")
+	}
+	log.Infof("Size: %s ( %d B )", utils.FormatBytes(meta.Size), meta.Size)
+	log.Infof("")
+
+	if meta.Compressed {
+		tmpFile := "/tmp/data.tmp"
+		log.Infof("Decompressing %s to %s ...", log.Bold(dataFile), tmpFile)
+
+		in, err := os.Open(dataFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer in.Close()
+
+		reader, err := gzip.NewReader(in)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		out, err := os.Create(tmpFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer out.Close()
+
+		if written, err := io.Copy(out, reader); err != nil {
+			log.Fatal(err)
+		} else {
+			log.Infof("Extracted file is %s ( %d B ).", utils.FormatBytes(uint64(written)), written)
+		}
+
+		dataFile = tmpFile
+	}
+
+	err, fileSize := utils.FileSize(dataFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Infof("Decrypting %s ...", log.Bold(dataFile))
+	in, err := os.Open(dataFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer in.Close()
+
+	file := make([]byte, fileSize)
+	read, err := io.ReadFull(in, file[:])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	/*
+	 * Needed for converting javascript binary strings to golang strings.
+	 */
+	buffer := make([]rune, 0)
+	sfile := string(file)
+	for len(sfile) > 0 {
+		r, size := utf8.DecodeRuneInString(sfile)
+		buffer = append(buffer, r)
+		sfile = sfile[size:]
+		l := len(sfile)
+
+		if l%1000 == 0 {
+			log.Infof("size: %d", l)
+		}
+	}
+	file = []byte(string(buffer))
+
+	log.Infof("Read %d bytes of file.", read)
+	log.Infof("runes %d", utf8.RuneCountInString(string(file)))
+
+	salt := file[0:saltSize]
+	iv := file[saltSize : saltSize+ivSize]
+	cipherText := file[saltSize+ivSize:]
+
+	cipherSize := fileSize - ivSize - saltSize
+	derivedKey := pbkdf2.Key([]byte(key), salt, pbkdfIterations, 32, sha256.New)
+	authData := []byte(authMessage)
+
+	log.Infof("")
+	log.Infof("SALT : %X", salt)
+	log.Infof("IV   : %X", iv)
+	log.Infof("KEY  : %X", derivedKey)
+	log.Infof("")
+	log.Infof("Decrypting ciphertext of %s ( %d B ) ...", utils.FormatBytes(uint64(cipherSize)), cipherSize)
+
+	block, _ := aes.NewCipher(derivedKey)
+	gcm, _ := cipher.NewGCMWithNonceSize(block, int(ivSize))
+
+	plain, err := gcm.Open(nil, iv, cipherText, authData)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("\n%s\n", string(plain))
+}

arcd/controllers/record.go

@@ -99,7 +99,6 @@ func GetRecordBuffer(c *gin.Context) {
 		// Let the client handle the decompression :P
 		if meta.Compressed {
 			c.Writer.Header().Set("Content-Encoding", "gzip")
-			c.Writer.Header().Set("Content-Type", "application/octet-stream")
 			c.Writer.Header().Set("Vary", "Accept-Encoding")
 		}
 

arcd/main.go

@@ -1,320 +0,0 @@
-/*
- * Arc - Copyleft of Simone 'evilsocket' Margaritelli.
- * evilsocket at protonmail dot com
- * https://www.evilsocket.net/
- *
- * See LICENSE.
- */
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"os/signal"
-	"path"
-	"regexp"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/evilsocket/arc/arcd/app"
-	"github.com/evilsocket/arc/arcd/config"
-	"github.com/evilsocket/arc/arcd/controllers"
-	"github.com/evilsocket/arc/arcd/db"
-	"github.com/evilsocket/arc/arcd/events"
-	"github.com/evilsocket/arc/arcd/log"
-	"github.com/evilsocket/arc/arcd/middlewares"
-	"github.com/evilsocket/arc/arcd/tls"
-	"github.com/evilsocket/arc/arcd/utils"
-
-	"github.com/gin-gonic/gin"
-)
-
-var (
-	signals    = make(chan os.Signal, 1)
-	appPath    = ""
-	confFile   = ""
-	debug      = false
-	logfile    = ""
-	noColors   = false
-	noAuth     = false
-	noUpdates  = false
-	export     = false
-	importFrom = ""
-	output     = "arc.tar"
-	dbIsNew    = false
-)
-
-func init() {
-	flag.StringVar(&appPath, "app", ".", "Path of the web application to serve.")
-	flag.StringVar(&confFile, "config", "", "JSON configuration file.")
-	flag.BoolVar(&noAuth, "no-auth", noAuth, "Disable authentication.")
-	flag.BoolVar(&noUpdates, "no-updates", noUpdates, "Disable updates check.")
-
-	flag.BoolVar(&debug, "log-debug", debug, "Enable debug logs.")
-	flag.StringVar(&logfile, "log-file", logfile, "Log messages to this file instead of standard error.")
-	flag.BoolVar(&noColors, "log-colors-off", noColors, "Disable colored output.")
-
-	flag.StringVar(&importFrom, "import", importFrom, "Import stores from this TAR export file.")
-	flag.BoolVar(&export, "export", export, "Export store to a TAR archive, requires --output parameter.")
-	flag.StringVar(&output, "output", output, "Export file name.")
-}
-
-func arcLoadApp(r *gin.Engine) *app.App {
-	err, webapp := app.Open(appPath)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	r.Use(middlewares.ServeStatic("/", webapp.Path, webapp.Manifest.Index))
-
-	return webapp
-}
-
-func arcBackupper() {
-	period := time.Duration(config.Conf.Backups.Period) * time.Second
-	filename := path.Join(config.Conf.Backups.Folder, "arc-backup.tar")
-
-	log.Debugf("Backup task started with a %v period to %s", period, filename)
-	for {
-
-		started := time.Now()
-		log.Infof("Backupping database to %s ...", filename)
-		if err := db.Export(filename); err != nil {
-			log.Errorf("Error while creating the backup file: %s.", err)
-		} else {
-			log.Infof("Backupped %s of data to %s in %s.", utils.FormatBytes(db.Size), log.Bold(filename), time.Since(started))
-		}
-
-		time.Sleep(period)
-	}
-}
-
-func arcScheduler() {
-	period := time.Duration(config.Conf.Scheduler.Period) * time.Second
-
-	log.Debugf("Scheduler started with a %v period.", period)
-
-	for {
-		time.Sleep(period)
-
-		db.Lock()
-
-		for _, store := range db.GetStores() {
-			for _, r := range store.Children() {
-				meta := r.Meta()
-				if r.Expired() {
-					if r.WasNotified() == false {
-						events.Add(events.RecordExpired(r))
-						r.SetNotified(true)
-					}
-
-					if meta.Prune {
-						log.Infof("Pruning record %d ( %s ) ...", meta.Id, meta.Title)
-						if _, err := store.Del(meta.Id); err != nil {
-							log.Errorf("Error while deleting record %d: %s.", meta.Id, err)
-						}
-					}
-				}
-			}
-		}
-
-		db.Unlock()
-	}
-}
-
-func arcUpdater() {
-	for {
-		log.Debugf("Checking for newer versions ...")
-
-		client := &http.Client{
-			CheckRedirect: func(req *http.Request, via []*http.Request) error {
-				return http.ErrUseLastResponse
-			},
-		}
-
-		req, _ := http.NewRequest("GET", "https://github.com/evilsocket/arc/releases/latest", nil)
-		resp, err := client.Do(req)
-		if err != nil {
-			if err := events.Setup(); err != nil {
-				log.Fatal(err)
-			}
-			log.Errorf("Error while checking latest version: %s.", err)
-			return
-		}
-		defer resp.Body.Close()
-
-		location := resp.Header.Get("Location")
-
-		log.Debugf("Location header = '%s'", location)
-
-		var verParser = regexp.MustCompile("^https://github\\.com/evilsocket/arc/releases/tag/v([\\d\\.a-z]+)$")
-		m := verParser.FindStringSubmatch(location)
-		if len(m) == 2 {
-			latest := m[1]
-			log.Debugf("Latest version is '%s'", latest)
-			if config.APP_VERSION != latest {
-				log.Importantf("Update to %s available at %s.", latest, location)
-				events.Add(events.UpdateAvailable(config.APP_VERSION, latest, location))
-			} else {
-				log.Debugf("No updates available.")
-			}
-		} else {
-			log.Warningf("Unexpected location header: '%s'.", location)
-		}
-
-		time.Sleep(time.Duration(60) * time.Minute)
-	}
-}
-
-func arcSignalHandler() {
-	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
-	s := <-signals
-	log.Raw("\n")
-	log.Importantf("RECEIVED SIGNAL: %s", s)
-	db.Flush()
-	os.Exit(1)
-}
-
-func setupRouter() *gin.Engine {
-	gin.SetMode(gin.ReleaseMode)
-
-	r := gin.New()
-
-	webapp := arcLoadApp(r)
-
-	api := r.Group("/api")
-	r.POST("/auth", controllers.Auth)
-
-	if noAuth == false {
-		api.Use(middlewares.AuthHandler())
-	} else {
-		log.Importantf("API authentication is disabled.")
-	}
-
-	controllers.App = webapp
-
-	api.GET("/status", controllers.GetStatus)
-	api.GET("/manifest", controllers.GetManifest)
-	api.GET("/config", controllers.GetConfig)
-
-	api.GET("/events/clear", controllers.ClearEvents)
-
-	api.GET("/stores", controllers.ListStores)
-	api.POST("/stores", controllers.CreateStore)
-	api.GET("/store/:id", controllers.GetStore)
-	api.PUT("/store/:id", controllers.UpdateStore)
-	api.DELETE("/store/:id", controllers.DeleteStore)
-
-	api.GET("/store/:id/records", controllers.ListRecords)
-	api.POST("/store/:id/records", controllers.CreateRecord)
-	api.GET("/store/:id/record/:r_id", controllers.GetRecord)
-	api.GET("/store/:id/record/:r_id/buffer", controllers.GetRecordBuffer)
-	api.PUT("/store/:id/record/:r_id", controllers.UpdateRecord)
-	api.DELETE("/store/:id/record/:r_id", controllers.DeleteRecord)
-
-	return r
-}
-
-func main() {
-	var err error
-
-	flag.Parse()
-
-	log.WithColors = !noColors
-
-	if logfile != "" {
-		log.Output, err = os.Create(logfile)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		defer log.Output.Close()
-	}
-
-	if debug == true {
-		log.MinLevel = log.DEBUG
-	} else {
-		log.MinLevel = log.INFO
-	}
-
-	log.Infof("%s (%s %s) is starting ...", log.Bold(config.APP_NAME+" v"+config.APP_VERSION), runtime.GOOS, runtime.GOARCH)
-
-	if confFile != "" {
-		if err = config.Load(confFile); err != nil {
-			log.Fatal(err)
-		}
-	}
-
-	if dbIsNew, err = db.Setup(); err != nil {
-		log.Fatal(err)
-	}
-
-	if export == true {
-		started := time.Now()
-		if err = db.Export(output); err != nil {
-			log.Fatal(err)
-		}
-		log.Infof("Archived %s of data in %s to %s.", utils.FormatBytes(db.Size), time.Since(started), log.Bold(output))
-		return
-	} else if importFrom != "" {
-		started := time.Now()
-		if err = db.Import(importFrom); err != nil {
-			log.Fatal(err)
-		}
-		log.Infof("Imported %s of data in %s.", utils.FormatBytes(db.Size), time.Since(started))
-		return
-	}
-
-	go arcSignalHandler()
-
-	if config.Conf.Scheduler.Enabled {
-		if err := events.Setup(); err != nil {
-			log.Fatal(err)
-		}
-
-		log.Debugf("Starting scheduler with a period of %ds ...", config.Conf.Scheduler.Period)
-		go arcScheduler()
-	} else {
-		log.Importantf("Scheduler is disabled.")
-	}
-
-	if config.Conf.Backups.Enabled {
-		log.Debugf("Starting backup task with a period of %ds ...", config.Conf.Backups.Period)
-		go arcBackupper()
-	} else {
-		log.Importantf("Backups are disabled.")
-	}
-
-	if noUpdates == false {
-		go arcUpdater()
-	}
-
-	address := fmt.Sprintf("%s:%d", config.Conf.Address, config.Conf.Port)
-
-	r := setupRouter()
-
-	if config.Conf.Certificate, err = utils.ExpandPath(config.Conf.Certificate); err != nil {
-		log.Fatal(err)
-	} else if config.Conf.Key, err = utils.ExpandPath(config.Conf.Key); err != nil {
-		log.Fatal(err)
-	}
-
-	if utils.Exists(config.Conf.Certificate) == false || utils.Exists(config.Conf.Key) == false {
-		log.Importantf("TLS certificate files not found, generating new ones ...")
-		if err = tls.Generate(&config.Conf); err != nil {
-			log.Fatal(err)
-		}
-		log.Infof("New RSA key and certificate have been generated, remember to add them as exceptions to your browser!")
-	}
-
-	if address[0] == ':' {
-		address = "0.0.0.0" + address
-	}
-	log.Infof("Running on %s ...", log.Bold("https://"+address+"/"))
-	if err = r.RunTLS(address, config.Conf.Certificate, config.Conf.Key); err != nil {
-		log.Fatal(err)
-	}
-}

arcd/main_test.go

@@ -1,24 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestMain(m *testing.M) {
-	confFile = "sample_config.json"
-	appPath = "../arc"
-}
-
-func TestHomeRoute(t *testing.T) {
-	router := setupRouter()
-
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/", nil)
-	router.ServeHTTP(w, req)
-
-	assert.Equal(t, 200, w.Code)
-}

arcd/utils/path.go

@@ -21,6 +21,14 @@ func Exists(path string) bool {
 	return true
 }
 
+func FileSize(filename string) (error, int64) {
+	s, e := os.Stat(filename)
+	if e != nil {
+		return e, 0
+	}
+	return nil, s.Size()
+}
+
 func ExpandPath(path string) (string, error) {
 	if strings.HasPrefix(path, "~") {
 		usr, err := user.Current()