init

README.md

@@ -0,0 +1,36 @@
+![Entropy logo](./entropy.png)
+
+> Paranoïd about having secrets leaked in your huge codebase? Entropy is here to help you find them!
+
+# Entropy
+
+Entropy is a CLI tool that will **scan your codebase for high entropy lines**, which are often secrets.
+
+## Installation
+
+If you have Go installed
+
+```bash
+go install github.com/EwenQuim/entropy@latest
+entropy
+```
+
+or in one line
+
+```bash
+go run github.com/EwenQuim/entropy@latest
+```
+
+### With brew
+
+WIP
+
+## Usage
+
+I don't want to maintain a documentation here, so just run
+
+```bash
+entropy -h
+```
+
+as it will always be up to date.

go.mod

@@ -0,0 +1,7 @@
+module entropy
+
+go 1.22.3
+
+require golang.org/x/term v0.20.0
+
+require golang.org/x/sys v0.20.0 // indirect

go.sum

@@ -0,0 +1,4 @@
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=

main.go

@@ -0,0 +1,225 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"math"
+	"os"
+	"slices"
+	"strings"
+	"sync"
+
+	"golang.org/x/term"
+)
+
+const (
+	minCharactersDefault      = 5
+	resultCountDefault        = 10
+	exploreHiddenDefault      = false
+	extensionsToIgnoreDefault = "pdf,png,jpg,jpeg,zip,mp4,gif"
+)
+
+var (
+	minCharacters      = minCharactersDefault // Minimum number of characters to consider computing entropy
+	resultCount        = resultCountDefault   // Number of results to display
+	exploreHidden      = exploreHiddenDefault // Ignore hidden files and folders
+	extensions         = []string{}           // List of file extensions to include. Empty string means all files
+	extensionsToIgnore = []string{}           // List of file extensions to ignore. Empty string means all files
+)
+
+type Entropy struct {
+	Entropy float64 // Entropy of the line
+	File    string  // File where the line is found
+	LineNum int     // Line number in the file
+	Line    string  // Line with high entropy
+}
+
+func main() {
+	minCharactersFlag := flag.Int("min", minCharactersDefault, "Minimum number of characters in the line to consider computing entropy")
+	resultCountFlag := flag.Int("top", resultCountDefault, "Number of results to display")
+	exploreHiddenFlag := flag.Bool("include-hidden", exploreHiddenDefault, "Search in hidden files and folders (.git, .env...). Slows down the search.")
+	extensionsFlag := flag.String("ext", "", "Search only in files with these extensions. Comma separated list, e.g. -ext go,py,js (default all files)")
+	extensionsToIgnoreFlag := flag.String("ignore-ext", extensionsToIgnoreDefault, "Ignore files with these extensions. Comma separated list, e.g. -ignore-ext pdf,png,jpg")
+
+	flag.CommandLine.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(), "%s [flags] file1 file2 file3 ...\n", os.Args[0])
+		fmt.Fprintf(flag.CommandLine.Output(), "Example: %s -top 10 -ext go,py,js .\n", os.Args[0])
+		fmt.Fprintln(flag.CommandLine.Output(), "Finds the highest entropy strings in files. The higher the entropy, the more random the string is. Useful for finding secrets (and alphabets, it seems).")
+		fmt.Fprintln(flag.CommandLine.Output(), "Please support me on GitHub: https://github.com/EwenQuim")
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+
+	// Apply flags
+	minCharacters = *minCharactersFlag
+	resultCount = *resultCountFlag
+	exploreHidden = *exploreHiddenFlag
+	extensions = strings.Split(*extensionsFlag, ",")
+	extensionsToIgnore = strings.Split(*extensionsToIgnoreFlag, ",")
+
+	// Read file names from cli
+	fileNames := flag.Args()
+	if len(fileNames) == 0 {
+		fmt.Println("No files provided, defaults to current folder.")
+		fileNames = []string{"."}
+	}
+	entropies := make([]Entropy, 0, 10*len(fileNames))
+	for _, fileName := range fileNames {
+		fileEntropies, err := readFile(fileName)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		entropies = append(entropies, fileEntropies...)
+	}
+
+	entropies = sortAndCutTop(entropies)
+
+	redMark := "\033[31m"
+	resetMark := "\033[0m"
+	if !term.IsTerminal(int(os.Stdout.Fd())) {
+		// If not a terminal, remove color
+		redMark = ""
+		resetMark = ""
+	}
+
+	for _, entropy := range entropies {
+		fmt.Printf("%.2f: %s%s:%d%s %s\n", entropy.Entropy, redMark, entropy.File, entropy.LineNum, resetMark, entropy.Line)
+	}
+}
+
+func readFile(fileName string) ([]Entropy, error) {
+	// If file is a folder, walk inside the folder
+	fileInfo, err := os.Stat(fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	if isFileHidden(fileInfo.Name()) && !exploreHidden {
+		return nil, nil
+	}
+
+	entropies := make([]Entropy, 0, 10)
+	if fileInfo.IsDir() {
+		// Walk through the folder and read all files
+		dir, err := os.ReadDir(fileName)
+		if err != nil {
+			return nil, err
+		}
+
+		entropiies := make([][]Entropy, len(dir))
+
+		var wg sync.WaitGroup
+		for i, file := range dir {
+			wg.Add(1)
+			go func(i int, file os.DirEntry) {
+				defer wg.Done()
+				fileEntropies, err := readFile(fileName + "/" + file.Name())
+				if err != nil {
+					panic(err)
+				}
+				entropiies[i] = fileEntropies
+			}(i, file)
+		}
+
+		wg.Wait()
+
+		for _, fileEntropies := range entropiies {
+			entropies = append(entropies, fileEntropies...)
+		}
+	}
+
+	if !isFileIncluded(fileInfo.Name()) {
+		return sortAndCutTop(entropies), nil
+	}
+
+	file, err := os.Open(fileName)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	i := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		i++
+		line := strings.TrimSpace(scanner.Text())
+
+		for _, field := range strings.Fields(line) {
+			if len(field) < minCharacters {
+				continue
+			}
+
+			entropies = append(entropies, Entropy{
+				Entropy: entropy(field),
+				File:    fileName,
+				LineNum: i,
+				Line:    field,
+			})
+		}
+	}
+
+	return sortAndCutTop(entropies), nil
+}
+
+func entropy(text string) float64 {
+	uniqueCharacters := make(map[rune]struct{}, len(text))
+	for _, r := range text {
+		uniqueCharacters[r] = struct{}{}
+	}
+
+	entropy := 0.0
+	for character := range uniqueCharacters {
+		res := float64(strings.Count(text, string(character))) / float64(len(text))
+		if res == 0 {
+			continue
+		}
+
+		entropy -= res * math.Log2(res)
+	}
+
+	return entropy
+}
+
+func isFileHidden(filename string) bool {
+	if filename == "." {
+		return false
+	}
+	filename = strings.TrimPrefix(filename, "./")
+
+	return strings.HasPrefix(filename, ".") || filename == "node_modules"
+}
+
+// isFileIncluded returns true if the file should be included in the search
+func isFileIncluded(filename string) bool {
+	for _, ext := range extensionsToIgnore {
+		if strings.HasSuffix(filename, ext) {
+			return false
+		}
+	}
+
+	if len(extensions) == 0 {
+		return true
+	}
+
+	for _, ext := range extensions {
+		if strings.HasSuffix(filename, ext) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func sortAndCutTop(entropies []Entropy) []Entropy {
+	slices.SortFunc(entropies, func(a, b Entropy) int {
+		return int((b.Entropy - a.Entropy) * 10000)
+	})
+
+	if len(entropies) > resultCount {
+		return entropies[:resultCount]
+	}
+
+	return entropies
+}

main_test.go

@@ -0,0 +1,171 @@
+package main
+
+import "testing"
+
+func BenchmarkFile(b *testing.B) {
+	for range b.N {
+		readFile("testdata")
+	}
+}
+
+func TestEntropy(t *testing.T) {
+	t.Run("empty", func(t *testing.T) {
+		Expect(t, entropy(""), 0.0)
+	})
+
+	t.Run("single character", func(t *testing.T) {
+		Expect(t, entropy("a"), 0.0)
+	})
+
+	t.Run("two same characters", func(t *testing.T) {
+		Expect(t, entropy("aa"), 0.0)
+	})
+
+	t.Run("three different characters", func(t *testing.T) {
+		ExpectFloat(t, entropy("abc"), 1.5849625007211563)
+	})
+
+	t.Run("three same characters", func(t *testing.T) {
+		Expect(t, entropy("aaa"), 0.0)
+	})
+
+	t.Run("four different characters", func(t *testing.T) {
+		Expect(t, entropy("abcd"), 2.0)
+	})
+
+	t.Run("four same characters", func(t *testing.T) {
+		Expect(t, entropy("aabb"), 1.0)
+	})
+
+	t.Run("12 characters", func(t *testing.T) {
+		ExpectFloat(t, entropy("aabbccddeeff"), 2.584962500721156)
+	})
+}
+
+func TestReadFile(t *testing.T) {
+	t.Run("random.js", func(t *testing.T) {
+		res, err := readFile("testdata/random.js")
+		if err != nil {
+			t.Errorf("expected nil, got %v", err)
+		}
+
+		Expect(t, len(res), 10)
+		ExpectFloat(t, res[0].Entropy, 5.53614242151549)
+		Expect(t, res[0].LineNum, 7) // The token is hidden here
+		ExpectFloat(t, res[4].Entropy, 3.321928094887362)
+	})
+
+	t.Run("testdata/folder", func(t *testing.T) {
+		res, err := readFile("testdata/folder")
+		if err != nil {
+			t.Errorf("expected nil, got %v", err)
+		}
+
+		Expect(t, len(res), 10)
+		ExpectFloat(t, res[0].Entropy, 3.7667029194153567)
+		Expect(t, res[0].LineNum, 7) // The token is hidden here
+		ExpectFloat(t, res[6].Entropy, 2.8553885422075336)
+	})
+}
+
+func TestSortAndCutTop(t *testing.T) {
+	resultCount = 5
+
+	t.Run("nil", func(t *testing.T) {
+		res := sortAndCutTop(nil)
+		if len(res) != 0 {
+			t.Errorf("expected 0, got %d", len(res))
+		}
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		res := sortAndCutTop([]Entropy{})
+		if len(res) != 0 {
+			t.Errorf("expected 0, got %d", len(res))
+		}
+	})
+
+	t.Run("less than resultCount", func(t *testing.T) {
+		res := sortAndCutTop([]Entropy{
+			{Entropy: 0.1},
+			{Entropy: 0.6},
+			{Entropy: 0.3},
+		})
+
+		Expect(t, len(res), 3)
+		Expect(t, res[0].Entropy, 0.6)
+		Expect(t, res[2].Entropy, 0.1)
+	})
+
+	t.Run("more than resultCount", func(t *testing.T) {
+		res := sortAndCutTop([]Entropy{
+			{Entropy: 0.1},
+			{Entropy: 0.6},
+			{Entropy: 0.3},
+			{Entropy: 0.7},
+			{Entropy: 0.4},
+			{Entropy: 0.5},
+			{Entropy: 0.2},
+		})
+
+		Expect(t, len(res), 5)
+		Expect(t, res[0].Entropy, 0.7)
+		Expect(t, res[4].Entropy, 0.3)
+	})
+}
+
+func TestIsFileIncluded(t *testing.T) {
+	t.Run("empty", func(t *testing.T) {
+		extensions = []string{}
+		Expect(t, isFileIncluded("main.go"), true)
+		Expect(t, isFileIncluded("main.py"), true)
+	})
+
+	t.Run("one element included", func(t *testing.T) {
+		extensions = []string{"go"}
+		Expect(t, isFileIncluded("main.py"), false)
+		Expect(t, isFileIncluded("main.go"), true)
+	})
+
+	t.Run("one element excluded", func(t *testing.T) {
+		extensions = []string{}
+		extensionsToIgnore = []string{"go"}
+		Expect(t, isFileIncluded("main.go"), false)
+		Expect(t, isFileIncluded("main.py"), true)
+	})
+
+	t.Run("multiple elements", func(t *testing.T) {
+		extensions = []string{"go", "py"}
+		extensionsToIgnore = []string{"pdf"}
+		Expect(t, isFileIncluded("main.go"), true)
+		Expect(t, isFileIncluded("main.py"), true)
+		Expect(t, isFileIncluded("main.pdf"), false)
+	})
+}
+
+func TestIsFileHidden(t *testing.T) {
+	Expect(t, isFileHidden("."), false)
+	Expect(t, isFileHidden("main.go"), false)
+	Expect(t, isFileHidden("main.py"), false)
+	Expect(t, isFileHidden("node_modules"), true)
+	Expect(t, isFileHidden("./.git"), true)
+	Expect(t, isFileHidden(".git"), true)
+	Expect(t, isFileHidden(".env"), true)
+}
+
+func Expect[T comparable](t *testing.T, got, expected T) {
+	t.Helper()
+	if got != expected {
+		t.Errorf("expected %v, got %v", expected, got)
+	}
+}
+
+func ExpectFloat(t *testing.T, got, expected float64) {
+	t.Helper()
+
+	gotInt := int(got * 10000)
+	expectedInt := int(expected * 10000)
+	if gotInt != expectedInt {
+		t.Errorf("expected %d, got %d", expectedInt, gotInt)
+	}
+}

testdata/.gitignore

@@ -0,0 +1,5 @@
+# repositories entropy is tested against
+kubernetes
+git
+cli
+etcd

testdata/folder/fileA.py

@@ -0,0 +1,4 @@
+def hiMom():
+	print("Hi Mom!")
+
+SUPER_SECRET = "dqizuydièqddQ9ZDYçDBzqdqj"

testdata/folder/fileB.go

@@ -0,0 +1,7 @@
+package folder
+
+func Hello() {
+	println("Hello, World!")
+}
+
+const SECRET = "x!çQDSDG9ZQdiiqbzdzqkdbqdzquid"

testdata/random.js

@@ -0,0 +1,7 @@
+function doThings() {
+  console.log("Doing innocent things");
+  return Math.random();
+}
+
+const DangerousToken =
+  "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkV3ZW4iLCJpYXQiOjE1MTYyMzkwMjJ9.BPqjdIG7zp-j9CUEZ-EcYQXplbhzo-2QNMG_QwcAgfk";