Merge pull request #6 from yamadashy/feature/secretlint

feat: Implement SecretLint integration for enhanced security checks
2025-06-11 00:25:54 +03:00 · 2024-07-23 12:17:26 +09:00
parent 97caac85c6 dd15c4ffdf
commit 80d16ff55f
8 changed files with 258 additions and 27 deletions
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 [![npm](https://img.shields.io/npm/l/repopack.svg?maxAge=1000)](https://github.com/yamadashy/repopack/blob/master/LICENSE.md)
 [![node](https://img.shields.io/node/v/repopack.svg?maxAge=1000)](https://www.npmjs.com/package/repopack)

-Repopack is a powerful tool that packs your entire repository into a single, AI-friendly file. Perfect for when you need to feed your codebase to Large Language Models (LLMs) or other AI tools.
+Repopack is a powerful tool that packs your entire repository into a single, AI-friendly file. Perfect for when you need to feed your codebase to Large Language Models (LLMs) or other AI tools. It now includes a security check feature to detect potentially sensitive information in your files.



@@ -15,6 +15,7 @@ Repopack is a powerful tool that packs your entire repository into a single, AI-
 - **Simple to Use**: Just one command to pack your entire repository.
 - **Customizable**: Easily configure what to include or exclude.
 - **Git-Aware**: Automatically respects your .gitignore files.
+- **Security Check**: Detects potentially sensitive information in your files.



@@ -91,6 +92,26 @@ npx repopack src



+## 🔍 Security Check
+
+Repopack now includes a security check feature that uses SecretLint to detect potentially sensitive information in your files. This feature helps you identify possible security risks before sharing your packed repository.
+
+The security check results will be displayed in the CLI output after the packing process is complete. If any suspicious files are detected, you'll see a list of these files along with a warning message.
+
+Example output:
+
+```
+🔍 Security Check:
+──────────────────
+2 suspicious file(s) detected:
+1. src/config.js
+2. tests/testData.json
+
+Please review these files for potential sensitive information.
+```
+
+
+
 ## ⚙️ Configuration

 Create a `repopack.config.json` file in your project root for custom configurations. Here's an explanation of the configuration options:
@@ -99,7 +120,7 @@ Create a `repopack.config.json` file in your project root for custom configurati
 |--------|-------------|---------|
 |`output.filePath`| The name of the output file | `"repopack-output.txt"` |
 |`output.headerText`| Custom text to include in the file header |`null`|
-|`output.removeComments`| Whether to remove comments from supported file types. Suppurts python  | `false` |
+|`output.removeComments`| Whether to remove comments from supported file types | `false` |
 |`output.topFilesLength`| Number of top files to display in the summary. If set to 0, no summary will be displayed |`5`|
 |`ignore.useDefaultPatterns`| Whether to use default ignore patterns |`true`|
 |`ignore.customPatterns`| Additional patterns to ignore |`[]`|
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,6 +9,8 @@
      "version": "0.1.7",
      "license": "MIT",
      "dependencies": {
+        "@secretlint/core": "^8.2.4",
+        "@secretlint/secretlint-rule-preset-recommend": "^8.2.4",
        "cli-spinners": "^2.9.2",
        "commander": "^7.1.0",
        "iconv-lite": "^0.6.3",
@@ -951,6 +953,41 @@
        "win32"
      ]
    },
+    "node_modules/@secretlint/core": {
+      "version": "8.2.4",
+      "resolved": "https://registry.npmjs.org/@secretlint/core/-/core-8.2.4.tgz",
+      "integrity": "sha512-Ws/jX/It7O5kRlvYXM6tHgvLmbSTvQTG7G+vQ0FWb1KjS14+5CbuRdxcQQLkD8shD/87tHu53lOmIlvn/Rc/YA==",
+      "dependencies": {
+        "@secretlint/profiler": "^8.2.4",
+        "@secretlint/types": "^8.2.4",
+        "debug": "^4.3.4",
+        "structured-source": "^4.0.0"
+      },
+      "engines": {
+        "node": "^14.13.1 || >=16.0.0"
+      }
+    },
+    "node_modules/@secretlint/profiler": {
+      "version": "8.2.4",
+      "resolved": "https://registry.npmjs.org/@secretlint/profiler/-/profiler-8.2.4.tgz",
+      "integrity": "sha512-KfRGWf7R4tAxJwk7Ojoa8n53yLh3zmM1hmG1Nh/xWkSXsatTF5qi7bfyi3+QjAxhBZRaz6aR9JbX8PS3JGon1w=="
+    },
+    "node_modules/@secretlint/secretlint-rule-preset-recommend": {
+      "version": "8.2.4",
+      "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-preset-recommend/-/secretlint-rule-preset-recommend-8.2.4.tgz",
+      "integrity": "sha512-Ifaz0ngkgP911TfJkkIXrP9dCpxQGCDAISZ/sG0mdOOg9KO8jF9pnGKzCOuVX4q97v6MDtELXjYAxnPa8xV4Ow==",
+      "engines": {
+        "node": "^14.13.1 || >=16.0.0"
+      }
+    },
+    "node_modules/@secretlint/types": {
+      "version": "8.2.4",
+      "resolved": "https://registry.npmjs.org/@secretlint/types/-/types-8.2.4.tgz",
+      "integrity": "sha512-Pf+ArQmx4+K75TpMRhUgqw2FL8DGNf0OkT9g6L1t4HoYESiVlfZw3quNTvO+GTWzRxssUjnWoQ4+sEg3MNgEHA==",
+      "engines": {
+        "node": "^14.13.1 || >=16.0.0"
+      }
+    },
    "node_modules/@types/eslint": {
      "version": "8.56.10",
      "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.10.tgz",
@@ -1548,6 +1585,11 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/boundary": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/boundary/-/boundary-2.0.0.tgz",
+      "integrity": "sha512-rJKn5ooC9u8q13IMCrW0RSp31pxBCHE3y9V/tp3TdWSLf8Em3p6Di4NBpfzbJge9YjjFEsD0RtFEjtvHL5VyEA=="
+    },
    "node_modules/brace-expansion": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -1788,7 +1830,6 @@
      "version": "4.3.5",
      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
      "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
-      "dev": true,
      "dependencies": {
        "ms": "2.1.2"
      },
@@ -3710,8 +3751,7 @@
    "node_modules/ms": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-      "dev": true
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
    },
    "node_modules/nanoid": {
      "version": "3.3.7",
@@ -4696,6 +4736,14 @@
        "url": "https://github.com/sponsors/antfu"
      }
    },
+    "node_modules/structured-source": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-4.0.0.tgz",
+      "integrity": "sha512-qGzRFNJDjFieQkl/sVOI2dUjHKRyL9dAJi2gCPGJLbJHBIkyOHxjuocpIEfbLioX+qSJpvbYdT49/YCdMznKxA==",
+      "dependencies": {
+        "boundary": "^2.0.0"
+      }
+    },
    "node_modules/supports-color": {
      "version": "7.2.0",
      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
--- a/package.json
+++ b/package.json
@@ -49,6 +49,8 @@
  },
  "type": "module",
  "dependencies": {
+    "@secretlint/core": "^8.2.4",
+    "@secretlint/secretlint-rule-preset-recommend": "^8.2.4",
    "cli-spinners": "^2.9.2",
    "commander": "^7.1.0",
    "iconv-lite": "^0.6.3",
@@ -62,8 +64,8 @@
  "devDependencies": {
    "@eslint/js": "^9.7.0",
    "@types/eslint": "~8.56.10",
-    "@types/eslint__js": "~8.42.3",
    "@types/eslint-config-prettier": "~6.11.3",
+    "@types/eslint__js": "~8.42.3",
    "@types/node": "^20.14.10",
    "@types/strip-comments": "^2.0.4",
    "@typescript-eslint/eslint-plugin": "^7.16.0",
--- a/src/cli/cliOutput.ts
+++ b/src/cli/cliOutput.ts
@@ -8,6 +8,21 @@ export function printSummary(totalFiles: number, totalCharacters: number, output
  console.log(`${pc.white('     Output:')} ${pc.white(outputPath)}`);
 }

+export function printSecurityCheck(suspiciousFiles: string[]) {
+  console.log(pc.white('🔎 Security Check:'));
+  console.log(pc.dim('──────────────────'));
+
+  if (suspiciousFiles.length === 0) {
+    console.log(pc.green('✔') + ' ' + pc.white('No suspicious files detected.'));
+  } else {
+    console.log(pc.yellow(`${suspiciousFiles.length} suspicious file(s) detected:`));
+    suspiciousFiles.forEach((file, index) => {
+      console.log(`${pc.white(`${index + 1}.`)} ${pc.white(file)}`);
+    });
+    console.log(pc.yellow('\nPlease review these files for potential sensitive information.'));
+  }
+}
+
 export function printTopFiles(fileCharCounts: Record<string, number>, topFilesLength: number) {
  console.log(pc.white(`📈 Top ${topFilesLength} Files by Character Count:`));
  console.log(pc.dim('──────────────────────────────────'));
--- a/src/cli/index.ts
+++ b/src/cli/index.ts
@@ -8,7 +8,7 @@ import { getVersion } from '../utils/packageJsonUtils.js';
 import Spinner from '../utils/spinner.js';
 import pc from 'picocolors';
 import { handleError } from '../utils/errorHandler.js';
-import { printSummary, printTopFiles, printCompletion } from './cliOutput.js';
+import { printSummary, printTopFiles, printCompletion, printSecurityCheck } from './cliOutput.js';

 interface CliOptions extends OptionValues {
  version?: boolean;
@@ -59,7 +59,7 @@ async function executeAction(directory: string, options: CliOptions) {
  spinner.start();

  try {
-    const { totalFiles, totalCharacters, fileCharCounts } = await pack(targetPath, config);
+    const { totalFiles, totalCharacters, fileCharCounts, suspiciousFiles } = await pack(targetPath, config);
    spinner.succeed('Packing completed successfully!');
    console.log('');

@@ -68,6 +68,9 @@ async function executeAction(directory: string, options: CliOptions) {
      console.log('');
    }

+    printSecurityCheck(suspiciousFiles);
+    console.log('');
+
    printSummary(totalFiles, totalCharacters, config.output.filePath);
    console.log('');

--- a/src/core/packager.ts
+++ b/src/core/packager.ts
@@ -9,6 +9,7 @@ import {
 } from '../utils/gitignoreUtils.js';
 import { generateOutput as defaultGenerateOutput } from './outputGenerator.js';
 import { defaultIgnoreList } from '../utils/defaultIgnore.js';
+import { checkFileWithSecretLint, createSecretLintConfig } from '../utils/secretLintUtils.js';

 export interface Dependencies {
  getGitignorePatterns: typeof defaultGetGitignorePatterns;
@@ -21,6 +22,7 @@ export interface PackResult {
  totalFiles: number;
  totalCharacters: number;
  fileCharCounts: Record<string, number>;
+  suspiciousFiles: string[];
 }

 export async function pack(
@@ -33,18 +35,24 @@ export async function pack(
    generateOutput: defaultGenerateOutput,
  },
 ): Promise<PackResult> {
+  // Get ignore patterns
  const gitignorePatterns = await deps.getGitignorePatterns(rootDir);
-
  const ignorePatterns = getIgnorePatterns(gitignorePatterns, config);
  const ignoreFilter = deps.createIgnoreFilter(ignorePatterns);

-  const packedFiles = await packDirectory(rootDir, '', config, ignoreFilter, deps);
+  // Get all file paths in the directory
+  const filePaths = await getFilePaths(rootDir, '', ignoreFilter);

-  const totalFiles = packedFiles.length;
-  const totalCharacters = packedFiles.reduce((sum, file) => sum + file.content.length, 0);
+  // Perform security check
+  const suspiciousFiles = await performSecurityCheck(filePaths, rootDir);

+  // Pack files and generate output
+  const packedFiles = await packFiles(filePaths, rootDir, config, deps);
  await deps.generateOutput(rootDir, config, packedFiles);

+  // Metrics
+  const totalFiles = packedFiles.length;
+  const totalCharacters = packedFiles.reduce((sum, file) => sum + file.content.length, 0);
  const fileCharCounts: Record<string, number> = {};
  packedFiles.forEach((file) => {
    fileCharCounts[file.path] = file.content.length;
@@ -54,6 +62,7 @@ export async function pack(
    totalFiles,
    totalCharacters,
    fileCharCounts,
+    suspiciousFiles,
  };
 }

@@ -68,30 +77,55 @@ function getIgnorePatterns(gitignorePatterns: string[], config: RepopackConfigMe
  return ignorePatterns;
 }

-async function packDirectory(
-  dir: string,
-  relativePath: string,
-  config: RepopackConfigMerged,
-  ignoreFilter: IgnoreFilter,
-  deps: Dependencies,
-): Promise<{ path: string; content: string }[]> {
+async function getFilePaths(dir: string, relativePath: string, ignoreFilter: IgnoreFilter): Promise<string[]> {
  const entries = await fs.readdir(dir, { withFileTypes: true });
-  const packedFiles: { path: string; content: string }[] = [];
+  const filePaths: string[] = [];

  for (const entry of entries) {
-    const fullPath = path.join(dir, entry.name);
    const entryRelativePath = path.join(relativePath, entry.name);

    if (!ignoreFilter(entryRelativePath)) continue;

    if (entry.isDirectory()) {
-      const subDirFiles = await packDirectory(fullPath, entryRelativePath, config, ignoreFilter, deps);
-      packedFiles.push(...subDirFiles);
+      const subDirPaths = await getFilePaths(path.join(dir, entry.name), entryRelativePath, ignoreFilter);
+      filePaths.push(...subDirPaths);
    } else {
-      const content = await deps.processFile(fullPath, config);
-      if (content) {
-        packedFiles.push({ path: entryRelativePath, content });
-      }
+      filePaths.push(entryRelativePath);
+    }
+  }
+
+  return filePaths;
+}
+
+async function performSecurityCheck(filePaths: string[], rootDir: string): Promise<string[]> {
+  const secretLintConfig = createSecretLintConfig();
+  const suspiciousFiles: string[] = [];
+
+  for (const filePath of filePaths) {
+    const fullPath = path.join(rootDir, filePath);
+    const content = await fs.readFile(fullPath, 'utf-8');
+    const isSuspicious = await checkFileWithSecretLint(fullPath, content, secretLintConfig);
+    if (isSuspicious) {
+      suspiciousFiles.push(filePath);
+    }
+  }
+
+  return suspiciousFiles;
+}
+
+async function packFiles(
+  filePaths: string[],
+  rootDir: string,
+  config: RepopackConfigMerged,
+  deps: Dependencies,
+): Promise<{ path: string; content: string }[]> {
+  const packedFiles: { path: string; content: string }[] = [];
+
+  for (const filePath of filePaths) {
+    const fullPath = path.join(rootDir, filePath);
+    const content = await deps.processFile(fullPath, config);
+    if (content) {
+      packedFiles.push({ path: filePath, content });
    }
  }

--- a/src/utils/secretLintUtils.ts
+++ b/src/utils/secretLintUtils.ts
@@ -0,0 +1,34 @@
+import type { SecretLintCoreConfig } from '@secretlint/types';
+import { lintSource } from '@secretlint/core';
+import { creator } from '@secretlint/secretlint-rule-preset-recommend';
+
+export async function checkFileWithSecretLint(
+  filePath: string,
+  content: string,
+  config: SecretLintCoreConfig,
+): Promise<boolean> {
+  const result = await lintSource({
+    source: {
+      filePath: filePath,
+      content: content,
+      ext: filePath.split('.').pop() || '',
+      contentType: 'text',
+    },
+    options: {
+      config: config,
+    },
+  });
+
+  return result.messages.length > 0;
+}
+
+export function createSecretLintConfig(): SecretLintCoreConfig {
+  return {
+    rules: [
+      {
+        id: '@secretlint/secretlint-rule-preset-recommend',
+        rule: creator,
+      },
+    ],
+  };
+}
--- a/tests/utils/secretLintUtils.test.ts
+++ b/tests/utils/secretLintUtils.test.ts
@@ -0,0 +1,74 @@
+import { expect, test, describe } from 'vitest';
+import { checkFileWithSecretLint, createSecretLintConfig } from '../../src/utils/secretLintUtils.js';
+import type { SecretLintCoreConfig } from '@secretlint/types';
+
+describe('secretLintUtils', () => {
+  const config: SecretLintCoreConfig = createSecretLintConfig();
+
+  test('should detect sensitive information', async () => {
+    const sensitiveContent = `
+# Secretlint Demo
+
+URL: https://user:pass@example.com
+
+GitHub Token: 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
+
+SendGrid: "SG.APhb3zgjtx3hajdas1TjBB.H7Sgbba3afgKSDyB442aDK0kpGO3SD332313-L5528Kewhere"
+
+AWS_SECRET_ACCESS_KEY = wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRETSKEY
+
+Slack:
+xoxa-23984754863-2348975623103
+xoxb-23984754863-2348975623103
+xoxo-23984754863-2348975623103
+
+Private Key:
+
+-----BEGIN RSA PRIVATE KEY-----
+MIICWwIBAAKBgQCYdGaf5uYMsilGHfnx/zxXtihdGFr3hCWwebHGhgEAVn0xlsTd
+1QwoKi+rpI1O6hzyVOuoQtboODsONGRlHbNl6yJ936Yhmr8PiNwpA5qIxZAdmFv2
+tqEllWr0dGPPm3B/2NbjuMpSiJNAcBQa46X++doG5yNMY8NCgTsjBZIBKwIDAQAB
+AoGAN+Pkg5aIm/rsurHeoeMqYhV7srVtE/S0RIA4tkkGMPOELhvRzGmAbXEZzNkk
+nNujBQww4JywYK3MqKZ4b8F1tMG3infs1w8V7INAYY/c8HzfrT3f+MVxijoKV2Fl
+JlUXCclztoZhxAxhCR+WC1Upe1wIrWNwad+JA0Vws/mwrEECQQDxiT/Q0lK+gYaa
+riFeZmOaqwhlFlYNSK2hCnLz0vbnvnZE5ITQoV+yiy2+BhpMktNFsYNCfb0pdKN
+D87x+jr7AkEAoZWITvqErh1RbMCXd26QXZEfZyrvVZMpYf8BmWFaBXIbrVGme0/Q
+d7amI6B8Vrowyt+qgcUk7rYYaA39jYB7kQJAdaX2sY5gw25v1Dlfe5Q5WYdYBJsv
+0alAGUrS2PVF69nJtRS1SDBUuedcVFsP+N2IlCoNmfhKk+vZXOBgWrkZ1QJAGJlE
+FAntUvhhofW72VG6ppPmPPV7VALARQvmOWxpoPSbJAqPFqyy5tamejv/UdCshuX/
+9huGINUV6BlhJT6PEQJAF/aqQTwZqJdwwJqYEQArSmyOW7UDAlQMmKMofjBbeBvd
+H4PSJT5bvaEhxRj7QCwonoX4ZpV0beTnzloS55Z65g==
+-----END RSA PRIVATE KEY-----
+    `;
+
+    const result = await checkFileWithSecretLint('test.md', sensitiveContent, config);
+    expect(result).toBe(true);
+  });
+
+  test('should not detect sensitive information in normal content', async () => {
+    const normalContent = `
+# Normal Content
+
+This is a regular markdown file with no sensitive information.
+
+Here's some code:
+
+\`\`\`javascript
+function greet(name) {
+  console.log(\`Hello, \${name}!\`);
+}
+\`\`\`
+
+And here's a list:
+
+1. Item 1
+2. Item 2
+3. Item 3
+
+That's all!
+    `;
+
+    const result = await checkFileWithSecretLint('normal.md', normalContent, config);
+    expect(result).toBe(false);
+  });
+});