feat: Initial implementation of Repopack

This commit is contained in:
Kazuki Yamada
2024-07-15 17:19:18 +09:00
parent ac621e8e3d
commit 582607829e
38 changed files with 7652 additions and 286 deletions

13
.editorconfig Normal file
View File

@@ -0,0 +1,13 @@
root = true
[*.*]
indent_style = space
indent_size = 2
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf
max_line_length = null
[*.md]
trim_trailing_whitespace = false

47
.github/workflows/test.yml vendored Normal file
View File

@@ -0,0 +1,47 @@
name: Test
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
workflow_dispatch:
jobs:
lint:
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup node
uses: actions/setup-node@v4
with:
node-version-file: .tool-versions
cache: npm
- name: Install dependencies
run: npm i
- name: Lint
run: yarn lint
test:
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup node
uses: actions/setup-node@v4
with:
node-version-file: .tool-versions
cache: npm
- name: Install dependencies
run: npm i
- name: Test
run: npm test

309
.gitignore vendored
View File

@@ -1,294 +1,31 @@
# ==================================================
# Generated by gibo
# $ gibo dump Node JetBrains VisualStudioCode macOS Windows
# ==================================================
### Generated by gibo (https://github.com/simonwhitaker/gibo)
### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Node.gitignore
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# Build output
lib/
# TypeScript cache
*.tsbuildinfo
# Logs
*.log
# Optional npm cache directory
.npm
# OS generated files
.DS_Store
# Optional eslint cache
# Editor directories and files
.vscode/
.idea/
# Test coverage
coverage/
# Temporary files
*.tmp
*.temp
# Repopack output
repopack-output.txt
# ESLint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
### Generated by gibo (https://github.com/simonwhitaker/gibo)
### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Global/JetBrains.gitignore
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# AWS User-specific
.idea/**/aws.xml
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# SonarLint plugin
.idea/sonarlint/
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### Generated by gibo (https://github.com/simonwhitaker/gibo)
### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Global/VisualStudioCode.gitignore
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix
### Generated by gibo (https://github.com/simonwhitaker/gibo)
### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Global/macOS.gitignore
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### Generated by gibo (https://github.com/simonwhitaker/gibo)
### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Global/Windows.gitignore
# Windows thumbnail cache files
Thumbs.db
Thumbs.db:encryptable
ehthumbs.db
ehthumbs_vista.db
# Dump file
*.stackdump
# Folder config file
[Dd]esktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
# ==================================================
# Custom
# ==================================================
.idea
# yarn
.yarn/

1
.node-version Normal file
View File

@@ -0,0 +1 @@
20.15.0

47
.npmignore Normal file
View File

@@ -0,0 +1,47 @@
# Source files
src/
# Test files
tests/
coverage/
# Configuration files
tsconfig.json
tsconfig.build.json
.eslintrc.js
eslint.config.mjs
prettier.config.mjs
vite.config.mts
# Git files
.gitignore
.git
# CI files
.github/
# Editor files
.vscode/
.idea/
# Logs
*.log
# Repopack output
repopack-output.txt
# Development scripts
scripts/
# Documentation files (except README and LICENSE)
docs/
CONTRIBUTING.md
CHANGELOG.md
# Temporary files
*.tmp
*.temp
# OS generated files
.DS_Store
Thumbs.db

1
.tool-versions Normal file
View File

@@ -0,0 +1 @@
nodejs 20.15.0

7
LICENSE Normal file
View File

@@ -0,0 +1,7 @@
Copyright 2024 Kazuki Yamada
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

99
README.md Normal file
View File

@@ -0,0 +1,99 @@
# 📦 Repopack
[![npm version](https://badge.fury.io/js/repopack.svg)](https://badge.fury.io/js/repopack)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
Repopack is a powerful tool that packs your entire repository into a single, AI-friendly file. Perfect for when you need to feed your codebase to Large Language Models (LLMs) or other AI tools.
## 🚀 Features
- **AI-Optimized**: Formats your codebase in a way that's easy for AI to understand and process.
- **Simple to Use**: Just one command to pack your entire repository.
- **Customizable**: Easily configure what to include or exclude.
- **Git-Aware**: Automatically respects your .gitignore files.
- **Verbose Mode**: Detailed logging for debugging and understanding the packing process.
## 🛠 Installation
```bash
npm install -g repopack
```
Or if you prefer using Yarn:
```bash
yarn global add repopack
```
## 📊 Usage
Navigate to your project directory and run:
```bash
repopack
```
This will create a `repopack-output.txt` file containing your entire codebase.
### Command Line Options
- `-o, --output <file>`: Specify the output file name (default: repopack-output.txt)
- `-i, --ignore <items>`: Comma-separated list of additional items to ignore
- `-c, --config <path>`: Path to a custom config file (default: repopack.config.js)
- `--no-default-ignore`: Disable the default ignore list
- `-v, --verbose`: Enable verbose logging
Example:
```bash
repopack -o custom-output.txt -i "*.log,tmp" -v
```
## ⚙️ Configuration
Create a `repopack.config.js` file in your project root for custom configurations:
```javascript
/** @type {import('repopack').RepopackConfig} */
const config = {
output: {
filePath: 'custom-output.txt',
headerText: 'Custom header information for the packed file',
},
ignore: {
useDefaultPatterns: true,
customPatterns: ['additional-folder', '*.log'],
},
};
export default config;
```
## 📄 Output Format
Repopack generates a single file with clear separators between different parts of your codebase:
```
================================================================
REPOPACK OUTPUT FILE
================================================================
(Metadata and usage instructions)
================================================================
Repository Files
================================================================
================
File: src/index.js
================
// File contents here
================
File: src/utils.js
================
// File contents here
```
This format ensures that AI tools can easily distinguish between different files in your codebase.
## 📜 License
MIT

7
bin/repopack.cjs Executable file
View File

@@ -0,0 +1,7 @@
#!/usr/bin/env node
'use strict';
(async () => {
const { run } = await import('../lib/cli/index.js');
run();
})();

14
bin/repopack.js Executable file
View File

@@ -0,0 +1,14 @@
#!/usr/bin/env node
/*
Add this file so we can use `node bin/repopack` or `node bin/repopack.js`
instead of `node bin/repopack.cjs`.
This file should only used for development.
*/
'use strict';
import { run } from '../lib/cli/index.js';
run();

42
eslint.config.mjs Normal file
View File

@@ -0,0 +1,42 @@
import js from '@eslint/js';
import typescriptEslintParser from '@typescript-eslint/parser';
import typescriptEslint from '@typescript-eslint/eslint-plugin';
import importPlugin from 'eslint-plugin-import';
import eslintPluginPrettier from 'eslint-plugin-prettier';
import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended';
import globals from 'globals';
export default [
{
files: ['**/*.ts'],
languageOptions: {
globals: {
...globals.es2021,
...globals.node,
...globals.browser,
},
parser: typescriptEslintParser,
},
plugins: {
'js': js,
'@typescript-eslint': typescriptEslint,
'import': importPlugin,
'prettier': eslintPluginPrettier,
},
rules: {
...js.configs.recommended.rules,
...typescriptEslint.configs.recommended.rules,
...typescriptEslint.configs.strict.rules,
...typescriptEslint.configs.stylistic.rules,
'@typescript-eslint/no-var-requires': 'off',
...importPlugin.configs.typescript.rules,
'import/no-unresolved': 'off',
'import/prefer-default-export': 'off',
...eslintPluginPrettierRecommended.rules,
'prettier/prettier': 'warn',
},
},
];

6241
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

82
package.json Normal file
View File

@@ -0,0 +1,82 @@
{
"name": "repopack",
"version": "0.1.0",
"description": "A tool to pack repository contents to single file for AI consumption",
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"exports": {
".": {
"types": "./lib/index.d.ts"
}
},
"bin": "./bin/repopack.cjs",
"scripts": {
"clean": "rimraf lib",
"build": "npm run clean && tsc -p tsconfig.build.json --sourceMap --declaration",
"lint": "eslint ./src ./tests --max-warnings 0 --cache --fix && tsc --noEmit",
"test": "vitest",
"test-coverage": "vitest run --coverage",
"cli-run": "npm run build && node --trace-warnings bin/repopack",
"npm-publish": "npm run lint && npm run test-coverage && npm run build && npm publish",
"npm-release-patch": "npm version patch && npm run npm-publish",
"npm-release-minor": "npm version minor && npm run npm-publish",
"npm-release-prerelease": "npm version prerelease && npm run npm-publish"
},
"keywords": [
"repository",
"ai",
"llm",
"source-code",
"code-analysis",
"codebase-packer",
"development-tool",
"ai-assistant",
"code-review"
],
"repository": {
"type": "git",
"url": "git://github.com/yamadashy/repopack.git"
},
"bugs": {
"url": "https://github.com/yamadashy/repopack/issues"
},
"author": "Kazuki Yamada <koukun0120@gmail.com>",
"homepage": "https://github.com/yamadashy/repopack",
"license": "MIT",
"publishConfig": {
"access": "public"
},
"type": "module",
"dependencies": {
"cli-spinners": "^2.9.2",
"commander": "^7.1.0",
"iconv-lite": "^0.6.3",
"ignore": "^5.2.0",
"is-binary-path": "^2.1.0",
"jschardet": "^3.1.3",
"log-update": "^6.0.0",
"picocolors": "^1.0.1"
},
"devDependencies": {
"@eslint/js": "^9.7.0",
"@types/eslint": "~8.56.10",
"@types/eslint__js": "~8.42.3",
"@types/eslint-config-prettier": "~6.11.3",
"@types/node": "^20.14.10",
"@typescript-eslint/eslint-plugin": "^7.16.0",
"@typescript-eslint/parser": "^7.16.0",
"@vitest/coverage-v8": "^2.0.2",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-prettier": "^5.1.3",
"rimraf": "^5.0.7",
"typescript": "^4.9.5",
"vite": "^4.1.4",
"vitest": "^2.0.2"
},
"engines": {
"node": ">=16.0.0",
"yarn": ">=1.0.0"
}
}

9
prettier.config.mjs Normal file
View File

@@ -0,0 +1,9 @@
export default {
tabWidth: 2,
semi: true,
singleQuote: true,
printWidth: 120,
bracketSpacing: true,
trailingComma: 'all',
quoteProps: 'consistent',
};

28
repopack.config.js Normal file
View File

@@ -0,0 +1,28 @@
// @ts-check
/** @type {import('repopack').RepopackConfig} */
const config = {
output: {
filePath: 'repopack-output.txt',
headerText: `
This repository contains the source code for the Repopack tool.
Repopack is designed to pack repository contents into a single file,
making it easier for AI systems to analyze and process the codebase.
Key Features:
- Configurable ignore patterns
- Custom header text support
- Efficient file processing and packing
Please refer to the README.md file for more detailed information on usage and configuration.
`,
},
ignore: {
useDefaultPatterns: true,
customPatterns: [
// Custom ignore patterns
],
},
};
export default config;

99
src/cli/index.ts Normal file
View File

@@ -0,0 +1,99 @@
import { program } from 'commander';
import Spinner from '../utils/spinner.js';
import { pack } from '../core/packager.js';
import { RepopackConfig } from '../types/index.js';
import { loadConfig, mergeConfigs } from '../config/configLoader.js';
import { logger } from '../utils/logger.js';
import { getVersion } from '../utils/packageJsonUtils.js';
import { handleError, RepopackError } from '../utils/errorHandler.js';
import pc from 'picocolors';
export async function run() {
try {
await runInternal();
} catch (error) {
handleError(error);
}
}
async function runInternal() {
const version = await getVersion();
console.log(pc.dim(`\n📦 Repopack v${version}\n`));
program
.version(version)
.description('Repopack - Pack your repository into a single AI-friendly file')
.option('-o, --output <file>', 'specify the output file name (default: repopack-output.txt)')
.option('-i, --ignore <items>', 'comma-separated list of additional items to ignore')
.option('-c, --config <path>', 'path to a custom config file (default: repopack.config.js)')
.option('--no-default-ignore', 'disable the default ignore list')
.option('-v, --verbose', 'enable verbose logging for detailed output')
.addHelpText(
'after',
`
Example calls:
$ repopack
$ repopack -o custom-output.txt
$ repopack -i "*.log,tmp" -v
$ repopack -c ./custom-config.js
For more information, visit: https://github.com/yamadashy/repopack`,
)
.parse(process.argv);
const options = program.opts();
logger.setVerbose(options.verbose);
logger.trace('Command line options:', options);
const fileConfig = await loadConfig(options.config);
logger.trace('Loaded file config:', fileConfig);
const cliConfig: Partial<RepopackConfig> = {
...(options.output && { output: { filePath: options.output } }),
ignore: {
useDefaultPatterns: options.defaultIgnore !== false,
customPatterns: options.ignore ? options.ignore.split(',') : undefined,
},
};
logger.trace('CLI config:', cliConfig);
const config = mergeConfigs(fileConfig, cliConfig);
logger.trace('Merged config:', config);
if (!config.output.filePath) {
throw new RepopackError(
'Output file is not specified. Please provide it in the config file or via command line option.',
);
}
console.log('');
const spinner = new Spinner('Packing files...');
spinner.start();
try {
const { totalFiles, totalCharacters } = await pack(process.cwd(), config);
spinner.succeed('Packing completed successfully!');
console.log('');
console.log(pc.white('📊 Pack Summary:'));
console.log(pc.dim('────────────────'));
console.log(`${pc.white('Total Files:')} ${pc.white(totalFiles.toString())}`);
console.log(`${pc.white('Total Chars:')} ${pc.white(totalCharacters.toString())}`);
console.log(`${pc.white(' Output:')} ${pc.white(config.output.filePath)}`);
console.log('');
console.log(pc.green('🎉 All Done!'));
console.log(pc.white('Your repository has been successfully packed.'));
} catch (error) {
spinner.fail('Error during packing');
throw error;
}
}

View File

@@ -0,0 +1,66 @@
import path from 'path';
import { RepopackConfig } from '../types/index.js';
import { defaultConfig } from './defaultConfig.js';
import { logger } from '../utils/logger.js';
import * as fs from 'fs/promises';
const defaultConfigPath = 'repopack.config.js';
export async function loadConfig(configPath: string | null): Promise<Partial<RepopackConfig>> {
let useDefaultConfig = false;
if (!configPath) {
useDefaultConfig = true;
configPath = defaultConfigPath;
}
const fullPath = path.resolve(process.cwd(), configPath);
logger.trace('Loading config from:', fullPath);
// Check file existence
const isFileExists = await fs
.stat(fullPath)
.then((stats) => stats.isFile())
.catch(() => false);
if (!isFileExists) {
if (useDefaultConfig) {
logger.note(
`No custom config found at ${configPath}.\nYou can add a config file for additional settings. Please check https://github.com/yamadashy/repopack for more information.`,
);
return {};
} else {
throw new Error(`Config file not found at ${configPath}`);
}
}
try {
const config = await import(fullPath);
return config.default || {};
} catch (error) {
if (error instanceof Error) {
throw new Error(`Error loading config from ${configPath}: ${error.message}`);
} else {
throw new Error(`Error loading config from ${configPath}`);
}
}
}
export function mergeConfigs(fileConfig: Partial<RepopackConfig>, cliConfig: Partial<RepopackConfig>): RepopackConfig {
return {
output: {
...defaultConfig.output,
...fileConfig.output,
...cliConfig.output,
},
ignore: {
...defaultConfig.ignore,
...fileConfig.ignore,
...cliConfig.ignore,
customPatterns: [
...(defaultConfig.ignore.customPatterns || []),
...(fileConfig.ignore?.customPatterns || []),
...(cliConfig.ignore?.customPatterns || []),
],
},
};
}

View File

@@ -0,0 +1,11 @@
import { RepopackConfig } from '../types/index.js';
export const defaultConfig: RepopackConfig = {
output: {
filePath: 'repopack-output.txt',
},
ignore: {
useDefaultPatterns: true,
customPatterns: [],
},
};

2
src/config/index.ts Normal file
View File

@@ -0,0 +1,2 @@
export { defaultConfig } from './defaultConfig.js';
export { loadConfig, mergeConfigs } from './configLoader.js';

View File

@@ -0,0 +1,92 @@
import { RepopackConfig } from '../types/index.js';
import * as fs from 'fs/promises';
import path from 'path';
const SEPARATOR = '='.repeat(16);
const LONG_SEPARATOR = '='.repeat(64);
export async function generateOutput(
rootDir: string,
config: RepopackConfig,
packedFiles: { path: string; content: string }[],
fsModule = fs,
): Promise<void> {
const output: string[] = [];
// Generate and add the header
const header = generateFileHeader(config);
output.push(header);
// Add packed files
for (const file of packedFiles) {
output.push(SEPARATOR);
output.push(`File: ${file.path}`);
output.push(SEPARATOR);
output.push(file.content);
output.push(''); // Add an empty line after each file content
}
const outputPath = path.resolve(rootDir, config.output.filePath);
await fsModule.writeFile(outputPath, output.join('\n'));
}
export function generateFileHeader(config: RepopackConfig): string {
const defaultHeader = `${LONG_SEPARATOR}
REPOPACK OUTPUT FILE
${LONG_SEPARATOR}
This file was generated by Repopack on: ${new Date().toISOString()}
Purpose:
--------
This file contains a packed representation of the entire repository's contents.
It is designed to be easily consumable by AI systems for analysis, code review,
or other automated processes.
File Format:
------------
The content is organized as follows:
1. This header section
2. Multiple file entries, each consisting of:
a. A separator line (${SEPARATOR})
b. The file path (File: path/to/file)
c. Another separator line
d. The full contents of the file
e. A blank line
Usage Guidelines:
-----------------
1. This file should be treated as read-only. Any changes should be made to the
original repository files, not this packed version.
2. When processing this file, use the separators and "File:" markers to
distinguish between different files in the repository.
3. Be aware that this file may contain sensitive information. Handle it with
the same level of security as you would the original repository.
Notes:
------
- Some files may have been excluded based on .gitignore rules and Repopack's
configuration.
- Binary files are not included in this packed representation.
For more information about Repopack, visit: https://github.com/yamadashy/repopack
`;
let headerText = defaultHeader;
if (config.output.headerText) {
headerText += `
Additional User-Provided Header:
--------------------------------
${config.output.headerText}
`;
}
headerText += `
${LONG_SEPARATOR}
Repository Files
${LONG_SEPARATOR}
`;
return headerText;
}

91
src/core/packager.ts Normal file
View File

@@ -0,0 +1,91 @@
import * as fs from 'fs/promises';
import path from 'path';
import { RepopackConfig } from '../types/index.js';
import { processFile as defaultProcessFile } from '../utils/fileHandler.js';
import {
getGitignorePatterns as defaultGetGitignorePatterns,
createIgnoreFilter as defaultCreateIgnoreFilter,
} from '../utils/gitignoreUtils.js';
import { generateOutput as defaultGenerateOutput } from './outputGenerator.js';
import { defaultIgnoreList } from '../utils/defaultIgnore.js';
export interface Dependencies {
getGitignorePatterns: typeof defaultGetGitignorePatterns;
createIgnoreFilter: typeof defaultCreateIgnoreFilter;
processFile: typeof defaultProcessFile;
generateOutput: typeof defaultGenerateOutput;
}
export interface PackResult {
totalFiles: number;
totalCharacters: number;
}
export async function pack(
rootDir: string,
config: RepopackConfig,
deps: Dependencies = {
getGitignorePatterns: defaultGetGitignorePatterns,
createIgnoreFilter: defaultCreateIgnoreFilter,
processFile: defaultProcessFile,
generateOutput: defaultGenerateOutput,
},
): Promise<PackResult> {
const gitignorePatterns = await deps.getGitignorePatterns(rootDir);
const ignorePatterns = getIgnorePatterns(gitignorePatterns, config);
const ignoreFilter = deps.createIgnoreFilter(ignorePatterns);
const packedFiles = await packDirectory(rootDir, '', config, ignoreFilter, deps);
const totalFiles = packedFiles.length;
const totalCharacters = packedFiles.reduce((sum, file) => sum + file.content.length, 0);
await deps.generateOutput(rootDir, config, packedFiles);
return {
totalFiles,
totalCharacters,
};
}
function getIgnorePatterns(gitignorePatterns: string[], config: RepopackConfig): string[] {
let ignorePatterns = [...gitignorePatterns];
if (config.ignore.useDefaultPatterns) {
ignorePatterns = [...ignorePatterns, ...defaultIgnoreList];
}
if (config.ignore.customPatterns) {
ignorePatterns = [...ignorePatterns, ...config.ignore.customPatterns];
}
return ignorePatterns;
}
async function packDirectory(
dir: string,
relativePath: string,
config: RepopackConfig,
ignoreFilter: (path: string) => boolean,
deps: Dependencies,
): Promise<{ path: string; content: string }[]> {
const entries = await fs.readdir(dir, { withFileTypes: true });
const packedFiles: { path: string; content: string }[] = [];
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
const entryRelativePath = path.join(relativePath, entry.name);
if (!ignoreFilter(entryRelativePath)) continue;
if (entry.isDirectory()) {
const subDirFiles = await packDirectory(fullPath, entryRelativePath, config, ignoreFilter, deps);
packedFiles.push(...subDirFiles);
} else {
const content = await deps.processFile(fullPath);
if (content) {
packedFiles.push({ path: entryRelativePath, content });
}
}
}
return packedFiles;
}

3
src/index.ts Normal file
View File

@@ -0,0 +1,3 @@
export { pack } from './core/packager.js';
export type { RepopackConfig } from './types/index.js';
export { run as cli } from './cli/index.js';

10
src/types/index.ts Normal file
View File

@@ -0,0 +1,10 @@
export interface RepopackConfig {
output: {
filePath: string;
headerText?: string;
};
ignore: {
useDefaultPatterns: boolean;
customPatterns?: string[];
};
}

123
src/utils/defaultIgnore.ts Normal file
View File

@@ -0,0 +1,123 @@
export const defaultIgnoreList = [
// Version control
'.git',
'.gitignore',
'.gitattributes',
'.hg',
'.hgignore',
'.svn',
// Dependency directories
'node_modules',
'bower_components',
// Logs
'logs',
'*.log',
'npm-debug.log*',
'yarn-debug.log*',
'yarn-error.log*',
// Runtime data
'pids',
'*.pid',
'*.seed',
'*.pid.lock',
// Directory for instrumented libs generated by jscoverage/JSCover
'lib-cov',
// Coverage directory used by tools like istanbul
'coverage',
// nyc test coverage
'.nyc_output',
// Grunt intermediate storage
'.grunt',
// Bower dependency directory
'bower_components',
// node-waf configuration
'.lock-wscript',
// Compiled binary addons
'build/Release',
// Dependency directories
'jspm_packages/',
// TypeScript v1 declaration files
'typings/',
// Optional npm cache directory
'.npm',
// Optional eslint cache
'.eslintcache',
// Optional REPL history
'.node_repl_history',
// Output of 'npm pack'
'*.tgz',
// Yarn files
'.yarn/*',
// Yarn Integrity file
'.yarn-integrity',
// dotenv environment variables file
'.env',
// next.js build output
'.next',
// nuxt.js build output
'.nuxt',
// vuepress build output
'.vuepress/dist',
// Serverless directories
'.serverless/',
// FuseBox cache
'.fusebox/',
// DynamoDB Local files
'.dynamodb/',
// TypeScript output
'dist',
// OS generated files
'.DS_Store',
'Thumbs.db',
// Editor directories and files
'.idea',
'.vscode',
'*.swp',
'*.swo',
'*.swn',
'*.bak',
// Package manager locks
'package-lock.json',
'yarn.lock',
'pnpm-lock.yaml',
// Build outputs
'build',
'out',
// Temporary files
'tmp',
'temp',
// repopack output
'repopack-output.txt',
];

22
src/utils/errorHandler.ts Normal file
View File

@@ -0,0 +1,22 @@
import { logger } from './logger.js';
export class RepopackError extends Error {
constructor(message: string) {
super(message);
this.name = 'RepopackError';
}
}
export function handleError(error: unknown): void {
if (error instanceof RepopackError) {
logger.error(`Error: ${error.message}`);
} else if (error instanceof Error) {
logger.error(`Unexpected error: ${error.message}`);
logger.debug('Stack trace:', error.stack);
} else {
logger.error('An unknown error occurred');
}
logger.info('For more help, please visit: https://github.com/yamadashy/repopack/issues');
process.exit(1);
}

39
src/utils/fileHandler.ts Normal file
View File

@@ -0,0 +1,39 @@
import * as fs from 'fs/promises';
import isBinaryPath from 'is-binary-path';
import jschardet from 'jschardet';
import iconv from 'iconv-lite';
export async function processFile(filePath: string, fsModule = fs): Promise<string | null> {
// Skip binary files
if (isBinaryPath(filePath)) {
return null;
}
try {
const buffer = await fsModule.readFile(filePath);
const encoding = jschardet.detect(buffer).encoding || 'utf-8';
const content = iconv.decode(buffer, encoding);
if (!isValidTextContent(content)) {
return null;
}
return preprocessContent(content);
} catch (error) {
console.warn(`Error processing file ${filePath}:`, error);
return null;
}
}
function isValidTextContent(content: string): boolean {
// Check the validity of the text
// If the percentage of non-printable characters is greater than a certain value, it is judged not to
const nonPrintableChars = content.replace(/[\x20-\x7E\n\r\t]/g, '');
// If the percentage of non-printable characters is greater than 10%, it is judged not to be text
return nonPrintableChars.length / content.length < 0.1;
}
export function preprocessContent(content: string): string {
return content.trim();
}

View File

@@ -0,0 +1,26 @@
import * as fs from 'fs/promises';
import path from 'path';
import ignore from 'ignore';
export async function getGitignorePatterns(rootDir: string, fsModule = fs): Promise<string[]> {
const gitignorePath = path.join(rootDir, '.gitignore');
try {
const gitignoreContent = await fsModule.readFile(gitignorePath, 'utf-8');
return parseGitignoreContent(gitignoreContent);
} catch (error) {
console.warn('No .gitignore file found or unable to read it.');
return [];
}
}
export function parseGitignoreContent(content: string): string[] {
return content
.split('\n')
.map((line) => line.trim())
.filter((line) => line && !line.startsWith('#'));
}
export function createIgnoreFilter(patterns: string[]): (path: string) => boolean {
const ig = ignore.default().add(patterns);
return (filePath: string) => !ig.ignores(filePath);
}

58
src/utils/logger.ts Normal file
View File

@@ -0,0 +1,58 @@
import pc from 'picocolors';
import util from 'util';
class Logger {
private isVerbose = false;
setVerbose(value: boolean) {
this.isVerbose = value;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
error(...args: any[]) {
console.error(pc.red(this.formatArgs(args)));
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
warn(...args: any[]) {
console.log(pc.yellow(this.formatArgs(args)));
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
success(...args: any[]) {
console.log(pc.green(this.formatArgs(args)));
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
info(...args: any[]) {
console.log(pc.cyan(this.formatArgs(args)));
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
note(...args: string[]) {
console.log(pc.dim(this.formatArgs(args)));
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
debug(...args: unknown[]) {
if (this.isVerbose) {
console.log(pc.blue(this.formatArgs(args)));
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
trace(...args: any[]) {
if (this.isVerbose) {
console.log(pc.gray(this.formatArgs(args)));
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private formatArgs(args: any[]): string {
return args
.map((arg) => (typeof arg === 'object' ? util.inspect(arg, { depth: null, colors: true }) : arg))
.join(' ');
}
}
export const logger = new Logger();

View File

@@ -0,0 +1,22 @@
import path from 'path';
import * as fs from 'fs/promises';
export async function getVersion(): Promise<string> {
try {
const packageJson = await getPackageJson();
return packageJson.version;
} catch (error) {
console.error('Error reading package.json:', error);
return 'unknown';
}
}
async function getPackageJson(): Promise<{
name: string;
version: string;
}> {
const packageJsonPath = path.join(import.meta.dirname, '..', '..', 'package.json');
const packageJsonFile = await fs.readFile(packageJsonPath, 'utf-8');
const packageJson = JSON.parse(packageJsonFile);
return packageJson;
}

41
src/utils/spinner.ts Normal file
View File

@@ -0,0 +1,41 @@
import cliSpinners from 'cli-spinners';
import logUpdate from 'log-update';
import pc from 'picocolors';
class Spinner {
private spinner = cliSpinners.dots;
private message: string;
private currentFrame = 0;
private interval: ReturnType<typeof setInterval> | null = null;
constructor(message: string) {
this.message = message;
}
start(): void {
this.interval = setInterval(() => {
const frame = this.spinner.frames[this.currentFrame];
logUpdate(`${pc.cyan(frame)} ${this.message}`);
this.currentFrame = (this.currentFrame + 1) % this.spinner.frames.length;
}, this.spinner.interval);
}
stop(finalMessage: string): void {
if (this.interval) {
clearInterval(this.interval);
this.interval = null;
}
logUpdate(finalMessage);
logUpdate.done();
}
succeed(message: string): void {
this.stop(`${pc.green('✔')} ${message}`);
}
fail(message: string): void {
this.stop(`${pc.red('✖')} ${message}`);
}
}
export default Spinner;

View File

@@ -0,0 +1,61 @@
import { expect, test, describe } from 'vitest';
import { mergeConfigs } from '../../src/config/configLoader.js';
import { RepopackConfig } from '../../src/types/index.js';
import { defaultConfig } from '../../src/config/defaultConfig.js';
describe('configLoader', () => {
test('mergeConfigs should correctly merge configs', () => {
const fileConfig: Partial<RepopackConfig> = {
output: {
filePath: 'file-output.txt',
headerText: 'File header',
},
ignore: {
useDefaultPatterns: true,
customPatterns: ['file-ignore'],
},
};
const cliConfig: Partial<RepopackConfig> = {
output: {
filePath: 'cli-output.txt',
},
ignore: {
useDefaultPatterns: true,
customPatterns: ['cli-ignore'],
},
};
const mergedConfig = mergeConfigs(fileConfig, cliConfig);
expect(mergedConfig).toEqual({
output: {
filePath: 'cli-output.txt',
headerText: 'File header',
},
ignore: {
useDefaultPatterns: true,
customPatterns: ['file-ignore', 'cli-ignore'],
},
});
});
test('mergeConfigs should use default values when not provided', () => {
const mergedConfig = mergeConfigs({}, {});
expect(mergedConfig).toEqual(defaultConfig);
});
test('mergeConfigs should override default headerText', () => {
const fileConfig: Partial<RepopackConfig> = {
output: {
filePath: 'file-output.txt',
headerText: 'Custom header',
},
};
const mergedConfig = mergeConfigs(fileConfig, {});
expect(mergedConfig.output.headerText).toBe('Custom header');
});
});

View File

@@ -0,0 +1,51 @@
import { expect, test, vi, describe, beforeEach } from 'vitest';
import { generateOutput, generateFileHeader } from '../../src/core/outputGenerator.js';
import { RepopackConfig } from '../../src/types/index.js';
import * as fs from 'fs/promises';
import path from 'path';
vi.mock('fs/promises');
describe('outputGenerator', () => {
beforeEach(() => {
vi.resetAllMocks();
});
test('generateOutput should write correct content to file', async () => {
const mockConfig: RepopackConfig = {
output: { filePath: 'output.txt' },
ignore: { useDefaultPatterns: true },
};
const mockPackedFiles = [
{ path: 'file1.txt', content: 'content1' },
{ path: 'dir/file2.txt', content: 'content2' },
];
await generateOutput('root', mockConfig, mockPackedFiles);
expect(fs.writeFile).toHaveBeenCalledTimes(1);
expect(vi.mocked(fs.writeFile).mock.calls[0][0]).toBe(path.resolve('root', 'output.txt'));
const writtenContent = vi.mocked(fs.writeFile).mock.calls[0][1] as string;
expect(writtenContent).toContain('REPOPACK OUTPUT FILE');
expect(writtenContent).toContain('File: file1.txt');
expect(writtenContent).toContain('content1');
expect(writtenContent).toContain('File: dir/file2.txt');
expect(writtenContent).toContain('content2');
});
test('generateFileHeader should include user-provided header text', () => {
const mockConfig: RepopackConfig = {
output: {
filePath: 'output.txt',
headerText: 'Custom header text',
},
ignore: { useDefaultPatterns: true },
};
const header = generateFileHeader(mockConfig);
expect(header).toContain('REPOPACK OUTPUT FILE');
expect(header).toContain('Custom header text');
});
});

View File

@@ -0,0 +1,47 @@
import { expect, test, vi, describe, beforeEach } from 'vitest';
import { pack, Dependencies } from '../../src/core/packager.js';
import { RepopackConfig } from '../../src/types/index.js';
import path from 'path';
import * as fs from 'fs/promises';
import { Dirent } from 'fs';
vi.mock('fs/promises');
describe('packager', () => {
let mockDeps: Dependencies;
beforeEach(() => {
vi.resetAllMocks();
mockDeps = {
getGitignorePatterns: vi.fn().mockResolvedValue([]),
createIgnoreFilter: vi.fn().mockReturnValue(() => true),
processFile: vi.fn().mockResolvedValue('processed content'),
generateOutput: vi.fn().mockResolvedValue(undefined),
};
});
test('pack should process files and generate output', async () => {
const mockConfig: RepopackConfig = {
output: { filePath: 'output.txt' },
ignore: { useDefaultPatterns: true },
};
vi.mocked(fs.readdir)
.mockResolvedValueOnce([
{ name: 'file1.txt', isDirectory: () => false },
{ name: 'dir1', isDirectory: () => true },
] as Dirent[])
.mockResolvedValueOnce([{ name: 'file2.txt', isDirectory: () => false }] as Dirent[]);
await pack('root', mockConfig, mockDeps);
expect(fs.readdir).toHaveBeenCalledTimes(2);
expect(vi.mocked(fs.readdir).mock.calls[0][0]).toBe('root');
expect(vi.mocked(fs.readdir).mock.calls[1][0]).toBe(path.join('root', 'dir1'));
expect(mockDeps.generateOutput).toHaveBeenCalledWith('root', mockConfig, [
{ path: 'file1.txt', content: 'processed content' },
{ path: path.join('dir1', 'file2.txt'), content: 'processed content' },
]);
});
});

View File

@@ -0,0 +1,28 @@
import { expect, test, vi, describe, beforeEach } from 'vitest';
import { processFile, preprocessContent } from '../../src/utils/fileHandler.js';
import * as fs from 'fs/promises';
vi.mock('fs/promises');
describe('fileHandler', () => {
beforeEach(() => {
vi.resetAllMocks();
});
test('processFile should read and preprocess file content', async () => {
const mockContent = ' Some file content \n';
vi.mocked(fs.readFile).mockResolvedValue(mockContent);
const result = await processFile('/path/to/file.txt');
expect(fs.readFile).toHaveBeenCalledWith('/path/to/file.txt');
expect(result).toBe('Some file content');
});
test('preprocessContent should trim content', () => {
const content = ' Some content with whitespace \n';
const result = preprocessContent(content);
expect(result).toBe('Some content with whitespace');
});
});

View File

@@ -0,0 +1,59 @@
import { expect, test, vi, describe, beforeEach } from 'vitest';
import { getGitignorePatterns, parseGitignoreContent, createIgnoreFilter } from '../../src/utils/gitignoreUtils.js';
import path from 'path';
import * as fs from 'fs/promises';
vi.mock('fs/promises');
describe('gitignoreUtils', () => {
beforeEach(() => {
vi.resetAllMocks();
});
test('getGitignorePatterns should read and parse .gitignore file', async () => {
const mockContent = `
# Comment
node_modules
*.log
.DS_Store
`;
vi.mocked(fs.readFile).mockResolvedValue(mockContent);
const patterns = await getGitignorePatterns('/mock/root');
expect(fs.readFile).toHaveBeenCalledWith(path.join('/mock/root', '.gitignore'), 'utf-8');
expect(patterns).toEqual(['node_modules', '*.log', '.DS_Store']);
});
test('getGitignorePatterns should return empty array if .gitignore is not found', async () => {
vi.mocked(fs.readFile).mockRejectedValue(new Error('File not found'));
const patterns = await getGitignorePatterns('/mock/root');
expect(patterns).toEqual([]);
});
test('parseGitignoreContent should correctly parse gitignore content', () => {
const content = `
# Comment
node_modules
*.log
.DS_Store
`;
const patterns = parseGitignoreContent(content);
expect(patterns).toEqual(['node_modules', '*.log', '.DS_Store']);
});
test('createIgnoreFilter should create a function that correctly filters paths', () => {
const patterns = ['node_modules', '*.log', '.DS_Store'];
const filter = createIgnoreFilter(patterns);
expect(filter('src/index.js')).toBe(true);
expect(filter('node_modules/package/index.js')).toBe(false);
expect(filter('logs/error.log')).toBe(false);
expect(filter('.DS_Store')).toBe(false);
});
});

7
tsconfig.build.json Normal file
View File

@@ -0,0 +1,7 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"rootDir": "./src"
},
"include": ["./src/**/*"]
}

21
tsconfig.json Normal file
View File

@@ -0,0 +1,21 @@
{
"compileOnSave": false,
"compilerOptions": {
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "es2016",
"outDir": "./lib",
"rootDir": ".",
"strict": true,
"esModuleInterop": true,
"noImplicitAny": true,
"skipLibCheck": true,
"lib": ["es2022"],
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"types": ["node", "picocolors"]
}
}

12
vite.config.mts Normal file
View File

@@ -0,0 +1,12 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
test: {
globals: true,
environment: 'node',
include: ['tests/**/*.test.ts'],
coverage: {
reporter: ['text', 'json', 'html'],
},
}
})