test(core): improve test coverage for core components

This commit is contained in:
Kazuki Yamada
2024-11-16 17:52:24 +09:00
parent 8a9424129d
commit e34a8dabd2
12 changed files with 790 additions and 86 deletions

View File

@@ -2,6 +2,7 @@
"$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
"files": {
"include": [
"./bin/**",
"./src/**",
"./tests/**",
"package.json",

View File

@@ -38,7 +38,7 @@ export const runInitAction = async (rootDir: string, isGlobal: boolean): Promise
}
};
export async function createConfigFile(rootDir: string, isGlobal: boolean): Promise<boolean> {
export const createConfigFile = async (rootDir: string, isGlobal: boolean): Promise<boolean> => {
const configPath = path.resolve(isGlobal ? getGlobalDirectory() : rootDir, 'repomix.config.json');
const isCreateConfig = await prompts.confirm({
@@ -121,9 +121,9 @@ export async function createConfigFile(rootDir: string, isGlobal: boolean): Prom
);
return true;
}
};
export async function createIgnoreFile(rootDir: string, isGlobal: boolean): Promise<boolean> {
export const createIgnoreFile = async (rootDir: string, isGlobal: boolean): Promise<boolean> => {
if (isGlobal) {
prompts.log.info(`Skipping ${pc.green('.repomixignore')} file creation for global configuration.`);
return false;
@@ -173,4 +173,4 @@ export async function createIgnoreFile(rootDir: string, isGlobal: boolean): Prom
);
return true;
}
};

View File

@@ -52,13 +52,13 @@ export const formatGitUrl = (url: string): string => {
return url;
};
const createTempDirectory = async (): Promise<string> => {
export const createTempDirectory = async (): Promise<string> => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'repomix-'));
logger.trace(`Created temporary directory. (path: ${pc.dim(tempDir)})`);
return tempDir;
};
const cloneRepository = async (url: string, directory: string): Promise<void> => {
export const cloneRepository = async (url: string, directory: string): Promise<void> => {
logger.log(`Clone repository: ${url} to temporary directory. ${pc.dim(`path: ${directory}`)}`);
logger.log('');
@@ -69,12 +69,12 @@ const cloneRepository = async (url: string, directory: string): Promise<void> =>
}
};
const cleanupTempDirectory = async (directory: string): Promise<void> => {
export const cleanupTempDirectory = async (directory: string): Promise<void> => {
logger.trace(`Cleaning up temporary directory: ${directory}`);
await fs.rm(directory, { recursive: true, force: true });
};
const checkGitInstallation = async (): Promise<boolean> => {
export const checkGitInstallation = async (): Promise<boolean> => {
try {
const result = await execAsync('git --version');
return !result.stderr;
@@ -84,7 +84,7 @@ const checkGitInstallation = async (): Promise<boolean> => {
}
};
const copyOutputToCurrentDirectory = async (
export const copyOutputToCurrentDirectory = async (
sourceDir: string,
targetDir: string,
outputFileName: string,

View File

@@ -26,7 +26,7 @@ export interface CliOptions extends OptionValues {
remote?: string;
}
export async function run() {
export const run = async () => {
try {
program
.description('Repomix - Pack your repository into a single AI-friendly file')
@@ -50,9 +50,9 @@ export async function run() {
} catch (error) {
handleError(error);
}
}
};
const executeAction = async (directory: string, cwd: string, options: CliOptions) => {
export const executeAction = async (directory: string, cwd: string, options: CliOptions) => {
logger.setVerbose(options.verbose || false);
if (options.version) {

View File

@@ -24,7 +24,7 @@ export class PermissionError extends Error {
}
}
export async function checkDirectoryPermissions(dirPath: string): Promise<PermissionCheckResult> {
export const checkDirectoryPermissions = async (dirPath: string): Promise<PermissionCheckResult> => {
try {
// First try to read directory contents
await fs.readdir(dirPath);
@@ -87,9 +87,9 @@ export async function checkDirectoryPermissions(dirPath: string): Promise<Permis
error: error instanceof Error ? error : new Error(String(error)),
};
}
}
};
function getMacOSPermissionMessage(dirPath: string, errorCode?: string): string {
const getMacOSPermissionMessage = (dirPath: string, errorCode?: string): string => {
if (platform() === 'darwin') {
return `Permission denied: Cannot access '${dirPath}', error code: ${errorCode}.
@@ -109,4 +109,4 @@ If your terminal app is not listed:
}
return `Permission denied: Cannot access '${dirPath}'`;
}
};

View File

@@ -1,43 +1,88 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { formatGitUrl } from '../../../src/cli/actions/remoteAction.js';
import * as fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import {
cleanupTempDirectory,
copyOutputToCurrentDirectory,
createTempDirectory,
formatGitUrl,
} from '../../../src/cli/actions/remoteAction.js';
vi.mock('node:fs/promises');
vi.mock('node:child_process');
vi.mock('../../../src/cli/actions/defaultAction.js');
vi.mock('../../../src/shared/logger.js');
vi.mock('node:fs/promises');
vi.mock('node:os');
vi.mock('../../../src/shared/logger');
describe('remoteAction', () => {
describe('remoteAction functions', () => {
beforeEach(() => {
vi.resetAllMocks();
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('formatGitUrl', () => {
it('should format GitHub shorthand correctly', () => {
test('should convert GitHub shorthand to full URL', () => {
expect(formatGitUrl('user/repo')).toBe('https://github.com/user/repo.git');
expect(formatGitUrl('user-name/repo-name')).toBe('https://github.com/user-name/repo-name.git');
expect(formatGitUrl('user_name/repo_name')).toBe('https://github.com/user_name/repo_name.git');
});
it('should add .git to HTTPS URLs if missing', () => {
test('should handle HTTPS URLs', () => {
expect(formatGitUrl('https://github.com/user/repo')).toBe('https://github.com/user/repo.git');
});
it('should not modify URLs that are already correctly formatted', () => {
expect(formatGitUrl('https://github.com/user/repo.git')).toBe('https://github.com/user/repo.git');
expect(formatGitUrl('git@github.com:user/repo.git')).toBe('git@github.com:user/repo.git');
});
it('should not modify SSH URLs', () => {
expect(formatGitUrl('git@github.com:user/repo.git')).toBe('git@github.com:user/repo.git');
test('should not modify SSH URLs', () => {
const sshUrl = 'git@github.com:user/repo.git';
expect(formatGitUrl(sshUrl)).toBe(sshUrl);
});
});
describe('createTempDirectory', () => {
test('should create temporary directory', async () => {
const mockTempDir = '/mock/temp/dir';
vi.mocked(os.tmpdir).mockReturnValue('/mock/temp');
vi.mocked(fs.mkdtemp).mockResolvedValue(mockTempDir);
const result = await createTempDirectory();
expect(result).toBe(mockTempDir);
expect(fs.mkdtemp).toHaveBeenCalledWith(path.join('/mock/temp', 'repomix-'));
});
});
describe('cleanupTempDirectory', () => {
test('should cleanup directory', async () => {
const mockDir = '/mock/temp/dir';
vi.mocked(fs.rm).mockResolvedValue();
await cleanupTempDirectory(mockDir);
expect(fs.rm).toHaveBeenCalledWith(mockDir, { recursive: true, force: true });
});
});
describe('copyOutputToCurrentDirectory', () => {
test('should copy output file', async () => {
const sourceDir = '/source/dir';
const targetDir = '/target/dir';
const fileName = 'output.txt';
vi.mocked(fs.copyFile).mockResolvedValue();
await copyOutputToCurrentDirectory(sourceDir, targetDir, fileName);
expect(fs.copyFile).toHaveBeenCalledWith(path.join(sourceDir, fileName), path.join(targetDir, fileName));
});
it('should not modify URLs from other Git hosting services', () => {
expect(formatGitUrl('https://gitlab.com/user/repo.git')).toBe('https://gitlab.com/user/repo.git');
expect(formatGitUrl('https://bitbucket.org/user/repo.git')).toBe('https://bitbucket.org/user/repo.git');
test('should throw error when copy fails', async () => {
const sourceDir = '/source/dir';
const targetDir = '/target/dir';
const fileName = 'output.txt';
vi.mocked(fs.copyFile).mockRejectedValue(new Error('Permission denied'));
await expect(copyOutputToCurrentDirectory(sourceDir, targetDir, fileName)).rejects.toThrow(
'Failed to copy output file',
);
});
});
});

118
tests/cli/cliRun.test.ts Normal file
View File

@@ -0,0 +1,118 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
import * as defaultAction from '../../src/cli/actions/defaultAction.js';
import * as initAction from '../../src/cli/actions/initAction.js';
import * as remoteAction from '../../src/cli/actions/remoteAction.js';
import * as versionAction from '../../src/cli/actions/versionAction.js';
import { executeAction, run } from '../../src/cli/cliRun.js';
import type { RepomixConfigMerged } from '../../src/config/configSchema.js';
import type { PackResult } from '../../src/core/packager.js';
import { logger } from '../../src/shared/logger.js';
vi.mock('../../src/shared/logger', () => ({
logger: {
log: vi.fn(),
trace: vi.fn(),
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
success: vi.fn(),
note: vi.fn(),
setVerbose: vi.fn(),
},
}));
vi.mock('commander', () => ({
program: {
description: vi.fn().mockReturnThis(),
arguments: vi.fn().mockReturnThis(),
option: vi.fn().mockReturnThis(),
action: vi.fn().mockReturnThis(),
parseAsync: vi.fn().mockResolvedValue(undefined),
},
}));
vi.mock('../../src/cli/actions/defaultAction');
vi.mock('../../src/cli/actions/initAction');
vi.mock('../../src/cli/actions/remoteAction');
vi.mock('../../src/cli/actions/versionAction');
describe('cliRun', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(defaultAction.runDefaultAction).mockResolvedValue({
config: {
cwd: process.cwd(),
output: {
filePath: 'repomix-output.txt',
style: 'plain',
topFilesLength: 5,
showLineNumbers: false,
removeComments: false,
removeEmptyLines: false,
copyToClipboard: false,
},
include: [],
ignore: {
useGitignore: true,
useDefaultPatterns: true,
customPatterns: [],
},
security: {
enableSecurityCheck: true,
},
} satisfies RepomixConfigMerged,
packResult: {
totalFiles: 0,
totalCharacters: 0,
totalTokens: 0,
fileCharCounts: {},
fileTokenCounts: {},
suspiciousFilesResults: [],
} satisfies PackResult,
});
vi.mocked(initAction.runInitAction).mockResolvedValue();
vi.mocked(remoteAction.runRemoteAction).mockResolvedValue();
vi.mocked(versionAction.runVersionAction).mockResolvedValue();
});
test('should run without arguments', async () => {
await expect(run()).resolves.not.toThrow();
});
describe('executeAction', () => {
test('should execute default action when no special options provided', async () => {
await executeAction('.', process.cwd(), {});
expect(defaultAction.runDefaultAction).toHaveBeenCalledWith('.', process.cwd(), expect.any(Object));
});
test('should enable verbose logging when verbose option is true', async () => {
await executeAction('.', process.cwd(), { verbose: true });
expect(logger.setVerbose).toHaveBeenCalledWith(true);
});
test('should execute version action when version option is true', async () => {
await executeAction('.', process.cwd(), { version: true });
expect(versionAction.runVersionAction).toHaveBeenCalled();
expect(defaultAction.runDefaultAction).not.toHaveBeenCalled();
});
test('should execute init action when init option is true', async () => {
await executeAction('.', process.cwd(), { init: true });
expect(initAction.runInitAction).toHaveBeenCalledWith(process.cwd(), false);
expect(defaultAction.runDefaultAction).not.toHaveBeenCalled();
});
test('should execute remote action when remote option is provided', async () => {
await executeAction('.', process.cwd(), { remote: 'yamadashy/repomix' });
expect(remoteAction.runRemoteAction).toHaveBeenCalledWith('yamadashy/repomix', expect.any(Object));
expect(defaultAction.runDefaultAction).not.toHaveBeenCalled();
});
});
});

View File

@@ -0,0 +1,242 @@
import { constants } from 'node:fs';
import * as fs from 'node:fs/promises';
import { platform } from 'node:os';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { PermissionError, checkDirectoryPermissions } from '../../../src/core/file/permissionCheck.js';
vi.mock('node:fs/promises');
vi.mock('node:os');
describe('permissionCheck', () => {
const testDirPath = '/test/directory';
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(platform).mockReturnValue('linux');
});
describe('successful cases', () => {
test('should return success when all permissions are available', async () => {
// Mock successful readdir
vi.mocked(fs.readdir).mockResolvedValue([]);
// Mock successful access checks
vi.mocked(fs.access).mockResolvedValue(undefined);
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: true,
details: {
read: true,
write: true,
execute: true,
},
});
// Verify all permission checks were called
expect(fs.access).toHaveBeenCalledWith(testDirPath, constants.R_OK);
expect(fs.access).toHaveBeenCalledWith(testDirPath, constants.W_OK);
expect(fs.access).toHaveBeenCalledWith(testDirPath, constants.X_OK);
});
test('should pass with only required permissions', async () => {
// Mock successful readdir
vi.mocked(fs.readdir).mockResolvedValue([]);
// Mock mixed permission check results
vi.mocked(fs.access).mockImplementation(async (path, mode) => {
if (mode === constants.R_OK || mode === constants.X_OK) {
return Promise.resolve(undefined);
}
return Promise.reject(new Error('Permission denied'));
});
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
details: {
read: true,
write: false,
execute: true,
},
});
});
});
describe('error cases', () => {
test('should handle EPERM error', async () => {
const error = new Error('Permission denied');
(error as NodeJS.ErrnoException).code = 'EPERM';
vi.mocked(fs.readdir).mockRejectedValue(error);
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
error: expect.any(PermissionError),
});
expect(result.error).toBeInstanceOf(PermissionError);
expect(result.error?.message).toContain('Permission denied');
});
test('should handle EACCES error', async () => {
const error = new Error('Access denied');
(error as NodeJS.ErrnoException).code = 'EACCES';
vi.mocked(fs.readdir).mockRejectedValue(error);
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
error: expect.any(PermissionError),
});
expect(result.error).toBeInstanceOf(PermissionError);
expect(result.error?.message).toContain('Permission denied');
});
test('should handle EISDIR error', async () => {
const error = new Error('Is a directory');
(error as NodeJS.ErrnoException).code = 'EISDIR';
vi.mocked(fs.readdir).mockRejectedValue(error);
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
error: expect.any(PermissionError),
});
});
test('should handle non-Error objects', async () => {
vi.mocked(fs.readdir).mockRejectedValue('String error');
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
error: new Error('String error'),
});
});
});
describe('platform specific behavior', () => {
test('should return macOS specific error message', async () => {
// Mock platform as macOS
vi.mocked(platform).mockReturnValue('darwin');
const error = new Error('Permission denied');
(error as NodeJS.ErrnoException).code = 'EACCES';
vi.mocked(fs.readdir).mockRejectedValue(error);
const result = await checkDirectoryPermissions(testDirPath);
expect(result.error).toBeInstanceOf(PermissionError);
expect(result.error?.message).toContain('macOS security restrictions');
expect(result.error?.message).toContain('System Settings');
expect(result.error?.message).toContain('Privacy & Security');
});
test('should return standard error message for non-macOS platforms', async () => {
// Mock platform as Windows
vi.mocked(platform).mockReturnValue('win32');
const error = new Error('Permission denied');
(error as NodeJS.ErrnoException).code = 'EACCES';
vi.mocked(fs.readdir).mockRejectedValue(error);
const result = await checkDirectoryPermissions(testDirPath);
expect(result.error).toBeInstanceOf(PermissionError);
expect(result.error?.message).toBe(`Permission denied: Cannot access '${testDirPath}'`);
expect(result.error?.message).not.toContain('macOS security restrictions');
});
});
describe('PermissionError class', () => {
test('should create PermissionError with correct properties', () => {
const message = 'Test error message';
const path = '/test/path';
const code = 'EACCES';
const error = new PermissionError(message, path, code);
expect(error).toBeInstanceOf(Error);
expect(error.name).toBe('PermissionError');
expect(error.message).toBe(message);
expect(error.path).toBe(path);
expect(error.code).toBe(code);
});
test('should create PermissionError without code', () => {
const message = 'Test error message';
const path = '/test/path';
const error = new PermissionError(message, path);
expect(error).toBeInstanceOf(Error);
expect(error.name).toBe('PermissionError');
expect(error.message).toBe(message);
expect(error.path).toBe(path);
expect(error.code).toBeUndefined();
});
});
describe('edge cases', () => {
test('should handle undefined error code', async () => {
const error = new Error('Permission denied');
vi.mocked(fs.readdir).mockRejectedValue(error);
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
error: error,
});
});
test('should handle partial permission checks failing', async () => {
// Mock successful readdir
vi.mocked(fs.readdir).mockResolvedValue([]);
// Mock access to fail for write permission only
vi.mocked(fs.access).mockImplementation(async (path, mode) => {
if (mode === constants.W_OK) {
throw new Error('Write permission denied');
}
return Promise.resolve(undefined);
});
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
details: {
read: true,
write: false,
execute: true,
},
});
});
test('should handle all permission checks failing', async () => {
// Mock successful readdir
vi.mocked(fs.readdir).mockResolvedValue([]);
// Mock all access checks to fail
vi.mocked(fs.access).mockRejectedValue(new Error('Permission denied'));
const result = await checkDirectoryPermissions(testDirPath);
expect(result).toEqual({
hasPermission: false,
details: {
read: false,
write: false,
execute: false,
},
});
});
});
});

View File

@@ -1,32 +1,202 @@
import process from 'node:process';
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { generateOutput } from '../../../../src/core/output/outputGenerate.js';
import { createMockConfig } from '../../../testing/testUtils.js';
vi.mock('fs/promises');
import Handlebars from 'handlebars';
import { describe, expect, test } from 'vitest';
import { getMarkdownTemplate } from '../../../../src/core/output/outputStyles/markdownStyle.js';
describe('markdownStyle', () => {
beforeEach(() => {
vi.resetAllMocks();
});
test('generateOutput for md should include user-provided header text', async () => {
const mockConfig = createMockConfig({
output: {
filePath: 'output.md',
style: 'markdown',
headerText: 'Custom header text',
topFilesLength: 2,
showLineNumbers: false,
removeComments: false,
removeEmptyLines: false,
},
describe('getMarkdownTemplate', () => {
test('should return valid markdown template', () => {
const template = getMarkdownTemplate();
expect(template).toContain('# File Summary');
expect(template).toContain('# Repository Structure');
expect(template).toContain('# Repository Files');
expect(template).toContain('{{#if instruction}}');
expect(template).toContain('# Instruction');
});
const output = await generateOutput(process.cwd(), mockConfig, [], []);
test('should correctly render template with basic data', () => {
const template = getMarkdownTemplate();
const compiledTemplate = Handlebars.compile(template);
const data = {
generationHeader: 'Generated Test Header',
summaryPurpose: 'Test Purpose',
summaryFileFormat: 'Test Format',
summaryUsageGuidelines: 'Test Guidelines',
summaryNotes: 'Test Notes',
summaryAdditionalInfo: 'Test Additional Info',
treeString: 'src/\n index.ts',
processedFiles: [
{
path: 'src/index.ts',
content: 'console.log("Hello");',
},
],
};
expect(output).toContain('# File Summary');
expect(output).toContain('# Repository Structure');
expect(output).toContain('# Repository Files');
const result = compiledTemplate(data);
expect(result).toContain('Generated Test Header');
expect(result).toContain('Test Purpose');
expect(result).toContain('Test Format');
expect(result).toContain('Test Guidelines');
expect(result).toContain('Test Notes');
expect(result).toContain('Test Additional Info');
expect(result).toContain('src/\n index.ts');
expect(result).toContain('## File: src/index.ts');
expect(result).toContain('console.log("Hello");');
});
test('should render optional header text when provided', () => {
const template = getMarkdownTemplate();
const compiledTemplate = Handlebars.compile(template);
const data = {
headerText: 'Custom Header Text',
processedFiles: [],
};
const result = compiledTemplate(data);
expect(result).toContain('### User Provided Header');
expect(result).toContain('Custom Header Text');
});
test('should not render header section when headerText is not provided', () => {
const template = getMarkdownTemplate();
const compiledTemplate = Handlebars.compile(template);
const data = {
processedFiles: [],
};
const result = compiledTemplate(data);
expect(result).not.toContain('### User Provided Header');
});
test('should render instruction section when provided', () => {
const template = getMarkdownTemplate();
const compiledTemplate = Handlebars.compile(template);
const data = {
instruction: 'Custom Instruction Text',
processedFiles: [],
};
const result = compiledTemplate(data);
expect(result).toContain('# Instruction');
expect(result).toContain('Custom Instruction Text');
});
});
describe('getFileExtension helper', () => {
// Helper to get extension mapping result
const getExtension = (filePath: string): string => {
const helper = Handlebars.helpers.getFileExtension as Handlebars.HelperDelegate;
return helper(filePath) as string;
};
// JavaScript variants
test('should handle JavaScript related extensions', () => {
expect(getExtension('file.js')).toBe('javascript');
expect(getExtension('file.jsx')).toBe('javascript');
expect(getExtension('file.ts')).toBe('typescript');
expect(getExtension('file.tsx')).toBe('typescript');
});
// Web technologies
test('should handle web technology extensions', () => {
expect(getExtension('file.html')).toBe('html');
expect(getExtension('file.css')).toBe('css');
expect(getExtension('file.scss')).toBe('scss');
expect(getExtension('file.sass')).toBe('scss');
expect(getExtension('file.vue')).toBe('vue');
});
// Backend languages
test('should handle backend language extensions', () => {
expect(getExtension('file.py')).toBe('python');
expect(getExtension('file.rb')).toBe('ruby');
expect(getExtension('file.php')).toBe('php');
expect(getExtension('file.java')).toBe('java');
expect(getExtension('file.go')).toBe('go');
});
// System programming languages
test('should handle system programming language extensions', () => {
expect(getExtension('file.c')).toBe('cpp');
expect(getExtension('file.cpp')).toBe('cpp');
expect(getExtension('file.rs')).toBe('rust');
expect(getExtension('file.swift')).toBe('swift');
expect(getExtension('file.kt')).toBe('kotlin');
});
// Configuration and data format files
test('should handle configuration and data format extensions', () => {
expect(getExtension('file.json')).toBe('json');
expect(getExtension('file.json5')).toBe('json5');
expect(getExtension('file.xml')).toBe('xml');
expect(getExtension('file.yaml')).toBe('yaml');
expect(getExtension('file.yml')).toBe('yaml');
expect(getExtension('file.toml')).toBe('toml');
});
// Shell and scripting
test('should handle shell and scripting extensions', () => {
expect(getExtension('file.sh')).toBe('bash');
expect(getExtension('file.bash')).toBe('bash');
expect(getExtension('file.ps1')).toBe('powershell');
});
// Database and query languages
test('should handle database related extensions', () => {
expect(getExtension('file.sql')).toBe('sql');
expect(getExtension('file.graphql')).toBe('graphql');
expect(getExtension('file.gql')).toBe('graphql');
});
// Functional programming languages
test('should handle functional programming language extensions', () => {
expect(getExtension('file.fs')).toBe('fsharp');
expect(getExtension('file.fsx')).toBe('fsharp');
expect(getExtension('file.hs')).toBe('haskell');
expect(getExtension('file.clj')).toBe('clojure');
expect(getExtension('file.cljs')).toBe('clojure');
});
// Other languages and tools
test('should handle other programming language extensions', () => {
expect(getExtension('file.scala')).toBe('scala');
expect(getExtension('file.dart')).toBe('dart');
expect(getExtension('file.ex')).toBe('elixir');
expect(getExtension('file.exs')).toBe('elixir');
expect(getExtension('file.erl')).toBe('erlang');
expect(getExtension('file.coffee')).toBe('coffeescript');
});
// Infrastructure and templating
test('should handle infrastructure and templating extensions', () => {
expect(getExtension('file.tf')).toBe('hcl');
expect(getExtension('file.tfvars')).toBe('hcl');
expect(getExtension('file.dockerfile')).toBe('dockerfile');
expect(getExtension('file.pug')).toBe('pug');
expect(getExtension('file.proto')).toBe('protobuf');
});
// Miscellaneous
test('should handle miscellaneous file extensions', () => {
expect(getExtension('file.md')).toBe('markdown');
expect(getExtension('file.r')).toBe('r');
expect(getExtension('file.pl')).toBe('perl');
expect(getExtension('file.pm')).toBe('perl');
expect(getExtension('file.lua')).toBe('lua');
expect(getExtension('file.groovy')).toBe('groovy');
expect(getExtension('file.vb')).toBe('vb');
});
// Edge cases
test('should handle edge cases', () => {
expect(getExtension('file')).toBe(''); // No extension
expect(getExtension('.gitignore')).toBe(''); // Dotfile
expect(getExtension('file.unknown')).toBe(''); // Unknown extension
expect(getExtension('path/to/file.js')).toBe('javascript'); // Path with directory
});
});
});

View File

@@ -1,40 +1,167 @@
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
import { type Tiktoken, get_encoding } from 'tiktoken';
import { type Mock, afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import { TokenCounter } from '../../../src/core/tokenCount/tokenCount.js';
import { logger } from '../../../src/shared/logger.js';
describe('tokenCount', () => {
vi.mock('tiktoken', () => ({
get_encoding: vi.fn(),
}));
vi.mock('../../../src/shared/logger');
describe('TokenCounter', () => {
let tokenCounter: TokenCounter;
let mockEncoder: {
encode: Mock;
free: Mock;
};
beforeAll(() => {
beforeEach(() => {
// Initialize mock encoder
mockEncoder = {
encode: vi.fn(),
free: vi.fn(),
};
// Setup mock encoder behavior
vi.mocked(get_encoding).mockReturnValue(mockEncoder as unknown as Tiktoken);
// Create new TokenCounter instance
tokenCounter = new TokenCounter();
});
afterAll(() => {
afterEach(() => {
tokenCounter.free();
vi.resetAllMocks();
});
test('should correctly count tokens', () => {
const testCases = [
{ input: 'Hello, world!', expectedTokens: 4 },
{ input: 'This is a longer sentence with more tokens.', expectedTokens: 9 },
{ input: 'Special characters like !@#$%^&*() should be handled correctly.', expectedTokens: 15 },
{ input: 'Numbers 123 and symbols @#$ might affect tokenization.', expectedTokens: 12 },
{ input: 'Multi-line\ntext\nshould\nwork\ntoo.', expectedTokens: 11 },
];
for (const { input, expectedTokens } of testCases) {
const tokenCount = tokenCounter.countTokens(input);
expect(tokenCount).toBe(expectedTokens);
}
test('should initialize with cl100k_base encoding', () => {
expect(get_encoding).toHaveBeenCalledWith('cl100k_base');
});
test('should handle empty input', () => {
const tokenCount = tokenCounter.countTokens('');
expect(tokenCount).toBe(0);
test('should correctly count tokens for simple text', () => {
const text = 'Hello, world!';
const mockTokens = [123, 456, 789]; // Example token IDs
mockEncoder.encode.mockReturnValue(mockTokens);
const count = tokenCounter.countTokens(text);
expect(count).toBe(3); // Length of mockTokens
expect(mockEncoder.encode).toHaveBeenCalledWith(text);
});
test('should handle very long input', () => {
const longText = 'a'.repeat(1000);
const tokenCount = tokenCounter.countTokens(longText);
expect(tokenCount).toBeGreaterThan(0);
test('should handle empty string', () => {
mockEncoder.encode.mockReturnValue([]);
const count = tokenCounter.countTokens('');
expect(count).toBe(0);
expect(mockEncoder.encode).toHaveBeenCalledWith('');
});
test('should handle multi-line text', () => {
const text = 'Line 1\nLine 2\nLine 3';
const mockTokens = [1, 2, 3, 4, 5, 6];
mockEncoder.encode.mockReturnValue(mockTokens);
const count = tokenCounter.countTokens(text);
expect(count).toBe(6);
expect(mockEncoder.encode).toHaveBeenCalledWith(text);
});
test('should handle special characters', () => {
const text = '!@#$%^&*()_+';
const mockTokens = [1, 2, 3];
mockEncoder.encode.mockReturnValue(mockTokens);
const count = tokenCounter.countTokens(text);
expect(count).toBe(3);
expect(mockEncoder.encode).toHaveBeenCalledWith(text);
});
test('should handle unicode characters', () => {
const text = '你好,世界!🌍';
const mockTokens = [1, 2, 3, 4];
mockEncoder.encode.mockReturnValue(mockTokens);
const count = tokenCounter.countTokens(text);
expect(count).toBe(4);
expect(mockEncoder.encode).toHaveBeenCalledWith(text);
});
test('should handle code snippets', () => {
const text = `
function hello() {
console.log("Hello, world!");
}
`;
const mockTokens = Array(10).fill(1); // 10 tokens
mockEncoder.encode.mockReturnValue(mockTokens);
const count = tokenCounter.countTokens(text);
expect(count).toBe(10);
expect(mockEncoder.encode).toHaveBeenCalledWith(text);
});
test('should handle markdown text', () => {
const text = `
# Heading
## Subheading
* List item 1
* List item 2
**Bold text** and _italic text_
`;
const mockTokens = Array(15).fill(1); // 15 tokens
mockEncoder.encode.mockReturnValue(mockTokens);
const count = tokenCounter.countTokens(text);
expect(count).toBe(15);
expect(mockEncoder.encode).toHaveBeenCalledWith(text);
});
test('should handle very long text', () => {
const text = 'a'.repeat(10000);
const mockTokens = Array(100).fill(1); // 100 tokens
mockEncoder.encode.mockReturnValue(mockTokens);
const count = tokenCounter.countTokens(text);
expect(count).toBe(100);
expect(mockEncoder.encode).toHaveBeenCalledWith(text);
});
test('should properly handle encoding errors without file path', () => {
const error = new Error('Encoding error');
mockEncoder.encode.mockImplementation(() => {
throw error;
});
const count = tokenCounter.countTokens('test content');
expect(count).toBe(0);
expect(logger.warn).toHaveBeenCalledWith('Failed to count tokens. error: Encoding error');
});
test('should properly handle encoding errors with file path', () => {
const error = new Error('Encoding error');
mockEncoder.encode.mockImplementation(() => {
throw error;
});
const count = tokenCounter.countTokens('test content', 'test.txt');
expect(count).toBe(0);
expect(logger.warn).toHaveBeenCalledWith('Failed to count tokens. path: test.txt, error: Encoding error');
});
test('should free encoder resources on cleanup', () => {
tokenCounter.free();
expect(mockEncoder.free).toHaveBeenCalled();
});
});

View File

@@ -7,6 +7,7 @@ export default defineConfig({
include: ['tests/**/*.test.ts'],
coverage: {
include: ['src/**/*'],
exclude: ['src/index.ts'],
reporter: ['text', 'json', 'html'],
},
},