[add] add tree-sitter-c-sharp and simplify tree-sitter dependency build (#147)

Signed-off-by: ShawnZheng <shawn.zheng@zilliz.com>
This commit is contained in:
Shawn Zheng
2025-08-11 11:22:03 +08:00
committed by GitHub
parent 944e07fb30
commit a7fd7fa2a9
6 changed files with 23 additions and 93 deletions

View File

@@ -32,13 +32,6 @@
"os-browserify": "^0.3.0",
"stream-http": "^3.2.0",
"tree-sitter-cli": "^0.25.6",
"tree-sitter-cpp": "^0.23.4",
"tree-sitter-go": "^0.23.4",
"tree-sitter-java": "^0.21.0",
"tree-sitter-javascript": "^0.23.1",
"tree-sitter-python": "^0.21.0",
"tree-sitter-rust": "^0.21.0",
"tree-sitter-typescript": "^0.23.2",
"typescript": "^5.8.3",
"url": "^0.11.4",
"vm-browserify": "^1.1.2"

View File

@@ -27,6 +27,7 @@
"tree-sitter-java": "^0.21.0",
"tree-sitter-javascript": "^0.21.0",
"tree-sitter-python": "^0.21.0",
"tree-sitter-c-sharp": "^0.21.0",
"tree-sitter-rust": "^0.21.0",
"tree-sitter-typescript": "^0.21.0",
"typescript": "^5.0.0",

View File

@@ -9,6 +9,7 @@ const Java = require('tree-sitter-java');
const Cpp = require('tree-sitter-cpp');
const Go = require('tree-sitter-go');
const Rust = require('tree-sitter-rust');
const CSharp = require('tree-sitter-c-sharp');
// Node types that represent logical code units
const SPLITTABLE_NODE_TYPES = {
@@ -18,7 +19,8 @@ const SPLITTABLE_NODE_TYPES = {
java: ['method_declaration', 'class_declaration', 'interface_declaration', 'constructor_declaration'],
cpp: ['function_definition', 'class_specifier', 'namespace_definition', 'declaration'],
go: ['function_declaration', 'method_declaration', 'type_declaration', 'var_declaration', 'const_declaration'],
rust: ['function_item', 'impl_item', 'struct_item', 'enum_item', 'trait_item', 'mod_item']
rust: ['function_item', 'impl_item', 'struct_item', 'enum_item', 'trait_item', 'mod_item'],
csharp: ['method_declaration', 'class_declaration', 'interface_declaration', 'struct_declaration', 'enum_declaration']
};
export class AstCodeSplitter implements Splitter {
@@ -93,7 +95,8 @@ export class AstCodeSplitter implements Splitter {
'c': { parser: Cpp, nodeTypes: SPLITTABLE_NODE_TYPES.cpp },
'go': { parser: Go, nodeTypes: SPLITTABLE_NODE_TYPES.go },
'rust': { parser: Rust, nodeTypes: SPLITTABLE_NODE_TYPES.rust },
'rs': { parser: Rust, nodeTypes: SPLITTABLE_NODE_TYPES.rust }
'rs': { parser: Rust, nodeTypes: SPLITTABLE_NODE_TYPES.rust },
'cs': { parser: CSharp, nodeTypes: SPLITTABLE_NODE_TYPES.csharp }
};
return langMap[language.toLowerCase()] || null;
@@ -256,7 +259,7 @@ export class AstCodeSplitter implements Splitter {
static isLanguageSupported(language: string): boolean {
const supportedLanguages = [
'javascript', 'js', 'typescript', 'ts', 'python', 'py',
'java', 'cpp', 'c++', 'c', 'go', 'rust', 'rs'
'java', 'cpp', 'c++', 'c', 'go', 'rust', 'rs', 'cs'
];
return supportedLanguages.includes(language.toLowerCase());
}

View File

@@ -26,7 +26,8 @@ const LANGUAGE_PARSERS = {
java: 'tree-sitter-java.wasm',
cpp: 'tree-sitter-cpp.wasm',
go: 'tree-sitter-go.wasm',
rust: 'tree-sitter-rust.wasm'
rust: 'tree-sitter-rust.wasm',
csharp: 'tree-sitter-c_sharp.wasm'
};
// Node types that represent logical code units
@@ -37,7 +38,8 @@ const SPLITTABLE_NODE_TYPES = {
java: ['method_declaration', 'class_declaration', 'interface_declaration', 'constructor_declaration'],
cpp: ['function_definition', 'class_specifier', 'namespace_definition', 'declaration'],
go: ['function_declaration', 'method_declaration', 'type_declaration', 'var_declaration', 'const_declaration'],
rust: ['function_item', 'impl_item', 'struct_item', 'enum_item', 'trait_item', 'mod_item']
rust: ['function_item', 'impl_item', 'struct_item', 'enum_item', 'trait_item', 'mod_item'],
csharp: ['method_declaration', 'class_declaration', 'interface_declaration', 'struct_declaration', 'enum_declaration']
};
class AstCodeSplitterStub {
@@ -146,7 +148,8 @@ class AstCodeSplitterStub {
'py': 'python',
'c++': 'cpp',
'c': 'cpp',
'rs': 'rust'
'rs': 'rust',
'cs': 'csharp'
};
return langMap[language.toLowerCase()] || language.toLowerCase();
}

90
pnpm-lock.yaml generated
View File

@@ -38,27 +38,6 @@ importers:
tree-sitter-cli:
specifier: ^0.25.6
version: 0.25.6
tree-sitter-cpp:
specifier: ^0.23.4
version: 0.23.4
tree-sitter-go:
specifier: ^0.23.4
version: 0.23.4
tree-sitter-java:
specifier: ^0.21.0
version: 0.21.0(tree-sitter@0.21.1)
tree-sitter-javascript:
specifier: ^0.23.1
version: 0.23.1
tree-sitter-python:
specifier: ^0.21.0
version: 0.21.0(tree-sitter@0.21.1)
tree-sitter-rust:
specifier: ^0.21.0
version: 0.21.0(tree-sitter@0.21.1)
tree-sitter-typescript:
specifier: ^0.23.2
version: 0.23.2
typescript:
specifier: ^5.8.3
version: 5.8.3
@@ -187,6 +166,9 @@ importers:
tree-sitter:
specifier: ^0.21.1
version: 0.21.1
tree-sitter-c-sharp:
specifier: ^0.21.0
version: 0.21.3(tree-sitter@0.21.1)
tree-sitter-cpp:
specifier: ^0.22.0
version: 0.22.3(tree-sitter@0.21.1)
@@ -3946,12 +3928,13 @@ packages:
tr46@0.0.3:
resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
tree-sitter-c@0.23.6:
resolution: {integrity: sha512-0dxXKznVyUA0s6PjNolJNs2yF87O5aL538A/eR6njA5oqX3C3vH4vnx3QdOKwuUdpKEcFdHuiDpRKLLCA/tjvQ==}
tree-sitter-c-sharp@0.21.3:
resolution: {integrity: sha512-TVsl5EhmqetO/mhzDPVnMK6TPFnpNMKP0OTNuAQIprshk5Hx672ODRxoIoG5WqvUUlsnBu8J0zmn35hmJqelsA==}
peerDependencies:
tree-sitter: ^0.22.1
tree-sitter: ^0.21.0
tree_sitter: '*'
peerDependenciesMeta:
tree-sitter:
tree_sitter:
optional: true
tree-sitter-cli@0.25.6:
@@ -3968,14 +3951,6 @@ packages:
tree_sitter:
optional: true
tree-sitter-cpp@0.23.4:
resolution: {integrity: sha512-qR5qUDyhZ5jJ6V8/umiBxokRbe89bCGmcq/dk94wI4kN86qfdV8k0GHIUEKaqWgcu42wKal5E97LKpLeVW8sKw==}
peerDependencies:
tree-sitter: ^0.21.1
peerDependenciesMeta:
tree-sitter:
optional: true
tree-sitter-go@0.21.2:
resolution: {integrity: sha512-aMFwjsB948nWhURiIxExK8QX29JYKs96P/IfXVvluVMRJZpL04SREHsdOZHYqJr1whkb7zr3/gWHqqvlkczmvw==}
peerDependencies:
@@ -3985,14 +3960,6 @@ packages:
tree_sitter:
optional: true
tree-sitter-go@0.23.4:
resolution: {integrity: sha512-iQaHEs4yMa/hMo/ZCGqLfG61F0miinULU1fFh+GZreCRtKylFLtvn798ocCZjO2r/ungNZgAY1s1hPFyAwkc7w==}
peerDependencies:
tree-sitter: ^0.21.1
peerDependenciesMeta:
tree-sitter:
optional: true
tree-sitter-java@0.21.0:
resolution: {integrity: sha512-CKJiTo1uc3SUsgEcaZgufGx8my6dzihy8JR/JsJH40Tj3uSe2/eFLk+0q+fpbosGAyY4YiXJtEoFB2O4bS2yOw==}
peerDependencies:
@@ -4011,14 +3978,6 @@ packages:
tree_sitter:
optional: true
tree-sitter-javascript@0.23.1:
resolution: {integrity: sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==}
peerDependencies:
tree-sitter: ^0.21.1
peerDependenciesMeta:
tree-sitter:
optional: true
tree-sitter-python@0.21.0:
resolution: {integrity: sha512-IUKx7JcTVbByUx1iHGFS/QsIjx7pqwTMHL9bl/NGyhyyydbfNrpruo2C7W6V4KZrbkkCOlX8QVrCoGOFW5qecg==}
peerDependencies:
@@ -4046,14 +4005,6 @@ packages:
tree_sitter:
optional: true
tree-sitter-typescript@0.23.2:
resolution: {integrity: sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==}
peerDependencies:
tree-sitter: ^0.21.0
peerDependenciesMeta:
tree-sitter:
optional: true
tree-sitter@0.21.1:
resolution: {integrity: sha512-7dxoA6kYvtgWw80265MyqJlkRl4yawIjO7S5MigytjELkX43fV2WsAXzsNfO7sBpPPCF5Gp0+XzHk0DwLCq3xQ==}
@@ -8628,10 +8579,11 @@ snapshots:
tr46@0.0.3: {}
tree-sitter-c@0.23.6:
tree-sitter-c-sharp@0.21.3(tree-sitter@0.21.1):
dependencies:
node-addon-api: 8.4.0
node-gyp-build: 4.8.4
tree-sitter: 0.21.1
tree-sitter-cli@0.25.6: {}
@@ -8641,23 +8593,12 @@ snapshots:
node-gyp-build: 4.8.4
tree-sitter: 0.21.1
tree-sitter-cpp@0.23.4:
dependencies:
node-addon-api: 8.4.0
node-gyp-build: 4.8.4
tree-sitter-c: 0.23.6
tree-sitter-go@0.21.2(tree-sitter@0.21.1):
dependencies:
node-addon-api: 8.4.0
node-gyp-build: 4.8.4
tree-sitter: 0.21.1
tree-sitter-go@0.23.4:
dependencies:
node-addon-api: 8.4.0
node-gyp-build: 4.8.4
tree-sitter-java@0.21.0(tree-sitter@0.21.1):
dependencies:
node-addon-api: 8.4.0
@@ -8670,11 +8611,6 @@ snapshots:
node-gyp-build: 4.8.4
tree-sitter: 0.21.1
tree-sitter-javascript@0.23.1:
dependencies:
node-addon-api: 8.4.0
node-gyp-build: 4.8.4
tree-sitter-python@0.21.0(tree-sitter@0.21.1):
dependencies:
node-addon-api: 7.1.1
@@ -8693,12 +8629,6 @@ snapshots:
node-gyp-build: 4.8.4
tree-sitter: 0.21.1
tree-sitter-typescript@0.23.2:
dependencies:
node-addon-api: 8.4.0
node-gyp-build: 4.8.4
tree-sitter-javascript: 0.23.1
tree-sitter@0.21.1:
dependencies:
node-addon-api: 8.4.0