mirror of
https://github.com/hhy-huang/HiRAG.git
synced 2025-09-16 23:52:00 +03:00
153 lines
4.3 KiB
Python
153 lines
4.3 KiB
Python
from dataclasses import dataclass, field
|
|
from typing import TypedDict, Union, Literal, Generic, TypeVar
|
|
from ._utils import EmbeddingFunc
|
|
import numpy as np
|
|
|
|
|
|
@dataclass
|
|
class QueryParam:
|
|
mode: Literal["hi_global", "hi_local", "hi_bridge", "hi_nobridge", "naive", "hi"] = "hi"
|
|
only_need_context: bool = False
|
|
response_type: str = "Multiple Paragraphs"
|
|
level: int = 2
|
|
top_k: int = 20 # retrieve top-k entities
|
|
top_m: int = 10 # retrieve top-m entities in each retrieved community
|
|
# naive search
|
|
naive_max_token_for_text_unit = 10000
|
|
# hi search
|
|
max_token_for_text_unit: int = 20000
|
|
max_token_for_local_context: int = 20000
|
|
max_token_for_bridge_knowledge: int = 12500
|
|
max_token_for_community_report: int = 12500
|
|
community_single_one: bool = False
|
|
|
|
|
|
TextChunkSchema = TypedDict(
|
|
"TextChunkSchema",
|
|
{"tokens": int, "content": str, "full_doc_id": str, "chunk_order_index": int},
|
|
)
|
|
|
|
SingleCommunitySchema = TypedDict(
|
|
"SingleCommunitySchema",
|
|
{
|
|
"level": int,
|
|
"title": str,
|
|
"edges": list[list[str, str]],
|
|
"nodes": list[str],
|
|
"chunk_ids": list[str],
|
|
"occurrence": float,
|
|
"sub_communities": list[str],
|
|
},
|
|
)
|
|
|
|
|
|
class CommunitySchema(SingleCommunitySchema):
|
|
report_string: str
|
|
report_json: dict
|
|
|
|
|
|
T = TypeVar("T")
|
|
|
|
|
|
@dataclass
|
|
class StorageNameSpace:
|
|
namespace: str
|
|
global_config: dict
|
|
|
|
async def index_start_callback(self):
|
|
"""commit the storage operations after indexing"""
|
|
pass
|
|
|
|
async def index_done_callback(self):
|
|
"""commit the storage operations after indexing"""
|
|
pass
|
|
|
|
async def query_done_callback(self):
|
|
"""commit the storage operations after querying"""
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class BaseVectorStorage(StorageNameSpace):
|
|
embedding_func: EmbeddingFunc
|
|
meta_fields: set = field(default_factory=set)
|
|
|
|
async def query(self, query: str, top_k: int) -> list[dict]:
|
|
raise NotImplementedError
|
|
|
|
async def upsert(self, data: dict[str, dict]):
|
|
"""Use 'content' field from value for embedding, use key as id.
|
|
If embedding_func is None, use 'embedding' field from value
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
@dataclass
|
|
class BaseKVStorage(Generic[T], StorageNameSpace):
|
|
async def all_keys(self) -> list[str]:
|
|
raise NotImplementedError
|
|
|
|
async def get_by_id(self, id: str) -> Union[T, None]:
|
|
raise NotImplementedError
|
|
|
|
async def get_by_ids(
|
|
self, ids: list[str], fields: Union[set[str], None] = None
|
|
) -> list[Union[T, None]]:
|
|
raise NotImplementedError
|
|
|
|
async def filter_keys(self, data: list[str]) -> set[str]:
|
|
"""return un-exist keys"""
|
|
raise NotImplementedError
|
|
|
|
async def upsert(self, data: dict[str, T]):
|
|
raise NotImplementedError
|
|
|
|
async def drop(self):
|
|
raise NotImplementedError
|
|
|
|
|
|
@dataclass
|
|
class BaseGraphStorage(StorageNameSpace):
|
|
async def has_node(self, node_id: str) -> bool:
|
|
raise NotImplementedError
|
|
|
|
async def has_edge(self, source_node_id: str, target_node_id: str) -> bool:
|
|
raise NotImplementedError
|
|
|
|
async def node_degree(self, node_id: str) -> int:
|
|
raise NotImplementedError
|
|
|
|
async def edge_degree(self, src_id: str, tgt_id: str) -> int:
|
|
raise NotImplementedError
|
|
|
|
async def get_node(self, node_id: str) -> Union[dict, None]:
|
|
raise NotImplementedError
|
|
|
|
async def get_edge(
|
|
self, source_node_id: str, target_node_id: str
|
|
) -> Union[dict, None]:
|
|
raise NotImplementedError
|
|
|
|
async def get_node_edges(
|
|
self, source_node_id: str
|
|
) -> Union[list[tuple[str, str]], None]:
|
|
raise NotImplementedError
|
|
|
|
async def upsert_node(self, node_id: str, node_data: dict[str, str]):
|
|
raise NotImplementedError
|
|
|
|
async def upsert_edge(
|
|
self, source_node_id: str, target_node_id: str, edge_data: dict[str, str]
|
|
):
|
|
raise NotImplementedError
|
|
|
|
async def clustering(self, algorithm: str):
|
|
raise NotImplementedError
|
|
|
|
async def community_schema(self) -> dict[str, SingleCommunitySchema]:
|
|
"""Return the community representation with report and nodes"""
|
|
raise NotImplementedError
|
|
|
|
async def embed_nodes(self, algorithm: str) -> tuple[np.ndarray, list[str]]:
|
|
raise NotImplementedError("Node embedding is not used in HiRAG.")
|