mirror of
https://github.com/BaranziniLab/KG_RAG.git
synced 2024-06-08 14:12:54 +03:00
adding true/fasle testing notebook
This commit is contained in:
root
@@ -36,8 +36,8 @@ LLM_TEMPERATURE : 0
|
||||
SAVE_RESULTS_PATH : '/data/somank/KG_RAG/data/analysis_results'
|
||||
|
||||
# File paths for test questions
|
||||
MCQ_PATH : '/data/somank/KG_RAG/data/benchmark_data/test_questions_two_hop_mcq_from_monarch_and_robokop.csv'
|
||||
TRUE_FALSE_PATH : '/data/somank/kg_rag_fork/KG_RAG/data/benchmark_data/test_questions_one_hop_true_false_v3.csv'
|
||||
MCQ_PATH : '/data/somank/KG_RAG/data/benchmark_data/mcq_questions.csv'
|
||||
TRUE_FALSE_PATH : '/data/somank/kg_rag_fork/KG_RAG/data/benchmark_data/true_false_questions.csv'
|
||||
SINGLE_DISEASE_ENTITY_FILE : '/data/somank/KG_RAG/data/hyperparam_tuning_data/single_disease_entity_prompts.csv'
|
||||
TWO_DISEASE_ENTITY_FILE : '/data/somank/KG_RAG/data/hyperparam_tuning_data/two_disease_entity_prompts.csv'
|
||||
|
||||
|
||||
56
notebooks/true_false_analysis.ipynb
Normal file
56
notebooks/true_false_analysis.ipynb
Normal file
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "e38c27d7-ebfe-406f-aa9f-07f9d6662d52",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a5723e54-a5ef-48cd-85d1-88d0fe7ea78e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data_path = '/data/somank/llm_data/analysis'\n",
|
||||
"llama_prompt_path = os.path.join(data_path, 'Llama_2_13b_chat_hf_prompt_based_true_false_binary_response.csv')\n",
|
||||
"llama_kg_rag_path = os.path.join(data_path, 'Llama_2_13b_chat_hf_PubMedBert_and_entity_recognition_based_node_retrieval_rag_based_true_false_binary_response.csv')\n",
|
||||
"\n",
|
||||
"gpt_35_prompt_path = os.path.join(data_path, 'gpt_35_turbo_prompt_based_true_false_binary_response.csv')\n",
|
||||
"gpt_35_kg_rag_path = os.path.join(data_path,)\n",
|
||||
"\n",
|
||||
"gpt_4_prompt_path = os.path.join(data_path, 'gpt_4_prompt_based_true_false_binary_response.csv')\n",
|
||||
"gpt_4_kg_rag_path = os.path.join(data_path, 'gpt_4_PubMedBert_and_entity_recognition_based_node_retrieval_rag_based_true_false_binary_response.csv')\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -13,10 +13,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 2,
|
||||
"id": "e1f43477-120b-4cc5-a588-22b5f18eee92",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/root/anaconda3/envs/kg_rag_test_2/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from kg_rag.utility import *\n",
|
||||
"from tqdm import tqdm\n"
|
||||
@@ -50,7 +59,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 4,
|
||||
"id": "155bda08-d15b-413c-89d1-d1f97f43bb30",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -86,7 +95,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 5,
|
||||
"id": "041b21f3-8746-47ff-b4f2-c3c29f2a0dcf",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -98,7 +107,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": 6,
|
||||
"id": "6f4632d9-6a60-4cfe-851d-4d65d4089a52",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@@ -106,10 +115,27 @@
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0it [01:21, ?it/s]\n",
|
||||
"\n",
|
||||
"KeyboardInterrupt\n",
|
||||
"\n"
|
||||
"9it [03:22, 22.46s/it]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "RetryError",
|
||||
"evalue": "RetryError[<Future at 0x7f5c7479b6d0 state=finished raised IndexError>]",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)",
|
||||
"File \u001b[0;32m~/anaconda3/envs/kg_rag_test_2/lib/python3.10/site-packages/tenacity/__init__.py:382\u001b[0m, in \u001b[0;36mRetrying.__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 381\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 382\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m: \u001b[38;5;66;03m# noqa: B902\u001b[39;00m\n",
|
||||
"File \u001b[0;32m/data/somank/kg_rag_fork/KG_RAG/kg_rag/utility.py:128\u001b[0m, in \u001b[0;36mget_context_using_spoke_api\u001b[0;34m(node_value)\u001b[0m\n\u001b[1;32m 127\u001b[0m context \u001b[38;5;241m=\u001b[39m merge_2\u001b[38;5;241m.\u001b[39mcontext\u001b[38;5;241m.\u001b[39mstr\u001b[38;5;241m.\u001b[39mcat(sep\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m--> 128\u001b[0m context \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m node_value \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m has a \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[43mnode_context\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproperties\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m identifier of \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m node_context[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproperties\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124midentifier\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m and Provenance of this is from \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m node_context[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproperties\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m context, merge_2\n",
|
||||
"\u001b[0;31mIndexError\u001b[0m: list index out of range",
|
||||
"\nThe above exception was the direct cause of the following exception:\n",
|
||||
"\u001b[0;31mRetryError\u001b[0m Traceback (most recent call last)",
|
||||
"File \u001b[0;32m<timed exec>:4\u001b[0m\n",
|
||||
"File \u001b[0;32m/data/somank/kg_rag_fork/KG_RAG/kg_rag/utility.py:275\u001b[0m, in \u001b[0;36mretrieve_context\u001b[0;34m(question, vectorstore, embedding_function, node_context_df, context_volume, context_sim_threshold, context_sim_min_threshold, edge_evidence, api)\u001b[0m\n\u001b[1;32m 273\u001b[0m node_context \u001b[38;5;241m=\u001b[39m node_context_df[node_context_df\u001b[38;5;241m.\u001b[39mnode_name \u001b[38;5;241m==\u001b[39m node_name]\u001b[38;5;241m.\u001b[39mnode_context\u001b[38;5;241m.\u001b[39mvalues[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 274\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 275\u001b[0m node_context,context_table \u001b[38;5;241m=\u001b[39m \u001b[43mget_context_using_spoke_api\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnode_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 276\u001b[0m node_context_list \u001b[38;5;241m=\u001b[39m node_context\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m\"\u001b[39m) \n\u001b[1;32m 277\u001b[0m node_context_embeddings \u001b[38;5;241m=\u001b[39m embedding_function\u001b[38;5;241m.\u001b[39membed_documents(node_context_list)\n",
|
||||
"File \u001b[0;32m~/anaconda3/envs/kg_rag_test_2/lib/python3.10/site-packages/tenacity/__init__.py:289\u001b[0m, in \u001b[0;36mBaseRetrying.wraps.<locals>.wrapped_f\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(f)\n\u001b[1;32m 288\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapped_f\u001b[39m(\u001b[38;5;241m*\u001b[39margs: t\u001b[38;5;241m.\u001b[39mAny, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw: t\u001b[38;5;241m.\u001b[39mAny) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m t\u001b[38;5;241m.\u001b[39mAny:\n\u001b[0;32m--> 289\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[0;32m~/anaconda3/envs/kg_rag_test_2/lib/python3.10/site-packages/tenacity/__init__.py:379\u001b[0m, in \u001b[0;36mRetrying.__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 377\u001b[0m retry_state \u001b[38;5;241m=\u001b[39m RetryCallState(retry_object\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m, fn\u001b[38;5;241m=\u001b[39mfn, args\u001b[38;5;241m=\u001b[39margs, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[1;32m 378\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 379\u001b[0m do \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mretry_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretry_state\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 380\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(do, DoAttempt):\n\u001b[1;32m 381\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
|
||||
"File \u001b[0;32m~/anaconda3/envs/kg_rag_test_2/lib/python3.10/site-packages/tenacity/__init__.py:326\u001b[0m, in \u001b[0;36mBaseRetrying.iter\u001b[0;34m(self, retry_state)\u001b[0m\n\u001b[1;32m 324\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreraise:\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m retry_exc\u001b[38;5;241m.\u001b[39mreraise()\n\u001b[0;32m--> 326\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m retry_exc \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mfut\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexception\u001b[39;00m()\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwait:\n\u001b[1;32m 329\u001b[0m sleep \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwait(retry_state)\n",
|
||||
"\u001b[0;31mRetryError\u001b[0m: RetryError[<Future at 0x7f5c7479b6d0 state=finished raised IndexError>]"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -130,7 +156,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bf710d7c-db58-4083-9762-03de0dd5eb1a",
|
||||
"id": "67850830-9d36-4586-b01b-488df7c36a15",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
|
||||
Reference in New Issue
Block a user