mirror of
https://github.com/anthropics/claude-cookbooks.git
synced 2025-10-06 01:00:28 +03:00
Skill - Classification V1.0
This commit is contained in:
Garvan Doyle
File diff suppressed because one or more lines are too long
@@ -1,26 +0,0 @@
|
||||
text label
|
||||
I received a bill for my auto policy renewal, but the amount seems higher than usual. Can you explain why it increased? Billing Inquiries
|
||||
I want to adjust my coverage limits and deductibles for my homeowners policy. What are my options? Policy Administration
|
||||
My parked car was hit by another vehicle, but I don't have the other driver's information. How should I proceed with filing a claim? Claims Assistance
|
||||
I'm confused about the difference between comprehensive and collision coverage. Can you clarify what each one includes? Coverage Explanations
|
||||
I'm considering switching my insurance to your company. Can you provide a quote that matches or beats my current coverage? Quotes and Proposals
|
||||
I tried to log in to my online account, but it says my password is incorrect. I never changed it, so what could be the issue? Account Management
|
||||
I noticed a charge on my credit card from your company that I don't recognize. It's not my usual premium amount. Can you look into this? Billing Disputes
|
||||
My claim for water damage to my kitchen was denied, but I believe it should be covered under my policy. How can I appeal this decision? Claims Disputes
|
||||
I'm debating between a high-deductible health plan and a traditional PPO. Can you help me understand the pros and cons of each option? Policy Comparisons
|
||||
I received a letter in the mail about some changes to my policy, but I'm not sure what they mean. Can you provide more details? General Inquiries
|
||||
I recently added a new vehicle to my policy, but my bill doesn't reflect the multi-car discount I was promised. Can you review this? Billing Inquiries
|
||||
I need to file a claim for hail damage to my roof, but I'm not sure if I should file it under my homeowners or separate windstorm policy. What do you recommend? Claims Assistance
|
||||
My auto policy includes rental car coverage, but I'm unclear on the specific terms. How much is covered per day and per incident? Coverage Explanations
|
||||
I'm shopping for life insurance and wondering if I should choose term or whole life. Can you explain the difference and help me decide? Quotes and Proposals
|
||||
I want to enroll in paperless billing and automatic payments. How do I set this up through my online account? Account Management
|
||||
I think I'm being double-charged for my motorcycle endorsement. My premium went up, but I don't see the charge itemized anywhere. Can you clarify? Billing Disputes
|
||||
The estimate for repairs to my vehicle after an accident is higher than the amount your adjuster approved. How can I dispute this discrepancy? Claims Disputes
|
||||
I'm comparing your small business insurance policies to those of a few other providers. Can you explain how your coverage and pricing stack up against theirs? Policy Comparisons
|
||||
My agent mentioned something about a "disappearing deductible" feature, but I'm not sure what that entails. Could you provide more information? General Inquiries
|
||||
I'm planning to retire soon and want to adjust my life insurance coverage accordingly. What changes should I consider making to my policy? Policy Administration
|
||||
My basement flooded due to a sump pump failure, and I'm not sure if this is covered under my homeowners policy. Can you clarify what water damage is included? Coverage Explanations
|
||||
I received a cancellation notice for non-payment, but I believe I paid my premium on time. Can you investigate what happened and reinstate my policy? Billing Disputes
|
||||
I took my car to the shop for repairs after an accident, but the repair estimate is much higher than what your appraiser quoted. How should I handle this situation? Claims Disputes
|
||||
I'm trying to decide between two different health plans with similar premiums but different out-of-pocket maximums. Which one would you recommend based on my medical history? Policy Comparisons
|
||||
Can you explain how your accident forgiveness program works and what the eligibility requirements are? General Inquiries
|
||||
|
Can't render this file because it contains an unexpected character in line 20 and column 38.
|
Binary file not shown.
@@ -1,26 +0,0 @@
|
||||
text,label,__expected
|
||||
"I received a bill for my auto policy renewal, but the amount seems higher than usual. Can you explain why it increased?",Billing Inquiries,contains:Billing Inquiries
|
||||
I want to adjust my coverage limits and deductibles for my homeowners policy. What are my options?,Policy Administration,contains:Policy Administration
|
||||
"My parked car was hit by another vehicle, but I don't have the other driver's information. How should I proceed with filing a claim?",Claims Assistance,contains:Claims Assistance
|
||||
I'm confused about the difference between comprehensive and collision coverage. Can you clarify what each one includes?,Coverage Explanations,contains:Coverage Explanations
|
||||
I'm considering switching my insurance to your company. Can you provide a quote that matches or beats my current coverage?,Quotes and Proposals,contains:Quotes and Proposals
|
||||
"I tried to log in to my online account, but it says my password is incorrect. I never changed it, so what could be the issue?",Account Management,contains:Account Management
|
||||
I noticed a charge on my credit card from your company that I don't recognize. It's not my usual premium amount. Can you look into this?,Billing Disputes,contains:Billing Disputes
|
||||
"My claim for water damage to my kitchen was denied, but I believe it should be covered under my policy. How can I appeal this decision?",Claims Disputes,contains:Claims Disputes
|
||||
I'm debating between a high-deductible health plan and a traditional PPO. Can you help me understand the pros and cons of each option?,Policy Comparisons,contains:Policy Comparisons
|
||||
"I received a letter in the mail about some changes to my policy, but I'm not sure what they mean. Can you provide more details?",General Inquiries,contains:General Inquiries
|
||||
"I recently added a new vehicle to my policy, but my bill doesn't reflect the multi-car discount I was promised. Can you review this?",Billing Inquiries,contains:Billing Inquiries
|
||||
"I need to file a claim for hail damage to my roof, but I'm not sure if I should file it under my homeowners or separate windstorm policy. What do you recommend?",Claims Assistance,contains:Claims Assistance
|
||||
"My auto policy includes rental car coverage, but I'm unclear on the specific terms. How much is covered per day and per incident?",Coverage Explanations,contains:Coverage Explanations
|
||||
I'm shopping for life insurance and wondering if I should choose term or whole life. Can you explain the difference and help me decide?,Quotes and Proposals,contains:Quotes and Proposals
|
||||
I want to enroll in paperless billing and automatic payments. How do I set this up through my online account?,Account Management,contains:Account Management
|
||||
"I think I'm being double-charged for my motorcycle endorsement. My premium went up, but I don't see the charge itemized anywhere. Can you clarify?",Billing Disputes,contains:Billing Disputes
|
||||
The estimate for repairs to my vehicle after an accident is higher than the amount your adjuster approved. How can I dispute this discrepancy?,Claims Disputes,contains:Claims Disputes
|
||||
I'm comparing your small business insurance policies to those of a few other providers. Can you explain how your coverage and pricing stack up against theirs?,Policy Comparisons,contains:Policy Comparisons
|
||||
"My agent mentioned something about a ""disappearing deductible"" feature, but I'm not sure what that entails. Could you provide more information?",General Inquiries,contains:General Inquiries
|
||||
I'm planning to retire soon and want to adjust my life insurance coverage accordingly. What changes should I consider making to my policy?,Policy Administration,contains:Policy Administration
|
||||
"My basement flooded due to a sump pump failure, and I'm not sure if this is covered under my homeowners policy. Can you clarify what water damage is included?",Coverage Explanations,contains:Coverage Explanations
|
||||
"I received a cancellation notice for non-payment, but I believe I paid my premium on time. Can you investigate what happened and reinstate my policy?",Billing Disputes,contains:Billing Disputes
|
||||
"I took my car to the shop for repairs after an accident, but the repair estimate is much higher than what your appraiser quoted. How should I handle this situation?",Claims Disputes,contains:Claims Disputes
|
||||
I'm trying to decide between two different health plans with similar premiums but different out-of-pocket maximums. Which one would you recommend based on my medical history?,Policy Comparisons,contains:Policy Comparisons
|
||||
Can you explain how your accident forgiveness program works and what the eligibility requirements are?,General Inquiries,contains:General Inquiries
|
||||
|
@@ -1,60 +0,0 @@
|
||||
You will classify a customer support ticket into one of the following categories
|
||||
<categories>
|
||||
<category>
|
||||
<label>Billing Inquiries</label>
|
||||
<content> Questions about invoices, charges, fees, and premiums Requests for clarification on billing statements Inquiries about payment methods and due dates
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Policy Administration</label>
|
||||
<content> Requests for policy changes, updates, or cancellations Questions about policy renewals and reinstatements Inquiries about adding or removing coverage options
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Claims Assistance</label>
|
||||
<content> Questions about the claims process and filing procedures Requests for help with submitting claim documentation Inquiries about claim status and payout timelines
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Coverage Explanations</label>
|
||||
<content> Questions about what is covered under specific policy types Requests for clarification on coverage limits and exclusions Inquiries about deductibles and out-of-pocket expenses
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Quotes and Proposals</label>
|
||||
<content> Requests for new policy quotes and price comparisons Questions about available discounts and bundling options Inquiries about switching from another insurer
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Account Management</label>
|
||||
<content> Requests for login credentials or password resets Questions about online account features and functionality Inquiries about updating contact or personal information
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Billing Disputes</label>
|
||||
<content> Complaints about unexpected or incorrect charges Requests for refunds or premium adjustments Inquiries about late fees or collection notices
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Claims Disputes</label>
|
||||
<content> Complaints about denied or underpaid claims Requests for reconsideration of claim decisions Inquiries about appealing a claim outcome
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>Policy Comparisons</label>
|
||||
<content> Questions about the differences between policy options Requests for help deciding between coverage levels Inquiries about how policies compare to competitors' offerings
|
||||
</content>
|
||||
</category>
|
||||
<category>
|
||||
<label>General Inquiries</label>
|
||||
<content> Questions about company contact information or hours of operation Requests for general information about products or services Inquiries that don't fit neatly into other categories
|
||||
</content>
|
||||
</category>
|
||||
</categories>
|
||||
|
||||
Here is the customer support ticket:
|
||||
<ticket>
|
||||
{{text}}
|
||||
</ticket>
|
||||
|
||||
Respond with just the label of the category between category tags.
|
||||
@@ -1,2 +0,0 @@
|
||||
def get_transform(output, context):
|
||||
return output.split("<category>")[1].split("</category>")[0].strip()
|
||||
File diff suppressed because one or more lines are too long
4431
skills/classification/data/results.csv
Normal file
4431
skills/classification/data/results.csv
Normal file
File diff suppressed because one or more lines are too long
69
skills/classification/data/test.tsv
Normal file
69
skills/classification/data/test.tsv
Normal file
@@ -0,0 +1,69 @@
|
||||
text label
|
||||
I'm confused about a charge on my recent auto insurance bill that's higher than my usual premium payment. Can you clarify what this extra fee is for and why it was added? Billing Inquiries
|
||||
My latest car insurance statement includes a charge that I don't recognize, and it's made my total payment go up quite a bit. Could you look into this and explain what the additional cost is related to? Billing Inquiries
|
||||
I noticed an unfamiliar line item on my auto policy billing that's causing my premium to be more expensive than normal. Would you mind investigating this charge and letting me know the reason behind it? Billing Inquiries
|
||||
I'd like to make some changes to the deductibles and coverage options on my home insurance policy. Can you outline the different choices I have and help me decide which ones might be best for my situation? Policy Administration
|
||||
I'm thinking about adjusting the coverage limits and deductibles associated with my homeowners insurance. What alternatives are available to me, and can you provide some guidance on selecting the most appropriate options? Policy Administration
|
||||
I want to explore the possibility of modifying my home insurance policy's deductibles and coverage amounts. Could you describe the various options I can consider and offer some advice on which ones might suit my needs best? Policy Administration
|
||||
I came out to my car after shopping and found that another vehicle had hit it, but the driver was nowhere to be found. What should my next steps be in terms of filing an insurance claim for the damage? Claims Assistance
|
||||
Someone apparently crashed into my parked car while I was away from it, but I have no information about who did it. Can you guide me through the process of submitting a claim in this type of hit-and-run scenario? Claims Assistance
|
||||
I discovered that my parked vehicle had been struck by another car, but the other driver left the scene without providing any contact details. How do I go about reporting this incident to insurance and getting the damage covered? Claims Assistance
|
||||
I'm a bit puzzled about what the difference is between collision and comprehensive coverage for car insurance. Would you be able to clarify the distinction and explain what each type of coverage protects against? Coverage Explanations
|
||||
Can you help me better understand the nuances between auto collision and comprehensive insurance? I want to make sure I'm clear on what kinds of situations and damages fall under the scope of each one. Coverage Explanations
|
||||
I'd like to learn more about the respective benefits of collision versus comprehensive car insurance coverage. Could you break down what sets them apart and describe the primary areas they each address? Coverage Explanations
|
||||
I'm in the process of shopping around for new insurance and am interested in seeing what your company can offer. Can you provide me with a quote that is competitive with or better than the coverage I currently have elsewhere? Quotes and Proposals
|
||||
As I compare insurance options from different providers, I'm curious to know how your company's rates and coverage stack up against my existing policy. Would you be able to prepare a quote that meets or beats what I have now? Quotes and Proposals
|
||||
I'm considering switching insurance carriers and want to find out if your company can give me a better deal than what I have at present. Can you put together a quote that showcases the value and coverage advantages you can offer compared to my current insurer? Quotes and Proposals
|
||||
I'm having trouble signing in to my online insurance account portal. When I enter my password, it says it's incorrect even though I know I'm using the right one. Can you help me troubleshoot this login issue? Account Management
|
||||
Every time I attempt to access my insurance account on your website, I get an error message indicating that my password is invalid. I haven't changed it recently, so I'm not sure what the problem could be. Can you assist me in resolving this? Account Management
|
||||
I can't seem to log in to my web-based insurance profile because the system keeps telling me my password is wrong, but I'm certain I'm entering it correctly. Would you be able to look into this and help me regain access to my account? Account Management
|
||||
There's a mysterious charge from your insurance company on my recent credit card statement that doesn't coincide with my regular premium payment. Can you investigate the nature of this charge and clarify why it was applied to my account? Billing Disputes
|
||||
I've encountered an odd fee from your company on my latest credit card bill, and it's not the same as my typical insurance premium amount. Could you do some digging to find out what this charge represents and why I'm being billed for it? Billing Disputes
|
||||
My credit card has been hit with an unexpected charge from your insurance agency that I don't understand, as it's separate from my standard premium. Would you be willing to research this matter and provide an explanation for the additional fee? Billing Disputes
|
||||
I recently filed a claim for some water damage that occurred in my kitchen, but I was surprised to learn that it was denied. I believe this type of incident should be covered under my homeowners policy, so how can I go about appealing the denial? Claims Disputes
|
||||
After submitting a claim for water-related damage to my kitchen, I was dismayed to receive a rejection notice stating that it wasn't covered. However, I feel strongly that my policy should apply to this situation. What recourse do I have to challenge this decision and get my claim approved? Claims Disputes
|
||||
I'm confused and frustrated by your company's refusal to cover the water damage claim I filed for my kitchen, as I'm fairly certain my homeowners insurance should extend to this type of event. Can you explain the process for disputing this denial and arguing my case for coverage? Claims Disputes
|
||||
I'm wavering between a high-deductible health plan and a more traditional PPO option, and I could use some assistance in weighing the relative merits and drawbacks of each. Can you offer a comparison of these two choices to help inform my decision? Policy Comparisons
|
||||
Choosing between a health insurance policy with a high deductible and a standard PPO plan has me feeling uncertain. Could you provide an overview of the key differences I should consider as I try to determine which one is the better fit for me? Policy Comparisons
|
||||
I received a letter outlining some forthcoming adjustments to my insurance policy, but I'm having trouble deciphering exactly what they mean for my coverage. Would you be able to walk me through the changes in greater detail so I can fully understand their implications? General Inquiries
|
||||
A piece of mail from your company arrived informing me of some impending updates to my insurance plan, but the specifics are a little hazy to me. Can you shed more light on what these modifications entail and how they will impact my policy moving forward? General Inquiries
|
||||
Your agency sent me a notice about some planned revisions to my insurance coverage, but I'm not entirely sure I grasp the practical effects of these changes. Could you provide a more thorough explanation of what I can expect and how my policy will be different as a result? General Inquiries
|
||||
I added another vehicle to my car insurance policy not long ago, with the understanding that I would receive a multi-car discount. However, this discount is nowhere to be found on my latest billing statement. Can you double-check that it's been properly applied to my account? Billing Inquiries
|
||||
When I recently expanded my auto insurance to cover an additional vehicle, I was told I'd qualify for a multi-car price break. Upon reviewing my most recent bill, though, I don't see any indication of this discount being included. Could you verify that the appropriate adjustment has been made to my premium? Billing Inquiries
|
||||
I was under the impression that adding a second car to my existing auto policy would trigger a multi-vehicle discount on my insurance costs. Strangely, this reduction in price isn't displayed on my current bill. Would you mind confirming that my account reflects the expected discount? Billing Inquiries
|
||||
Following a hailstorm that left my home with some roof damage, I need to initiate an insurance claim. However, I'm unsure whether this falls under my primary homeowners policy or if I should instead file it against my separate windstorm coverage. Can you advise me on the correct course of action? Claims Assistance
|
||||
I'm ready to begin the claims process after discovering hail damage to my home's roof, but I'm uncertain about which policy to invoke - my standard homeowners insurance or the windstorm rider I have. What would you recommend as the most appropriate path forward? Claims Assistance
|
||||
My house sustained hail damage during a recent storm, and I'm planning to submit an insurance claim to cover the necessary roof repairs. I'm just not clear on whether this should be handled through my main homeowners policy or my additional windstorm coverage. Can you point me in the right direction? Claims Assistance
|
||||
My auto insurance policy includes provisions for rental car coverage in certain situations, but I'm a bit fuzzy on the specifics of what's included. Can you fill me in on the exact daily and per-incident allowances for rental expenses under my plan? Coverage Explanations
|
||||
I'm aware that my car insurance offers some degree of rental vehicle coverage, but I don't have a complete grasp of the details involved. Would you be able to clarify for me the precise daily and per-event limits that apply to rental reimbursements through my policy? Coverage Explanations
|
||||
While I know rental cars are covered to some extent by my auto insurance, I'm not entirely confident about the particular limits and restrictions that come into play. Could you spell out for me the exact amounts covered per day and per incident when I need to rent a vehicle? Coverage Explanations
|
||||
As I explore life insurance options, I find myself torn between term and whole life policies. Can you help me understand the core differences between these two choices and provide some guidance on which route might be most advantageous for my individual circumstances? Quotes and Proposals
|
||||
I'm actively seeking life insurance coverage but feel uncertain about whether to pursue a term policy or a whole life plan. Could you elaborate on the key distinctions I should be aware of and share your perspective on which might be the better fit considering my personal situation? Quotes and Proposals
|
||||
I'd like to make the switch to paperless billing for my insurance premiums and set up automatic withdrawal for my regular payments. What steps do I need to take within my online account portal to enable these convenient features? Account Management
|
||||
To streamline my insurance billing and payment processes, I'm hoping to transition to electronic statements and recurring automatic charges. Can you outline for me the specific actions required to establish these preferences through my web-based account interface? Account Management
|
||||
My goal is to eliminate paper copies of my insurance bills and have my premiums automatically drafted from my designated account. Would you be able to provide a step-by-step walkthrough of how to configure these settings within my online profile? Account Management
|
||||
It appears that I may have been charged twice for the motorcycle endorsement associated with my auto insurance policy. While my overall premium has increased, I don't see a line item clearly identifying the endorsement fee. Can you clarify what's behind this price hike? Billing Disputes
|
||||
I have a sneaking suspicion that I'm being double-billed for my motorcycle endorsement, as my car insurance rates have gone up without a transparent breakdown of the charges. Could you investigate whether I'm being inadvertently overcharged for this add-on coverage? Billing Disputes
|
||||
My auto insurance premium has recently risen, and I'm concerned that it might be due to a redundant charge for my motorcycle endorsement, since the cost isn't plainly stated on my bill. Would you be willing to look into this and confirm whether or not I'm paying twice for the same thing? Billing Disputes
|
||||
In the aftermath of an accident, the repair estimate I received from the body shop significantly exceeds the amount that your claims adjuster has authorized. What's the most effective way to challenge this lowball offer and make sure the full scope of necessary repairs is covered? Claims Disputes
|
||||
Having obtained a repair quote following a crash, I'm dismayed to find that it's markedly higher than what your insurance adjuster has agreed to pay. How should I proceed in disputing this inadequate settlement and advocating for more comprehensive coverage of the damage? Claims Disputes
|
||||
After bringing my vehicle to a repair facility post-accident, I've been handed an estimate that puts the cost of fixes well above the figure presented by your adjuster. Can you advise me on the optimal approach for contesting your company's position and securing a fair resolution? Claims Disputes
|
||||
"During a recent conversation with my insurance agent, the concept of a ""diminishing deductible"" was mentioned, but I don't fully comprehend the mechanics of how it works. Could you expand on this topic and elucidate the advantages of opting for such a feature?" General Inquiries
|
||||
"I'm intrigued by the idea of a ""disappearing deductible"" that my insurance rep alluded to, but I'm still a bit hazy on the particulars. Would you be able to furnish a more extensive explanation of this option and describe how it could potentially benefit me as a policyholder?" General Inquiries
|
||||
"The notion of a ""vanishing deductible"" came up in a recent discussion with my insurance agent, piquing my curiosity. However, I'm not confident that I wholly grasp the nuances of this provision. Might you be able to shed additional light on how it functions and what value it could bring to my coverage?" General Inquiries
|
||||
As I contemplate the impact of my upcoming retirement on my insurance needs, I'm wondering what modifications to my life insurance policy might be prudent. What guidance can you share about tailoring my coverage to align with my evolving financial objectives and risk profile in this new chapter? Policy Administration
|
||||
With my retirement on the horizon, I'm keen to ensure that my life insurance policy remains well-suited to my changing circumstances. Can you recommend any adjustments I should consider to optimize my coverage and better safeguard my loved ones' financial security during this transitional phase? Policy Administration
|
||||
On the cusp of retiring, I recognize the importance of reassessing my life insurance strategy to account for my new realities and priorities. What words of wisdom can you impart about fine-tuning my policy to provide the most appropriate level of protection as I navigate this major life milestone? Policy Administration
|
||||
I'm wrestling with how to proceed after a sump pump malfunction led to extensive water damage in my basement, and I'm not certain whether this type of event falls within the scope of my homeowners insurance. Can you clarify the extent to which water-related incidents are typically covered under a standard policy? Coverage Explanations
|
||||
A recent sump pump failure at my home resulted in significant flooding and water damage to my basement. I'm unsure if I can expect my homeowners insurance to cover the repairs, as I'm fuzzy on exactly what kinds of water damage are included. Could you elucidate the common parameters around such coverage? Coverage Explanations
|
||||
Much to my chagrin, my basement sustained considerable water damage following a sump pump breakdown, and I'm grappling with whether this scenario is likely to be covered by my homeowners policy. Would you be able to shed some light on the usual inclusions and exclusions pertaining to water-related damage? Coverage Explanations
|
||||
I've received a troubling notification that my insurance policy is on the brink of cancellation due to an allegedly missed payment. However, I'm quite certain I submitted the premium by the specified deadline. Could you please investigate this discrepancy and ensure that my coverage remains in force if there's been an error on your end? Billing Disputes
|
||||
According to a recent communication from your company, my insurance coverage is poised to lapse because of a perceived failure to remit payment in a timely fashion. I'm confident, though, that I did indeed tender the amount owed before the stated due date. Might you look into this matter and reinstate my policy if a mistake has occurred? Billing Disputes
|
||||
I find myself in a perplexing situation, having been informed that my insurance is set to be terminated for non-payment despite my belief that I had punctually rendered the requisite premium. Would you be so kind as to delve into this quandary and reactive my coverage post-haste if there has been an administrative snafu? Billing Disputes
|
||||
The repair shop I visited in the wake of my automobile accident has furnished me with an estimate that eclipses the value assessment put forth by your designated appraiser. I'm struggling to reconcile this pronounced disparity and would greatly appreciate your counsel on how to bridge the gap and reach an equitable resolution. Claims Disputes
|
||||
Upon entrusting my vehicle to a reputable mechanic for post-collision repairs, I've been presented with a projected cost that markedly outstrips the appraisal tendered by your representative. Confronted with this substantial discrepancy, I'm keen to solicit your guidance on navigating the impasse and achieving a mutually agreeable outcome. Claims Disputes
|
||||
Having sought out a repair facility to mend the damage inflicted upon my car in a recent mishap, I now find myself at odds with the estimate they've proffered, which towers above the figure quoted by your appraiser. I'd be most grateful for your insight into surmounting this incongruity and charting a course toward a fair and satisfactory settlement. Claims Disputes
|
||||
As I endeavor to select between two health insurance policies with comparable premium structures but contrasting out-of-pocket maximum thresholds, I'm grappling with how my personal medical history should inform my decision. Might you be able to proffer some sage advice on which option would be most judicious given my unique health profile? Policy Comparisons
|
||||
I'm vacillating between a pair of health insurance plans that boast similar monthly premiums but diverge in their out-of-pocket maximum provisions. Taking into account my individualized medical background, which path would you counsel me to pursue to optimize my coverage and safeguard my financial well-being? Policy Comparisons
|
||||
Confronted with a binary choice between two health insurance policies that mirror each other in premium costs but differ in their out-of-pocket maximum stipulations, I find myself at a crossroads. Could you impart some wisdom on which alternative would be most propitious for me, bearing in mind my distinctive healthcare chronicles? Policy Comparisons
|
||||
I'd be much obliged if you could furnish me with a more exhaustive elucidation of your company's accident forgiveness program, detailing both its operational intricacies and the eligibility criteria that policyholders must satisfy to avail themselves of this enticing benefit. General Inquiries
|
||||
|
BIN
skills/classification/data/vector_db.pkl
Normal file
BIN
skills/classification/data/vector_db.pkl
Normal file
Binary file not shown.
@@ -1,41 +1,48 @@
|
||||
# Evaluations with Promptfoo
|
||||
|
||||
### Pre-requisities
|
||||
To use promptfoo you will need to have node.js & npm installed on your system. For more information follow [this guide](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)
|
||||
|
||||
You can install promptfoo using npm or run it directly using npx.
|
||||
|
||||
### Pre-requisities
|
||||
To use Promptfoo you will need to have node.js & npm installed on your system. For more information follow [this guide](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)
|
||||
|
||||
You can install promptfoo using npm or run it directly using npx. In this guide we will use npx.
|
||||
|
||||
*Note: For this example you will not need to run `npx promptfoo@latest init` there is already an initialized `promptfooconfig.yaml` file in this directory*
|
||||
|
||||
See the official docs [here](https://www.promptfoo.dev/docs/getting-started)
|
||||
|
||||
In this guide we will use npx.
|
||||
|
||||
|
||||
### Getting Started
|
||||
The evaluation is orchestrated by the `promptfooconfig.yaml` file. In this file we define the following sections:
|
||||
|
||||
- Prompts
|
||||
- Promptfoo enables you to import prompts in many different formats. You can read more about this [here](https://www.promptfoo.dev/docs/configuration/parameters).
|
||||
- In this example we will load 3 prompts - the same used in `guide.ipynb` in two seperate formats:
|
||||
- For the `simple_classify` prompt we will load it directly from the `prompt.txt` file as the content is static between examples.
|
||||
- For the `rag_classify` and `rag_chain_of_thought_classify` we will load the prompts from a python file called `prompts.py`. You can read more about prompt functions [here](https://www.promptfoo.dev/docs/configuration/parameters#prompt-functions). Using python allows us to reuse the VectorDB class which is necessary for RAG, this is defined in `vectordb.py`.
|
||||
- In this example we will load 3 prompts - the same used in `guide.ipynb` from the `prompts.py` file:
|
||||
- The functions are identical to those used in `guide.ipynb` except that instead of calling the Anthropic API they just return the prompt. Promptfoo then handles the orchestration of calling the API and storing the results.
|
||||
- You can read more about prompt functions [here](https://www.promptfoo.dev/docs/configuration/parameters#prompt-functions). Using python allows us to reuse the VectorDB class which is necessary for RAG, this is defined in `vectordb.py`.
|
||||
- Providers
|
||||
- With Promptfoo you can connect to many different LLMs from different platforms, see [here for more](https://www.promptfoo.dev/docs/providers). In `guide.ipynb` we used Haiku with default temperature 0.0. We will use promptfoo to experiment with an array of different temperature settings to identify the optimal choice for our use case.
|
||||
- With Promptfoo you can connect to many different LLMs from different platforms, see [here for more](https://www.promptfoo.dev/docs/providers). In `guide.ipynb` we used Haiku with default temperature 0.0. We will use Promptfoo to experiment with an array of different temperature settings to identify the optimal choice for our use case.
|
||||
- Tests
|
||||
- We will use the same data that was used in `guide.ipynb` which can be found in this [Google Sheet](https://docs.google.com/spreadsheets/d/1UwbrWCWsTFGVshyOfY2ywtf5BEt7pUcJEGYZDkfkufU/edit#gid=0).
|
||||
- Promptfoo has a wide array of built in tests which can be found [here](https://www.promptfoo.dev/docs/configuration/expected-outputs/deterministic).
|
||||
- In this example we will define a test in our `dataset.csv` as the conditions of our evaluation change with each row and a test in the `promptfooconfig.yaml` for conditions that are consistent across all test cases.
|
||||
- In this example we will define a test in our `dataset.csv` as the conditions of our evaluation change with each row and a test in the `promptfooconfig.yaml` for conditions that are consistent across all test cases. Read more about this [here](https://www.promptfoo.dev/docs/configuration/parameters/#import-from-csv)
|
||||
- Transform
|
||||
- In the `defaultTest` section we define a transform function. This is a python function which extracts the specific output we want to test from the LLM response.
|
||||
- Output
|
||||
- We define the path for the output file. Promptfoo can output results in many formats, [see here](https://www.promptfoo.dev/docs/configuration/parameters/#output-file). You can also use Promptfoo's web UI, [see here](https://www.promptfoo.dev/docs/usage/web-ui).
|
||||
- We define the path for the output file. Promptfoo can output results in many formats, [see here](https://www.promptfoo.dev/docs/configuration/parameters/#output-file). Alternatively you can use Promptfoo's web UI, [see here](https://www.promptfoo.dev/docs/usage/web-ui).
|
||||
|
||||
|
||||
### Run an eval
|
||||
### Run the eval
|
||||
|
||||
Before running your evaluation you must deinfe the following enviroment variables:
|
||||
To get started with Promptfoo open your terminal and navigate to this directory (`./evaluation`).
|
||||
|
||||
Before running your evaluation you must define the following enviroment variables:
|
||||
|
||||
`export ANTHROPIC_API_KEY=YOUR_API_KEY`
|
||||
`export VOYAGE_API_KE=YOUR_API_KEY`
|
||||
`export VOYAGE_API_KEY=YOUR_API_KEY`
|
||||
|
||||
Assuming you in the `evaluation` directory, run the following command.
|
||||
From the `evaluation` directory, run the following command.
|
||||
|
||||
`npx promptfoo@latest eval`
|
||||
|
||||
69
skills/classification/evaluation/dataset.csv
Normal file
69
skills/classification/evaluation/dataset.csv
Normal file
@@ -0,0 +1,69 @@
|
||||
text,label,__expected
|
||||
I'm confused about a charge on my recent auto insurance bill that's higher than my usual premium payment. Can you clarify what this extra fee is for and why it was added?,Billing Inquiries,contains:Billing Inquiries
|
||||
"My latest car insurance statement includes a charge that I don't recognize, and it's made my total payment go up quite a bit. Could you look into this and explain what the additional cost is related to?",Billing Inquiries,contains:Billing Inquiries
|
||||
I noticed an unfamiliar line item on my auto policy billing that's causing my premium to be more expensive than normal. Would you mind investigating this charge and letting me know the reason behind it?,Billing Inquiries,contains:Billing Inquiries
|
||||
I'd like to make some changes to the deductibles and coverage options on my home insurance policy. Can you outline the different choices I have and help me decide which ones might be best for my situation?,Policy Administration,contains:Policy Administration
|
||||
"I'm thinking about adjusting the coverage limits and deductibles associated with my homeowners insurance. What alternatives are available to me, and can you provide some guidance on selecting the most appropriate options?",Policy Administration,contains:Policy Administration
|
||||
I want to explore the possibility of modifying my home insurance policy's deductibles and coverage amounts. Could you describe the various options I can consider and offer some advice on which ones might suit my needs best?,Policy Administration,contains:Policy Administration
|
||||
"I came out to my car after shopping and found that another vehicle had hit it, but the driver was nowhere to be found. What should my next steps be in terms of filing an insurance claim for the damage?",Claims Assistance,contains:Claims Assistance
|
||||
"Someone apparently crashed into my parked car while I was away from it, but I have no information about who did it. Can you guide me through the process of submitting a claim in this type of hit-and-run scenario?",Claims Assistance,contains:Claims Assistance
|
||||
"I discovered that my parked vehicle had been struck by another car, but the other driver left the scene without providing any contact details. How do I go about reporting this incident to insurance and getting the damage covered?",Claims Assistance,contains:Claims Assistance
|
||||
I'm a bit puzzled about what the difference is between collision and comprehensive coverage for car insurance. Would you be able to clarify the distinction and explain what each type of coverage protects against?,Coverage Explanations,contains:Coverage Explanations
|
||||
Can you help me better understand the nuances between auto collision and comprehensive insurance? I want to make sure I'm clear on what kinds of situations and damages fall under the scope of each one.,Coverage Explanations,contains:Coverage Explanations
|
||||
I'd like to learn more about the respective benefits of collision versus comprehensive car insurance coverage. Could you break down what sets them apart and describe the primary areas they each address?,Coverage Explanations,contains:Coverage Explanations
|
||||
I'm in the process of shopping around for new insurance and am interested in seeing what your company can offer. Can you provide me with a quote that is competitive with or better than the coverage I currently have elsewhere?,Quotes and Proposals,contains:Quotes and Proposals
|
||||
"As I compare insurance options from different providers, I'm curious to know how your company's rates and coverage stack up against my existing policy. Would you be able to prepare a quote that meets or beats what I have now?",Quotes and Proposals,contains:Quotes and Proposals
|
||||
I'm considering switching insurance carriers and want to find out if your company can give me a better deal than what I have at present. Can you put together a quote that showcases the value and coverage advantages you can offer compared to my current insurer?,Quotes and Proposals,contains:Quotes and Proposals
|
||||
"I'm having trouble signing in to my online insurance account portal. When I enter my password, it says it's incorrect even though I know I'm using the right one. Can you help me troubleshoot this login issue?",Account Management,contains:Account Management
|
||||
"Every time I attempt to access my insurance account on your website, I get an error message indicating that my password is invalid. I haven't changed it recently, so I'm not sure what the problem could be. Can you assist me in resolving this?",Account Management,contains:Account Management
|
||||
"I can't seem to log in to my web-based insurance profile because the system keeps telling me my password is wrong, but I'm certain I'm entering it correctly. Would you be able to look into this and help me regain access to my account?",Account Management,contains:Account Management
|
||||
There's a mysterious charge from your insurance company on my recent credit card statement that doesn't coincide with my regular premium payment. Can you investigate the nature of this charge and clarify why it was applied to my account?,Billing Disputes,contains:Billing Disputes
|
||||
"I've encountered an odd fee from your company on my latest credit card bill, and it's not the same as my typical insurance premium amount. Could you do some digging to find out what this charge represents and why I'm being billed for it?",Billing Disputes,contains:Billing Disputes
|
||||
"My credit card has been hit with an unexpected charge from your insurance agency that I don't understand, as it's separate from my standard premium. Would you be willing to research this matter and provide an explanation for the additional fee?",Billing Disputes,contains:Billing Disputes
|
||||
"I recently filed a claim for some water damage that occurred in my kitchen, but I was surprised to learn that it was denied. I believe this type of incident should be covered under my homeowners policy, so how can I go about appealing the denial?",Claims Disputes,contains:Claims Disputes
|
||||
"After submitting a claim for water-related damage to my kitchen, I was dismayed to receive a rejection notice stating that it wasn't covered. However, I feel strongly that my policy should apply to this situation. What recourse do I have to challenge this decision and get my claim approved?",Claims Disputes,contains:Claims Disputes
|
||||
"I'm confused and frustrated by your company's refusal to cover the water damage claim I filed for my kitchen, as I'm fairly certain my homeowners insurance should extend to this type of event. Can you explain the process for disputing this denial and arguing my case for coverage?",Claims Disputes,contains:Claims Disputes
|
||||
"I'm wavering between a high-deductible health plan and a more traditional PPO option, and I could use some assistance in weighing the relative merits and drawbacks of each. Can you offer a comparison of these two choices to help inform my decision?",Policy Comparisons,contains:Policy Comparisons
|
||||
Choosing between a health insurance policy with a high deductible and a standard PPO plan has me feeling uncertain. Could you provide an overview of the key differences I should consider as I try to determine which one is the better fit for me?,Policy Comparisons,contains:Policy Comparisons
|
||||
"I received a letter outlining some forthcoming adjustments to my insurance policy, but I'm having trouble deciphering exactly what they mean for my coverage. Would you be able to walk me through the changes in greater detail so I can fully understand their implications?",General Inquiries,contains:General Inquiries
|
||||
"A piece of mail from your company arrived informing me of some impending updates to my insurance plan, but the specifics are a little hazy to me. Can you shed more light on what these modifications entail and how they will impact my policy moving forward?",General Inquiries,contains:General Inquiries
|
||||
"Your agency sent me a notice about some planned revisions to my insurance coverage, but I'm not entirely sure I grasp the practical effects of these changes. Could you provide a more thorough explanation of what I can expect and how my policy will be different as a result?",General Inquiries,contains:General Inquiries
|
||||
"I added another vehicle to my car insurance policy not long ago, with the understanding that I would receive a multi-car discount. However, this discount is nowhere to be found on my latest billing statement. Can you double-check that it's been properly applied to my account?",Billing Inquiries,contains:Billing Inquiries
|
||||
"When I recently expanded my auto insurance to cover an additional vehicle, I was told I'd qualify for a multi-car price break. Upon reviewing my most recent bill, though, I don't see any indication of this discount being included. Could you verify that the appropriate adjustment has been made to my premium?",Billing Inquiries,contains:Billing Inquiries
|
||||
"I was under the impression that adding a second car to my existing auto policy would trigger a multi-vehicle discount on my insurance costs. Strangely, this reduction in price isn't displayed on my current bill. Would you mind confirming that my account reflects the expected discount?",Billing Inquiries,contains:Billing Inquiries
|
||||
"Following a hailstorm that left my home with some roof damage, I need to initiate an insurance claim. However, I'm unsure whether this falls under my primary homeowners policy or if I should instead file it against my separate windstorm coverage. Can you advise me on the correct course of action?",Claims Assistance,contains:Claims Assistance
|
||||
"I'm ready to begin the claims process after discovering hail damage to my home's roof, but I'm uncertain about which policy to invoke - my standard homeowners insurance or the windstorm rider I have. What would you recommend as the most appropriate path forward?",Claims Assistance,contains:Claims Assistance
|
||||
"My house sustained hail damage during a recent storm, and I'm planning to submit an insurance claim to cover the necessary roof repairs. I'm just not clear on whether this should be handled through my main homeowners policy or my additional windstorm coverage. Can you point me in the right direction?",Claims Assistance,contains:Claims Assistance
|
||||
"My auto insurance policy includes provisions for rental car coverage in certain situations, but I'm a bit fuzzy on the specifics of what's included. Can you fill me in on the exact daily and per-incident allowances for rental expenses under my plan?",Coverage Explanations,contains:Coverage Explanations
|
||||
"I'm aware that my car insurance offers some degree of rental vehicle coverage, but I don't have a complete grasp of the details involved. Would you be able to clarify for me the precise daily and per-event limits that apply to rental reimbursements through my policy?",Coverage Explanations,contains:Coverage Explanations
|
||||
"While I know rental cars are covered to some extent by my auto insurance, I'm not entirely confident about the particular limits and restrictions that come into play. Could you spell out for me the exact amounts covered per day and per incident when I need to rent a vehicle?",Coverage Explanations,contains:Coverage Explanations
|
||||
"As I explore life insurance options, I find myself torn between term and whole life policies. Can you help me understand the core differences between these two choices and provide some guidance on which route might be most advantageous for my individual circumstances?",Quotes and Proposals,contains:Quotes and Proposals
|
||||
I'm actively seeking life insurance coverage but feel uncertain about whether to pursue a term policy or a whole life plan. Could you elaborate on the key distinctions I should be aware of and share your perspective on which might be the better fit considering my personal situation?,Quotes and Proposals,contains:Quotes and Proposals
|
||||
I'd like to make the switch to paperless billing for my insurance premiums and set up automatic withdrawal for my regular payments. What steps do I need to take within my online account portal to enable these convenient features?,Account Management,contains:Account Management
|
||||
"To streamline my insurance billing and payment processes, I'm hoping to transition to electronic statements and recurring automatic charges. Can you outline for me the specific actions required to establish these preferences through my web-based account interface?",Account Management,contains:Account Management
|
||||
My goal is to eliminate paper copies of my insurance bills and have my premiums automatically drafted from my designated account. Would you be able to provide a step-by-step walkthrough of how to configure these settings within my online profile?,Account Management,contains:Account Management
|
||||
"It appears that I may have been charged twice for the motorcycle endorsement associated with my auto insurance policy. While my overall premium has increased, I don't see a line item clearly identifying the endorsement fee. Can you clarify what's behind this price hike?",Billing Disputes,contains:Billing Disputes
|
||||
"I have a sneaking suspicion that I'm being double-billed for my motorcycle endorsement, as my car insurance rates have gone up without a transparent breakdown of the charges. Could you investigate whether I'm being inadvertently overcharged for this add-on coverage?",Billing Disputes,contains:Billing Disputes
|
||||
"My auto insurance premium has recently risen, and I'm concerned that it might be due to a redundant charge for my motorcycle endorsement, since the cost isn't plainly stated on my bill. Would you be willing to look into this and confirm whether or not I'm paying twice for the same thing?",Billing Disputes,contains:Billing Disputes
|
||||
"In the aftermath of an accident, the repair estimate I received from the body shop significantly exceeds the amount that your claims adjuster has authorized. What's the most effective way to challenge this lowball offer and make sure the full scope of necessary repairs is covered?",Claims Disputes,contains:Claims Disputes
|
||||
"Having obtained a repair quote following a crash, I'm dismayed to find that it's markedly higher than what your insurance adjuster has agreed to pay. How should I proceed in disputing this inadequate settlement and advocating for more comprehensive coverage of the damage?",Claims Disputes,contains:Claims Disputes
|
||||
"After bringing my vehicle to a repair facility post-accident, I've been handed an estimate that puts the cost of fixes well above the figure presented by your adjuster. Can you advise me on the optimal approach for contesting your company's position and securing a fair resolution?",Claims Disputes,contains:Claims Disputes
|
||||
"During a recent conversation with my insurance agent, the concept of a ""diminishing deductible"" was mentioned, but I don't fully comprehend the mechanics of how it works. Could you expand on this topic and elucidate the advantages of opting for such a feature?",General Inquiries,contains:General Inquiries
|
||||
"I'm intrigued by the idea of a ""disappearing deductible"" that my insurance rep alluded to, but I'm still a bit hazy on the particulars. Would you be able to furnish a more extensive explanation of this option and describe how it could potentially benefit me as a policyholder?",General Inquiries,contains:General Inquiries
|
||||
"The notion of a ""vanishing deductible"" came up in a recent discussion with my insurance agent, piquing my curiosity. However, I'm not confident that I wholly grasp the nuances of this provision. Might you be able to shed additional light on how it functions and what value it could bring to my coverage?",General Inquiries,contains:General Inquiries
|
||||
"As I contemplate the impact of my upcoming retirement on my insurance needs, I'm wondering what modifications to my life insurance policy might be prudent. What guidance can you share about tailoring my coverage to align with my evolving financial objectives and risk profile in this new chapter?",Policy Administration,contains:Policy Administration
|
||||
"With my retirement on the horizon, I'm keen to ensure that my life insurance policy remains well-suited to my changing circumstances. Can you recommend any adjustments I should consider to optimize my coverage and better safeguard my loved ones' financial security during this transitional phase?",Policy Administration,contains:Policy Administration
|
||||
"On the cusp of retiring, I recognize the importance of reassessing my life insurance strategy to account for my new realities and priorities. What words of wisdom can you impart about fine-tuning my policy to provide the most appropriate level of protection as I navigate this major life milestone?",Policy Administration,contains:Policy Administration
|
||||
"I'm wrestling with how to proceed after a sump pump malfunction led to extensive water damage in my basement, and I'm not certain whether this type of event falls within the scope of my homeowners insurance. Can you clarify the extent to which water-related incidents are typically covered under a standard policy?",Coverage Explanations,contains:Coverage Explanations
|
||||
"A recent sump pump failure at my home resulted in significant flooding and water damage to my basement. I'm unsure if I can expect my homeowners insurance to cover the repairs, as I'm fuzzy on exactly what kinds of water damage are included. Could you elucidate the common parameters around such coverage?",Coverage Explanations,contains:Coverage Explanations
|
||||
"Much to my chagrin, my basement sustained considerable water damage following a sump pump breakdown, and I'm grappling with whether this scenario is likely to be covered by my homeowners policy. Would you be able to shed some light on the usual inclusions and exclusions pertaining to water-related damage?",Coverage Explanations,contains:Coverage Explanations
|
||||
"I've received a troubling notification that my insurance policy is on the brink of cancellation due to an allegedly missed payment. However, I'm quite certain I submitted the premium by the specified deadline. Could you please investigate this discrepancy and ensure that my coverage remains in force if there's been an error on your end?",Billing Disputes,contains:Billing Disputes
|
||||
"According to a recent communication from your company, my insurance coverage is poised to lapse because of a perceived failure to remit payment in a timely fashion. I'm confident, though, that I did indeed tender the amount owed before the stated due date. Might you look into this matter and reinstate my policy if a mistake has occurred?",Billing Disputes,contains:Billing Disputes
|
||||
"I find myself in a perplexing situation, having been informed that my insurance is set to be terminated for non-payment despite my belief that I had punctually rendered the requisite premium. Would you be so kind as to delve into this quandary and reactive my coverage post-haste if there has been an administrative snafu?",Billing Disputes,contains:Billing Disputes
|
||||
The repair shop I visited in the wake of my automobile accident has furnished me with an estimate that eclipses the value assessment put forth by your designated appraiser. I'm struggling to reconcile this pronounced disparity and would greatly appreciate your counsel on how to bridge the gap and reach an equitable resolution.,Claims Disputes,contains:Claims Disputes
|
||||
"Upon entrusting my vehicle to a reputable mechanic for post-collision repairs, I've been presented with a projected cost that markedly outstrips the appraisal tendered by your representative. Confronted with this substantial discrepancy, I'm keen to solicit your guidance on navigating the impasse and achieving a mutually agreeable outcome.",Claims Disputes,contains:Claims Disputes
|
||||
"Having sought out a repair facility to mend the damage inflicted upon my car in a recent mishap, I now find myself at odds with the estimate they've proffered, which towers above the figure quoted by your appraiser. I'd be most grateful for your insight into surmounting this incongruity and charting a course toward a fair and satisfactory settlement.",Claims Disputes,contains:Claims Disputes
|
||||
"As I endeavor to select between two health insurance policies with comparable premium structures but contrasting out-of-pocket maximum thresholds, I'm grappling with how my personal medical history should inform my decision. Might you be able to proffer some sage advice on which option would be most judicious given my unique health profile?",Policy Comparisons,contains:Policy Comparisons
|
||||
"I'm vacillating between a pair of health insurance plans that boast similar monthly premiums but diverge in their out-of-pocket maximum provisions. Taking into account my individualized medical background, which path would you counsel me to pursue to optimize my coverage and safeguard my financial well-being?",Policy Comparisons,contains:Policy Comparisons
|
||||
"Confronted with a binary choice between two health insurance policies that mirror each other in premium costs but differ in their out-of-pocket maximum stipulations, I find myself at a crossroads. Could you impart some wisdom on which alternative would be most propitious for me, bearing in mind my distinctive healthcare chronicles?",Policy Comparisons,contains:Policy Comparisons
|
||||
"I'd be much obliged if you could furnish me with a more exhaustive elucidation of your company's accident forgiveness program, detailing both its operational intricacies and the eligibility criteria that policyholders must satisfy to avail themselves of this enticing benefit.",General Inquiries,contains:General Inquiries
|
||||
|
@@ -2,7 +2,7 @@ description: 'Classification'
|
||||
|
||||
# Prompts defined in the prompts.py file
|
||||
prompts:
|
||||
- prompt.txt
|
||||
- prompts.py:simple_classify
|
||||
- prompts.py:rag_classify
|
||||
- prompts.py:rag_chain_of_thought_classify
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from vectordb import VectorDB
|
||||
|
||||
import textwrap
|
||||
vectordb = VectorDB()
|
||||
# Load the vector database
|
||||
vectordb.load_db()
|
||||
@@ -55,62 +55,91 @@ categories = """<category>
|
||||
</content>
|
||||
</category>"""
|
||||
|
||||
|
||||
def simple_classify(context: dict):
|
||||
X = context['vars']['text']
|
||||
prompt = textwrap.dedent("""
|
||||
You will classify a customer support ticket into one of the following categories:
|
||||
<categories>
|
||||
{{categories}}
|
||||
</categories>
|
||||
|
||||
Here is the customer support ticket:
|
||||
<ticket>
|
||||
{{ticket}}
|
||||
</ticket>
|
||||
|
||||
Respond with just the label of the category between category tags.
|
||||
""").replace("{{categories}}", categories).replace("{{ticket}}", X)
|
||||
return prompt
|
||||
|
||||
|
||||
|
||||
def rag_classify(context: dict):
|
||||
X = context['vars']['text']
|
||||
rag = vectordb.search(X,25)
|
||||
rag = vectordb.search(X,5)
|
||||
rag_string = ""
|
||||
for example in rag:
|
||||
rag_string += f"""
|
||||
rag_string += textwrap.dedent(f"""
|
||||
<example>
|
||||
<query>
|
||||
{example["metadata"]["text"]}
|
||||
"{example["metadata"]["text"]}"
|
||||
</query>
|
||||
<label>
|
||||
{example["metadata"]["label"]}
|
||||
</label>
|
||||
</example>
|
||||
"""
|
||||
prompt = """
|
||||
You are an accurate classifier.
|
||||
Given a query, classify it into one of the provided labels.
|
||||
<query>
|
||||
<QUERY>
|
||||
</query>
|
||||
""")
|
||||
prompt = textwrap.dedent("""
|
||||
You will classify a customer support ticket into one of the following categories:
|
||||
<categories>
|
||||
{{categories}}
|
||||
</categories>
|
||||
|
||||
<labels>
|
||||
<LABELS>
|
||||
</labels>
|
||||
Here is the customer support ticket:
|
||||
<ticket>
|
||||
{{ticket}}
|
||||
</ticket>
|
||||
|
||||
Use the following examples to help you classify the query:
|
||||
<examples>
|
||||
<RAG>
|
||||
{{examples}}
|
||||
</examples>
|
||||
|
||||
Respond with just the label of the category between category tags.
|
||||
""".replace("<LABELS>", categories).replace("<QUERY>", X).replace("<RAG>", rag_string)
|
||||
""").replace("{{categories}}", categories).replace("{{ticket}}", X).replace("{{examples}}", rag_string)
|
||||
return prompt
|
||||
|
||||
|
||||
def rag_chain_of_thought_classify(context: dict):
|
||||
X = context['vars']['text']
|
||||
rag = vectordb.search(X,25)
|
||||
rag = vectordb.search(X,5)
|
||||
rag_string = ""
|
||||
for example in rag:
|
||||
rag_string += f'<example>\n<query>\n"{example["metadata"]["text"]}"\n</query>\n<label>\n{example["metadata"]["label"]}\n</label>\n</example>\n'
|
||||
prompt = """
|
||||
You are an accurate classifier.
|
||||
Given a query, classify it into one of the provided labels.
|
||||
<query>
|
||||
<QUERY>
|
||||
</query>
|
||||
rag_string += textwrap.dedent(f"""
|
||||
<example>
|
||||
<query>
|
||||
"{example["metadata"]["text"]}"
|
||||
</query>
|
||||
<label>
|
||||
{example["metadata"]["label"]}
|
||||
</label>
|
||||
</example>
|
||||
""")
|
||||
prompt = textwrap.dedent("""
|
||||
You will classify a customer support ticket into one of the following categories:
|
||||
<categories>
|
||||
{{categories}}
|
||||
</categories>
|
||||
|
||||
<labels>
|
||||
<LABELS>
|
||||
</labels>
|
||||
Here is the customer support ticket:
|
||||
<ticket>
|
||||
{{ticket}}
|
||||
</ticket>
|
||||
|
||||
Use the following examples to help you classify the query:
|
||||
<examples>
|
||||
<RAG>
|
||||
{{examples}}
|
||||
</examples>
|
||||
|
||||
First you will think step-by-step about the problem in scratchpad tags.
|
||||
@@ -121,5 +150,5 @@ def rag_chain_of_thought_classify(context: dict):
|
||||
<scratchpad>Your thoughts and analysis go here</scratchpad>
|
||||
<category>The category label you chose goes here</category>
|
||||
</response>
|
||||
""".replace("<LABELS>", categories).replace("<QUERY>", X).replace("<RAG>", rag_string)
|
||||
""").replace("{{categories}}", categories).replace("{{ticket}}", X).replace("{{examples}}", rag_string)
|
||||
return prompt
|
||||
6
skills/classification/evaluation/transform.py
Normal file
6
skills/classification/evaluation/transform.py
Normal file
@@ -0,0 +1,6 @@
|
||||
def get_transform(output, context):
|
||||
try:
|
||||
return output.split("<category>")[1].split("</category>")[0].strip()
|
||||
except Exception as e:
|
||||
print(f"Error in get_transform: {e}")
|
||||
return output
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import numpy as np
|
||||
import voyageai
|
||||
import pickle
|
||||
import json
|
||||
|
||||
class VectorDB:
|
||||
def __init__(self, api_key=None):
|
||||
@@ -31,8 +32,7 @@ class VectorDB:
|
||||
result = [
|
||||
self.client.embed(
|
||||
texts[i : i + batch_size],
|
||||
model="voyage-large-2-instruct",
|
||||
input_type="document",
|
||||
model="voyage-2"
|
||||
).embeddings
|
||||
for i in range(0, len(texts), batch_size)
|
||||
]
|
||||
@@ -40,7 +40,6 @@ class VectorDB:
|
||||
# Flatten the embeddings
|
||||
self.embeddings = [embedding for batch in result for embedding in batch]
|
||||
self.metadata = [item for item in data]
|
||||
self.save_db()
|
||||
# Save the vector database to disk
|
||||
print("Vector database loaded and saved.")
|
||||
|
||||
@@ -49,7 +48,7 @@ class VectorDB:
|
||||
if query in self.query_cache:
|
||||
query_embedding = self.query_cache[query]
|
||||
else:
|
||||
query_embedding = self.client.embed([query], model="voyage-large-2-instruct", input_type="query").embeddings[0]
|
||||
query_embedding = self.client.embed([query], model="voyage-2").embeddings[0]
|
||||
self.query_cache[query] = query_embedding
|
||||
|
||||
if not self.embeddings:
|
||||
@@ -71,14 +70,6 @@ class VectorDB:
|
||||
break
|
||||
|
||||
return top_examples
|
||||
|
||||
def save_db(self):
|
||||
data = {
|
||||
"embeddings": self.embeddings,
|
||||
"metadata": self.metadata
|
||||
}
|
||||
with open(self.db_path, "wb") as file:
|
||||
pickle.dump(data, file)
|
||||
|
||||
def load_db(self):
|
||||
if not os.path.exists(self.db_path):
|
||||
@@ -87,4 +78,5 @@ class VectorDB:
|
||||
with open(self.db_path, "rb") as file:
|
||||
data = pickle.load(file)
|
||||
self.embeddings = data["embeddings"]
|
||||
self.metadata = data["metadata"]
|
||||
self.metadata = data["metadata"]
|
||||
self.query_cache = json.loads(data["query_cache"])
|
||||
1099
skills/classification/guide.ipynb
Normal file
1099
skills/classification/guide.ipynb
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user