creates data pipeline

fingpt/FinGPT_Forecaster/data.py

@@ -0,0 +1,294 @@
+import os
+import re
+import csv
+import math
+import time
+import json
+import finnhub
+from tqdm import tqdm
+import pandas as pd
+import yfinance as yf
+from datetime import datetime
+from collections import defaultdict
+import datasets
+from datasets import Dataset
+from openai import OpenAI
+
+from indices import *
+from prompt import get_all_prompts
+
+finnhub_client = finnhub.Client(api_key=os.environ.get("FINNHUB_KEY"))
+client = OpenAI(api_key=os.environ.get("OPENAI_KEY"))
+
+
+# ----------------------------------------------------------------------------------- #
+# ---------------------------- RAW FINANCIAL ACQUISITION ---------------------------- #
+# ----------------------------------------------------------------------------------- #
+
+def bin_mapping(ret):
+    
+    up_down = 'U' if ret >= 0 else 'D'
+    integer = math.ceil(abs(100 * ret))
+    
+    return up_down + (str(integer) if integer <= 5 else '5+')
+
+
+def get_returns(stock_symbol, start_date, end_date):
+    # TODO: likely to be merged with get_stock_data
+    
+    # Download historical stock data
+    stock_data = yf.download(stock_symbol, start=start_date, end=end_date)
+    
+    weekly_data = stock_data['Adj Close'].resample('W').ffill()
+    weekly_returns = weekly_data.pct_change()[1:]
+    weekly_start_prices = weekly_data[:-1]
+    weekly_end_prices = weekly_data[1:]
+
+    weekly_data = pd.DataFrame({
+        'Start Date': weekly_start_prices.index,
+        'Start Price': weekly_start_prices.values,
+        'End Date': weekly_end_prices.index,
+        'End Price': weekly_end_prices.values,
+        'Weekly Returns': weekly_returns.values
+    })
+    
+    weekly_data['Bin Label'] = weekly_data['Weekly Returns'].map(bin_mapping)
+
+    return weekly_data
+
+
+def get_news(symbol, data):
+    
+    news_list = []
+    
+    for end_date, row in data.iterrows():
+        start_date = row['Start Date'].strftime('%Y-%m-%d')
+        end_date = row['End Date'].strftime('%Y-%m-%d')
+        # print(symbol, ': ', start_date, ' - ', end_date)
+        time.sleep(1) # control qpm
+        weekly_news = finnhub_client.company_news(symbol, _from=start_date, to=end_date)
+        weekly_news = [
+            {
+                "date": datetime.fromtimestamp(n['datetime']).strftime('%Y%m%d%H%M%S'),
+                "headline": n['headline'],
+                "summary": n['summary'],
+            } for n in weekly_news
+        ]
+        weekly_news.sort(key=lambda x: x['date'])
+        news_list.append(json.dumps(weekly_news))
+    
+    data['News'] = news_list
+    
+    return data
+
+
+def get_basics(symbol, data, start_date, always=False):
+    
+    basic_financials = finnhub_client.company_basic_financials(symbol, 'all')
+    
+    final_basics, basic_list, basic_dict = [], [], defaultdict(dict)
+    
+    for metric, value_list in basic_financials['series']['quarterly'].items():
+        for value in value_list:
+            basic_dict[value['period']].update({metric: value['v']})
+
+    for k, v in basic_dict.items():
+        v.update({'period': k})
+        basic_list.append(v)
+        
+    basic_list.sort(key=lambda x: x['period'])
+            
+    for i, row in data.iterrows():
+        
+        start_date = row['End Date'].strftime('%Y-%m-%d')
+        last_start_date = start_date if i < 2 else data.loc[i-2, 'Start Date'].strftime('%Y-%m-%d')
+        
+        used_basic = {}
+        for basic in basic_list[::-1]:
+            if (always and basic['period'] < start_date) or (last_start_date <= basic['period'] < start_date):
+                used_basic = basic
+                break
+        final_basics.append(json.dumps(used_basic))
+        
+    data['Basics'] = final_basics
+    
+    return data
+    
+
+def prepare_data_for_symbol(symbol, data_dir, start_date, end_date, with_basics=True):
+    
+    data = get_returns(symbol, start_date, end_date)
+    data = get_news(symbol, data)
+    
+    if with_basics:
+        data = get_basics(symbol, data, start_date)
+        data.to_csv(f"{data_dir}/{symbol}_{start_date}_{end_date}.csv")
+    else:
+        data['Basics'] = [json.dumps({})] * len(data)
+        data.to_csv(f"{data_dir}/{symbol}_{start_date}_{end_date}_nobasics.csv")
+    
+    return data
+
+
+# ----------------------------------------------------------------------------------- #
+# ---------------------------------- GPT4 ANALYSIS ---------------------------------- #
+# ----------------------------------------------------------------------------------- #
+
+
+def append_to_csv(filename, input_data, output_data):
+    
+    with open(filename, mode='a', newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow([input_data, output_data])
+
+        
+def initialize_csv(filename):
+    
+    with open(filename, mode='w', newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow(["prompt", "answer"])
+
+
+def query_gpt4(symbol_list, data_dir, start_date, end_date, min_past_weeks=1, max_past_weeks=3, with_basics=True):
+
+    for symbol in tqdm(symbol_list):
+        
+        csv_file = f'{data_dir}/{symbol}_{start_date}_{end_date}_gpt-4.csv' if with_basics else \
+                   f'{data_dir}/{symbol}_{start_date}_{end_date}_nobasics_gpt-4.csv'
+        
+        if not os.path.exists(csv_file):
+            initialize_csv(csv_file)
+            pre_done = 0
+        else:
+            df = pd.read_csv(csv_file)
+            pre_done = len(df)
+
+        prompts = get_all_prompts(symbol, data_dir, start_date, end_date, min_past_weeks, max_past_weeks, with_basics)
+        system_prompt = SYSTEM_PROMPTS["crypto"] if symbol in CRYPTO else SYSTEM_PROMPTS["company"]
+        for i, prompt in enumerate(prompts):
+            
+            if i < pre_done:
+                continue
+
+            # print(f"{symbol} - {i}")
+            
+            cnt = 0
+            while cnt < 5:
+                try:
+                    completion = client.chat.completions.create(
+                        model="gpt-4",
+                        messages=[
+                            {"role": "system", "content": system_prompt},
+                            {"role": "user", "content": prompt}
+                          ]
+                    )
+                    break    
+                except Exception:
+                    cnt += 1
+                    print(f'retry cnt {cnt}')
+            
+            answer = completion.choices[0].message.content if cnt < 5 else ""
+            append_to_csv(csv_file, prompt, answer)
+
+
+
+# ----------------------------------------------------------------------------------- #
+# -------------------------- TRANSFORM INTO TRAINING FORMAT ------------------------- #
+# ----------------------------------------------------------------------------------- #
+
+B_INST, E_INST = "[INST]", "[/INST]"
+B_SYS, E_SYS = "<<SYS>>\n", "\n<</SYS>>\n\n"
+
+SYSTEM_PROMPTS = {
+    "company": "You are a seasoned stock market analyst. Your task is to list the positive developments and potential concerns for companies based on relevant news and basic financials from the past weeks, then provide an analysis and prediction for the companies' stock price movement for the upcoming week. " \
+    "Your answer format should be as follows:\n\n[Positive Developments]:\n1. ...\n\n[Potential Concerns]:\n1. ...\n\n[Prediction & Analysis]:\n...\n",
+
+    "crypto": "You are a seasoned crypto market analyst. Your task is to list the positive developments and potential concerns for cryptocurrencies based on relevant news and basic financials from the past weeks, then provide an analysis and prediction for the cryptocurrencies price movement for the upcoming week. " \
+    "Your answer format should be as follows:\n\n[Positive Developments]:\n1. ...\n\n[Potential Concerns]:\n1. ...\n\n[Prediction & Analysis]:\n...\n",
+}
+
+def gpt4_to_llama(symbol, data_dir, start_date, end_date, with_basics=True):
+
+    csv_file = f'{data_dir}/{symbol}_{start_date}_{end_date}_gpt-4.csv' if with_basics else \
+                   f'{data_dir}/{symbol}_{start_date}_{end_date}_nobasics_gpt-4.csv'
+    
+    df = pd.read_csv(csv_file)
+    
+    prompts, answers, periods, labels = [], [], [], []
+    
+    for i, row in df.iterrows():
+        
+        prompt, answer = row['prompt'], row['answer']
+        
+        res = re.search(r"Then let's assume your prediction for next week \((.*)\) is ((:?up|down) by .*%).", prompt)
+        
+        period, label = res.group(1), res.group(2)
+#         label = label.replace('more than 5', '5+')
+        
+        prompt = re.sub(
+            r"Then let's assume your prediction for next week \((.*)\) is (up|down) by ((:?.*)%). Provide a summary analysis to support your prediction. The prediction result need to be inferred from your analysis at the end, and thus not appearing as a foundational factor of your analysis.", 
+            f"Then make your prediction of the {symbol} cryptocurrency price movement for next week ({period}). Provide a summary analysis to support your prediction.",
+            prompt
+        )
+        try:
+            answer = re.sub(
+                r"\[Prediction & Analysis\]:\s*",
+                f"[Prediction & Analysis]:\nPrediction: {label.capitalize()}\nAnalysis: ",
+                answer
+            )
+        except Exception:
+            print(symbol, i)
+            print(label)
+            print(answer)
+            continue
+            
+        system_prompt = SYSTEM_PROMPTS["crypto"] if symbol in CRYPTO else SYSTEM_PROMPTS["company"]
+        new_system_prompt = system_prompt.replace(':\n...', '\nPrediction: ...\nAnalysis: ...')
+#         new_system_prompt = SYSTEM_PROMPT.replace(':\n...', '\nPrediction: {Up|Down} by {1-2|2-3|3-4|4-5|5+}%\nAnalysis: ...')
+        
+        prompt = B_INST + B_SYS + new_system_prompt + E_SYS + prompt + E_INST
+        
+        prompts.append(prompt)
+        answers.append(answer)
+        periods.append(period)
+        labels.append(label)
+        
+    return {
+        "prompt": prompts,
+        "answer": answers,
+        "period": periods,
+        "label": labels,
+    }
+
+
+def create_dataset(symbol_list, data_dir, start_date, end_date, train_ratio=0.8, with_basics=True):
+
+    train_dataset_list = []
+    test_dataset_list = []
+
+    for symbol in symbol_list:
+
+        data_dict = gpt4_to_llama(symbol, data_dir, start_date, end_date,  with_basics)
+#         print(data_dict['prompt'][-1])
+#         print(data_dict['answer'][-1])
+        symbols = [symbol] * len(data_dict['label'])
+        data_dict.update({"symbol": symbols})
+
+        dataset = Dataset.from_dict(data_dict)
+        train_size = round(train_ratio * len(dataset))
+
+        train_dataset_list.append(dataset.select(range(train_size)))
+        if train_size >= len(dataset):
+            continue
+        test_dataset_list.append(dataset.select(range(train_size, len(dataset))))
+
+    train_dataset = datasets.concatenate_datasets(train_dataset_list)
+    test_dataset = datasets.concatenate_datasets(test_dataset_list)
+
+    dataset = datasets.DatasetDict({
+        'train': train_dataset,
+        'test': test_dataset
+    })
+    
+    return dataset
+   

fingpt/FinGPT_Forecaster/data_infererence_fetch.py

@@ -0,0 +1,119 @@
+import os
+import finnhub
+import yfinance as yf
+import pandas as pd
+from datetime import date, datetime, timedelta
+from collections import defaultdict
+
+from data import get_news
+from prompt import get_company_prompt, get_prompt_by_row, sample_news
+
+finnhub_client = finnhub.Client(api_key=os.environ.get("FINNHUB_KEY"))
+
+
+def get_curday():
+    
+    return date.today().strftime("%Y-%m-%d")
+
+
+def n_weeks_before(date_string, n):
+    
+    date = datetime.strptime(date_string, "%Y-%m-%d") - timedelta(days=7*n)
+    
+    return date.strftime("%Y-%m-%d")
+
+
+def get_stock_data(stock_symbol, steps):
+
+    stock_data = yf.download(stock_symbol, steps[0], steps[-1])
+    
+#     print(stock_data)
+    
+    dates, prices = [], []
+    available_dates = stock_data.index.format()
+    
+    for date in steps[:-1]:
+        for i in range(len(stock_data)):
+            if available_dates[i] >= date:
+                prices.append(stock_data['Close'][i])
+                dates.append(datetime.strptime(available_dates[i], "%Y-%m-%d"))
+                break
+
+    dates.append(datetime.strptime(available_dates[-1], "%Y-%m-%d"))
+    prices.append(stock_data['Close'][-1])
+    
+    return pd.DataFrame({
+        "Start Date": dates[:-1], "End Date": dates[1:],
+        "Start Price": prices[:-1], "End Price": prices[1:]
+    })
+
+
+def get_current_basics(symbol, curday):
+
+    basic_financials = finnhub_client.company_basic_financials(symbol, 'all')
+    
+    final_basics, basic_list, basic_dict = [], [], defaultdict(dict)
+    
+    for metric, value_list in basic_financials['series']['quarterly'].items():
+        for value in value_list:
+            basic_dict[value['period']].update({metric: value['v']})
+
+    for k, v in basic_dict.items():
+        v.update({'period': k})
+        basic_list.append(v)
+        
+    basic_list.sort(key=lambda x: x['period'])
+    
+    for basic in basic_list[::-1]:
+        if basic['period'] <= curday:
+            break
+            
+    return basic
+
+
+def fetch_all_data(symbol, curday, n_weeks=3):
+
+    steps = [n_weeks_before(curday, i) for i in range(n_weeks+1)][::-1]
+
+    data = get_stock_data(symbol, steps)
+    data = get_news(symbol, data)
+
+    return data
+    
+
+def get_all_prompts_online(symbol, data, curday, with_basics=True):
+
+    company_prompt = get_company_prompt(symbol)
+
+    prev_rows = []
+
+    for row_idx, row in data.iterrows():
+        head, news, _ = get_prompt_by_row(symbol, row)
+        prev_rows.append((head, news, None))
+        
+    prompt = ""
+    for i in range(-len(prev_rows), 0):
+        prompt += "\n" + prev_rows[i][0]
+        sampled_news = sample_news(
+            prev_rows[i][1],
+            min(5, len(prev_rows[i][1]))
+        )
+        if sampled_news:
+            prompt += "\n".join(sampled_news)
+        else:
+            prompt += "No relative news reported."
+        
+    period = "{} to {}".format(curday, n_weeks_before(curday, -1))
+    
+    if with_basics:
+        basics = get_current_basics(symbol, curday)
+        basics = "Some recent basic financials of {}, reported at {}, are presented below:\n\n[Basic Financials]:\n\n".format(
+            symbol, basics['period']) + "\n".join(f"{k}: {v}" for k, v in basics.items() if k != 'period')
+    else:
+        basics = "[Basic Financials]:\n\nNo basic financial reported."
+
+    info = company_prompt + '\n' + prompt + '\n' + basics
+    prompt = info + f"\n\nBased on all the information before {curday}, let's first analyze the positive developments and potential concerns for {symbol}. Come up with 2-4 most important factors respectively and keep them concise. Most factors should be inferred from company related news. " \
+        f"Then make your prediction of the {symbol} stock price movement for next week ({period}). Provide a summary analysis to support your prediction."
+        
+    return info, prompt

fingpt/FinGPT_Forecaster/data_pipeline.py

@@ -0,0 +1,69 @@
+import os 
+import json
+from tqdm import tqdm
+import argparse
+
+from indices import *
+from data import  prepare_data_for_symbol, query_gpt4, create_dataset
+from prompt import get_all_prompts
+from data_infererence_fetch import get_curday, fetch_all_data, get_all_prompts_online
+
+
+def main(args):
+
+    index_name = args['index_name']
+    start_date = args['start_date']
+    end_date = args['end_date']
+    min_past_weeks = args['min_past_weeks']
+    max_past_weeks = args['max_past_weeks']
+    train_ratio = args['train_ratio']
+
+    with_basics = True
+    if index_name == "dow":
+        index_name = "DOW-30"
+        index = DOW_30
+    elif index_name == "euro":
+        index_name = "EURO-STOXX-50"
+        index = EURO_STOXX_50
+    elif index_name == "crypto":
+        index_name = "CRYPTO"
+        index = CRYPTO
+        with_basics = False
+    else:
+        raise ValueError("Invalid index name")
+    
+    data_dir = f"./data/{index_name}_{start_date}_{end_date}"
+    os.makedirs(data_dir, exist_ok=True)
+    
+    # Acquire data
+    print("Acquiring data")
+    for symbol in tqdm(index):
+        print(f"Processing {symbol}")
+        prepare_data_for_symbol(symbol, data_dir, start_date, end_date, with_basics=with_basics)
+
+    # Generate prompt and query GPT-4
+    print("Generating prompts and querying GPT-4")
+    query_gpt4(index, data_dir, start_date, end_date, min_past_weeks, max_past_weeks, with_basics=with_basics)
+
+    # Transform into training format
+    print("Transforming into training format")
+    dataset = create_dataset(index, data_dir, start_date, end_date, train_ratio, with_basics=with_basics)
+
+    # Save dataset
+    dataset.save_to_disk(
+        f"./data/fingpt-forecaster-{index_name.lower()}-{start_date.replace('-', '')}-{end_date.replace('-', '')}-{min_past_weeks}-{max_past_weeks}-{str(train_ratio).replace('.', '')}"
+    )
+
+
+if __name__ == "__main__":
+
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--index_name", default="crypto", choices=["dow", "euro", "crypto"], help="index name")
+    ap.add_argument("--start_date", default="2022-12-31", help="start date")
+    ap.add_argument("--end_date", default="2023-12-31", help="end date")
+    ap.add_argument("--min_past_weeks", default=1, help="min past weeks")
+    ap.add_argument("--max_past_weeks", default=4, help="max past weeks")
+    ap.add_argument("--train_ratio", default=0.6, help="train ratio")
+    args = vars(ap.parse_args())
+
+    main(args)

fingpt/FinGPT_Forecaster/indices.py

@@ -0,0 +1,27 @@
+DOW_30 = [
+    "AXP", "AMGN", "AAPL", "BA", "CAT", "CSCO", "CVX", "GS", "HD", "HON",
+    "IBM", "INTC", "JNJ", "KO", "JPM", "MCD", "MMM", "MRK", "MSFT", "NKE",
+    "PG", "TRV", "UNH", "CRM", "VZ", "V", "WBA", "WMT", "DIS", "DOW"
+]
+
+EURO_STOXX_50 = [
+    "ADS.DE", "ADYEN.AS", "AD.AS", "AI.PA", "AIR.PA", "ALV.DE", 
+    "ABI.BR", "ASML.AS", "CS.PA", "BAS.DE", "BAYN.DE", "BBVA.MC", 
+    "SAN.MC", "BMW.DE", "BNP.PA", "BN.PA", "DAI.DE", "DPW.DE", "DTE.DE", 
+    "ENEL.MI", "ENGI.PA", "EL.PA", "FRE.DE", "IBE.MC", "ITX.MC", "IFX.DE",
+    "INGA.AS", "ISP.MI", "KER.PA", "AD.AS", "PHIA.AS", "OR.PA", "LIN.DE",
+    "MC.PA", "MUV2.DE", "NOKIA.SE", "ORA.PA", "RI.PA", "SAF.PA", "SAN.PA",
+    "SAP.DE", "SU.PA", "SIE.DE", "GLE.PA", "STM.PA", "TEF.MC", "TTE.PA", 
+    "UNA.AS", "DG.PA", "VOW3.DE"]
+
+CRYPTO = [
+    "BTC-USD", 
+    "ETH-USD",
+    "ADA-USD",
+    "XRP-USD",
+    "SOL-USD",
+    "DOT-USD",
+    "AVAX-USD",
+    "DOGE-USD",
+    "TRX-USD"
+]

fingpt/FinGPT_Forecaster/prompt.py

@@ -0,0 +1,164 @@
+import os
+import json
+import random
+import finnhub
+import yfinance as yf
+import pandas as pd
+from openai import OpenAI
+
+from indices import *
+
+finnhub_client = finnhub.Client(api_key=os.environ.get("FINNHUB_KEY"))
+
+
+def get_company_prompt(symbol):
+    
+    profile = finnhub_client.company_profile2(symbol=symbol)
+
+    company_template = "[Company Introduction]:\n\n{name} is a leading entity in the {finnhubIndustry} sector. Incorporated and publicly traded since {ipo}, the company has established its reputation as one of the key players in the market. As of today, {name} has a market capitalization of {marketCapitalization:.2f} in {currency}, with {shareOutstanding:.2f} shares outstanding." \
+        "\n\n{name} operates primarily in the {country}, trading under the ticker {ticker} on the {exchange}. As a dominant force in the {finnhubIndustry} space, the company continues to innovate and drive progress within the industry."
+
+    formatted_str = company_template.format(**profile)
+    
+    return formatted_str
+
+
+def get_crypto_prompt(symbol):
+
+    profile = yf.Ticker(symbol).info
+
+    crpyto_template = """[Cryptocurrency Introduction]: {description}. It has a market capilization of {marketCap}."""
+    
+    formatted_str = crpyto_template.format(**profile)
+    
+    return formatted_str
+
+
+def get_prompt_by_row(symbol, row):
+
+    start_date = row['Start Date'] if isinstance(row['Start Date'], str) else row['Start Date'].strftime('%Y-%m-%d')
+    end_date = row['End Date'] if isinstance(row['End Date'], str) else row['End Date'].strftime('%Y-%m-%d')
+    term = 'increased' if row['End Price'] > row['Start Price'] else 'decreased'
+    head = "From {} to {}, {}'s stock price {} from {:.2f} to {:.2f}. News during this period are listed below:\n\n".format(
+        start_date, end_date, symbol, term, row['Start Price'], row['End Price'])
+    
+    news = json.loads(row["News"])
+    news = ["[Headline]: {}\n[Summary]: {}\n".format(
+        n['headline'], n['summary']) for n in news if n['date'][:8] <= end_date.replace('-', '') and \
+        not n['summary'].startswith("Looking for stock market analysis and research with proves results?")]
+
+    basics = json.loads(row['Basics'])
+    if basics:
+        basics = "Some recent basic financials of {}, reported at {}, are presented below:\n\n[Basic Financials]:\n\n".format(
+            symbol, basics['period']) + "\n".join(f"{k}: {v}" for k, v in basics.items() if k != 'period')
+    else:
+        basics = "[Basic Financials]:\n\nNo basic financial reported."
+    
+    return head, news, basics
+
+
+def get_crypto_prompt_by_row(symbol, row):
+
+    start_date = row['Start Date'] if isinstance(row['Start Date'], str) else row['Start Date'].strftime('%Y-%m-%d')
+    end_date = row['End Date'] if isinstance(row['End Date'], str) else row['End Date'].strftime('%Y-%m-%d')
+    term = 'increased' if row['End Price'] > row['Start Price'] else 'decreased'
+    head = "From {} to {}, {}'s stock price {} from {:.2f} to {:.2f}. News during this period are listed below:\n\n".format(
+        start_date, end_date, symbol, term, row['Start Price'], row['End Price'])
+    
+    news = json.loads(row["News"])
+    news = ["[Headline]: {}\n[Summary]: {}\n".format(
+        n['headline'], n['summary']) for n in news if n['date'][:8] <= end_date.replace('-', '') and \
+        not n['summary'].startswith("Looking for stock market analysis and research with proves results?")]
+
+    return head, news, None
+
+
+def sample_news(news, k=5):
+    
+    return [news[i] for i in sorted(random.sample(range(len(news)), k))]
+
+
+def map_bin_label(bin_lb):
+    
+    lb = bin_lb.replace('U', 'up by ')
+    lb = lb.replace('D', 'down by ')
+    lb = lb.replace('1', '0-1%')
+    lb = lb.replace('2', '1-2%')
+    lb = lb.replace('3', '2-3%')
+    lb = lb.replace('4', '3-4%')
+    if lb.endswith('+'):
+        lb = lb.replace('5+', 'more than 5%')
+#         lb = lb.replace('5+', '5+%')
+    else:
+        lb = lb.replace('5', '4-5%')
+    
+    return lb
+
+PROMPT_END = {
+    "company": "\n\nBased on all the information before {start_date}, let's first analyze the positive developments and potential concerns for {symbol}. Come up with 2-4 most important factors respectively and keep them concise. Most factors should be inferred from company related news. " \
+        "Then let's assume your prediction for next week ({start_date} to {end_date}) is {prediction}. Provide a summary analysis to support your prediction. The prediction result need to be inferred from your analysis at the end, and thus not appearing as a foundational factor of your analysis.",
+
+    "crypto": "\n\nBased on all the information before {start_date}, let's first analyze the positive developments and potential concerns for {symbol}. Come up with 2-4 most important factors respectively and keep them concise. Most factors should be inferred from cryptocurrencies related news. " \
+        "Then let's assume your prediction for next week ({start_date} to {end_date}) is {prediction}. Provide a summary analysis to support your prediction. The prediction result need to be inferred from your analysis at the end, and thus not appearing as a foundational factor of your analysis."
+}
+
+def get_all_prompts(symbol, data_dir, start_date, end_date, min_past_weeks=1, max_past_weeks=3, with_basics=True):
+
+    
+    if with_basics:
+        df = pd.read_csv(f'{data_dir}/{symbol}_{start_date}_{end_date}.csv')
+    else:
+        df = pd.read_csv(f'{data_dir}/{symbol}_{start_date}_{end_date}_nobasics.csv')
+    
+    if symbol in CRYPTO:
+        info_prompt = get_crypto_prompt(symbol)
+    else:
+        info_prompt = get_company_prompt(symbol)
+
+    prev_rows = []
+    all_prompts = []
+
+    for row_idx, row in df.iterrows():
+
+        prompt = ""
+        if len(prev_rows) >= min_past_weeks:
+            idx = min(random.choice(range(min_past_weeks, max_past_weeks+1)), len(prev_rows))
+            for i in range(-idx, 0):
+                # Add Price Movement (Head)
+                prompt += "\n" + prev_rows[i][0]
+                # Add News of previous weeks
+                sampled_news = sample_news(
+                    prev_rows[i][1],
+                    min(5, len(prev_rows[i][1]))
+                )
+                if sampled_news:
+                    prompt += "\n".join(sampled_news)
+                else:
+                    prompt += "No relative news reported."
+
+        if symbol in CRYPTO:
+            head, news, basics = get_crypto_prompt_by_row(symbol, row)
+        else:
+            head, news, basics = get_prompt_by_row(symbol, row)
+
+        prev_rows.append((head, news, basics))
+        if len(prev_rows) > max_past_weeks:
+            prev_rows.pop(0)  
+
+        if not prompt:
+            continue
+
+        prediction = map_bin_label(row['Bin Label'])
+        
+        prompt = info_prompt + '\n' + prompt + '\n' + basics
+
+        prompt += PROMPT_END['crypto' if symbol in CRYPTO else 'company'].format(
+            start_date=row['Start Date'],
+            end_date=row['End Date'],
+            prediction=prediction,
+            symbol=symbol
+        )
+
+        all_prompts.append(prompt.strip())
+    
+    return all_prompts

fingpt/FinGPT_Forecaster/train_lora.py

@@ -27,7 +27,7 @@ from peft import (
 )
 
 # Replace with your own api_key and project name
-os.environ['WANDB_API_KEY'] = 'ecf1e5e4f47441d46822d38a3249d62e8fc94db4'
+os.environ['WANDB_API_KEY'] = ''    # TODO: Replace with your environment variable
 os.environ['WANDB_PROJECT'] = 'fingpt-forecaster'
 
 
@@ -97,12 +97,14 @@ def main(args):
     tokenizer.padding_side = "right"
     
     # load data
-    dataset_list = load_dataset(args.dataset, args.from_remote)
+    dataset_fname = "./data/" + args.dataset
+    dataset_list = load_dataset(dataset_fname, args.from_remote)
     
     dataset_train = datasets.concatenate_datasets([d['train'] for d in dataset_list]).shuffle(seed=42)
     
     if args.test_dataset:
-        dataset_list = load_dataset(args.test_dataset, args.from_remote)
+        test_dataset_fname = "./data/" + args.test_dataset
+        dataset_list = load_dataset(test_dataset_fname, args.from_remote)
             
     dataset_test = datasets.concatenate_datasets([d['test'] for d in dataset_list])
     

fingpt/FinGPT_Forecaster/utils.py

@@ -49,7 +49,7 @@ def parse_model_name(name, from_remote=False):
     if name == 'chatglm2':
         return 'THUDM/chatglm2-6b' if from_remote else 'base_models/chatglm2-6b'
     elif name == 'llama2':
-        return 'meta-llama/Llama-2-7b-chat-hf' if from_remote else 'base_models/Llama-2-7b-chat-hf'
+        return 'meta-llama/Llama-2-7b-chat-hf' # if from_remote else 'base_models/Llama-2-7b-chat-hf'
     else:
         raise ValueError(f"Undefined base model {name}")
         
@@ -75,11 +75,11 @@ def load_dataset(names, from_remote=False):
 
 def parse_answer(answer):
     
-    match_res = re.match(r"^\s*\[Positive Developments\]:\s*(.*)\s*\[Potential Concerns\]:\s*(.*)\s*\[Prediction & Analysis\]:\s*(.*)\s*$", answer, flags=re.DOTALL)
+    match_res = re.match(r"^\s*\[Positive Developments\]:\s*(.*)\s*\[Potential Concerns\]:\s*(.*)\s*\[Prediction (&|and) Analysis\]:\s*(.*)\s*$", answer, flags=re.DOTALL)
     if not match_res:
         return None
     
-    pros, cons, pna = match_res.group(1), match_res.group(2), match_res.group(3)
+    pros, cons, pna = match_res.group(1), match_res.group(2), match_res.group(4)
         
     match_res = re.match(r'^Prediction:\s*(.*)\s*Analysis:\s*(.*)\s*$', pna, flags=re.DOTALL)
     if not match_res: