{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lxml import etree\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What is the sentiment of this news? Answer:{very negative/negative/neutral/positive/very positive}\n"
     ]
    }
   ],
   "source": [
    "# The base_path should be the results with labels generated in ../data_preparations/add_labels.py\n",
    "base_path = r\"C:\\Users\\Olive\\python_projects\\FinRL-Meta\\FinGPT-ChatGLM-Fineturning_old\\Chinese_News\\scrape\\content_with_labels\"\n",
    "columns = [\"instruction\",\"input\",\"output\"]\n",
    "instruction = \"What is the sentiment of this news? Answer:{very negative/negative/neutral/positive/very positive}\"\n",
    "\n",
    "file_list = os.listdir(base_path)\n",
    "print(instruction)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Title/标题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 216/216 [00:33<00:00,  6.47it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset = pd.DataFrame(columns= columns)\n",
    "for file_name in tqdm(file_list):\n",
    "    df = pd.read_csv(os.path.join(base_path, file_name))\n",
    "    df[\"input\"] = df.apply(lambda x: f'今天的日期为：{x[\"publish_date\"]}。\\n新闻标题为：\\\"{x[\"post_title\"]}\\\"。', axis = 1)\n",
    "    df[\"instruction\"] = instruction\n",
    "    df['output'] = df[\"label\"]\n",
    "    tmp = df[columns+[\"date\"]]\n",
    "    dataset = pd.concat([dataset, tmp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2023-04-07。\\n新闻标题为：\"平安银行：融资净买入3199.17万元...</td>\n",
       "      <td>No data</td>\n",
       "      <td>2023-04-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2023-04-07。\\n新闻标题为：\"平安银行今日大宗交易折价成交198万股...</td>\n",
       "      <td>No data</td>\n",
       "      <td>2023-04-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2023-04-06。\\n新闻标题为：\"平安银行：融资净偿还1627.94万元...</td>\n",
       "      <td>No data</td>\n",
       "      <td>2023-04-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2023-04-05。\\n新闻标题为：\"平安银行04月04日被深股通减持301...</td>\n",
       "      <td>No data</td>\n",
       "      <td>2023-04-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2023-04-02。\\n新闻标题为：\"平安银行本周深股通持股市值增加1.09...</td>\n",
       "      <td>No data</td>\n",
       "      <td>2023-04-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>977</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2021-04-22。\\n新闻标题为：\"德业股份一字涨停，年报业绩同比增加47...</td>\n",
       "      <td>very negative</td>\n",
       "      <td>2021-04-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>978</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2021-04-21。\\n新闻标题为：\"德业股份今日正式登陆沪市主板 多元化发...</td>\n",
       "      <td>negative</td>\n",
       "      <td>2021-04-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>979</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2021-04-21。\\n新闻标题为：\"N德业（605117）龙虎榜数据（04...</td>\n",
       "      <td>negative</td>\n",
       "      <td>2021-04-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2021-04-21。\\n新闻标题为：\"德业股份封涨停板 三机构卖出64万元\"。</td>\n",
       "      <td>negative</td>\n",
       "      <td>2021-04-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>981</th>\n",
       "      <td>What is the sentiment of this news?\\nAnswer:{v...</td>\n",
       "      <td>今天的日期为：2021-04-21。\\n新闻标题为：\"德业股份今日上市交易 首日达到44%的...</td>\n",
       "      <td>negative</td>\n",
       "      <td>2021-04-20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495601 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           instruction  \\\n",
       "0    What is the sentiment of this news?\\nAnswer:{v...   \n",
       "1    What is the sentiment of this news?\\nAnswer:{v...   \n",
       "2    What is the sentiment of this news?\\nAnswer:{v...   \n",
       "3    What is the sentiment of this news?\\nAnswer:{v...   \n",
       "4    What is the sentiment of this news?\\nAnswer:{v...   \n",
       "..                                                 ...   \n",
       "977  What is the sentiment of this news?\\nAnswer:{v...   \n",
       "978  What is the sentiment of this news?\\nAnswer:{v...   \n",
       "979  What is the sentiment of this news?\\nAnswer:{v...   \n",
       "980  What is the sentiment of this news?\\nAnswer:{v...   \n",
       "981  What is the sentiment of this news?\\nAnswer:{v...   \n",
       "\n",
       "                                                 input         output  \\\n",
       "0    今天的日期为：2023-04-07。\\n新闻标题为：\"平安银行：融资净买入3199.17万元...        No data   \n",
       "1    今天的日期为：2023-04-07。\\n新闻标题为：\"平安银行今日大宗交易折价成交198万股...        No data   \n",
       "2    今天的日期为：2023-04-06。\\n新闻标题为：\"平安银行：融资净偿还1627.94万元...        No data   \n",
       "3    今天的日期为：2023-04-05。\\n新闻标题为：\"平安银行04月04日被深股通减持301...        No data   \n",
       "4    今天的日期为：2023-04-02。\\n新闻标题为：\"平安银行本周深股通持股市值增加1.09...        No data   \n",
       "..                                                 ...            ...   \n",
       "977  今天的日期为：2021-04-22。\\n新闻标题为：\"德业股份一字涨停，年报业绩同比增加47...  very negative   \n",
       "978  今天的日期为：2021-04-21。\\n新闻标题为：\"德业股份今日正式登陆沪市主板 多元化发...       negative   \n",
       "979  今天的日期为：2021-04-21。\\n新闻标题为：\"N德业（605117）龙虎榜数据（04...       negative   \n",
       "980    今天的日期为：2021-04-21。\\n新闻标题为：\"德业股份封涨停板 三机构卖出64万元\"。       negative   \n",
       "981  今天的日期为：2021-04-21。\\n新闻标题为：\"德业股份今日上市交易 首日达到44%的...       negative   \n",
       "\n",
       "           date  \n",
       "0    2023-04-07  \n",
       "1    2023-04-06  \n",
       "2    2023-04-06  \n",
       "3    2023-04-05  \n",
       "4    2023-04-02  \n",
       "..          ...  \n",
       "977  2021-04-21  \n",
       "978  2021-04-20  \n",
       "979  2021-04-20  \n",
       "980  2021-04-20  \n",
       "981  2021-04-20  \n",
       "\n",
       "[495601 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset[dataset.output != \"No data\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2011-05-12', '2011-05-13', '2011-05-16', ..., '2023-03-29',\n",
       "       '2023-03-30', '2023-03-31'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.date.sort_values().unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='date'>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 19072 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset.groupby(\"date\").date.count().plot()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  Test from 2022-04-01 till 2023-03-31\n",
    "#####  Train & valid from 2011-05-12 till 2022-03-31"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(354608, 4)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[dataset.date<\"2022-04-01\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(492549, 4)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "今天的日期为：2023-04-08。\n",
      "新闻标题为：\"德业股份：融资净偿还759.19万元，融资余额4.73亿元（04-07）\"。\n"
     ]
    }
   ],
   "source": [
    "print(df[\"input\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train & valid\n",
    "train_valid = dataset[dataset.date < \"2022-04-01\"][columns]\n",
    "train_valid.to_csv(\"content/dataset_title_train_and_valid.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test\n",
    "test = dataset[dataset.date >= \"2022-04-01\"][columns]\n",
    "test.to_csv(\"content/dataset_title_test.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_list = []\n",
    "for item in train_valid.itertuples():\n",
    "    tmp = {}\n",
    "    tmp[\"instruction\"] = item.instruction\n",
    "    tmp[\"input\"] = item.input\n",
    "    tmp[\"output\"] = item.output\n",
    "    data_list.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Json\n",
    "import json\n",
    "with open(\"content/dataset_title_train_and_valid.json\", \"w+\", encoding='utf-8') as f:\n",
    "    json.dump(data_list, f, ensure_ascii= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Json\n",
    "import json\n",
    "with open(\"content/dataset_title_test.json\", \"w+\", encoding='utf-8') as f:\n",
    "    json.dump(data_list, f, ensure_ascii= False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Contetn/内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_content(x):\n",
    "    try:\n",
    "        tree = etree.HTML(x)\n",
    "        content = tree.xpath(\"//*[name(.)!='style']/text()\")\n",
    "        content = \"\".join(content)\n",
    "        content = content.replace(\"\\u3000\", '')\n",
    "        return content\n",
    "    except:\n",
    "        return np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 216/216 [01:13<00:00,  2.94it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset_content = pd.DataFrame(columns= columns)\n",
    "for file_name in tqdm(file_list):\n",
    "    df = pd.read_csv(os.path.join(base_path, file_name))  \n",
    "    df['input'] = df['post_content'].apply(get_content)\n",
    "    df = df.dropna(subset=['input'])\n",
    "    df[\"input\"] = df.apply(lambda x: f'今天的日期为：{x[\"publish_date\"]}。\\n新闻内容为：\\\"{x[\"input\"]}\\\"。', axis = 1)\n",
    "    df[\"instruction\"] = instruction\n",
    "    df['output'] = df[\"label\"]\n",
    "    tmp = df[columns+[\"date\"]]\n",
    "    dataset_content = pd.concat([dataset_content, tmp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "今天的日期为：2023-04-08。\n",
      "新闻内容为：\"来源：东方财富Choice数据 作者：财智星 德业股份融资融券信息显示，2023年4月7日融资净偿还759.19万元；融资余额4.73亿元，较前一日下降1.58%。融资方面，当日融资买入1179.84万元，融资偿还1939.03万元，融资净偿还759.19万元。融券方面，融券卖出6.12万股，融券偿还1.77万股，融券余量41.18万股，融券余额1.02亿元。融资融券余额合计5.75亿元。德业股份融资融券交易明细（04-07）德业股份历史融资融券数据一览免责声明：本文基于大数据生产，仅供参考，不构成任何投资建议，据此操作风险自担。（文章来源：东方财富Choice数据） [点击查看原文]\"。\n"
     ]
    }
   ],
   "source": [
    "print(df[\"input\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(492527, 4)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_content = dataset_content[dataset_content.output != \"No data\"]\n",
    "dataset_content.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(492523, 5)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_content[\"total_len\"] = dataset_content[\"input\"].apply(lambda x:len(x.split(\" \")))\n",
    "dataset_content = dataset_content[dataset_content[\"total_len\"] < 1000]\n",
    "dataset_content.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "796"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_content[\"total_len\"].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train & valid\n",
    "train_valid = dataset_content[dataset_content.date < \"2022-04-01\"][columns]\n",
    "train_valid.to_csv(\"content/dataset_content_train_and_valid.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test\n",
    "test = dataset_content[dataset_content.date >= \"2022-04-01\"][columns]\n",
    "test.to_csv(\"content/dataset_content_test.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_list = []\n",
    "for item in train_valid.itertuples():\n",
    "    tmp = {}\n",
    "    tmp[\"instruction\"] = item.instruction\n",
    "    tmp[\"input\"] = item.input\n",
    "    tmp[\"output\"] = item.output\n",
    "    data_list.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"content/dataset_content_train_and_valid.json\", \"w+\", encoding='utf-8') as f:\n",
    "    json.dump(data_list, f, ensure_ascii= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>date</th>\n",
       "      <th>total_len</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>output</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>negative</th>\n",
       "      <td>74125</td>\n",
       "      <td>74125</td>\n",
       "      <td>74125</td>\n",
       "      <td>74125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>114563</td>\n",
       "      <td>114563</td>\n",
       "      <td>114563</td>\n",
       "      <td>114563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>positive</th>\n",
       "      <td>64071</td>\n",
       "      <td>64071</td>\n",
       "      <td>64071</td>\n",
       "      <td>64071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>very negative</th>\n",
       "      <td>45513</td>\n",
       "      <td>45513</td>\n",
       "      <td>45513</td>\n",
       "      <td>45513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>very positive</th>\n",
       "      <td>55632</td>\n",
       "      <td>55632</td>\n",
       "      <td>55632</td>\n",
       "      <td>55632</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               instruction   input    date  total_len\n",
       "output                                               \n",
       "negative             74125   74125   74125      74125\n",
       "neutral             114563  114563  114563     114563\n",
       "positive             64071   64071   64071      64071\n",
       "very negative        45513   45513   45513      45513\n",
       "very positive        55632   55632   55632      55632"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_content[dataset_content.date < \"2022-04-01\"].groupby(\"output\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 9792 characters]",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset_content[dataset_content.date < \"2022-04-01\"].output.hist()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "finrl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
