{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "current_directory = os.getcwd()\n",
    "sys.path.append(os.path.join(current_directory, \"FinNLP\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from lxml import etree\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Scrape main page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = 'https://seekingalpha.com/api/v3/symbols/dnkn/news'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',\n",
    "    'Referer': 'https://seekingalpha.com/symbol/DNKN/news',\n",
    "    'Cookie': 'machine_cookie=0427091031005; _gcl_au=1.1.1005531817.1683650457; _pcid=%7B%22browserId%22%3A%22lhgi0f449va50asj%22%7D; _pcus=eyJ1c2VyU2VnbWVudHMiOm51bGx9; _ga=GA1.1.571982864.1683650457; _pctx=%7Bu%7DN4IgrgzgpgThIC4B2YA2qA05owMoBcBDfSREQpAeyRCwgEt8oBJAE0RXQF8g; g_state={\"i_p\":1683657760251,\"i_l\":1}; __pnahc=255; __pat=-14400000; _hjFirstSeen=1; _hjIncludedInSessionSample_65666=0; _hjSession_65666=eyJpZCI6ImIzMjA5MDc4LWE5MGYtNDQ0Yy04ODBkLTM3OWY1YjNkNjVhOSIsImNyZWF0ZWQiOjE2ODk3NzkzNjQyOTcsImluU2FtcGxlIjpmYWxzZX0=; pxcts=3fac07b0-2646-11ee-bff1-616f70657664; _pxvid=478e1d92-ee88-11ed-b1b8-3ca69df203a9; __pvi=eyJpZCI6InYtMjAyMy0wNy0xOS0yMy0wOS0zMi02Njktc1JHZER1QkgzR252VHQxYy01MjI3MWRmODU3ZWE4NzMwNjczYmQ2M2JkZTI4ODhhNiIsImRvbWFpbiI6Ii5zZWVraW5nYWxwaGEuY29tIiwidGltZSI6MTY4OTc3OTM3MjY2OX0%3D; __tbc=%7Bkpex%7DDrVMv-S2Lu44MbwIrOYGeU7FfHdrk2vbdbHOR3NF62cwA75AfFmA_eSFgY7p3f_X; xbc=%7Bkpex%7DTP8YvFWJqW8JPT1Sd_4S5w-w4GKfXlC35UKXSyc2ZW4fGAzntp-ZdjfHYH-ipMBSlxNIenCc6UQx83ynAHa1ZzbEQf1mPw4k0lmSIgJwrwhjUl2t2vEr9zUmIAwmb1K2vC-VfvwNTuZ6mMy34ViltC6KerStsrZOgAc-srfgeb8xu2WP1sWHOtEcQlI3X_tl0qE2kLzVHF9XS7k59oFocAIsMt9MqEdldpZBjT8a1NZW20szyRBIppg2pxiECSOFyDgI2UzgKuKqjDgQQuz_ovq_oFS6m-5kfxXl02r0bTA; sailthru_pageviews=2; _uetsid=3ede2e80264611eebc148d442b00d4cf; _uetvid=46ad8ae0ee8811ed8be87bcb9b56b00b; _hjSessionUser_65666=eyJpZCI6IjA4ZGUwZmQwLWY2NzAtNTNiOS1iMzU0LTZjNGFlZmU3MTBlZiIsImNyZWF0ZWQiOjE2ODk3NzkzNjQyOTEsImV4aXN0aW5nIjp0cnVlfQ==; sailthru_content=eaca8a3ebde49b34d28768f72fae972f3c5f57e561d9315a8a9a86be8720511e40f35d5811aadb491568a010d9e8412955f840c37c2c459dd8a3af459c70dc7c; sailthru_visitor=f3dfa04b-a344-4fc4-82dd-4165853dce73; _conv_v=vi%3A1*sc%3A1*cs%3A1689779364*fs%3A1689779364*pv%3A2*exp%3A%7B100037059.%7Bv.1000217424-g.%7B100030295.1-100030296.1%7D%7D-100037060.%7Bv.1000217426-g.%7B%7D%7D%7D; _conv_s=si%3A1*sh%3A1689779364047-0.31875808241036085*pv%3A2; _px=qy8b0Ak1J37TryVQ0s49/AS7z4OgQKvCYu6G3QgfA1AQdP8tx5RJhanpuA5DwliVFww7T8GOntqWAl3juuwCAw==:1000:P/s5wj9cep4grNwRVMlpfY1F1KXSnTKhiCBhAa5Bg+oYJxLo+YQpQS98wYDGvux9YqFR4lmhBNlXuhiDZsVipN4iFg0VH14Yu+nqxdNDXcWWors4Ju3iXP6CRVBz7JZNV4db3lhg83WHgI+SlkzpbAZQdlN6xo2YiIky0fW1kaSvzwDIbLv+9pgZ6PEN9BYeJASjjSDSHZB41vtV/uhW7v/liaMB+wkMsEyxhNDJiGJc7a5sVEFooI45nq6AQo6jCnCpkWga/jNvpkL22r1JSg==; _px2=eyJ1IjoiNDU0OGE4ZTAtMjY0Ni0xMWVlLWFkMjYtYWY5OTAwZWFiNzFmIiwidiI6IjQ3OGUxZDkyLWVlODgtMTFlZC1iMWI4LTNjYTY5ZGYyMDNhOSIsInQiOjE2ODk3ODAwMDYxMzcsImgiOiJiMzcyNDI0ODNmZmVmNzUzOGVjMTlhNTA1NGIxNTZkNTU5MjQ1MDYzYzNkNmMzYWMzMTIzOGIwOTAyOWNhOThmIn0=; _pxde=0056db64e1c15fff119cb3115ba5d2755dfb655597f3309250fce29f2494d80f:eyJ0aW1lc3RhbXAiOjE2ODk3Nzk1MDYxMzcsImZfa2IiOjB9; _ga_KGRFF2R2C5=GS1.1.1689779364.3.1.1689779565.60.0.0; LAST_VISITED_PAGE=%7B%22pageKey%22%3A%224e91f493-684f-4c49-a8e1-b5947b434c4e%22%7D'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    'filter[since]': '0',\n",
    "    'filter[until]': '0',\n",
    "    'id': 'dnkn',\n",
    "    'include': 'author,primaryTickers,secondaryTickers,sentiments',\n",
    "    'isMounting': 'true',\n",
    "    'page[size]': '40',\n",
    "    'page[number]': '1',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "200"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = requests.get(url = base_url, data = data, headers = headers)\n",
    "res.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = res.json()\n",
    "data = pd.DataFrame(res[\"data\"])\n",
    "# make new features\n",
    "new_columns = [\"publishOn\", \"isLockedPro\", \"commentCount\", \"gettyImageUrl\", \"videoPreviewUrl\", \"themes\", \"title\", \"isPaywalled\"]\n",
    "data[new_columns] = data.apply(lambda x:list(x.attributes.values()), axis = 1,result_type =\"expand\" )\n",
    "new_columns = [\"author\", \"sentiments\", \"primaryTickers\", \"secondaryTickers\", \"otherTags\"]\n",
    "data[new_columns] = data.apply(lambda x:list(x.relationships.values()), axis = 1,result_type =\"expand\" )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Scrape content page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "200"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_url = data.head(1).links[0][\"self\"]\n",
    "new_url = \"https://seekingalpha.com\"+ new_url\n",
    "res = requests.get(url = new_url, headers = headers)\n",
    "res.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = etree.HTML(res.text)\n",
    "scripts= tree.xpath(\"//script\")\n",
    "res = scripts[3].xpath(\".//text()\")\n",
    "res = ' '.join(res)\n",
    "js = json.loads(res[18:-1])\n",
    "tre = etree.HTML(js['article']['response']['data']['attributes']['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hispanolistic/E+ via Getty Images Starting today, New York City will impose a vaccination requirement on dining, fitness clubs and performances held indoors, making it the first major U.S. city to implement such restrictions. Employees of those venues will also be required to be vaccinated, and  following a grace period of a few weeks, enforcement will begin on Sept. 13. The policy is gaining traction, with San Francisco instituting its own vaccination requirement for indoor activities that will go into effect on Aug. 20 (Los Angeles is mulling a similar plan). Bigger picture:  The developments could spell trouble for an industry that has been one of the hardest hit by a national labor shortage. The U.S. unemployment rate for restaurants was 8.4% in July, according to the Bureau of Labor Statistics, and many eateries have been turning to pay hikes and retention bonuses to attract new workers. However, some feel that restaurant operators could see some benefits from the vaccine mandate, like less workers that call out sick or stemming the rise in new COVID cases. \"I just don't think that we're gonna be the vaccination police,\"  said  Mary Josephine Generoso, manager at Pasticceria Rocco's of Bay Ridge. \"That should be up to the mayor's office, it's up to the health department to figure out, but it certainly shouldn't be the burden of store owners, bars and gyms to be regulating that.\" Medical or religious exemptions could also present difficulties in implementing the mandate, as well as tourists, whose foreign vaccine barcodes or documents aren't recognized in the U.S. On the corporate side:  As safety restrictions and vaccine mandates evolve, Booking Holdings'  (NASDAQ: BKNG )  OpenTable has  rolled out  a feature that allows restaurants to display COVID-19 vaccine requirements to diners. The reservation service additionally plans to launch a national list, updated in real-time, of restaurants that are currently requiring proof of vaccination. Related:  MCD ,  SBUX ,  YUM ,  SHAK ,  DRI ,  QSR ,  CMG ,  DPZ ,  BLMN ,  PZZA ,  DNKN ,  DNUT ,  PLNT\n"
     ]
    }
   ],
   "source": [
    "str_list = tre.xpath('//text()')\n",
    "str_list = [s for s in str_list if s != ' ']\n",
    "str_list = ' '.join(str_list)\n",
    "print(str_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "finrl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
