{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter id:M.1517194272.A.97B.html\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "articleid=input(\"Enter id:\")\n",
    "text_list=[]\n",
    "\n",
    "url=\"https://www.ptt.cc/bbs/Japan_Travel/\"+articleid\n",
    "response = requests.get(url, cookies={'over18': '1'})\n",
    "result = response.text\n",
    "soup = BeautifulSoup(result, 'html.parser')\n",
    "soup = soup.find(id=\"main-content\")\n",
    "if soup != None:\n",
    "            for span in soup('span'):\n",
    "                span.decompose()\n",
    "            for div in soup('div'):\n",
    "                div.decompose()\n",
    "            for a in soup('a'):\n",
    "                a.decompose()\n",
    "            text_list.append(soup.text)\n",
    "else:\n",
    "    print(\"sorry, no page\")\n",
    "\n",
    "# print out what you get\n",
    "\n",
    "import re\n",
    "# Break into sentences\n",
    "temp_list=[]\n",
    "sentence_list=[]\n",
    "pattern = r',|\\.|/|;|\\'|`|\\[|\\]|<|>|\\?|:|\"|\\{|\\}|\\~|!|@|#|\\$|%|\\^|&|\\(|\\)|-|=|\\_|\\+|，|。|、|；|‘|’|【|】|·|！| |…|（|）' \n",
    "    \n",
    "for element in text_list:\n",
    "    element = re.split(pattern,element)\n",
    "    for t in element:\n",
    "        if t != '':\n",
    "            temp_list.append(t)\n",
    "    \n",
    "# Remove \\n in list\n",
    "temp_list = [x for x in temp_list if x != '\\n']\n",
    "for item in temp_list:\n",
    "    item = item.replace(\"\\n\",\"\")\n",
    "    sentence_list.append(item)\n",
    "\n",
    "import jieba\n",
    "\n",
    "# Process into wrods by Jieba\n",
    "print(\"Run word, powered by JieBa\")\n",
    "\n",
    "jieba.set_dictionary('dict.txt')\n",
    "stops = open('stop.txt', 'r', encoding='utf8') \n",
    "stopword = stops.read().split('\\n')\n",
    "stops.close()\n",
    "    \n",
    "# Update stop words\n",
    "stoplist=['\\n',' ','']\n",
    "for element in stoplist:\n",
    "    stopword.append(element)\n",
    "\n",
    "# Remove any stop words from list\n",
    "new_list=[]\n",
    "i = 0\n",
    "while i < len(sentence_list):\n",
    "    element = sentence_list[i]\n",
    "    new_list.append([t for t in jieba.cut(element) if t not in stopword])\n",
    "    i+=1\n",
    "    \n",
    "# Return a bag of words\n",
    "jieba_list=[]\n",
    "j = 0\n",
    "while j < len(new_list):\n",
    "    jieba_list += new_list[j]\n",
    "    j += 1\n",
    "\n",
    "# Pool all words\n",
    "print(\"Run Pool\")\n",
    "return_string = ','.join(jieba_list)\n",
    "\n",
    "import sqlite3\n",
    "\n",
    "# Update database\n",
    "print(\"Update database\")\n",
    "articleid = articleid\n",
    "articlecontent = return_string\n",
    "conn = sqlite3.connect('ptt.db')\n",
    "c = conn.cursor()\n",
    "c.execute(\"UPDATE article_large SET content = ? where id=?\",(articlecontent,articleid))\n",
    "conn.commit()\n",
    "print(\"Update database done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
