#!/usr/bin/env python
# coding: utf-8

# excellent guide to study leetcode
# 
# https://www.1point3acres.com/bbs/thread-678970-1-1.html

# In[1]:


import urllib, requests
from bs4 import BeautifulSoup
import re
import urllib.parse
import pandas as pd


# In[2]:


re_leetcode = re.compile(r'.*(https://leetcode.com/problems/.*/)')
re_lintcode = re.compile(r'.*(https://www.lintcode.com/problem/.*/description)')

color2priority = {'#9932cc':'p1', '#4169e1':'p2', '#2e8b57':'p3', '#ffa500':'p4'}


# In[3]:


headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"} 

urls = ['https://www.1point3acres.com/bbs/forum.php?mod=viewthread&tid=678970&page=1&authorid=682747',
        'https://www.1point3acres.com/bbs/forum.php?mod=viewthread&tid=678970&extra=&authorid=682747&page=2',
#         'https://www.1point3acres.com/bbs/forum.php?mod=viewthread&tid=678970&extra=&authorid=682747&page=3', # no question 
       ]


# In[4]:


soups = []
for url in urls:
    response = requests.get(url=url, headers=headers)
    response.encoding = 'utf-8'
    soup = BeautifulSoup(response.text, features='lxml')
    soups.append(soup)


# In[39]:


indices = []
titles = []
priorities = []
links = []
prev_obj = None

for soup in soups:
    all_a = soup.findAll('a', target=True, href=True, rel=True)
    for a in all_a:
        if re_leetcode.findall(urllib.parse.unquote(a['href'])):
            links.append(re_leetcode.findall(urllib.parse.unquote(a['href']))[0])
        elif re_lintcode.findall(urllib.parse.unquote(a['href'])):
            links.append(re_lintcode.findall(urllib.parse.unquote(a['href']))[0])
        else:
            continue
        if a['href'] == 'https://link.1point3acres.com/?url=https%3A%2F%2Fleetcode.com%2Fproblems%2Fsearch-in-rotated-sorted-array%2F':
            obj = a.find_previous_sibling('strong').find_previous_sibling('strong')      
        else:
            obj = a.find_previous_sibling('strong')
        if obj == prev_obj:
            prev_obj = obj
            obj = a.find_previous_sibling('font')
            index, title = obj.text.split('. ')
            indices.append(index)
            titles.append(title)
            priority = color2priority[obj['color']]
            priorities.append(priority)
        else:
            try:
                index, title = obj.text.split('. ')
                indices.append(index)
                titles.append(title)
                priority = color2priority[obj.font['color']]
                priorities.append(priority)
            except:
                obj = a.find_previous_sibling('font')
                index, title = obj.text.split('. ')
                indices.append(index)
                titles.append(title)
                priority = color2priority[obj['color']]
                priorities.append(priority)
            prev_obj = obj


# In[43]:


df = pd.DataFrame()
df['index'] = indices
df['title'] = titles
df['priority'] = priorities
df['link'] = links
df.to_csv('outputs.csv')
# df


# In[ ]:




