Snippet revisions — Bitbucket

+#!/usr/local/bin/python

+import os

+import requests

+import sqlite3

+from contextlib import closing

+from bs4 import BeautifulSoup as BS4

+URLTEMPLATE = (r'http://www.pathofexile.com/forum/view-forum'

+               r'/%(forumid)s/page/%(page)s')

+FORUMS = {

+    'duelist': 40,

+    'marauder': 23,

+    'ranger': 24,

+    'scion': 436,

+    'shadow': 303,

+    'templar': 41,

+    'witch': 22,

+}

+DBFILE = os.path.abspath(os.path.join(

+    os.path.dirname(__file__), os.path.splitext(__file__)[0] + '.db',

+))

+if os.path.exists(DBFILE):

+    os.unlink(DBFILE)

+dbi = sqlite3.connect(DBFILE)

+with closing(dbi.cursor()) as csr:

+    csr.execute('''

+create table if not exists threads(

+    forum text not null,

+    title text not null,

+    views integer not null,

+    replies integer not null,

+    page integer not null,

+    url text primary key

+);

+    ''')

+    csr.execute(

+        'create index if not exists threads_idx_forum on threads(forum);')

+    csr.execute(

+        'create index if not exists threads_idx_title on threads(title);')

+    csr.execute(

+        'create index if not exists threads_idx_views on threads(views);')

+    csr.execute(

+        'create index if not exists threads_idx_replies on threads(replies);')

+    csr.execute(

+        'create index if not exists threads_idx_page on threads(page);')

+    csr.execute('begin;')

+for forum, forumid in FORUMS.items():

+    pageno = 1

+    while True:

+        url = URLTEMPLATE % {'page': pageno, 'forumid': forumid}

+        body = requests.get(url).text

+        if 'No Threads' in body:

+            break

+        soup = BS4(body)

+        table = soup.find('table', id='view_forum_table')

+        for row in table.find_all('tr'):

+            thread = row.find('td', class_='thread')

+            if thread:

+                link = thread.find(class_='title').find('a')

+                url = link['href']

+                if url.startswith('/'):

+                    url = 'http://www.pathofexile.com%s' % url

+                title = link.text

+                views = int(row.find('td', class_='views').text)

+                replies = int(row.find('td', class_='replies').text)

+                with closing(dbi.cursor()) as csr:

+                    csr.execute(

+                        'insert into threads values (?, ?, ?, ?, ?, ?)',

+                        (forum, title, views, replies, pageno, url))

+        print 'FORUM %s: PAGE %s' % (forum, pageno)

+        pageno += 1

+with closing(dbi.cursor()) as csr:

+    try:

+        csr.execute('commit;')

+    except:

+        pass

+try:

+    dbi.close()

+except:

+    pass

Snippets