from twill.commands import *
import time
import re
import BeautifulSoup
import rpy2.robjects as R
import cPickle

#Construct a browser object.
b = get_browser()
b.go("http://forums.offtopic.com")
#Fill field 'vb_login_username' of form 1 with "ByteMining", the username.
fv("1", "vb_login_username", "ByteMining")
#Fill field 'vb_login_password' of form 1 with "yeah_right", the password.
fv("1", "vb_login_password", "yeah_right")
#Click the fifth submit button (called 4)
b.submit('4')
#Bypass the "Please wait while we log you in..." prompt
b.go("http://forums.offtopic.com")

#Extract the HTML table containing the actual data. There are multiple tables.
patt = re.compile('(<table class="tborder" cellpadding="6" cellspacing="1" ' +
    'border="0" width="100%" align="center"><tr align="center">.' +
    '<td class="thead" align="left" nowrap="nowrap">' +
    '<a href="memberlist\.php.*?</table>)',re.MULTILINE)

#Map variable names to column numbers in the HTML table.
fieldnames = {'username': 0, 'join_date': 4, 'posts': 5, 'last_visit': 6,
    'birthday': 7, 'age': 8}    #Removed im: 2 for now.

#Initialize what will become the R data.frame.
data = {}
for field in fieldnames:
    data[field] = []

for i in xrange(1195):
    b.go("http://forums.offtopic.com/memberlist.php?do=getall&page=%s" % str(i+1))
    content = b.get_html()
    content = re.sub(r'\r|\n', '', content)
    the_table = re.findall(patt, content)[0]
    the_table = re.sub(r'\t', '', the_table)
    bs = BeautifulSoup.BeautifulSoup(the_table)
    rows = bs.table.contents
    rows = rows[1:-1] #Eliminate header and footer of table.
    for row in rows:
        cols = row.contents
        for field in fieldnames:
            if field == "username":
                data[field].append(cols[fieldnames[field]].a.contents[0])
            else:
                try:
                    data[field].append(cols[fieldnames[field]].contents[0].replace('&nbsp;','NA'))
                except AttributeError:
                    data[field].append('NA')
                except IndexError:
                    data[field].append('NA')

OUT = open("data.pickle", "wb")
cPickle.dump(data, OUT)
OUT.close()