#!/usr/bin/python

import os
import time
import sys
import psycopg2
import urllib2
import random

import threading
import Queue

import socket
socket.setdefaulttimeout(120)
import urllib
import sc
sc = sc.SystemConfiguration()

import re
links = re.compile(r'<a .*?href="(.*?)".*?>(.*?)</a>',re.DOTALL)
#postliste = re.compile(r'<a .*?href="(.*?)".*?>.*?postliste.*?</a>',re.DOTALL)
#postjournal = re.compile(r'<a .*?href="(.*?)".*?>.*?postjournal.*?</a>',re.DOTALL)
generator = re.compile(r'<meta.*name="generator".*content="(.*?)".*>',re.DOTALL)


con = psycopg2.connect(user=sc.urlrepusername, database=sc.urlrepdatabase, password=sc.urlreppassword)
cur = con.cursor()
cur.execute('select distinct domain,random() from site where isactive=TRUE order by random();')


municipalitylists = [i[0] for i in cur.fetchall()]

#Hack
municipalitylists = municipalitylists
#End of hack

#municipalitylists = ['www.lillesand.kommune.no']

#random.shuffle(municipalitylists)


#print 'Municipality, Has Postlist,Service'#, Videooverføring, Innsyn'

def unquote(url,site):
    if not url:
        return ''
    if type(url) == type([]):
        url = url[0]
    if not url.startswith('http'):
        url = url.lstrip('/')
        url = 'http://'+site+'/' + url
        url = url.replace('&amp;','&')
        url = urllib.unquote(url)
    return url



results = []

class ReqestServer(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.dataqueue=Queue.Queue()
        self.results = []

    def push(self,url):
        self.dataqueue.put((url))

    def stop(self):
        self.dataqueue.put(None)

    def checkforpostliste(self,site):
        #returns postjournal,viedooverføring,innsyn
        #print 'Downloading index page'

        self.data = ''
        try:
            pre = time.time()
            self.data = urllib2.urlopen('http://%(site)s/'%locals()).read().lower()
            print time.time()-pre
        except:# urllib2.URLerror:
            return ('Error')#,'Error','Error')
        self.result = []
        self.postlisturl = ''
        self.theselinks = []
        if 'postliste' in self.data or 'postjournal' in self.data:
            self.result.append(True)
            self.theselinks = links.findall(self.data)
            res = [i[0] for i in self.theselinks if ('innsyn' in i[1] or 'postliste' in i[1] or 'postjournal' in i[1] or 'postlister' in i[1])]
            res = unquote(res,site)
            self.postlisturl = res
            self.result.append(res)
        else:
            self.result.append(False)
            self.result.append('')
        if self.theselinks:
            somelink = random.sample([i for i in self.theselinks if not ('innsyn' in i[1] or 'postliste' in i[1] or 'postjournal' in i[1])],1)[0][0]
            somelink = unquote(somelink,site)
            try:
                #self.data = urllib2.urlopen(somelink).read().lower()
                self.data = ''
            except:
                self.data = ''

            if '<iframe' in self.data or '<frame' in self.data or '<frameset' in self.data:
               #print' Ignoring'
               self.data = ''


            #print 'Notpostliste',site,somelink
            #f = open('./notpostlists/'+site+'3','w')
            #f.write(self.data)
            #f.close()
        if self.postlisturl:# and 0==1:
            #print 'Downloadinging first postlist'
            try:
               pre = time.time()
               self.data = urllib2.urlopen(self.postlisturl).read().lower()
               print time.time()-pre
            except:
               self.data = ''
            #newpostlisteurl = postliste.findall(self.data.lower())
            self.theselinks = links.findall(self.data)
            res = [i[0] for i in self.theselinks if ('innsyn' in i[1] or 'postliste' in i[1] or 'postjournal' in i[1] or 'postlister' in i[1])]
            res = unquote(res,site)
            newpostlisteurl = res

            if newpostlisteurl:
                #self.postlisturl = newpostlisteurl[0]
                
                if not self.postlisturl.startswith('http'):
                    self.postlisturl = 'http://%(site)s/'%locals()+self.postlisturl
                
                #print 'Downloading second postlist'
                try:
                   pre = time.time()
                   self.data = urllib2.urlopen(self.postlisturl).read().lower()
                   print time.time()-pre
                except:
                   pass

            if '<iframe' in self.data or '<frame' in self.data or '<frameset' in self.data:
               #print' Ignoring'
               self.data = ''

            #print 'Postliste',site,self.postlisturl

            #f = open('./postlists/'+site,'w')
            #f.write(self.data)
            #f.close()
            thisgenerator = generator.findall(self.data)
            self.result.append(thisgenerator)
        else:
            self.result.append('')
        #if 'ephorma as' in self.data:
        #    self.result.append('ephorma')
        #else:
        #    self.result.append('Unkown')
        return self.result


    def run(self):
        while True:
            element = self.dataqueue.get()
            if not element:
                return
            else:
                site = element
                sys.stdout.flush()
                #print site,self.checkforpostliste(site)
                tempres = self.checkforpostliste(site) 
                self.results.append((element,tempres[0],tempres[1],tempres[2]))#,tempres[2]))


rservers = [ReqestServer() for i in range(50)]
for r in rservers:
    r.start()

num = 0
for url in municipalitylists:
    num += 1
    random.sample(rservers,1)[0].push(url)

for r in rservers:
    r.stop()
    r.join()

results = []

for r in rservers:
    for res in r.results:
        results.append(res)

#for res in results:
#    print ','.join([str(i) for i in res])