Threading with webkit

In a previous post I showed how to scrape a list of webpages. That is fine for small crawls but will take too long otherwise. Here is an updated example that downloads the content in multiple threads.

import sys
from collections import deque # threadsafe datatype
from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyQt4.QtWebKit import *
NUM_THREADS = 3 # how many threads to use

class Render(QWebView):
    active = deque() # track how many threads are still active
    data = {} # store the data

    def __init__(self, urls):
        self.urls = urls

    def crawl(self):
            url = self.urls.pop()
            print 'downloading', url
        except IndexError:
            # no more urls to process
            if not
                # no more threads downloading
                print 'finished'

    def _loadFinished(self, result):
        # process the downloaded html
        frame =
        url = str(frame.url().toString())[url] = frame.toHtml()
        self.crawl() # crawl next URL in the list

app = QApplication(sys.argv) # can only instantiate this once so must move outside class 
urls = deque(['', '',
    '', ''])
renders = [Render(urls) for i in range(NUM_THREADS)]
app.exec_() # will execute qt loop until class calls close event

Source code is available at my bitbucket account.

blog comments powered by Disqus