Mercurial > hg > index.fcgi > www > www-1

annotate myrss/myrss_parser.py @ 40:62464a0034d1

find changesets by author, revision, files, or words in the commit message
add threaded url opener
author paulo
date Thu, 31 Jan 2013 02:19:39 -0800
parents 915032dd35f4
children 5f9bc02e9caf
rev   line source
paulo@39 1 import os
paulo@40 2 import sys
paulo@39 3 import re
paulo@40 4 import urllib2
paulo@40 5 import threading
paulo@40 6 import Queue
paulo@39 7
paulo@39 8 import html
paulo@39 9 import xml.etree.ElementTree
paulo@39 10
paulo@39 11
paulo@39 12 MAX_ITEMS = 30
paulo@39 13 MAX_LINK_Z = 4
paulo@40 14 MAX_THREADS = 20
paulo@39 15
paulo@39 16
paulo@39 17 _PARSE_ROOT_TAG_RE = re.compile(r"(\{(.+)\})?(.+)")
paulo@39 18
paulo@39 19 def _parse_root_tag(root_tag):
paulo@39 20 re_match = _PARSE_ROOT_TAG_RE.match(root_tag)
paulo@39 21
paulo@39 22 if re_match is None:
paulo@39 23 return (None, None)
paulo@39 24 else:
paulo@39 25 return re_match.group(2, 3)
paulo@39 26
paulo@39 27
paulo@39 28 def _go_rss(elementTree):
paulo@39 29 title = elementTree.find("channel/title").text.strip()
paulo@39 30 link = elementTree.find("channel/link").text
paulo@39 31
paulo@39 32 items = []
paulo@39 33
paulo@39 34 for i in elementTree.findall("channel/item")[:MAX_ITEMS]:
paulo@39 35 it_title = i.find("title").text.strip()
paulo@39 36 it_link = i.find("link").text
paulo@39 37
paulo@39 38 items.append((it_title, it_link))
paulo@39 39
paulo@39 40 return (title, link, items)
paulo@39 41
paulo@39 42
paulo@39 43 def _go_atom(elementTree):
paulo@39 44 ns = "http://www.w3.org/2005/Atom"
paulo@39 45
paulo@39 46 title = elementTree.find("{%s}title" % ns).text.strip()
paulo@39 47 link = ''
paulo@39 48
paulo@39 49 for i in elementTree.findall("{%s}link" % ns):
paulo@39 50 if i.get("type") == "text/html" and i.get("rel") == "alternate":
paulo@39 51 link = i.get("href")
paulo@39 52 break
paulo@39 53
paulo@39 54 items = []
paulo@39 55
paulo@39 56 for i in elementTree.findall("{%s}entry" % ns)[:MAX_ITEMS]:
paulo@39 57 it_title = i.find("{%s}title" % ns).text.strip()
paulo@39 58 it_link = ''
paulo@39 59
paulo@39 60 for j in i.findall("{%s}link" % ns):
paulo@39 61 if j.get("type") == "text/html" and j.get("rel") == "alternate":
paulo@39 62 it_link = j.get("href")
paulo@39 63 break
paulo@39 64
paulo@39 65 items.append((it_title, it_link))
paulo@39 66
paulo@39 67 return (title, link, items)
paulo@39 68
paulo@39 69
paulo@39 70 def _to_html(docstruct):
paulo@39 71 root = html.HTML()
paulo@39 72
paulo@39 73 header = root.header
paulo@39 74 header.title("myrss")
paulo@39 75 header.link(rel="stylesheet", type="text/css", href="index.css")
paulo@39 76
paulo@39 77 link_z = 0
paulo@39 78
paulo@39 79 for feed in docstruct:
paulo@40 80 if feed is None:
paulo@40 81 continue
paulo@40 82
paulo@39 83 (title, link, items) = feed
paulo@39 84
paulo@39 85 root.h1.a(title, href=link, klass="z%d" % (link_z % MAX_LINK_Z))
paulo@39 86 link_z += 1
paulo@39 87 p = root.p
paulo@39 88
paulo@39 89 for (i, (it_title, it_link)) in enumerate(items):
paulo@39 90 if i > 0:
paulo@39 91 p += " - "
paulo@39 92
paulo@39 93 p.a(it_title, href=it_link, klass="z%d" % (link_z % MAX_LINK_Z))
paulo@39 94 link_z += 1
paulo@39 95
paulo@39 96 return unicode(root).encode("utf-8")
paulo@39 97
paulo@39 98
paulo@40 99 def _process_url(url):
paulo@40 100 ret = None
paulo@40 101
paulo@40 102 try:
paulo@40 103 print >> sys.stderr, "--> processing %s" % url
paulo@40 104 feed = urllib2.urlopen(url)
paulo@40 105 except urllib2.HTTPError as e:
paulo@40 106 print >> sys.stderr, "--> (%s) %s" % (url, e)
paulo@40 107 return ret
paulo@40 108
paulo@40 109 elementTree = xml.etree.ElementTree.parse(feed)
paulo@40 110 root = elementTree.getroot()
paulo@40 111
paulo@40 112 parsed_root_tag = _parse_root_tag(root.tag)
paulo@40 113
paulo@40 114 if parsed_root_tag == (None, "rss"):
paulo@40 115 version = float(root.get("version", 0.0))
paulo@40 116 if version >= 2.0:
paulo@40 117 ret = _go_rss(elementTree)
paulo@40 118 else:
paulo@40 119 raise NotImplementedError("Unsupported rss version")
paulo@40 120 elif parsed_root_tag == ("http://www.w3.org/2005/Atom", "feed"):
paulo@40 121 ret = _go_atom(elementTree)
paulo@40 122 else:
paulo@40 123 raise NotImplementedError("Unknown root tag")
paulo@40 124
paulo@40 125 return ret
paulo@40 126
paulo@40 127
paulo@40 128 class WorkerThread(threading.Thread):
paulo@40 129 def __init__(self, *args, **kwargs):
paulo@40 130 self._input_queue = kwargs.pop("input_queue")
paulo@40 131 self._output_queue = kwargs.pop("output_queue")
paulo@40 132 threading.Thread.__init__(self, *args, **kwargs)
paulo@40 133 self.daemon = True
paulo@40 134
paulo@40 135 def run(self):
paulo@40 136 while True:
paulo@40 137 (idx, url) = self._input_queue.get()
paulo@40 138 docfeed = None
paulo@40 139 try:
paulo@40 140 docfeed = _process_url(url)
paulo@40 141 except Exception as e:
paulo@40 142 print >> sys.stderr, "--> (%s) exception: %s" % (url, e)
paulo@40 143 self._output_queue.put((idx, docfeed))
paulo@40 144 self._input_queue.task_done()
paulo@40 145
paulo@40 146
paulo@39 147 if __name__ == "__main__":
paulo@40 148 with open("FEEDS") as feeds_file:
paulo@40 149 feedlines = feeds_file.readlines()
paulo@39 150
paulo@40 151 docstruct = [None]*len(feedlines)
paulo@40 152 iq = Queue.Queue(feedlines)
paulo@40 153 oq = Queue.Queue(feedlines)
paulo@39 154
paulo@40 155 for _ in range(MAX_THREADS):
paulo@40 156 WorkerThread(input_queue=iq, output_queue=oq).start()
paulo@39 157
paulo@40 158 for (i, l) in enumerate(feedlines):
paulo@40 159 if l[0] != '#':
paulo@40 160 l = l.strip()
paulo@40 161 iq.put((i, l))
paulo@40 162
paulo@40 163 iq.join()
paulo@40 164
paulo@40 165 while True:
paulo@40 166 try:
paulo@40 167 (idx, docfeed) = oq.get_nowait()
paulo@40 168 docstruct[idx] = docfeed
paulo@40 169 except Queue.Empty:
paulo@40 170 break
paulo@40 171
paulo@40 172 print _to_html(docstruct)