Mercurial > hg > index.fcgi > www > www-1

view myrss/myrss_parser.py @ 42:a1456ecd25b9

find changesets by author, revision, files, or words in the commit message
fix bug with threadpool; create WSGI app and test server; use logging
author paulo
date Mon, 04 Feb 2013 23:58:02 -0800
parents 5f9bc02e9caf
children
line source
1 import os
2 import sys
3 import re
4 import urllib2
5 import threading
6 import Queue
7 import datetime
8 import time
9 import logging
10 logging.basicConfig(level=logging.INFO)
11
12 import html
13 import xml.etree.ElementTree
14
15
16 FEEDS_FILE = "FEEDS"
17 CACHE_HTML_FILE = "__cache__.html"
18
19 #CACHE_LIFE = 1200 # [seconds]
20 CACHE_LIFE = 30 # [seconds]
21 MAX_ITEMS = 30
22 MAX_LINK_Z = 4
23 MAX_THREADS = 20
24
25
26 _PARSE_ROOT_TAG_RE = re.compile(r"(\{(.+)\})?(.+)")
27
28 def _parse_root_tag(root_tag):
29 re_match = _PARSE_ROOT_TAG_RE.match(root_tag)
30
31 if re_match is None:
32 return (None, None)
33 else:
34 return re_match.group(2, 3)
35
36
37 def _go_rss(elementTree):
38 title = elementTree.find("channel/title").text.strip()
39 link = elementTree.find("channel/link").text
40
41 items = []
42
43 for i in elementTree.findall("channel/item")[:MAX_ITEMS]:
44 it_title = i.find("title").text.strip()
45 it_link = i.find("link").text
46
47 items.append((it_title, it_link))
48
49 return (title, link, items)
50
51
52 def _go_atom(elementTree):
53 ns = "http://www.w3.org/2005/Atom"
54
55 title = elementTree.find("{%s}title" % ns).text.strip()
56 link = ''
57
58 for i in elementTree.findall("{%s}link" % ns):
59 if i.get("type") == "text/html" and i.get("rel") == "alternate":
60 link = i.get("href")
61 break
62
63 items = []
64
65 for i in elementTree.findall("{%s}entry" % ns)[:MAX_ITEMS]:
66 it_title = i.find("{%s}title" % ns).text.strip()
67 it_link = ''
68
69 for j in i.findall("{%s}link" % ns):
70 if j.get("type") == "text/html" and j.get("rel") == "alternate":
71 it_link = j.get("href")
72 break
73
74 items.append((it_title, it_link))
75
76 return (title, link, items)
77
78
79 def _to_html(dtnow, docstruct):
80 datetime_str = dtnow.strftime("%Y-%m-%d %H:%M %Z")
81 page_title = "myrss -- %s" % datetime_str
82
83 root = html.HTML("html")
84
85 header = root.header
86 header.title(page_title)
87 header.link(rel="stylesheet", type="text/css", href="index.css")
88
89 body = root.body
90 body.h1(page_title)
91
92 link_z = 0
93
94 for feed in docstruct:
95 if feed is None:
96 continue
97
98 (title, link, items) = feed
99
100 body.h2.a(title, href=link, klass="z%d" % (link_z % MAX_LINK_Z))
101 link_z += 1
102 p = body.p
103
104 for (i, (it_title, it_link)) in enumerate(items):
105 if i > 0:
106 p += " - "
107
108 p.a(it_title, href=it_link, klass="z%d" % (link_z % MAX_LINK_Z))
109 link_z += 1
110
111 return unicode(root).encode("utf-8")
112
113
114 def _process_url(url):
115 ret = None
116
117 try:
118 logging.info("processing %s" % url)
119 feed = urllib2.urlopen(urllib2.Request(url, headers={"User-Agent": ''}))
120 except urllib2.HTTPError as e:
121 logging.info("(%s) %s" % (url, e))
122 return ret
123
124 elementTree = xml.etree.ElementTree.parse(feed)
125 root = elementTree.getroot()
126
127 parsed_root_tag = _parse_root_tag(root.tag)
128
129 if parsed_root_tag == (None, "rss"):
130 version = float(root.get("version", 0.0))
131 if version >= 2.0:
132 ret = _go_rss(elementTree)
133 else:
134 raise NotImplementedError("Unsupported rss version")
135 elif parsed_root_tag == ("http://www.w3.org/2005/Atom", "feed"):
136 ret = _go_atom(elementTree)
137 else:
138 raise NotImplementedError("Unknown root tag")
139
140 return ret
141
142
143 class WorkerThread(threading.Thread):
144 def __init__(self, *args, **kwargs):
145 self._input_queue = kwargs.pop("input_queue")
146 self._output_queue = kwargs.pop("output_queue")
147 threading.Thread.__init__(self, *args, **kwargs)
148 self.daemon = True
149
150 def run(self):
151 while True:
152 (idx, url) = self._input_queue.get()
153 docfeed = None
154 try:
155 docfeed = _process_url(url)
156 except Exception as e:
157 logging.info("(%s) exception: %s" % (url, e))
158 self._output_queue.put((idx, docfeed))
159
160
161 def main(input_queue, output_queue):
162 ret = ''
163
164 epoch_now = time.time()
165 dtnow = datetime.datetime.fromtimestamp(epoch_now)
166
167 if os.path.exists(CACHE_HTML_FILE) and (epoch_now - os.stat(CACHE_HTML_FILE).st_mtime) < float(CACHE_LIFE):
168 with open(CACHE_HTML_FILE) as cache_html_file:
169 ret = cache_html_file.read()
170
171 else:
172 with open(FEEDS_FILE) as feeds_file:
173 feedlines = feeds_file.readlines()
174
175 docstruct = [None]*len(feedlines)
176 num_input = 0
177 for (i, l) in enumerate(feedlines):
178 if l[0] != '#':
179 l = l.strip()
180 input_queue.put((i, l))
181 num_input += 1
182
183 for _ in range(num_input):
184 (idx, docfeed) = output_queue.get()
185 docstruct[idx] = docfeed
186
187 ret = _to_html(dtnow, docstruct)
188
189 with open(CACHE_HTML_FILE, 'w') as cache_html_file:
190 cache_html_file.write(ret)
191
192 return ret
193
194
195 class MyRssApp:
196 def __init__(self):
197 self._iq = Queue.Queue(MAX_THREADS)
198 self._oq = Queue.Queue(MAX_THREADS)
199
200 for _ in range(MAX_THREADS):
201 WorkerThread(input_queue=self._iq, output_queue=self._oq).start()
202
203 def __call__(self, environ, start_response):
204 response_body = main(self._iq, self._oq)
205 response_headers = [
206 ("Content-Type", "text/html"),
207 ("Content-Length", str(len(response_body))),
208 ]
209 start_response("200 OK", response_headers)
210
211 return [response_body]
212
213