Mercurial > hg > index.fcgi > www > www-1
view myrss/myrss_app.py @ 69:ae0f2f438a95
myrss: add support for new "purl" RSS type
| author | paulo |
|---|---|
| date | Thu, 11 Jun 2015 22:03:34 -0700 |
| parents | 66a232bae83c |
| children | 3456dd3e8660 |
line source
1 import os
2 import sys
3 import re
4 import urllib2
5 import threading
6 import Queue
7 import datetime
8 import time
10 import logging
11 logging.basicConfig(
12 level=logging.INFO,
13 #filename="_LOG",
14 #format="%(asctime)s %(levelname)-8s %(message)s",
15 )
17 import xml.etree.ElementTree
18 import HTMLParser
20 import html
23 FEEDS_FILE = "FEEDS"
24 CACHE_HTML_FILE = "__cache__.html"
26 CACHE_LIFE = 1200 # [seconds]
27 MAX_ITEMS = 50
28 MAX_LINK_Z = 4
29 MAX_THREADS = 20
30 URLOPEN_TIMEOUT = 60 # [seconds]
33 _PARSE_ROOT_TAG_RE = re.compile(r"(\{(.+)\})?(.+)")
35 def _parse_root_tag(root_tag):
36 re_match = _PARSE_ROOT_TAG_RE.match(root_tag)
38 if re_match is None:
39 return (None, None)
40 else:
41 return re_match.group(2, 3)
44 def _strip_if_not_none(txt):
45 return txt.strip() if txt is not None else ''
48 def _go_rss(elementTree):
49 title = _strip_if_not_none(elementTree.find("channel/title").text)
50 link = elementTree.find("channel/link").text
52 items = []
54 for i in elementTree.findall("channel/item")[:MAX_ITEMS]:
55 it_title = _strip_if_not_none(i.find("title").text)
56 it_link = i.find("link").text
58 items.append((it_title, it_link))
60 return (title, link, items)
63 def _go_atom(elementTree):
64 ns = "http://www.w3.org/2005/Atom"
66 title = _strip_if_not_none(elementTree.find("{%s}title" % ns).text)
67 link = ''
69 for i in elementTree.findall("{%s}link" % ns):
70 if i.get("type") == "text/html" and i.get("rel") == "alternate":
71 link = i.get("href")
72 break
74 items = []
76 for i in elementTree.findall("{%s}entry" % ns)[:MAX_ITEMS]:
77 it_title = _strip_if_not_none(i.find("{%s}title" % ns).text)
78 it_link = ''
80 for j in i.findall("{%s}link" % ns):
81 if j.get("type") == "text/html" and j.get("rel") == "alternate":
82 it_link = j.get("href")
83 break
85 items.append((it_title, it_link))
87 return (title, link, items)
90 def _go_purl_rss(elementTree):
91 ns = "http://purl.org/rss/1.0/"
93 title = _strip_if_not_none(elementTree.find("{%s}channel/{%s}title" % (ns, ns)).text)
94 link = elementTree.find("{%s}channel/{%s}link" % (ns, ns)).text
96 items = []
98 for i in elementTree.findall("{%s}item" % ns)[:MAX_ITEMS]:
99 it_title = _strip_if_not_none(i.find("{%s}title" % ns).text)
100 it_link = i.find("{%s}link" % ns).text
102 items.append((it_title, it_link))
104 return (title, link, items)
107 _STRIP_HTML_RE = re.compile(r"<.*?>")
108 _htmlParser = HTMLParser.HTMLParser()
110 def _strip_html(txt):
111 return _htmlParser.unescape(_STRIP_HTML_RE.sub('', txt))
114 def _to_html(dtnow, docstruct):
115 datetime_str = dtnow.strftime("%Y-%m-%d %H:%M %Z")
116 page_title = "myrss -- %s" % datetime_str
118 root = html.HTML("html")
120 header = root.header
121 header.title(page_title)
122 header.link(rel="stylesheet", type="text/css", href="index.css")
124 body = root.body
125 body.h1(page_title)
127 link_z = 0
129 for feed in docstruct:
130 if feed is None:
131 continue
133 (title, link, items) = feed
135 body.h2.a(_strip_html(title), href=link, klass="z%d" % (link_z % MAX_LINK_Z))
136 link_z += 1
137 p = body.p
139 for (i, (it_title, it_link)) in enumerate(items):
140 if i > 0:
141 p += " - "
143 p.a(_strip_html(it_title), href=it_link, klass="z%d" % (link_z % MAX_LINK_Z))
144 link_z += 1
146 dtdelta = datetime.datetime.now() - dtnow
147 root.div("%.3f" % (dtdelta.days*86400 + dtdelta.seconds + dtdelta.microseconds/1e6), klass="debug")
149 return unicode(root).encode("utf-8")
152 def _fetch_url(url):
153 try:
154 logging.info("processing %s" % url)
155 feed = urllib2.urlopen(urllib2.Request(url, headers={"User-Agent": ''}), timeout=URLOPEN_TIMEOUT)
156 except urllib2.HTTPError as e:
157 logging.info("(%s) %s" % (url, e))
158 return None
160 return feed
163 def _process_feed(feed):
164 ret = None
166 elementTree = xml.etree.ElementTree.parse(feed)
167 root = elementTree.getroot()
169 parsed_root_tag = _parse_root_tag(root.tag)
171 if parsed_root_tag == (None, "rss"):
172 version = float(root.get("version", 0.0))
173 if version >= 2.0:
174 ret = _go_rss(elementTree)
175 else:
176 raise NotImplementedError("Unsupported rss version")
177 elif parsed_root_tag == ("http://www.w3.org/2005/Atom", "feed"):
178 ret = _go_atom(elementTree)
179 elif parsed_root_tag == ("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "RDF"):
180 ret = _go_purl_rss(elementTree)
181 else:
182 raise NotImplementedError("Unknown root tag")
184 return ret
187 class WorkerThread(threading.Thread):
188 def __init__(self, *args, **kwargs):
189 self._input_queue = kwargs.pop("input_queue")
190 self._output_queue = kwargs.pop("output_queue")
191 threading.Thread.__init__(self, *args, **kwargs)
192 self.daemon = True
194 def run(self):
195 while True:
196 (idx, url) = self._input_queue.get()
197 docfeed = None
198 try:
199 feed = _fetch_url(url)
200 if feed is not None:
201 docfeed = _process_feed(feed)
202 except Exception as e:
203 logging.info("(%s) exception: %s" % (url, e))
204 self._output_queue.put((idx, docfeed))
207 def main(input_queue, output_queue, lock):
208 ret = ''
210 with lock:
211 epoch_now = time.time()
212 dtnow = datetime.datetime.fromtimestamp(epoch_now)
214 if os.path.exists(CACHE_HTML_FILE) and (epoch_now - os.stat(CACHE_HTML_FILE).st_mtime) < float(CACHE_LIFE):
215 with open(CACHE_HTML_FILE) as cache_html_file:
216 ret = cache_html_file.read()
218 else:
219 with open(FEEDS_FILE) as feeds_file:
220 feedlines = feeds_file.readlines()
222 docstruct = [None]*len(feedlines)
223 num_input = 0
224 for (i, l) in enumerate(feedlines):
225 if l[0] != '#':
226 l = l.strip()
227 input_queue.put((i, l))
228 num_input += 1
230 for _ in range(num_input):
231 (idx, docfeed) = output_queue.get()
232 docstruct[idx] = docfeed
234 ret = _to_html(dtnow, docstruct)
236 with open(CACHE_HTML_FILE, 'w') as cache_html_file:
237 cache_html_file.write(ret)
239 return ret
242 class MyRssApp:
243 def __init__(self):
244 self._iq = Queue.Queue(MAX_THREADS)
245 self._oq = Queue.Queue(MAX_THREADS)
246 self._main_lock = threading.Lock()
248 for _ in range(MAX_THREADS):
249 WorkerThread(input_queue=self._iq, output_queue=self._oq).start()
251 def __call__(self, environ, start_response):
252 response_body = main(self._iq, self._oq, self._main_lock)
253 response_headers = [
254 ("Content-Type", "text/html; charset=UTF-8"),
255 ("Content-Length", str(len(response_body))),
256 ]
257 start_response("200 OK", response_headers)
259 return [response_body]