Mercurial > hg > index.fcgi > www > www-1
view myrss/myrss_app.py @ 44:c673e9e9c4ca
add threading.Lock to main() to support threaded server
| author | paulo |
|---|---|
| date | Tue, 05 Feb 2013 00:57:37 -0800 |
| parents | df6a1a347584 |
| children | aca02ce71274 |
line source
1 import os
2 import sys
3 import re
4 import urllib2
5 import threading
6 import Queue
7 import datetime
8 import time
9 import logging
10 logging.basicConfig(level=logging.INFO)
12 import html
13 import xml.etree.ElementTree
16 FEEDS_FILE = "FEEDS"
17 CACHE_HTML_FILE = "__cache__.html"
19 CACHE_LIFE = 1200 # [seconds]
20 MAX_ITEMS = 30
21 MAX_LINK_Z = 4
22 MAX_THREADS = 20
25 _PARSE_ROOT_TAG_RE = re.compile(r"(\{(.+)\})?(.+)")
27 def _parse_root_tag(root_tag):
28 re_match = _PARSE_ROOT_TAG_RE.match(root_tag)
30 if re_match is None:
31 return (None, None)
32 else:
33 return re_match.group(2, 3)
36 def _go_rss(elementTree):
37 title = elementTree.find("channel/title").text.strip()
38 link = elementTree.find("channel/link").text
40 items = []
42 for i in elementTree.findall("channel/item")[:MAX_ITEMS]:
43 it_title = i.find("title").text.strip()
44 it_link = i.find("link").text
46 items.append((it_title, it_link))
48 return (title, link, items)
51 def _go_atom(elementTree):
52 ns = "http://www.w3.org/2005/Atom"
54 title = elementTree.find("{%s}title" % ns).text.strip()
55 link = ''
57 for i in elementTree.findall("{%s}link" % ns):
58 if i.get("type") == "text/html" and i.get("rel") == "alternate":
59 link = i.get("href")
60 break
62 items = []
64 for i in elementTree.findall("{%s}entry" % ns)[:MAX_ITEMS]:
65 it_title = i.find("{%s}title" % ns).text.strip()
66 it_link = ''
68 for j in i.findall("{%s}link" % ns):
69 if j.get("type") == "text/html" and j.get("rel") == "alternate":
70 it_link = j.get("href")
71 break
73 items.append((it_title, it_link))
75 return (title, link, items)
78 def _to_html(dtnow, docstruct):
79 datetime_str = dtnow.strftime("%Y-%m-%d %H:%M %Z")
80 page_title = "myrss -- %s" % datetime_str
82 root = html.HTML("html")
84 header = root.header
85 header.title(page_title)
86 header.link(rel="stylesheet", type="text/css", href="index.css")
88 body = root.body
89 body.h1(page_title)
91 link_z = 0
93 for feed in docstruct:
94 if feed is None:
95 continue
97 (title, link, items) = feed
99 body.h2.a(title, href=link, klass="z%d" % (link_z % MAX_LINK_Z))
100 link_z += 1
101 p = body.p
103 for (i, (it_title, it_link)) in enumerate(items):
104 if i > 0:
105 p += " - "
107 p.a(it_title, href=it_link, klass="z%d" % (link_z % MAX_LINK_Z))
108 link_z += 1
110 return unicode(root).encode("utf-8")
113 def _process_url(url):
114 ret = None
116 try:
117 logging.info("processing %s" % url)
118 feed = urllib2.urlopen(urllib2.Request(url, headers={"User-Agent": ''}))
119 except urllib2.HTTPError as e:
120 logging.info("(%s) %s" % (url, e))
121 return ret
123 elementTree = xml.etree.ElementTree.parse(feed)
124 root = elementTree.getroot()
126 parsed_root_tag = _parse_root_tag(root.tag)
128 if parsed_root_tag == (None, "rss"):
129 version = float(root.get("version", 0.0))
130 if version >= 2.0:
131 ret = _go_rss(elementTree)
132 else:
133 raise NotImplementedError("Unsupported rss version")
134 elif parsed_root_tag == ("http://www.w3.org/2005/Atom", "feed"):
135 ret = _go_atom(elementTree)
136 else:
137 raise NotImplementedError("Unknown root tag")
139 return ret
142 class WorkerThread(threading.Thread):
143 def __init__(self, *args, **kwargs):
144 self._input_queue = kwargs.pop("input_queue")
145 self._output_queue = kwargs.pop("output_queue")
146 threading.Thread.__init__(self, *args, **kwargs)
147 self.daemon = True
149 def run(self):
150 while True:
151 (idx, url) = self._input_queue.get()
152 docfeed = None
153 try:
154 docfeed = _process_url(url)
155 except Exception as e:
156 logging.info("(%s) exception: %s" % (url, e))
157 self._output_queue.put((idx, docfeed))
160 def main(input_queue, output_queue, lock):
161 ret = ''
163 with lock:
164 epoch_now = time.time()
165 dtnow = datetime.datetime.fromtimestamp(epoch_now)
167 if os.path.exists(CACHE_HTML_FILE) and (epoch_now - os.stat(CACHE_HTML_FILE).st_mtime) < float(CACHE_LIFE):
168 with open(CACHE_HTML_FILE) as cache_html_file:
169 ret = cache_html_file.read()
171 else:
172 with open(FEEDS_FILE) as feeds_file:
173 feedlines = feeds_file.readlines()
175 docstruct = [None]*len(feedlines)
176 num_input = 0
177 for (i, l) in enumerate(feedlines):
178 if l[0] != '#':
179 l = l.strip()
180 input_queue.put((i, l))
181 num_input += 1
183 for _ in range(num_input):
184 (idx, docfeed) = output_queue.get()
185 docstruct[idx] = docfeed
187 ret = _to_html(dtnow, docstruct)
189 with open(CACHE_HTML_FILE, 'w') as cache_html_file:
190 cache_html_file.write(ret)
192 return ret
195 class MyRssApp:
196 def __init__(self):
197 self._iq = Queue.Queue(MAX_THREADS)
198 self._oq = Queue.Queue(MAX_THREADS)
199 self._main_lock = threading.Lock()
201 for _ in range(MAX_THREADS):
202 WorkerThread(input_queue=self._iq, output_queue=self._oq).start()
204 def __call__(self, environ, start_response):
205 response_body = main(self._iq, self._oq, self._main_lock)
206 response_headers = [
207 ("Content-Type", "text/html"),
208 ("Content-Length", str(len(response_body))),
209 ]
210 start_response("200 OK", response_headers)
212 return [response_body]