Mercurial > hg > index.fcgi > www > www-1
view myrss2/myrss_app.py @ 144:90f3021e3137
myrss2: FEEDS: Remove longform.org; add propublic.org
| author | paulo |
|---|---|
| date | Tue, 28 May 2024 06:23:58 +0000 |
| parents | 1a5c0fc5627a |
| children |
line source
1 import datetime
2 import gzip
3 import io
4 import os
5 import queue
6 import re
7 import sys
8 import threading
9 import time
10 import traceback
11 import urllib.error
12 import urllib.request
14 import logging
15 LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
16 logging.basicConfig(
17 level=getattr(logging, LOG_LEVEL),
18 format="%(asctime)s %(levelname)-8s %(message)s",
19 )
21 import xml.etree.ElementTree
22 import html
24 from html3.html3 import HTML
27 FEEDS_FILE = "FEEDS"
28 CACHE_HTML_FILE = "__cache__.html"
30 CACHE_LIFE = 1200 # [seconds]
31 MAX_ITEMS = 50
32 MAX_LINK_Z = 4
33 MAX_THREADS = 20
34 URLOPEN_TIMEOUT = 10 # [seconds]
37 _PARSE_ROOT_TAG_RE = re.compile(r"(\{(.+)\})?(.+)")
39 def _parse_root_tag(root_tag):
40 re_match = _PARSE_ROOT_TAG_RE.match(root_tag)
42 if re_match is None:
43 return (None, None)
44 else:
45 return re_match.group(2, 3)
48 def _strip_if_not_none(txt):
49 return txt.strip() if txt is not None else ''
52 def _go_rss(elementTree):
53 title = _strip_if_not_none(elementTree.find("channel/title").text)
54 link = elementTree.find("channel/link").text
56 items = []
58 for i in elementTree.findall("channel/item")[:MAX_ITEMS]:
59 it_title = _strip_if_not_none(i.find("title").text)
60 it_link = i.find("link").text
62 items.append((it_title, it_link))
64 return (title, link, items)
67 def _go_atom(elementTree):
68 ns = "http://www.w3.org/2005/Atom"
70 title = _strip_if_not_none(elementTree.find("{%s}title" % ns).text)
71 link = ''
73 links = elementTree.findall("{%s}link" % ns)
74 for i in links:
75 if len(links) == 1 or i.get("rel") == "alternate":
76 link = i.get("href")
77 break
79 items = []
81 for i in elementTree.findall("{%s}entry" % ns)[:MAX_ITEMS]:
82 it_title = _strip_if_not_none(i.find("{%s}title" % ns).text)
83 it_link = ''
85 it_links = i.findall("{%s}link" % ns)
86 for j in it_links:
87 if len(it_links) == 1 or j.get("rel") == "alternate":
88 it_link = j.get("href")
89 break
91 items.append((it_title, it_link))
93 return (title, link, items)
96 def _go_purl_rss(elementTree):
97 ns = "http://purl.org/rss/1.0/"
99 title = _strip_if_not_none(elementTree.find("{%s}channel/{%s}title" % (ns, ns)).text)
100 link = elementTree.find("{%s}channel/{%s}link" % (ns, ns)).text
102 items = []
104 for i in elementTree.findall("{%s}item" % ns)[:MAX_ITEMS]:
105 it_title = _strip_if_not_none(i.find("{%s}title" % ns).text)
106 it_link = i.find("{%s}link" % ns).text
108 items.append((it_title, it_link))
110 return (title, link, items)
113 _STRIP_HTML_RE = re.compile(r"<.*?>")
115 def _strip_html(txt):
116 return html.unescape(_STRIP_HTML_RE.sub('', txt))
119 def _to_html(dtnow, docstruct):
120 datetime_str = dtnow.strftime("%Y-%m-%d %H:%M %Z")
121 page_title = "myrss -- %s" % datetime_str
123 root = HTML("html")
125 header = root.head
126 header.meta(name="viewport", content="width=device-width, initial-scale=1")
127 header.title(page_title)
128 header.link(rel="stylesheet", type="text/css", href="static/index.css")
130 body = root.body
131 body.h1(page_title)
133 link_z = 0
135 for feed in docstruct:
136 if feed is None:
137 continue
139 (title, link, items) = feed
141 if title is None:
142 title = "(No title)"
143 if link is None:
144 link = ""
146 logging.debug("title: %s", title)
147 body.h2.a(_strip_html(title), href=link, klass="z%d" % (link_z % MAX_LINK_Z))
148 link_z += 1
149 p = body.p
151 for (i, (it_title, it_link)) in enumerate(items):
152 if i > 0:
153 p += " - "
155 if not it_title:
156 it_title = "(missing title)"
157 if it_link is not None:
158 p.a(_strip_html(it_title), href=it_link, klass="z%d" % (link_z % MAX_LINK_Z))
159 else:
160 p += _strip_html(it_title)
162 link_z += 1
164 dtdelta = datetime.datetime.now() - dtnow
165 root.div("%.3f" % (dtdelta.days*86400 + dtdelta.seconds + dtdelta.microseconds/1e6), klass="debug")
167 return str(root)
170 def _fetch_url(url):
171 try:
172 logging.info("processing %s" % url)
173 feed = urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 Browser"}), timeout=URLOPEN_TIMEOUT)
174 response_headers = feed.info().as_string().splitlines()
175 if 'Content-Encoding: gzip' in response_headers:
176 body = gzip.decompress(feed.read())
177 else:
178 body = feed.read()
180 except urllib.error.HTTPError as e:
181 logging.info("(%s) %s" % (url, e))
182 return None
184 return str(body, encoding="utf-8")
187 def _filter_feed(feed):
188 ret = feed.strip()
190 filter_out = ["\x16"]
191 for i in filter_out:
192 ret = ret.replace(i, "")
194 return ret
197 def _process_feed(feed):
198 ret = None
200 feed_sio = io.StringIO(feed)
201 elementTree = xml.etree.ElementTree.parse(feed_sio)
202 root = elementTree.getroot()
204 parsed_root_tag = _parse_root_tag(root.tag)
206 if parsed_root_tag == (None, "rss"):
207 version = float(root.get("version", 0.0))
208 if version >= 2.0:
209 ret = _go_rss(elementTree)
210 else:
211 raise NotImplementedError("Unsupported rss version")
212 elif parsed_root_tag == ("http://www.w3.org/2005/Atom", "feed"):
213 ret = _go_atom(elementTree)
214 elif parsed_root_tag == ("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "RDF"):
215 ret = _go_purl_rss(elementTree)
216 else:
217 raise NotImplementedError("Unknown root tag")
219 return ret
222 class WorkerThread(threading.Thread):
223 def __init__(self, *args, **kwargs):
224 self._input_queue = kwargs.pop("input_queue")
225 self._output_queue = kwargs.pop("output_queue")
226 threading.Thread.__init__(self, *args, **kwargs)
227 self.daemon = True
229 def run(self):
230 while True:
231 (idx, url) = self._input_queue.get()
232 docfeed = None
233 try:
234 feed = _fetch_url(url)
235 if feed is not None:
236 docfeed = _process_feed(_filter_feed(feed))
237 except Exception as e:
238 logging.info("(%s) exception: (%s) %s" % (url, type(e), e))
239 self._output_queue.put((idx, docfeed))
242 def main(input_queue, output_queue, lock):
243 ret = ''
245 with lock:
246 logging.debug("main() started")
247 epoch_now = time.time()
248 dtnow = datetime.datetime.fromtimestamp(epoch_now)
250 if os.path.exists(CACHE_HTML_FILE) and (epoch_now - os.stat(CACHE_HTML_FILE).st_mtime) < float(CACHE_LIFE):
251 with open(CACHE_HTML_FILE) as cache_html_file:
252 ret = cache_html_file.read()
254 else:
255 with open(FEEDS_FILE) as feeds_file:
256 feedlines = feeds_file.readlines()
258 docstruct = [None]*len(feedlines)
259 num_input = 0
260 for (i, l) in enumerate(feedlines):
261 if l[0] != '#':
262 l = l.strip()
263 input_queue.put((i, l))
264 num_input += 1
266 for _ in range(num_input):
267 (idx, docfeed) = output_queue.get()
268 docstruct[idx] = docfeed
270 ret = _to_html(dtnow, docstruct)
272 with open(CACHE_HTML_FILE, 'w') as cache_html_file:
273 cache_html_file.write(ret)
274 logging.debug("main() ended")
276 return ret
279 class MyRssApp:
280 def __init__(self):
281 logging.debug("MyRssApp.__init__() called")
282 self._iq = queue.Queue(MAX_THREADS)
283 self._oq = queue.Queue(MAX_THREADS)
284 self._main_lock = threading.Lock()
286 for i in range(MAX_THREADS):
287 logging.debug("Starting thread: %d" % i)
288 WorkerThread(input_queue=self._iq, output_queue=self._oq).start()
290 # Raw WSGI
291 def __call__(self, environ, start_response):
292 response_code = "500 Internal Server Error"
293 response_type = "text/plain; charset=UTF-8"
295 try:
296 response_body = main(self._iq, self._oq, self._main_lock)
297 response_code = "200 OK"
298 response_type = "text/html; charset=UTF-8"
299 except:
300 response_body = traceback.format_exc()
302 response_headers = [
303 ("Content-Type", response_type),
304 ("Content-Length", str(len(response_body))),
305 ]
306 start_response(response_code, response_headers)
308 return [bytes(response_body, encoding="utf-8")]
310 def call(self):
311 return main(self._iq, self._oq, self._main_lock)