Mercurial > hg > index.fcgi > www > www-1

annotate myrss/myrss_app.py @ 68:66a232bae83c

find changesets by author, revision, files, or words in the commit message
myrss: add commented out debug logging code
author paulo
date Thu, 11 Jun 2015 21:25:03 -0700
parents 315afeb47e52
children ae0f2f438a95
rev   line source
paulo@39 1 import os
paulo@40 2 import sys
paulo@39 3 import re
paulo@40 4 import urllib2
paulo@40 5 import threading
paulo@40 6 import Queue
paulo@41 7 import datetime
paulo@41 8 import time
paulo@47 9
paulo@42 10 import logging
paulo@68 11 logging.basicConfig(
paulo@68 12 level=logging.INFO,
paulo@68 13 #filename="_LOG",
paulo@68 14 #format="%(asctime)s %(levelname)-8s %(message)s",
paulo@68 15 )
paulo@39 16
paulo@47 17 import xml.etree.ElementTree
paulo@47 18 import HTMLParser
paulo@47 19
paulo@39 20 import html
paulo@39 21
paulo@39 22
paulo@41 23 FEEDS_FILE = "FEEDS"
paulo@41 24 CACHE_HTML_FILE = "__cache__.html"
paulo@41 25
paulo@44 26 CACHE_LIFE = 1200 # [seconds]
paulo@47 27 MAX_ITEMS = 50
paulo@39 28 MAX_LINK_Z = 4
paulo@40 29 MAX_THREADS = 20
paulo@46 30 URLOPEN_TIMEOUT = 60 # [seconds]
paulo@39 31
paulo@39 32
paulo@39 33 _PARSE_ROOT_TAG_RE = re.compile(r"(\{(.+)\})?(.+)")
paulo@39 34
paulo@39 35 def _parse_root_tag(root_tag):
paulo@39 36 re_match = _PARSE_ROOT_TAG_RE.match(root_tag)
paulo@39 37
paulo@39 38 if re_match is None:
paulo@39 39 return (None, None)
paulo@39 40 else:
paulo@39 41 return re_match.group(2, 3)
paulo@39 42
paulo@39 43
paulo@47 44 def _strip_if_not_none(txt):
paulo@47 45 return txt.strip() if txt is not None else ''
paulo@47 46
paulo@47 47
paulo@39 48 def _go_rss(elementTree):
paulo@47 49 title = _strip_if_not_none(elementTree.find("channel/title").text)
paulo@39 50 link = elementTree.find("channel/link").text
paulo@39 51
paulo@39 52 items = []
paulo@39 53
paulo@39 54 for i in elementTree.findall("channel/item")[:MAX_ITEMS]:
paulo@47 55 it_title = _strip_if_not_none(i.find("title").text)
paulo@39 56 it_link = i.find("link").text
paulo@39 57
paulo@39 58 items.append((it_title, it_link))
paulo@39 59
paulo@39 60 return (title, link, items)
paulo@39 61
paulo@39 62
paulo@39 63 def _go_atom(elementTree):
paulo@39 64 ns = "http://www.w3.org/2005/Atom"
paulo@39 65
paulo@47 66 title = _strip_if_not_none(elementTree.find("{%s}title" % ns).text)
paulo@39 67 link = ''
paulo@39 68
paulo@39 69 for i in elementTree.findall("{%s}link" % ns):
paulo@39 70 if i.get("type") == "text/html" and i.get("rel") == "alternate":
paulo@39 71 link = i.get("href")
paulo@39 72 break
paulo@39 73
paulo@39 74 items = []
paulo@39 75
paulo@39 76 for i in elementTree.findall("{%s}entry" % ns)[:MAX_ITEMS]:
paulo@47 77 it_title = _strip_if_not_none(i.find("{%s}title" % ns).text)
paulo@39 78 it_link = ''
paulo@39 79
paulo@39 80 for j in i.findall("{%s}link" % ns):
paulo@39 81 if j.get("type") == "text/html" and j.get("rel") == "alternate":
paulo@39 82 it_link = j.get("href")
paulo@39 83 break
paulo@39 84
paulo@39 85 items.append((it_title, it_link))
paulo@39 86
paulo@39 87 return (title, link, items)
paulo@39 88
paulo@39 89
paulo@47 90 _STRIP_HTML_RE = re.compile(r"<.*?>")
paulo@47 91 _htmlParser = HTMLParser.HTMLParser()
paulo@47 92
paulo@47 93 def _strip_html(txt):
paulo@47 94 return _htmlParser.unescape(_STRIP_HTML_RE.sub('', txt))
paulo@47 95
paulo@47 96
paulo@41 97 def _to_html(dtnow, docstruct):
paulo@41 98 datetime_str = dtnow.strftime("%Y-%m-%d %H:%M %Z")
paulo@41 99 page_title = "myrss -- %s" % datetime_str
paulo@41 100
paulo@42 101 root = html.HTML("html")
paulo@39 102
paulo@39 103 header = root.header
paulo@41 104 header.title(page_title)
paulo@39 105 header.link(rel="stylesheet", type="text/css", href="index.css")
paulo@39 106
paulo@41 107 body = root.body
paulo@41 108 body.h1(page_title)
paulo@41 109
paulo@39 110 link_z = 0
paulo@39 111
paulo@39 112 for feed in docstruct:
paulo@40 113 if feed is None:
paulo@40 114 continue
paulo@40 115
paulo@39 116 (title, link, items) = feed
paulo@39 117
paulo@47 118 body.h2.a(_strip_html(title), href=link, klass="z%d" % (link_z % MAX_LINK_Z))
paulo@39 119 link_z += 1
paulo@41 120 p = body.p
paulo@39 121
paulo@39 122 for (i, (it_title, it_link)) in enumerate(items):
paulo@39 123 if i > 0:
paulo@39 124 p += " - "
paulo@39 125
paulo@47 126 p.a(_strip_html(it_title), href=it_link, klass="z%d" % (link_z % MAX_LINK_Z))
paulo@39 127 link_z += 1
paulo@39 128
paulo@46 129 dtdelta = datetime.datetime.now() - dtnow
paulo@46 130 root.div("%.3f" % (dtdelta.days*86400 + dtdelta.seconds + dtdelta.microseconds/1e6), klass="debug")
paulo@46 131
paulo@39 132 return unicode(root).encode("utf-8")
paulo@39 133
paulo@39 134
paulo@47 135 def _fetch_url(url):
paulo@40 136 try:
paulo@42 137 logging.info("processing %s" % url)
paulo@46 138 feed = urllib2.urlopen(urllib2.Request(url, headers={"User-Agent": ''}), timeout=URLOPEN_TIMEOUT)
paulo@40 139 except urllib2.HTTPError as e:
paulo@42 140 logging.info("(%s) %s" % (url, e))
paulo@47 141 return None
paulo@47 142
paulo@47 143 return feed
paulo@47 144
paulo@47 145
paulo@47 146 def _process_feed(feed):
paulo@47 147 ret = None
paulo@40 148
paulo@40 149 elementTree = xml.etree.ElementTree.parse(feed)
paulo@40 150 root = elementTree.getroot()
paulo@40 151
paulo@40 152 parsed_root_tag = _parse_root_tag(root.tag)
paulo@40 153
paulo@40 154 if parsed_root_tag == (None, "rss"):
paulo@40 155 version = float(root.get("version", 0.0))
paulo@40 156 if version >= 2.0:
paulo@40 157 ret = _go_rss(elementTree)
paulo@40 158 else:
paulo@40 159 raise NotImplementedError("Unsupported rss version")
paulo@40 160 elif parsed_root_tag == ("http://www.w3.org/2005/Atom", "feed"):
paulo@40 161 ret = _go_atom(elementTree)
paulo@40 162 else:
paulo@40 163 raise NotImplementedError("Unknown root tag")
paulo@40 164
paulo@40 165 return ret
paulo@40 166
paulo@40 167
paulo@40 168 class WorkerThread(threading.Thread):
paulo@40 169 def __init__(self, *args, **kwargs):
paulo@40 170 self._input_queue = kwargs.pop("input_queue")
paulo@40 171 self._output_queue = kwargs.pop("output_queue")
paulo@40 172 threading.Thread.__init__(self, *args, **kwargs)
paulo@40 173 self.daemon = True
paulo@40 174
paulo@40 175 def run(self):
paulo@40 176 while True:
paulo@40 177 (idx, url) = self._input_queue.get()
paulo@40 178 docfeed = None
paulo@40 179 try:
paulo@47 180 feed = _fetch_url(url)
paulo@47 181 if feed is not None:
paulo@47 182 docfeed = _process_feed(feed)
paulo@40 183 except Exception as e:
paulo@42 184 logging.info("(%s) exception: %s" % (url, e))
paulo@40 185 self._output_queue.put((idx, docfeed))
paulo@40 186
paulo@40 187
paulo@44 188 def main(input_queue, output_queue, lock):
paulo@41 189 ret = ''
paulo@41 190
paulo@44 191 with lock:
paulo@44 192 epoch_now = time.time()
paulo@44 193 dtnow = datetime.datetime.fromtimestamp(epoch_now)
paulo@41 194
paulo@44 195 if os.path.exists(CACHE_HTML_FILE) and (epoch_now - os.stat(CACHE_HTML_FILE).st_mtime) < float(CACHE_LIFE):
paulo@44 196 with open(CACHE_HTML_FILE) as cache_html_file:
paulo@44 197 ret = cache_html_file.read()
paulo@41 198
paulo@44 199 else:
paulo@44 200 with open(FEEDS_FILE) as feeds_file:
paulo@44 201 feedlines = feeds_file.readlines()
paulo@41 202
paulo@44 203 docstruct = [None]*len(feedlines)
paulo@44 204 num_input = 0
paulo@44 205 for (i, l) in enumerate(feedlines):
paulo@44 206 if l[0] != '#':
paulo@44 207 l = l.strip()
paulo@44 208 input_queue.put((i, l))
paulo@44 209 num_input += 1
paulo@41 210
paulo@44 211 for _ in range(num_input):
paulo@44 212 (idx, docfeed) = output_queue.get()
paulo@44 213 docstruct[idx] = docfeed
paulo@41 214
paulo@44 215 ret = _to_html(dtnow, docstruct)
paulo@41 216
paulo@44 217 with open(CACHE_HTML_FILE, 'w') as cache_html_file:
paulo@44 218 cache_html_file.write(ret)
paulo@41 219
paulo@41 220 return ret
paulo@41 221
paulo@41 222
paulo@42 223 class MyRssApp:
paulo@42 224 def __init__(self):
paulo@42 225 self._iq = Queue.Queue(MAX_THREADS)
paulo@42 226 self._oq = Queue.Queue(MAX_THREADS)
paulo@44 227 self._main_lock = threading.Lock()
paulo@39 228
paulo@42 229 for _ in range(MAX_THREADS):
paulo@42 230 WorkerThread(input_queue=self._iq, output_queue=self._oq).start()
paulo@42 231
paulo@42 232 def __call__(self, environ, start_response):
paulo@44 233 response_body = main(self._iq, self._oq, self._main_lock)
paulo@42 234 response_headers = [
paulo@47 235 ("Content-Type", "text/html; charset=UTF-8"),
paulo@42 236 ("Content-Length", str(len(response_body))),
paulo@42 237 ]
paulo@42 238 start_response("200 OK", response_headers)
paulo@42 239
paulo@42 240 return [response_body]