Mercurial > hg > index.fcgi > www > www-1

changeset 18:3551de5fef13

find changesets by author, revision, files, or words in the commit message
update FEEDS; parallelize feed processing
author paulo@thepaulopc
date Wed, 10 Nov 2010 00:05:53 -0800
parents 5a0c2ee709f8
children fce61e65e3ae
files myrss/FEEDS myrss/myrss_update.sh
diffstat 2 files changed, 22 insertions(+), 4 deletions(-) [+]
line diff
     1.1 --- a/myrss/FEEDS	Wed Sep 29 22:04:46 2010 -0700
     1.2 +++ b/myrss/FEEDS	Wed Nov 10 00:05:53 2010 -0800
     1.3 @@ -19,9 +19,12 @@
     1.4  http://feeds.gawker.com/jalopnik/full
     1.5  http://feeds.gawker.com/gawker/full
     1.6  http://www.xkcd.com/rss.xml
     1.7 +http://thebigcaption.com/rss
     1.8  http://feeds.kottke.org/main
     1.9  http://feeds.boingboing.net/boingboing/iBag
    1.10  http://feeds.feedburner.com/codinghorror
    1.11 +http://syndication.thedailywtf.com/TheDailyWtf
    1.12  http://www.rottentomatoes.com/syndication/rss/in_theaters.xml
    1.13  http://sandiego.craigslist.org/search/cta?query=scion+xb&catAbb=cto&srchType=T&minAsk=&maxAsk=&format=rss
    1.14  http://sandiego.craigslist.org/search/cta?query=scion+xb&catAbb=cta&srchType=T&minAsk=&maxAsk=&format=rss
    1.15 +http://sandiego.craigslist.org/search/?areaID=8&subAreaID=&query=unicycle&catAbb=sss&format=rss

     2.1 --- a/myrss/myrss_update.sh	Wed Sep 29 22:04:46 2010 -0700
     2.2 +++ b/myrss/myrss_update.sh	Wed Nov 10 00:05:53 2010 -0800
     2.3 @@ -1,6 +1,22 @@
     2.4  #!/bin/sh
     2.5  
     2.6  TIMESTAMP=$(date '+%FT%T%:z')
     2.7 +TIMEOUT=60
     2.8 +
     2.9 +i=0
    2.10 +for URL in $(grep -v '^#' FEEDS)
    2.11 +do
    2.12 +	FN=$((i++)).feedtmp
    2.13 +	wget -q -T ${TIMEOUT} -U '' -O - ${URL} | xsltproc step1.xsl - | xsltproc step2.xsl - > ${FN} &
    2.14 +done 
    2.15 +
    2.16 +wait
    2.17 +
    2.18 +BODY=''
    2.19 +for j in $(seq 0 ${i})
    2.20 +do
    2.21 +	BODY=${BODY}$(cat ${j}.feedtmp)
    2.22 +done
    2.23  
    2.24  echo '<HTML>'
    2.25  echo '<HEAD>'
    2.26 @@ -11,10 +27,9 @@
    2.27  echo '<BODY>'
    2.28  echo "<H1>${TIMESTAMP}</H1>"
    2.29  
    2.30 -for URL in $(grep -v '^#' FEEDS)
    2.31 -do
    2.32 -	wget -q -U '' -O - ${URL} | xsltproc step1.xsl - | xsltproc step2.xsl - 
    2.33 -done 
    2.34 +echo ${BODY}
    2.35  
    2.36  echo '</BODY>'
    2.37  echo '</HTML>'
    2.38 +
    2.39 +rm *.feedtmp