home *** CD-ROM | disk | FTP | other *** search
/ Chip 2011 November / CHIP_2011_11.iso / Programy / Narzedzia / Calibre / calibre-0.8.18.msi / file_280 / modoros.recipe < prev    next >
Text File  |  2011-09-09  |  4KB  |  90 lines
  1. import re
  2. from calibre.web.feeds.recipes import BasicNewsRecipe
  3. from calibre.constants import config_dir, CONFIG_DIR_MODE
  4. import os, os.path, urllib
  5. from hashlib import md5
  6.  
  7. class ModorosBlogHu(BasicNewsRecipe):
  8.     __author__              = 'Zsolt Botykai'
  9.     title                   = u'Modoros Blog'
  10.     description             = u"Modoros.blog.hu"
  11.     oldest_article          = 10000
  12.     max_articles_per_feed   = 10000
  13.     reverse_article_order   = True
  14.     language                = 'hu'
  15.     remove_javascript       = True
  16.     remove_empty_feeds      = True
  17.     no_stylesheets          = True
  18.     feeds                   = [(u'Modoros Blog', u'http://modoros.blog.hu/rss')]
  19.     remove_javascript       = True
  20.     use_embedded_content    = False
  21.     preprocess_regexps      = [
  22.         (re.compile(r'<!--megosztas -->.*?</body>', re.DOTALL|re.IGNORECASE),
  23.          lambda match: '</body>'),
  24.         (re.compile(r'<p align="left"'), lambda m: '<p'),
  25.         (re.compile(r'<noscript.+?noscript>', re.DOTALL|re.IGNORECASE), lambda m: ''),
  26.         (re.compile(r'<img style="position: absolute;top:-10px.+?>', re.DOTALL|re.IGNORECASE), lambda m: ''),
  27.         (re.compile(r'<p>( | )*?</p>', re.DOTALL|re.IGNORECASE), lambda match: ''),
  28.     ]
  29.     extra_css = '''
  30.                     body { background-color: white; color: black }
  31.                 '''
  32.  
  33.  
  34.     remove_tags = [
  35.                        dict(name='div', attrs={'id':['csucs']}) ,
  36.                        dict(name='img', attrs={'style':['position: absolute;top:-10px;left:-10px;']}) ,
  37.                        dict(name='div', attrs={'class':['tovabb-is-van', \
  38.                                                         'page-break', \
  39.                                                         'clear']}) ,
  40.                        dict(name='span', attrs={'class':['hozzaszolas-szamlalo']})
  41.                   ]
  42.  
  43.     masthead_url='http://modoros.blog.hu/media/skins/modoros-neon/img/modorosblog-felirat.png'
  44.  
  45.     def get_cover_url(self):
  46.         return 'http://modoros.blog.hu/media/skins/modoros-neon/img/modorosblog-felirat.png'
  47.  
  48.     # As seen here: http://www.mobileread.com/forums/showpost.php?p=1295505&postcount=10
  49.     def parse_feeds(self):
  50.         recipe_dir = os.path.join(config_dir,'recipes')
  51.         hash_dir = os.path.join(recipe_dir,'recipe_storage')
  52.         feed_dir = os.path.join(hash_dir,self.title.encode('utf-8').replace('/',':'))
  53.         if not os.path.isdir(feed_dir):
  54.             os.makedirs(feed_dir,mode=CONFIG_DIR_MODE)
  55.  
  56.         feeds = BasicNewsRecipe.parse_feeds(self)
  57.  
  58.         for feed in feeds:
  59.             feed_hash = urllib.quote(feed.title.encode('utf-8'),safe='')
  60.             feed_fn = os.path.join(feed_dir,feed_hash)
  61.  
  62.             past_items = set()
  63.             if os.path.exists(feed_fn):
  64.                with file(feed_fn) as f:
  65.                    for h in f:
  66.                        past_items.add(h.strip())
  67.  
  68.             cur_items = set()
  69.             for article in feed.articles[:]:
  70.                 item_hash = md5()
  71.                 if article.content: item_hash.update(article.content.encode('utf-8'))
  72.                 if article.summary: item_hash.update(article.summary.encode('utf-8'))
  73.                 item_hash = item_hash.hexdigest()
  74.                 if article.url:
  75.                     item_hash = article.url + ':' + item_hash
  76.                 cur_items.add(item_hash)
  77.                 if item_hash in past_items:
  78.                     feed.articles.remove(article)
  79.             with file(feed_fn,'w') as f:
  80.                 for h in cur_items:
  81.                     f.write(h+'\n')
  82.  
  83.         remove = [f for f in feeds if len(f) == 0 and
  84.                 self.remove_empty_feeds]
  85.         for f in remove:
  86.             feeds.remove(f)
  87.  
  88.         return feeds
  89.  
  90.