home *** CD-ROM | disk | FTP | other *** search
/ Chip 2011 November / CHIP_2011_11.iso / Programy / Narzedzia / Calibre / calibre-0.8.18.msi / file_280 / new_scientist.recipe < prev    next >
Text File  |  2011-09-09  |  5KB  |  104 lines
  1. __license__   = 'GPL v3'
  2. __copyright__ = '2008-2010, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
  3. '''
  4. newscientist.com
  5. '''
  6.  
  7. import re
  8. import urllib
  9. from calibre.web.feeds.news import BasicNewsRecipe
  10.  
  11. class NewScientist(BasicNewsRecipe):
  12.     title                 = 'New Scientist - Online News w. subscription'
  13.     __author__            = 'Darko Miletic'
  14.     description           = 'Science news and science articles from New Scientist.'
  15.     language              = 'en'
  16.     publisher             = 'Reed Business Information Ltd.'
  17.     category              = 'science news, science articles, science jobs, drugs, cancer, depression, computer software'
  18.     oldest_article        = 7
  19.     max_articles_per_feed = 100
  20.     no_stylesheets        = True
  21.     use_embedded_content  = False
  22.     cover_url             = 'http://www.newscientist.com/currentcover.jpg'
  23.     masthead_url          = 'http://www.newscientist.com/img/misc/ns_logo.jpg'
  24.     encoding              = 'utf-8'
  25.     needs_subscription    = 'optional'
  26.     extra_css             = """
  27.                                  body{font-family: Arial,sans-serif}
  28.                                  img{margin-bottom: 0.8em; display: block}
  29.                                  .quotebx{font-size: x-large; font-weight: bold; margin-right: 2em; margin-left: 2em}
  30.                             """
  31.  
  32.     conversion_options = {
  33.                           'comment'          : description
  34.                         , 'tags'             : category
  35.                         , 'publisher'        : publisher
  36.                         , 'language'         : language
  37.                         }
  38.     preprocess_regexps = [(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')]
  39.  
  40.     keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
  41.  
  42.     def get_browser(self):
  43.         br = BasicNewsRecipe.get_browser()
  44.         br.open('http://www.newscientist.com/')
  45.         if self.username is not None and self.password is not None:        
  46.             br.open('https://www.newscientist.com/user/login')
  47.             data = urllib.urlencode({ 'source':'form'
  48.                                      ,'redirectURL':''
  49.                                      ,'loginId':self.username
  50.                                      ,'password':self.password
  51.                                    })
  52.             br.open('https://www.newscientist.com/user/login',data)
  53.         return br
  54.  
  55.     remove_tags = [
  56.                      dict(name='div'  , attrs={'class':['hldBd','adline','pnl','infotext' ]})
  57.                     ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools','comments','blgsocial','sharebtns']})
  58.                     ,dict(name='p'    , attrs={'class':['marker','infotext'               ]})
  59.                     ,dict(name='meta' , attrs={'name' :'description'                       })
  60.                     ,dict(name='a'    , attrs={'rel'  :'tag'                               })
  61.                     ,dict(name='ul'   , attrs={'class':'markerlist'                        })
  62.                     ,dict(name=['link','base','meta','iframe','object','embed'])
  63.                   ]
  64.     remove_tags_after = dict(attrs={'class':['nbpcopy','comments']})
  65.     remove_attributes = ['height','width','lang','onclick']
  66.  
  67.     feeds          = [
  68.                         (u'Latest Headlines'        , u'http://feeds.newscientist.com/science-news'       )
  69.                        ,(u'Magazine'                , u'http://feeds.newscientist.com/magazine'           )
  70.                        ,(u'Health'                  , u'http://feeds.newscientist.com/health'             )
  71.                        ,(u'Life'                    , u'http://feeds.newscientist.com/life'               )
  72.                        ,(u'Space'                   , u'http://feeds.newscientist.com/space'              )
  73.                        ,(u'Physics and Mathematics' , u'http://feeds.newscientist.com/physics-math'       )
  74.                        ,(u'Environment'             , u'http://feeds.newscientist.com/environment'        )
  75.                        ,(u'Science in Society'      , u'http://feeds.newscientist.com/science-in-society' )
  76.                        ,(u'Tech'                    , u'http://feeds.newscientist.com/tech'               )
  77.                      ]
  78.  
  79.     def get_article_url(self, article):
  80.         return article.get('guid',  None)
  81.  
  82.     def print_version(self, url):
  83.         return url + '?full=true&print=true'
  84.  
  85.     def preprocess_html(self, soup):
  86.         if soup.html.has_key('id'):
  87.            del soup.html['id']
  88.         for item in soup.findAll(style=True):
  89.             del item['style']
  90.         for item in soup.findAll(['quote','quotetext']):
  91.             item.name='p'
  92.         for item in soup.findAll(['xref','figref']):
  93.             tstr = item.string
  94.             item.replaceWith(tstr)            
  95.         for tg in soup.findAll('a'):
  96.             if tg.string == 'Home':
  97.                 tg.parent.extract()
  98.             else:
  99.                 if tg.string is not None:
  100.                    tstr = tg.string
  101.                    tg.replaceWith(tstr)
  102.         return soup
  103.  
  104.