home *** CD-ROM | disk | FTP | other *** search
/ Chip 2011 November / CHIP_2011_11.iso / Programy / Narzedzia / Calibre / calibre-0.8.18.msi / file_280 / sunday_times.recipe < prev    next >
Text File  |  2011-09-09  |  6KB  |  116 lines
  1.  
  2. __license__   = 'GPL v3'
  3. __copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
  4. '''
  5. www.thesundaytimes.co.uk
  6. '''
  7. import urllib
  8. from calibre.web.feeds.news import BasicNewsRecipe
  9.  
  10. class TimesOnline(BasicNewsRecipe):
  11.     title                 = 'The Sunday Times UK'
  12.     __author__            = 'Darko Miletic'
  13.     description           = 'news from United Kingdom and World'
  14.     language              = 'en_GB'
  15.     publisher             = 'Times Newspapers Ltd'
  16.     category              = 'news, politics, UK'
  17.     oldest_article        = 3
  18.     max_articles_per_feed = 100
  19.     no_stylesheets        = True
  20.     use_embedded_content  = False
  21.     encoding              = 'utf-8'
  22.     delay                 = 1
  23.     needs_subscription    = True
  24.     publication_type      = 'newspaper'
  25.     masthead_url          = 'http://www.thesundaytimes.co.uk/sto/public/images/logos/logo-home.gif'
  26.     INDEX                 = 'http://www.thesundaytimes.co.uk'
  27.     PREFIX                = u'http://www.thesundaytimes.co.uk/sto/'
  28.     extra_css             = """
  29.                                 .author-name,.authorName{font-style: italic}
  30.                                 .published-date,.multi-position-photo-text{font-family: Arial,Helvetica,sans-serif;
  31.                                                                            font-size: small; color: gray;
  32.                                                                            display:block; margin-bottom: 0.5em}
  33.                                 body{font-family: Georgia,"Times New Roman",Times,serif}
  34.                             """
  35.  
  36.     conversion_options = {
  37.                           'comment'    : description
  38.                         , 'tags'       : category
  39.                         , 'publisher'  : publisher
  40.                         , 'language'   : language
  41.                         }
  42.  
  43.  
  44.     def get_browser(self):
  45.         br = BasicNewsRecipe.get_browser()
  46.         br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thesundaytimes.co.uk/sto/')
  47.         if self.username is not None and self.password is not None:
  48.             data = urllib.urlencode({ 'userName':self.username
  49.                                      ,'password':self.password
  50.                                      ,'keepMeLoggedIn':'false'
  51.                                    })
  52.             br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
  53.         return br
  54.  
  55.     remove_tags      = [
  56.                            dict(name=['object','link','iframe','base','meta'])
  57.                           ,dict(attrs={'class':'tools comments-parent' })
  58.                         ]
  59.     remove_attributes=['lang']
  60.     keep_only_tags   = [
  61.                           dict(attrs={'class':'standard-content'})
  62.                          ,dict(attrs={'class':'f-author'})
  63.                          ,dict(attrs={'id':'bodycopy'})
  64.                        ]
  65.     remove_tags_after=dict(attrs={'class':'tools_border'})
  66.  
  67.     feeds = [
  68.                 (u'UK News'     , PREFIX + u'news/uk_news/'        )
  69.                ,(u'World'       , PREFIX + u'news/world_news/'     )
  70.                ,(u'Politics'    , PREFIX + u'news/Politics/'       )
  71.                ,(u'Focus'       , PREFIX + u'news/focus/'          )
  72.                ,(u'Insight'     , PREFIX + u'news/insight/'        )
  73.                ,(u'Ireland'     , PREFIX + u'news/ireland/'        )
  74.                ,(u'Columns'     , PREFIX + u'comment/columns/'     )
  75.                ,(u'Arts'        , PREFIX + u'culture/arts/'        )
  76.                ,(u'Books'       , PREFIX + u'culture/books/'       )
  77.                ,(u'Film and TV' , PREFIX + u'culture/film_and_tv/' )
  78.                ,(u'Sport'       , PREFIX + u'sport/'               )
  79.                ,(u'Business'    , PREFIX + u'business'             )
  80.                ,(u'Money'       , PREFIX + u'business/money/'      )
  81.                ,(u'Style'       , PREFIX + u'style/'               )
  82.                ,(u'Travel'      , PREFIX + u'travel/'              )
  83.                ,(u'Clarkson'    , PREFIX + u'ingear/clarkson/'     )
  84.                ,(u'Cars'        , PREFIX + u'ingear/cars/'         )
  85.                ,(u'Bikes'       , PREFIX + u'ingear/2_Wheels/'     )
  86.                ,(u'Tech'        , PREFIX + u'ingear/Tech___Games/' )
  87.                ,(u'Magazine'    , PREFIX + u'Magazine/'            )
  88.             ]
  89.  
  90.     def preprocess_html(self, soup):
  91.         for item in soup.findAll(style=True):
  92.             del item['style']
  93.         return self.adeify_images(soup)
  94.  
  95.     def parse_index(self):
  96.         totalfeeds = []
  97.         lfeeds = self.get_feeds()
  98.         for feedobj in lfeeds:
  99.             feedtitle, feedurl = feedobj
  100.             self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
  101.             articles = []
  102.             soup = self.index_to_soup(feedurl)
  103.             for atag in soup.findAll('a',href=True):
  104.                 parentName = atag.parent.name
  105.                 title      = self.tag_to_string(atag).strip()
  106.                 if (parentName == 'h2' or parentName == 'h3') and title is not None and title != '':
  107.                     url           = self.INDEX + atag['href']
  108.                     articles.append({
  109.                                           'title'      :title
  110.                                          ,'date'       :''
  111.                                          ,'url'        :url
  112.                                          ,'description':''
  113.                                         })
  114.             totalfeeds.append((feedtitle, articles))
  115.         return totalfeeds
  116.