home *** CD-ROM | disk | FTP | other *** search
/ Chip 2011 November / CHIP_2011_11.iso / Programy / Narzedzia / Calibre / calibre-0.8.18.msi / file_280 / nikkei_news.recipe < prev    next >
Text File  |  2011-09-09  |  4KB  |  94 lines
  1. from calibre.web.feeds.recipes import BasicNewsRecipe
  2. import re
  3.  
  4. #import pprint, sys
  5. #pp = pprint.PrettyPrinter(indent=4)
  6.  
  7. class NikkeiNet_paper_subscription(BasicNewsRecipe):
  8.     title           = u'\u65E5\u672C\u7D4C\u6E08\u65B0\u805E\uFF08\u671D\u520A\u30FB\u5915\u520A\uFF09'
  9.     __author__      = 'Ado Nishimura'
  10.     description     = u'\u65E5\u7D4C\u96FB\u5B50\u7248\u306B\u3088\u308B\u65E5\u672C\u7D4C\u6E08\u65B0\u805E\u3002\u671D\u520A\u30FB\u5915\u520A\u306F\u53D6\u5F97\u6642\u9593\u306B\u3088\u308A\u5207\u308A\u66FF\u308F\u308A\u307E\u3059\u3002\u8981\u8CFC\u8AAD'
  11.     needs_subscription = True
  12.     oldest_article  = 1
  13.     max_articles_per_feed = 30
  14.     language        = 'ja'
  15.     no_stylesheets  = True
  16.     cover_url       = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg'
  17.     masthead_url    = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg'
  18.  
  19.     remove_tags_before = {'class':"cmn-indent"}
  20.     remove_tags = [
  21. #                       {'class':"cmn-article_move"},
  22. #                       {'class':"cmn-pr_list"},
  23. #                       {'class':"cmnc-zoom"},
  24.                        {'class':"cmn-hide"},
  25.                        {'name':'form'},
  26.                   ]
  27.     remove_tags_after = {'class':"cmn-indent"}
  28.  
  29.     def get_browser(self):
  30.         br = BasicNewsRecipe.get_browser()
  31.  
  32.         #pp.pprint(self.parse_index())
  33.         #exit(1)
  34.  
  35.         #br.set_debug_http(True)
  36.         #br.set_debug_redirects(True)
  37.         #br.set_debug_responses(True)
  38.  
  39.         if self.username is not None and self.password is not None:
  40.             print "-------------------------open top page-------------------------------------"
  41.             br.open('http://www.nikkei.com/')
  42.             print "-------------------------open first login form-----------------------------"
  43.             link = br.links(url_regex="www.nikkei.com/etc/accounts/login").next()
  44.             br.follow_link(link)
  45.             #response = br.response()
  46.             #print response.get_data()
  47.             print "-------------------------JS redirect(send autoPostForm)--------------------"
  48.             br.select_form(name='autoPostForm')
  49.             br.submit()
  50.             #response = br.response()
  51.             print "-------------------------got login form------------------------------------"
  52.             br.select_form(name='LA0210Form01')
  53.             br['LA0210Form01:LA0210Email']    = self.username
  54.             br['LA0210Form01:LA0210Password'] = self.password
  55.             br.submit()
  56.             #response = br.response()
  57.             print "-------------------------JS redirect---------------------------------------"
  58.             br.select_form(nr=0)
  59.             br.submit()
  60.  
  61.             #br.set_debug_http(False)
  62.             #br.set_debug_redirects(False)
  63.             #br.set_debug_responses(False)
  64.         return br
  65.  
  66.     def cleanup(self):
  67.         print "-------------------------logout--------------------------------------------"
  68.         self.browser.open('https://regist.nikkei.com/ds/etc/accounts/logout')
  69.  
  70.     def parse_index(self):
  71.         print "-------------------------get index of paper--------------------------------"
  72.         result = []
  73.         soup = self.index_to_soup('http://www.nikkei.com/paper/')
  74.         #soup = self.index_to_soup(self.test_data())
  75.         sections = soup.findAll('div', 'cmn-section kn-special JSID_baseSection')
  76.     if len(sections) == 0:
  77.             sections = soup.findAll('div', 'cmn-section kn-special')
  78.         for sect in sections:
  79.             sect_title = sect.find('h3', 'cmnc-title').string
  80.             sect_result = []
  81.             for elem in sect.findAll(attrs={'class':['cmn-article_title']}):
  82.                 if elem.span.a == None  or  elem.span.a['href'].startswith('javascript') :
  83.                     continue
  84.                 url = 'http://www.nikkei.com' + elem.span.a['href']
  85.                 url = re.sub("/article/", "/print-article/", url) # print version.
  86.                 span = elem.span.a.span
  87.                 if ((span is not None) and (len(span.contents) > 1)):
  88.                     title = span.contents[1].string
  89.                     sect_result.append(dict(title=title, url=url, date='',
  90.                                             description='', content=''))
  91.             result.append([sect_title, sect_result])
  92.         return result
  93.  
  94.