Chip 2011 November/CHIP_2011_11.iso/Programy/Narzedzia/Calibre/calibre-0.8.18.msi/file_262/robotparser.pyo

Python Compiled Bytecode | 2011-09-09 | 5.8 KB | 206 lines

# Source Generated with Decompyle++
# File: in.pyo (Python 2.7)

import urlparse
import urllib
__all__ = [
    'RobotFileParser']

class RobotFileParser:

    def __init__(self, url = ''):
        self.entries = []
        self.default_entry = None
        self.disallow_all = False
        self.allow_all = False
        self.set_url(url)
        self.last_checked = 0


    def mtime(self):
        return self.last_checked


    def modified(self):
        import time
        self.last_checked = time.time()


    def set_url(self, url):
        self.url = url
        (self.host, self.path) = urlparse.urlparse(url)[1:3]


    def read(self):
        opener = URLopener()
        f = opener.open(self.url)
        lines = [ line.strip() for line in f ]
        f.close()
        self.errcode = opener.errcode
        if self.errcode in (401, 403):
            self.disallow_all = True
        elif self.errcode >= 400:
            self.allow_all = True
        elif self.errcode == 200 and lines:
            self.parse(lines)


    def _add_entry(self, entry):
        if '*' in entry.useragents or self.default_entry is None:
            self.default_entry = entry

        self.entries.append(entry)


    def parse(self, lines):
        state = 0
        linenumber = 0
        entry = Entry()
        for line in lines:
            linenumber += 1
            if not line:
                if state == 1:
                    entry = Entry()
                    state = 0
                elif state == 2:
                    self._add_entry(entry)
                    entry = Entry()
                    state = 0

            i = line.find('#')
            if i >= 0:
                line = line[:i]
            line = line.strip()
            if not line:
                continue
            line = line.split(':', 1)
            if len(line) == 2:
                line[0] = line[0].strip().lower()
                line[1] = urllib.unquote(line[1].strip())
                if line[0] == 'user-agent':
                    if state == 2:
                        self._add_entry(entry)
                        entry = Entry()
                    entry.useragents.append(line[1])
                    state = 1
                elif line[0] == 'disallow' or state != 0:
                    entry.rulelines.append(RuleLine(line[1], False))
                    state = 2

            elif line[0] == 'allow':
                if state != 0:
                    entry.rulelines.append(RuleLine(line[1], True))
                    state = 2



        if state == 2:
            self._add_entry(entry)


    def can_fetch(self, useragent, url):
        if self.disallow_all:
            return False
        if None.allow_all:
            return True
        parsed_url = None.urlparse(urllib.unquote(url))
        url = urlparse.urlunparse(('', '', parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment))
        url = urllib.quote(url)
        if not url:
            url = '/'
        for entry in self.entries:
            if entry.applies_to(useragent):
                return entry.allowance(url)

        if self.default_entry:
            return self.default_entry.allowance(url)


    def __str__(self):
        return ''.join([ str(entry) + '\n' for entry in self.entries ])



class RuleLine:

    def __init__(self, path, allowance):
        if path == '' and not allowance:
            allowance = True
        self.path = urllib.quote(path)
        self.allowance = allowance


    def applies_to(self, filename):
        if not self.path == '*':
            pass
        return filename.startswith(self.path)


    def __str__(self):
        if not self.allowance or 'Allow':
            pass
        return 'Disallow' + ': ' + self.path



class Entry:

    def __init__(self):
        self.useragents = []
        self.rulelines = []


    def __str__(self):
        ret = []
        for agent in self.useragents:
            ret.extend([
                'User-agent: ',
                agent,
                '\n'])

        for line in self.rulelines:
            ret.extend([
                str(line),
                '\n'])

        return ''.join(ret)


    def applies_to(self, useragent):
        useragent = useragent.split('/')[0].lower()
        for agent in self.useragents:
            if agent == '*':
                return True
            agent = None.lower()
            if agent in useragent:
                return True

        return False


    def allowance(self, filename):
        for line in self.rulelines:
            if line.applies_to(filename):
                return line.allowance

        return True



class URLopener(urllib.FancyURLopener):

    def __init__(self, *args):
        urllib.FancyURLopener.__init__(self, *args)
        self.errcode = 200


    def prompt_user_passwd(self, host, realm):
        return (None, None)


    def http_error_default(self, url, fp, errcode, errmsg, headers):
        self.errcode = errcode
        return urllib.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers)