home *** CD-ROM | disk | FTP | other *** search
/ Personal Computer World 2008 February / PCWFEB08.iso / Software / Freeware / Miro 1.0 / Miro_Installer.exe / xulrunner / python / downloader.py < prev    next >
Encoding:
Python Source  |  2007-11-12  |  22.4 KB  |  644 lines
  1. # Miro - an RSS based video player application
  2. # Copyright (C) 2005-2007 Participatory Culture Foundation
  3. #
  4. # This program is free software; you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation; either version 2 of the License, or
  7. # (at your option) any later version.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12. # GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program; if not, write to the Free Software
  16. # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  17.  
  18. from base64 import b64encode
  19. from gtcache import gettext as _
  20. from threading import RLock
  21. import os
  22. import re
  23. import shutil
  24.  
  25. from database import DDBObject, defaultDatabase
  26. from dl_daemon import daemon, command
  27. from download_utils import nextFreeFilename, getFileURLPath, filterDirectoryName
  28. from util import getTorrentInfoHash, returnsUnicode, checkU, returnsFilename, unicodify, checkF, stringify
  29. from platformutils import FilenameType
  30. import app
  31. import config
  32. import httpclient
  33. import indexes
  34. import prefs
  35. import random
  36. import views
  37. import platformutils
  38. import flashscraper
  39. import logging
  40. import traceback
  41. import templatehelper
  42. import fileutil
  43.  
  44. # a hash of download ids that the server knows about.
  45. _downloads = {}
  46.  
  47. # Returns an HTTP auth object corresponding to the given host, path or
  48. # None if it doesn't exist
  49. def findHTTPAuth(host,path,realm = None,scheme = None):
  50.     checkU(host)
  51.     checkU(path)
  52.     if realm:
  53.         checkU(realm)
  54.     if scheme:
  55.         checkU(scheme)
  56.     #print "Trying to find HTTPAuth with host %s, path %s, realm %s, and scheme %s" %(host,path,realm,scheme)
  57.     defaultDatabase.confirmDBThread()
  58.     for obj in views.httpauths:
  59.         if (obj.host == host and path.startswith(obj.path) and
  60.             (realm is None or obj.realm == realm) and
  61.             (scheme is None or obj.authScheme == scheme)):
  62.             return obj
  63.     return None
  64.  
  65.  
  66. class HTTPAuthPassword(DDBObject):
  67.     def __init__(self,username,password,host, realm, path, authScheme=u"Basic"):
  68.         checkU(username)
  69.         checkU(password)
  70.         checkU(host)
  71.         checkU(realm)
  72.         checkU(path)
  73.         checkU(authScheme)
  74.         oldAuth = findHTTPAuth(host,path,realm,authScheme)
  75.         while not oldAuth is None:
  76.             oldAuth.remove()
  77.             oldAuth = findHTTPAuth(host,path,realm,authScheme)
  78.         self.username = username
  79.         self.password = password
  80.         self.host = host
  81.         self.realm = realm
  82.         self.path = os.path.dirname(path)
  83.         self.authScheme = authScheme
  84.         DDBObject.__init__(self)
  85.  
  86.     def getAuthToken(self):
  87.         authString = u':'
  88.         self.confirmDBThread()
  89.         authString = self.username+u':'+self.password
  90.         return b64encode(authString)
  91.  
  92.     def getAuthScheme(self):
  93.         self.confirmDBThread()
  94.         return self.authScheme
  95.  
  96. totalUpRate = 0
  97. totalDownRate = 0
  98.  
  99. def _getDownloader (dlid):
  100.     return views.remoteDownloads.getItemWithIndex(indexes.downloadsByDLID, dlid)
  101.  
  102. @returnsUnicode
  103. def generateDownloadID():
  104.     dlid = u"download%08d" % random.randint(0,99999999)
  105.     while _getDownloader (dlid=dlid):
  106.         dlid = u"download%08d" % random.randint(0,99999999)
  107.     return dlid
  108.  
  109. class RemoteDownloader(DDBObject):
  110.     """Download a file using the downloader daemon."""
  111.  
  112.     def __init__(self, url, item, contentType = None, channelName = None):
  113.         checkU(url)
  114.         if contentType:
  115.             checkU(contentType)
  116.         self.origURL = self.url = url
  117.         self.itemList = [item]
  118.         self.dlid = generateDownloadID()
  119.         self.status = {}
  120.         if contentType is None:
  121.             # HACK:  Some servers report the wrong content-type for torrent
  122.             # files.  We try to work around that by assuming if the enclosure
  123.             # states that something is a torrent, it's a torrent.
  124.             # Thanks to j@v2v.cc
  125.             enclosureContentType = item.getFirstVideoEnclosureType()
  126.             if enclosureContentType == u'application/x-bittorrent':
  127.                 contentType = enclosureContentType
  128.         self.contentType = u""
  129.         self.deleteFiles = True
  130.         self.channelName = channelName
  131.         self.manualUpload = False
  132.         DDBObject.__init__(self)
  133.         if contentType is None:
  134.             self.contentType = u""
  135.         else:
  136.             self.contentType = contentType
  137.  
  138.         if self.contentType == u'':
  139.             self.getContentType()
  140.         else:
  141.             self.runDownloader()
  142.  
  143.     def signalChange (self, needsSave=True, needsSignalItem=True):
  144.         if needsSignalItem:
  145.             for item in self.itemList:
  146.                 item.signalChange(needsSave=False)
  147.         DDBObject.signalChange (self, needsSave=needsSave)
  148.  
  149.     def onContentType (self, info):
  150.         if not self.idExists():
  151.             return
  152.  
  153.         if info['status'] == 200:
  154.             self.url = info['updated-url'].decode('ascii','replace')
  155.             self.contentType = None
  156.             try:
  157.                 self.contentType = info['content-type'].decode('ascii','replace')
  158.             except:
  159.                 self.contentType = None
  160.             self.runDownloader()
  161.         else:
  162.             error = httpclient.UnexpectedStatusCode(info['status'])
  163.             self.onContentTypeError(error)
  164.  
  165.     def onContentTypeError (self, error):
  166.         if not self.idExists():
  167.             return
  168.  
  169.         self.status['state'] = u"failed"
  170.         self.status['shortReasonFailed'] = error.getFriendlyDescription()
  171.         self.status['reasonFailed'] = error.getLongDescription()
  172.         self.signalChange()
  173.  
  174.     def getContentType(self):
  175.         httpclient.grabHeaders(self.url, self.onContentType, self.onContentTypeError)
  176.  
  177.     @classmethod
  178.     def initializeDaemon(cls):
  179.         RemoteDownloader.dldaemon = daemon.ControllerDaemon()
  180.  
  181.     def _getRates(self):
  182.         state = self.getState()
  183.         if state == u'downloading':
  184.             return (self.status.get('rate', 0), self.status.get('upRate', 0))
  185.         if state == u'uploading':
  186.             return (0, self.status.get('upRate', 0))
  187.         return (0, 0)
  188.  
  189.     @classmethod
  190.     def updateStatus(cls, data):
  191.         for field in data:
  192.             if field not in ['filename','shortFilename','channelName','metainfo','fastResumeData']:
  193.                 data[field] = unicodify(data[field])
  194.         self = _getDownloader (dlid=data['dlid'])
  195.         # print data
  196.         if self is not None:
  197.             try:
  198.                 if self.status == data:
  199.                     return
  200.             except Exception, e:
  201.                 # This is a known bug with the way we used to save fast resume
  202.                 # data
  203.                 print "WARNING exception when comparing status: %s" % e
  204.  
  205.             wasFinished = self.isFinished()
  206.             global totalDownRate
  207.             global totalUpRate
  208.             rates = self._getRates()
  209.             totalDownRate -= rates[0]
  210.             totalUpRate -= rates[1]
  211.  
  212.             # FIXME: how do we get all of the possible bit torrent
  213.             # activity strings into gettext? --NN
  214.             if data.has_key('activity') and data['activity']:
  215.                 data['activity'] = _(data['activity'])
  216.  
  217.             self.status = data
  218.  
  219.             # Store the time the download finished
  220.             finished = self.isFinished() and not wasFinished
  221.             rates = self._getRates()
  222.             totalDownRate += rates[0]
  223.             totalUpRate += rates[1]
  224.  
  225.             if self.getState() == u'uploading' and not self.manualUpload and self.getUploadRatio() > 1.5:
  226.                 self.stopUpload()
  227.  
  228.             self.signalChange(needsSignalItem=not finished)
  229.             if finished:
  230.                 for item in self.itemList:
  231.                     item.onDownloadFinished()
  232.  
  233.     ##
  234.     # This is the actual download thread.
  235.     def runDownloader(self):
  236.         flashscraper.tryScrapingURL(self.url, self._runDownloader)
  237.  
  238.     def _runDownloader(self, url, contentType = None):
  239.         if not self.idExists():
  240.             return # we got deleted while we were doing the flash scraping
  241.         if contentType is not None:
  242.             self.contentType = contentType
  243.         if url is not None:
  244.             self.url = url
  245.             c = command.StartNewDownloadCommand(RemoteDownloader.dldaemon,
  246.                                                 self.url, self.dlid, self.contentType, self.channelName)
  247.             c.send()
  248.             _downloads[self.dlid] = self
  249.         else:
  250.             self.status["state"] = u'failed'
  251.             self.status["shortReasonFailed"] = _('File not found')
  252.             self.status["reasonFailed"] = _('Flash URL Scraping Error')
  253.         self.signalChange()
  254.  
  255.     ##
  256.     # Pauses the download.
  257.     def pause(self, block=False):
  258.         if _downloads.has_key(self.dlid):
  259.             c = command.PauseDownloadCommand(RemoteDownloader.dldaemon,
  260.                                              self.dlid)
  261.             c.send()
  262.         else:
  263.             self.status["state"] = u"paused"
  264.             self.signalChange()
  265.  
  266.     ##
  267.     # Stops the download and removes the partially downloaded
  268.     # file.
  269.     def stop(self, delete):
  270.         if ((self.getState() in [u'downloading',u'uploading', u'paused'])):
  271.             if _downloads.has_key(self.dlid):
  272.                 c = command.StopDownloadCommand(RemoteDownloader.dldaemon,
  273.                                                 self.dlid, delete)
  274.                 c.send()
  275.                 del _downloads[self.dlid]
  276.         else:
  277.             if delete:
  278.                 self.delete()
  279.             self.status["state"] = u"stopped"
  280.             self.signalChange()
  281.  
  282.     def delete(self):
  283.         try:
  284.             filename = self.status['filename']
  285.         except KeyError:
  286.             return
  287.         try:
  288.             fileutil.delete(filename)
  289.         except:
  290.             logging.warn("Error deleting downloaded file: %s\n%s" % 
  291.                     (templatehelper.toUni(stringify(filename)), traceback.format_exc()))
  292.  
  293.         parent = os.path.join(filename, os.path.pardir)
  294.         parent = os.path.normpath(parent)
  295.         moviesDir = config.get(prefs.MOVIES_DIRECTORY)
  296.         if (os.path.exists(parent) and os.path.exists(moviesDir) and
  297.             not platformutils.samefile(parent, moviesDir) and
  298.             len(os.listdir(parent)) == 0):
  299.             try:
  300.                 os.rmdir(parent)
  301.             except:
  302.                 logging.warn("Error deleting empty download directory: %s\n%s" %
  303.                         (templatehelper.toUni(parent), traceback.format_exc()))
  304.  
  305.     ##
  306.     # Continues a paused, stopped, or failed download thread
  307.     def start(self):
  308.         if self.getState() == u'failed':
  309.             if _downloads.has_key (self.dlid):
  310.                 del _downloads[self.dlid]
  311.             self.dlid = generateDownloadID()
  312.             views.remoteDownloads.recomputeIndex(indexes.downloadsByDLID)
  313.             self.status = {}
  314.             if self.contentType == u"":
  315.                 self.getContentType()
  316.             else:
  317.                 self.runDownloader()
  318.             self.signalChange()
  319.         elif self.getState() in (u'stopped', u'paused', u'offline'):
  320.             if _downloads.has_key(self.dlid):
  321.                 c = command.StartDownloadCommand(RemoteDownloader.dldaemon,
  322.                                                  self.dlid)
  323.                 c.send()
  324.             else:
  325.                 self.status['state'] = u'downloading'
  326.                 self.restart()
  327.                 self.signalChange()
  328.  
  329.     def migrate(self, directory):
  330.         if _downloads.has_key(self.dlid):
  331.             c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
  332.                                                self.dlid, directory)
  333.             c.send()
  334.         else:
  335.             # downloader doesn't have our dlid.  Move the file ourself.
  336.             try:
  337.                 shortFilename = self.status['shortFilename']
  338.             except KeyError:
  339.                 print """\
  340. WARNING: can't migrate download because we don't have a shortFilename!
  341. URL was %s""" % self.url
  342.                 return
  343.             try:
  344.                 filename = self.status['filename']
  345.             except KeyError:
  346.                 print """\
  347. WARNING: can't migrate download because we don't have a filename!
  348. URL was %s""" % self.url
  349.                 return
  350.             if os.path.exists(filename):
  351.                 if 'channelName' in self.status and self.status['channelName'] is not None:
  352.                     channelName = filterDirectoryName(self.status['channelName'])
  353.                     directory = os.path.join (directory, channelName)
  354.                 try:
  355.                     os.makedirs(directory)
  356.                 except:
  357.                     pass
  358.                 newfilename = os.path.join(directory, shortFilename)
  359.                 if newfilename == filename:
  360.                     return
  361.                 newfilename = nextFreeFilename(newfilename)
  362.                 def callback():
  363.                     self.status['filename'] = newfilename
  364.                     self.signalChange()
  365.                 fileutil.migrate_file(filename, newfilename, callback)
  366.         for i in self.itemList:
  367.             i.migrateChildren(directory)
  368.  
  369.     def setDeleteFiles(self, deleteFiles):
  370.         self.deleteFiles = deleteFiles
  371.  
  372.     def setChannelName(self, channelName):
  373.         if self.channelName is None:
  374.             if channelName:
  375.                 checkF(channelName)
  376.             self.channelName = channelName
  377.  
  378.     ##
  379.     # Removes downloader from the database and deletes the file.
  380.     def remove(self):
  381.         global totalDownRate
  382.         global totalUpRate
  383.         rates = self._getRates()
  384.         totalDownRate -= rates[0]
  385.         totalUpRate -= rates[1]
  386.         self.stop(self.deleteFiles)
  387.         DDBObject.remove(self)
  388.  
  389.     def getType(self):
  390.         """Get the type of download.  Will return either "http" or
  391.         "bittorrent".
  392.         """
  393.         self.confirmDBThread()
  394.         if self.contentType == u'application/x-bittorrent':
  395.             return u"bittorrent"
  396.         else:
  397.             return u"http"
  398.  
  399.     ##
  400.     # In case multiple downloaders are getting the same file, we can support
  401.     # multiple items
  402.     def addItem(self,item):
  403.         if item not in self.itemList:
  404.             self.itemList.append(item)
  405.  
  406.     def removeItem(self, item):
  407.         self.itemList.remove(item)
  408.         if len (self.itemList) == 0:
  409.             self.remove()
  410.  
  411.     def getRate(self):
  412.         self.confirmDBThread()
  413.         return self.status.get('rate', 0)
  414.  
  415.     def getETA(self):
  416.         self.confirmDBThread()
  417.         return self.status.get('eta', 0)
  418.  
  419.     @returnsUnicode
  420.     def getStartupActivity(self):
  421.         self.confirmDBThread()
  422.         activity = self.status.get('activity')
  423.         if activity is None:
  424.             return _("starting up")
  425.         else:
  426.             return activity
  427.  
  428.     ##
  429.     # Returns the reason for the failure of this download
  430.     # This should only be called when the download is in the failed state
  431.     @returnsUnicode
  432.     def getReasonFailed(self):
  433.         if not self.getState() == u'failed':
  434.             msg = u"getReasonFailed() called on a non-failed downloader"
  435.             raise ValueError(msg)
  436.         self.confirmDBThread()
  437.         return self.status['reasonFailed']
  438.  
  439.     @returnsUnicode
  440.     def getShortReasonFailed(self):
  441.         if not self.getState() == u'failed':
  442.             msg = u"getShortReasonFailed() called on a non-failed downloader"
  443.             raise ValueError(msg)
  444.         self.confirmDBThread()
  445.         return self.status['shortReasonFailed']
  446.     ##
  447.     # Returns the URL we're downloading
  448.     @returnsUnicode
  449.     def getURL(self):
  450.         self.confirmDBThread()
  451.         return self.url
  452.  
  453.     ##    
  454.     # Returns the state of the download: downloading, paused, stopped,
  455.     # failed, or finished
  456.     @returnsUnicode    
  457.     def getState(self):
  458.         self.confirmDBThread()
  459.         return self.status.get('state', u'downloading')
  460.  
  461.     def isFinished(self):
  462.         return self.getState() in (u'finished', u'uploading')
  463.  
  464.     ##
  465.     # Returns the total size of the download in bytes
  466.     def getTotalSize(self):
  467.         self.confirmDBThread()
  468.         return self.status.get(u'totalSize', -1)
  469.  
  470.     ##
  471.     # Returns the current amount downloaded in bytes
  472.     def getCurrentSize(self):
  473.         self.confirmDBThread()
  474.         return self.status.get(u'currentSize', 0)
  475.  
  476.     ##
  477.     # Returns the filename that we're downloading to. Should not be
  478.     # called until state is "finished."
  479.     @returnsFilename
  480.     def getFilename(self):
  481.         self.confirmDBThread()
  482.         return self.status.get('filename', FilenameType(''))
  483.  
  484.     def onRestore(self):
  485.         self.deleteFiles = True
  486.         self.itemList = []
  487.         if self.dlid == 'noid':
  488.             # this won't happen nowadays, but it can for old databases
  489.             self.dlid = generateDownloadID()
  490.         self.status['rate'] = 0
  491.         self.status['upRate'] = 0
  492.         self.status['eta'] = 0
  493.  
  494.     def getUploadRatio(self):
  495.         size = self.getCurrentSize()
  496.         if size == 0:
  497.             return 0
  498.         return self.status.get('uploaded', 0) * 1024 * 1024 / size
  499.     
  500.     def restartIfNeeded(self):
  501.         if self.getState() in (u'downloading',u'offline'):
  502.             self.restart()
  503.         if self.getState() in (u'uploading'):
  504.             if self.manualUpload or self.getUploadRatio() < 1.5:
  505.                 self.restart()
  506.             else:
  507.                 self.stopUpload()
  508.  
  509.     def restart(self):
  510.         if len(self.status) == 0 or self.status.get('dlerType') is None:
  511.             if self.contentType == u"":
  512.                 self.getContentType()
  513.             else:
  514.                 self.runDownloader()
  515.         else:
  516.             _downloads[self.dlid] = self
  517.             c = command.RestoreDownloaderCommand(RemoteDownloader.dldaemon, 
  518.                                                  self.status)
  519.             c.send()
  520.  
  521.     def startUpload(self):
  522.         if self.getState() != u'finished' or self.getType() != u'bittorrent':
  523.             return
  524.         self.manualUpload = True
  525.         if _downloads.has_key(self.dlid):
  526.             c = command.StartDownloadCommand(RemoteDownloader.dldaemon,
  527.                                              self.dlid)
  528.             c.send()
  529.         else:
  530.             self.status['state'] = u'uploading'
  531.             self.restart()
  532.             self.signalChange()
  533.  
  534.     def stopUpload(self):
  535.         if self.getState() != u"uploading":
  536.             return
  537.         if _downloads.has_key(self.dlid):
  538.             c = command.StopUploadCommand(RemoteDownloader.dldaemon,
  539.                                           self.dlid)
  540.             c.send()
  541.             del _downloads[self.dlid]
  542.         self.status["state"] = u"finished"
  543.         self.signalChange()
  544.  
  545. def cleanupIncompleteDownloads():
  546.     downloadDir = os.path.join(config.get(prefs.MOVIES_DIRECTORY),
  547.             'Incomplete Downloads')
  548.     if not os.path.exists(downloadDir):
  549.         return
  550.  
  551.     filesInUse = set()
  552.     views.remoteDownloads.confirmDBThread()
  553.     for downloader in views.remoteDownloads:
  554.         if downloader.getState() in ('downloading', 'paused', 'offline'):
  555.             filename = downloader.getFilename()
  556.             if len(filename) > 0:
  557.                 if not os.path.isabs(filename):
  558.                     filename = os.path.join(downloadDir, filename)
  559.                 filesInUse.add(filename)
  560.  
  561.     for f in os.listdir(downloadDir):
  562.         f = os.path.join(downloadDir, f)
  563.         if f not in filesInUse:
  564.             try:
  565.                 if os.path.isfile(f):
  566.                     os.remove (f)
  567.                 elif os.path.isdir(f):
  568.                     shutil.rmtree (f)
  569.             except:
  570.                 # FIXME - maybe a permissions error?
  571.                 pass
  572.  
  573. def restartDownloads():
  574.     views.remoteDownloads.confirmDBThread()
  575.     for downloader in views.remoteDownloads:
  576.         downloader.restartIfNeeded()
  577.  
  578. def killUploaders(*args):
  579.     torrent_limit = config.get(prefs.UPSTREAM_TORRENT_LIMIT)
  580.     while (views.autoUploads.len() > torrent_limit):
  581.         views.autoUploads[0].stopUpload()
  582.  
  583. def configChangeUploaders(key, value):
  584.     if key == prefs.UPSTREAM_TORRENT_LIMIT.key:
  585.         killUploaders()
  586.  
  587. def limitUploaders():
  588.     views.autoUploads.addAddCallback(killUploaders)
  589.     config.addChangeCallback(configChangeUploaders)
  590.     killUploaders()
  591.         
  592.  
  593. def startupDownloader():
  594.     """Initialize the downloaders.
  595.  
  596.     This method currently does 2 things.  It deletes any stale files self in
  597.     Incomplete Downloads, then it restarts downloads that have been restored
  598.     from the database.  It must be called before any RemoteDownloader objects
  599.     get created.
  600.     """
  601.  
  602.     cleanupIncompleteDownloads()
  603.     RemoteDownloader.initializeDaemon()
  604.     limitUploaders()
  605.     restartDownloads()
  606.  
  607. def shutdownDownloader(callback = None):
  608.     if hasattr(RemoteDownloader, 'dldaemon') and RemoteDownloader.dldaemon is not None:
  609.         RemoteDownloader.dldaemon.shutdownDownloaderDaemon(callback=callback)
  610.  
  611. def lookupDownloader(url):
  612.     return views.remoteDownloads.getItemWithIndex(indexes.downloadsByURL, url)
  613.  
  614. def getExistingDownloaderByURL(url):
  615.     downloader = lookupDownloader(url)
  616.     return downloader
  617.  
  618. def getExistingDownloader(item):
  619.     downloader = lookupDownloader(item.getURL())
  620.     if downloader:
  621.         downloader.addItem(item)
  622.     return downloader
  623.  
  624. def getDownloader(item):
  625.     existing = getExistingDownloader(item)
  626.     if existing:
  627.         return existing
  628.     url = item.getURL()
  629.     channelName = platformutils.unicodeToFilename(item.getChannelTitle(True))
  630.     if not channelName:
  631.         channelName = None
  632.     if url.startswith(u'file://'):
  633.         path = getFileURLPath(url)
  634.         try:
  635.             getTorrentInfoHash(path)
  636.         except ValueError:
  637.             raise ValueError("Don't know how to handle %s" % url)
  638.         except IOError:
  639.             return None
  640.         else:
  641.             return RemoteDownloader(url, item, u'application/x-bittorrent', channelName=channelName)
  642.     else:
  643.         return RemoteDownloader(url, item, channelName=channelName)
  644.