diff options
Diffstat (limited to 'scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py')
| -rw-r--r-- | scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py | 1477 |
1 files changed, 1477 insertions, 0 deletions
diff --git a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py new file mode 100644 index 0000000000..fefdab36f6 --- /dev/null +++ b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/grabber.py @@ -0,0 +1,1477 @@ +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, +# Boston, MA 02111-1307 USA + +# This file is part of urlgrabber, a high-level cross-protocol url-grabber +# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko + +"""A high-level cross-protocol url-grabber. + +GENERAL ARGUMENTS (kwargs) + + Where possible, the module-level default is indicated, and legal + values are provided. + + copy_local = 0 [0|1] + + ignored except for file:// urls, in which case it specifies + whether urlgrab should still make a copy of the file, or simply + point to the existing copy. The module level default for this + option is 0. + + close_connection = 0 [0|1] + + tells URLGrabber to close the connection after a file has been + transfered. This is ignored unless the download happens with the + http keepalive handler (keepalive=1). Otherwise, the connection + is left open for further use. The module level default for this + option is 0 (keepalive connections will not be closed). + + keepalive = 1 [0|1] + + specifies whether keepalive should be used for HTTP/1.1 servers + that support it. The module level default for this option is 1 + (keepalive is enabled). + + progress_obj = None + + a class instance that supports the following methods: + po.start(filename, url, basename, length, text) + # length will be None if unknown + po.update(read) # read == bytes read so far + po.end() + + text = None + + specifies an alternativ text item in the beginning of the progress + bar line. If not given, the basename of the file is used. + + throttle = 1.0 + + a number - if it's an int, it's the bytes/second throttle limit. + If it's a float, it is first multiplied by bandwidth. If throttle + == 0, throttling is disabled. If None, the module-level default + (which can be set on default_grabber.throttle) is used. See + BANDWIDTH THROTTLING for more information. + + timeout = None + + a positive float expressing the number of seconds to wait for socket + operations. If the value is None or 0.0, socket operations will block + forever. Setting this option causes urlgrabber to call the settimeout + method on the Socket object used for the request. See the Python + documentation on settimeout for more information. + http://www.python.org/doc/current/lib/socket-objects.html + + bandwidth = 0 + + the nominal max bandwidth in bytes/second. If throttle is a float + and bandwidth == 0, throttling is disabled. If None, the + module-level default (which can be set on + default_grabber.bandwidth) is used. See BANDWIDTH THROTTLING for + more information. + + range = None + + a tuple of the form (first_byte, last_byte) describing a byte + range to retrieve. Either or both of the values may set to + None. If first_byte is None, byte offset 0 is assumed. If + last_byte is None, the last byte available is assumed. Note that + the range specification is python-like in that (0,10) will yeild + the first 10 bytes of the file. + + If set to None, no range will be used. + + reget = None [None|'simple'|'check_timestamp'] + + whether to attempt to reget a partially-downloaded file. Reget + only applies to .urlgrab and (obviously) only if there is a + partially downloaded file. Reget has two modes: + + 'simple' -- the local file will always be trusted. If there + are 100 bytes in the local file, then the download will always + begin 100 bytes into the requested file. + + 'check_timestamp' -- the timestamp of the server file will be + compared to the timestamp of the local file. ONLY if the + local file is newer than or the same age as the server file + will reget be used. If the server file is newer, or the + timestamp is not returned, the entire file will be fetched. + + NOTE: urlgrabber can do very little to verify that the partial + file on disk is identical to the beginning of the remote file. + You may want to either employ a custom "checkfunc" or simply avoid + using reget in situations where corruption is a concern. + + user_agent = 'urlgrabber/VERSION' + + a string, usually of the form 'AGENT/VERSION' that is provided to + HTTP servers in the User-agent header. The module level default + for this option is "urlgrabber/VERSION". + + http_headers = None + + a tuple of 2-tuples, each containing a header and value. These + will be used for http and https requests only. For example, you + can do + http_headers = (('Pragma', 'no-cache'),) + + ftp_headers = None + + this is just like http_headers, but will be used for ftp requests. + + proxies = None + + a dictionary that maps protocol schemes to proxy hosts. For + example, to use a proxy server on host "foo" port 3128 for http + and https URLs: + proxies={ 'http' : 'http://foo:3128', 'https' : 'http://foo:3128' } + note that proxy authentication information may be provided using + normal URL constructs: + proxies={ 'http' : 'http://user:host@foo:3128' } + Lastly, if proxies is None, the default environment settings will + be used. + + prefix = None + + a url prefix that will be prepended to all requested urls. For + example: + g = URLGrabber(prefix='http://foo.com/mirror/') + g.urlgrab('some/file.txt') + ## this will fetch 'http://foo.com/mirror/some/file.txt' + This option exists primarily to allow identical behavior to + MirrorGroup (and derived) instances. Note: a '/' will be inserted + if necessary, so you cannot specify a prefix that ends with a + partial file or directory name. + + opener = None + + Overrides the default urllib2.OpenerDirector provided to urllib2 + when making requests. This option exists so that the urllib2 + handler chain may be customized. Note that the range, reget, + proxy, and keepalive features require that custom handlers be + provided to urllib2 in order to function properly. If an opener + option is provided, no attempt is made by urlgrabber to ensure + chain integrity. You are responsible for ensuring that any + extension handlers are present if said features are required. + + data = None + + Only relevant for the HTTP family (and ignored for other + protocols), this allows HTTP POSTs. When the data kwarg is + present (and not None), an HTTP request will automatically become + a POST rather than GET. This is done by direct passthrough to + urllib2. If you use this, you may also want to set the + 'Content-length' and 'Content-type' headers with the http_headers + option. Note that python 2.2 handles the case of these + badly and if you do not use the proper case (shown here), your + values will be overridden with the defaults. + + +RETRY RELATED ARGUMENTS + + retry = None + + the number of times to retry the grab before bailing. If this is + zero, it will retry forever. This was intentional... really, it + was :). If this value is not supplied or is supplied but is None + retrying does not occur. + + retrycodes = [-1,2,4,5,6,7] + + a sequence of errorcodes (values of e.errno) for which it should + retry. See the doc on URLGrabError for more details on this. You + might consider modifying a copy of the default codes rather than + building yours from scratch so that if the list is extended in the + future (or one code is split into two) you can still enjoy the + benefits of the default list. You can do that with something like + this: + + retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes + if 12 not in retrycodes: + retrycodes.append(12) + + checkfunc = None + + a function to do additional checks. This defaults to None, which + means no additional checking. The function should simply return + on a successful check. It should raise URLGrabError on an + unsuccessful check. Raising of any other exception will be + considered immediate failure and no retries will occur. + + If it raises URLGrabError, the error code will determine the retry + behavior. Negative error numbers are reserved for use by these + passed in functions, so you can use many negative numbers for + different types of failure. By default, -1 results in a retry, + but this can be customized with retrycodes. + + If you simply pass in a function, it will be given exactly one + argument: a CallbackObject instance with the .url attribute + defined and either .filename (for urlgrab) or .data (for urlread). + For urlgrab, .filename is the name of the local file. For + urlread, .data is the actual string data. If you need other + arguments passed to the callback (program state of some sort), you + can do so like this: + + checkfunc=(function, ('arg1', 2), {'kwarg': 3}) + + if the downloaded file has filename /tmp/stuff, then this will + result in this call (for urlgrab): + + function(obj, 'arg1', 2, kwarg=3) + # obj.filename = '/tmp/stuff' + # obj.url = 'http://foo.com/stuff' + + NOTE: both the "args" tuple and "kwargs" dict must be present if + you use this syntax, but either (or both) can be empty. + + failure_callback = None + + The callback that gets called during retries when an attempt to + fetch a file fails. The syntax for specifying the callback is + identical to checkfunc, except for the attributes defined in the + CallbackObject instance. The attributes for failure_callback are: + + exception = the raised exception + url = the url we're trying to fetch + tries = the number of tries so far (including this one) + retry = the value of the retry option + + The callback is present primarily to inform the calling program of + the failure, but if it raises an exception (including the one it's + passed) that exception will NOT be caught and will therefore cause + future retries to be aborted. + + The callback is called for EVERY failure, including the last one. + On the last try, the callback can raise an alternate exception, + but it cannot (without severe trickiness) prevent the exception + from being raised. + + interrupt_callback = None + + This callback is called if KeyboardInterrupt is received at any + point in the transfer. Basically, this callback can have three + impacts on the fetch process based on the way it exits: + + 1) raise no exception: the current fetch will be aborted, but + any further retries will still take place + + 2) raise a URLGrabError: if you're using a MirrorGroup, then + this will prompt a failover to the next mirror according to + the behavior of the MirrorGroup subclass. It is recommended + that you raise URLGrabError with code 15, 'user abort'. If + you are NOT using a MirrorGroup subclass, then this is the + same as (3). + + 3) raise some other exception (such as KeyboardInterrupt), which + will not be caught at either the grabber or mirror levels. + That is, it will be raised up all the way to the caller. + + This callback is very similar to failure_callback. They are + passed the same arguments, so you could use the same function for + both. + + urlparser = URLParser() + + The URLParser class handles pre-processing of URLs, including + auth-handling for user/pass encoded in http urls, file handing + (that is, filenames not sent as a URL), and URL quoting. If you + want to override any of this behavior, you can pass in a + replacement instance. See also the 'quote' option. + + quote = None + + Whether or not to quote the path portion of a url. + quote = 1 -> quote the URLs (they're not quoted yet) + quote = 0 -> do not quote them (they're already quoted) + quote = None -> guess what to do + + This option only affects proper urls like 'file:///etc/passwd'; it + does not affect 'raw' filenames like '/etc/passwd'. The latter + will always be quoted as they are converted to URLs. Also, only + the path part of a url is quoted. If you need more fine-grained + control, you should probably subclass URLParser and pass it in via + the 'urlparser' option. + +BANDWIDTH THROTTLING + + urlgrabber supports throttling via two values: throttle and + bandwidth Between the two, you can either specify and absolute + throttle threshold or specify a theshold as a fraction of maximum + available bandwidth. + + throttle is a number - if it's an int, it's the bytes/second + throttle limit. If it's a float, it is first multiplied by + bandwidth. If throttle == 0, throttling is disabled. If None, the + module-level default (which can be set with set_throttle) is used. + + bandwidth is the nominal max bandwidth in bytes/second. If throttle + is a float and bandwidth == 0, throttling is disabled. If None, the + module-level default (which can be set with set_bandwidth) is used. + + THROTTLING EXAMPLES: + + Lets say you have a 100 Mbps connection. This is (about) 10^8 bits + per second, or 12,500,000 Bytes per second. You have a number of + throttling options: + + *) set_bandwidth(12500000); set_throttle(0.5) # throttle is a float + + This will limit urlgrab to use half of your available bandwidth. + + *) set_throttle(6250000) # throttle is an int + + This will also limit urlgrab to use half of your available + bandwidth, regardless of what bandwidth is set to. + + *) set_throttle(6250000); set_throttle(1.0) # float + + Use half your bandwidth + + *) set_throttle(6250000); set_throttle(2.0) # float + + Use up to 12,500,000 Bytes per second (your nominal max bandwidth) + + *) set_throttle(6250000); set_throttle(0) # throttle = 0 + + Disable throttling - this is more efficient than a very large + throttle setting. + + *) set_throttle(0); set_throttle(1.0) # throttle is float, bandwidth = 0 + + Disable throttling - this is the default when the module is loaded. + + SUGGESTED AUTHOR IMPLEMENTATION (THROTTLING) + + While this is flexible, it's not extremely obvious to the user. I + suggest you implement a float throttle as a percent to make the + distinction between absolute and relative throttling very explicit. + + Also, you may want to convert the units to something more convenient + than bytes/second, such as kbps or kB/s, etc. + +""" + +# $Id: grabber.py,v 1.48 2006/09/22 00:58:05 mstenner Exp $ + +import os +import os.path +import sys +import urlparse +import rfc822 +import time +import string +import urllib +import urllib2 +from stat import * # S_* and ST_* + +######################################################################## +# MODULE INITIALIZATION +######################################################################## +try: + exec('from ' + (__name__.split('.'))[0] + ' import __version__') +except: + __version__ = '???' + +import sslfactory + +auth_handler = urllib2.HTTPBasicAuthHandler( \ + urllib2.HTTPPasswordMgrWithDefaultRealm()) + +try: + from i18n import _ +except ImportError, msg: + def _(st): return st + +try: + from httplib import HTTPException +except ImportError, msg: + HTTPException = None + +try: + # This is a convenient way to make keepalive optional. + # Just rename the module so it can't be imported. + import keepalive + from keepalive import HTTPHandler, HTTPSHandler + have_keepalive = True +except ImportError, msg: + have_keepalive = False + +try: + # add in range support conditionally too + import byterange + from byterange import HTTPRangeHandler, HTTPSRangeHandler, \ + FileRangeHandler, FTPRangeHandler, range_tuple_normalize, \ + range_tuple_to_header, RangeError +except ImportError, msg: + range_handlers = () + RangeError = None + have_range = 0 +else: + range_handlers = (HTTPRangeHandler(), HTTPSRangeHandler(), + FileRangeHandler(), FTPRangeHandler()) + have_range = 1 + + +# check whether socket timeout support is available (Python >= 2.3) +import socket +try: + TimeoutError = socket.timeout + have_socket_timeout = True +except AttributeError: + TimeoutError = None + have_socket_timeout = False + +######################################################################## +# functions for debugging output. These functions are here because they +# are also part of the module initialization. +DEBUG = None +def set_logger(DBOBJ): + """Set the DEBUG object. This is called by _init_default_logger when + the environment variable URLGRABBER_DEBUG is set, but can also be + called by a calling program. Basically, if the calling program uses + the logging module and would like to incorporate urlgrabber logging, + then it can do so this way. It's probably not necessary as most + internal logging is only for debugging purposes. + + The passed-in object should be a logging.Logger instance. It will + be pushed into the keepalive and byterange modules if they're + being used. The mirror module pulls this object in on import, so + you will need to manually push into it. In fact, you may find it + tidier to simply push your logging object (or objects) into each + of these modules independently. + """ + + global DEBUG + DEBUG = DBOBJ + if have_keepalive and keepalive.DEBUG is None: + keepalive.DEBUG = DBOBJ + if have_range and byterange.DEBUG is None: + byterange.DEBUG = DBOBJ + if sslfactory.DEBUG is None: + sslfactory.DEBUG = DBOBJ + +def _init_default_logger(): + '''Examines the environment variable URLGRABBER_DEBUG and creates + a logging object (logging.logger) based on the contents. It takes + the form + + URLGRABBER_DEBUG=level,filename + + where "level" can be either an integer or a log level from the + logging module (DEBUG, INFO, etc). If the integer is zero or + less, logging will be disabled. Filename is the filename where + logs will be sent. If it is "-", then stdout will be used. If + the filename is empty or missing, stderr will be used. If the + variable cannot be processed or the logging module cannot be + imported (python < 2.3) then logging will be disabled. Here are + some examples: + + URLGRABBER_DEBUG=1,debug.txt # log everything to debug.txt + URLGRABBER_DEBUG=WARNING,- # log warning and higher to stdout + URLGRABBER_DEBUG=INFO # log info and higher to stderr + + This funtion is called during module initialization. It is not + intended to be called from outside. The only reason it is a + function at all is to keep the module-level namespace tidy and to + collect the code into a nice block.''' + + try: + dbinfo = os.environ['URLGRABBER_DEBUG'].split(',') + import logging + level = logging._levelNames.get(dbinfo[0], int(dbinfo[0])) + if level < 1: raise ValueError() + + formatter = logging.Formatter('%(asctime)s %(message)s') + if len(dbinfo) > 1: filename = dbinfo[1] + else: filename = '' + if filename == '': handler = logging.StreamHandler(sys.stderr) + elif filename == '-': handler = logging.StreamHandler(sys.stdout) + else: handler = logging.FileHandler(filename) + handler.setFormatter(formatter) + DBOBJ = logging.getLogger('urlgrabber') + DBOBJ.addHandler(handler) + DBOBJ.setLevel(level) + except (KeyError, ImportError, ValueError): + DBOBJ = None + set_logger(DBOBJ) + +_init_default_logger() +######################################################################## +# END MODULE INITIALIZATION +######################################################################## + + + +class URLGrabError(IOError): + """ + URLGrabError error codes: + + URLGrabber error codes (0 -- 255) + 0 - everything looks good (you should never see this) + 1 - malformed url + 2 - local file doesn't exist + 3 - request for non-file local file (dir, etc) + 4 - IOError on fetch + 5 - OSError on fetch + 6 - no content length header when we expected one + 7 - HTTPException + 8 - Exceeded read limit (for urlread) + 9 - Requested byte range not satisfiable. + 10 - Byte range requested, but range support unavailable + 11 - Illegal reget mode + 12 - Socket timeout + 13 - malformed proxy url + 14 - HTTPError (includes .code and .exception attributes) + 15 - user abort + + MirrorGroup error codes (256 -- 511) + 256 - No more mirrors left to try + + Custom (non-builtin) classes derived from MirrorGroup (512 -- 767) + [ this range reserved for application-specific error codes ] + + Retry codes (< 0) + -1 - retry the download, unknown reason + + Note: to test which group a code is in, you can simply do integer + division by 256: e.errno / 256 + + Negative codes are reserved for use by functions passed in to + retrygrab with checkfunc. The value -1 is built in as a generic + retry code and is already included in the retrycodes list. + Therefore, you can create a custom check function that simply + returns -1 and the fetch will be re-tried. For more customized + retries, you can use other negative number and include them in + retry-codes. This is nice for outputting useful messages about + what failed. + + You can use these error codes like so: + try: urlgrab(url) + except URLGrabError, e: + if e.errno == 3: ... + # or + print e.strerror + # or simply + print e #### print '[Errno %i] %s' % (e.errno, e.strerror) + """ + pass + +class CallbackObject: + """Container for returned callback data. + + This is currently a dummy class into which urlgrabber can stuff + information for passing to callbacks. This way, the prototype for + all callbacks is the same, regardless of the data that will be + passed back. Any function that accepts a callback function as an + argument SHOULD document what it will define in this object. + + It is possible that this class will have some greater + functionality in the future. + """ + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + +def urlgrab(url, filename=None, **kwargs): + """grab the file at <url> and make a local copy at <filename> + If filename is none, the basename of the url is used. + urlgrab returns the filename of the local file, which may be different + from the passed-in filename if the copy_local kwarg == 0. + + See module documentation for a description of possible kwargs. + """ + return default_grabber.urlgrab(url, filename, **kwargs) + +def urlopen(url, **kwargs): + """open the url and return a file object + If a progress object or throttle specifications exist, then + a special file object will be returned that supports them. + The file object can be treated like any other file object. + + See module documentation for a description of possible kwargs. + """ + return default_grabber.urlopen(url, **kwargs) + +def urlread(url, limit=None, **kwargs): + """read the url into a string, up to 'limit' bytes + If the limit is exceeded, an exception will be thrown. Note that urlread + is NOT intended to be used as a way of saying "I want the first N bytes" + but rather 'read the whole file into memory, but don't use too much' + + See module documentation for a description of possible kwargs. + """ + return default_grabber.urlread(url, limit, **kwargs) + + +class URLParser: + """Process the URLs before passing them to urllib2. + + This class does several things: + + * add any prefix + * translate a "raw" file to a proper file: url + * handle any http or https auth that's encoded within the url + * quote the url + + Only the "parse" method is called directly, and it calls sub-methods. + + An instance of this class is held in the options object, which + means that it's easy to change the behavior by sub-classing and + passing the replacement in. It need only have a method like: + + url, parts = urlparser.parse(url, opts) + """ + + def parse(self, url, opts): + """parse the url and return the (modified) url and its parts + + Note: a raw file WILL be quoted when it's converted to a URL. + However, other urls (ones which come with a proper scheme) may + or may not be quoted according to opts.quote + + opts.quote = 1 --> quote it + opts.quote = 0 --> do not quote it + opts.quote = None --> guess + """ + quote = opts.quote + + if opts.prefix: + url = self.add_prefix(url, opts.prefix) + + parts = urlparse.urlparse(url) + (scheme, host, path, parm, query, frag) = parts + + if not scheme or (len(scheme) == 1 and scheme in string.letters): + # if a scheme isn't specified, we guess that it's "file:" + if url[0] not in '/\\': url = os.path.abspath(url) + url = 'file:' + urllib.pathname2url(url) + parts = urlparse.urlparse(url) + quote = 0 # pathname2url quotes, so we won't do it again + + if scheme in ['http', 'https']: + parts = self.process_http(parts) + + if quote is None: + quote = self.guess_should_quote(parts) + if quote: + parts = self.quote(parts) + + url = urlparse.urlunparse(parts) + return url, parts + + def add_prefix(self, url, prefix): + if prefix[-1] == '/' or url[0] == '/': + url = prefix + url + else: + url = prefix + '/' + url + return url + + def process_http(self, parts): + (scheme, host, path, parm, query, frag) = parts + + if '@' in host and auth_handler: + try: + user_pass, host = host.split('@', 1) + if ':' in user_pass: + user, password = user_pass.split(':', 1) + except ValueError, e: + raise URLGrabError(1, _('Bad URL: %s') % url) + if DEBUG: DEBUG.info('adding HTTP auth: %s, XXXXXXXX', user) + auth_handler.add_password(None, host, user, password) + + return (scheme, host, path, parm, query, frag) + + def quote(self, parts): + """quote the URL + + This method quotes ONLY the path part. If you need to quote + other parts, you should override this and pass in your derived + class. The other alternative is to quote other parts before + passing into urlgrabber. + """ + (scheme, host, path, parm, query, frag) = parts + path = urllib.quote(path) + return (scheme, host, path, parm, query, frag) + + hexvals = '0123456789ABCDEF' + def guess_should_quote(self, parts): + """ + Guess whether we should quote a path. This amounts to + guessing whether it's already quoted. + + find ' ' -> 1 + find '%' -> 1 + find '%XX' -> 0 + else -> 1 + """ + (scheme, host, path, parm, query, frag) = parts + if ' ' in path: + return 1 + ind = string.find(path, '%') + if ind > -1: + while ind > -1: + if len(path) < ind+3: + return 1 + code = path[ind+1:ind+3].upper() + if code[0] not in self.hexvals or \ + code[1] not in self.hexvals: + return 1 + ind = string.find(path, '%', ind+1) + return 0 + return 1 + +class URLGrabberOptions: + """Class to ease kwargs handling.""" + + def __init__(self, delegate=None, **kwargs): + """Initialize URLGrabberOptions object. + Set default values for all options and then update options specified + in kwargs. + """ + self.delegate = delegate + if delegate is None: + self._set_defaults() + self._set_attributes(**kwargs) + + def __getattr__(self, name): + if self.delegate and hasattr(self.delegate, name): + return getattr(self.delegate, name) + raise AttributeError, name + + def raw_throttle(self): + """Calculate raw throttle value from throttle and bandwidth + values. + """ + if self.throttle <= 0: + return 0 + elif type(self.throttle) == type(0): + return float(self.throttle) + else: # throttle is a float + return self.bandwidth * self.throttle + + def derive(self, **kwargs): + """Create a derived URLGrabberOptions instance. + This method creates a new instance and overrides the + options specified in kwargs. + """ + return URLGrabberOptions(delegate=self, **kwargs) + + def _set_attributes(self, **kwargs): + """Update object attributes with those provided in kwargs.""" + self.__dict__.update(kwargs) + if have_range and kwargs.has_key('range'): + # normalize the supplied range value + self.range = range_tuple_normalize(self.range) + if not self.reget in [None, 'simple', 'check_timestamp']: + raise URLGrabError(11, _('Illegal reget mode: %s') \ + % (self.reget, )) + + def _set_defaults(self): + """Set all options to their default values. + When adding new options, make sure a default is + provided here. + """ + self.progress_obj = None + self.throttle = 1.0 + self.bandwidth = 0 + self.retry = None + self.retrycodes = [-1,2,4,5,6,7] + self.checkfunc = None + self.copy_local = 0 + self.close_connection = 0 + self.range = None + self.user_agent = 'urlgrabber/%s' % __version__ + self.keepalive = 1 + self.proxies = None + self.reget = None + self.failure_callback = None + self.interrupt_callback = None + self.prefix = None + self.opener = None + self.cache_openers = True + self.timeout = None + self.text = None + self.http_headers = None + self.ftp_headers = None + self.data = None + self.urlparser = URLParser() + self.quote = None + self.ssl_ca_cert = None + self.ssl_context = None + +class URLGrabber: + """Provides easy opening of URLs with a variety of options. + + All options are specified as kwargs. Options may be specified when + the class is created and may be overridden on a per request basis. + + New objects inherit default values from default_grabber. + """ + + def __init__(self, **kwargs): + self.opts = URLGrabberOptions(**kwargs) + + def _retry(self, opts, func, *args): + tries = 0 + while 1: + # there are only two ways out of this loop. The second has + # several "sub-ways" + # 1) via the return in the "try" block + # 2) by some exception being raised + # a) an excepton is raised that we don't "except" + # b) a callback raises ANY exception + # c) we're not retry-ing or have run out of retries + # d) the URLGrabError code is not in retrycodes + # beware of infinite loops :) + tries = tries + 1 + exception = None + retrycode = None + callback = None + if DEBUG: DEBUG.info('attempt %i/%s: %s', + tries, opts.retry, args[0]) + try: + r = apply(func, (opts,) + args, {}) + if DEBUG: DEBUG.info('success') + return r + except URLGrabError, e: + exception = e + callback = opts.failure_callback + retrycode = e.errno + except KeyboardInterrupt, e: + exception = e + callback = opts.interrupt_callback + + if DEBUG: DEBUG.info('exception: %s', exception) + if callback: + if DEBUG: DEBUG.info('calling callback: %s', callback) + cb_func, cb_args, cb_kwargs = self._make_callback(callback) + obj = CallbackObject(exception=exception, url=args[0], + tries=tries, retry=opts.retry) + cb_func(obj, *cb_args, **cb_kwargs) + + if (opts.retry is None) or (tries == opts.retry): + if DEBUG: DEBUG.info('retries exceeded, re-raising') + raise + + if (retrycode is not None) and (retrycode not in opts.retrycodes): + if DEBUG: DEBUG.info('retrycode (%i) not in list %s, re-raising', + retrycode, opts.retrycodes) + raise + + def urlopen(self, url, **kwargs): + """open the url and return a file object + If a progress object or throttle value specified when this + object was created, then a special file object will be + returned that supports them. The file object can be treated + like any other file object. + """ + opts = self.opts.derive(**kwargs) + (url,parts) = opts.urlparser.parse(url, opts) + def retryfunc(opts, url): + return URLGrabberFileObject(url, filename=None, opts=opts) + return self._retry(opts, retryfunc, url) + + def urlgrab(self, url, filename=None, **kwargs): + """grab the file at <url> and make a local copy at <filename> + If filename is none, the basename of the url is used. + urlgrab returns the filename of the local file, which may be + different from the passed-in filename if copy_local == 0. + """ + opts = self.opts.derive(**kwargs) + (url,parts) = opts.urlparser.parse(url, opts) + (scheme, host, path, parm, query, frag) = parts + if filename is None: + filename = os.path.basename( urllib.unquote(path) ) + if scheme == 'file' and not opts.copy_local: + # just return the name of the local file - don't make a + # copy currently + path = urllib.url2pathname(path) + if host: + path = os.path.normpath('//' + host + path) + if not os.path.exists(path): + raise URLGrabError(2, + _('Local file does not exist: %s') % (path, )) + elif not os.path.isfile(path): + raise URLGrabError(3, + _('Not a normal file: %s') % (path, )) + elif not opts.range: + return path + + def retryfunc(opts, url, filename): + fo = URLGrabberFileObject(url, filename, opts) + try: + fo._do_grab() + if not opts.checkfunc is None: + cb_func, cb_args, cb_kwargs = \ + self._make_callback(opts.checkfunc) + obj = CallbackObject() + obj.filename = filename + obj.url = url + apply(cb_func, (obj, )+cb_args, cb_kwargs) + finally: + fo.close() + return filename + + return self._retry(opts, retryfunc, url, filename) + + def urlread(self, url, limit=None, **kwargs): + """read the url into a string, up to 'limit' bytes + If the limit is exceeded, an exception will be thrown. Note + that urlread is NOT intended to be used as a way of saying + "I want the first N bytes" but rather 'read the whole file + into memory, but don't use too much' + """ + opts = self.opts.derive(**kwargs) + (url,parts) = opts.urlparser.parse(url, opts) + if limit is not None: + limit = limit + 1 + + def retryfunc(opts, url, limit): + fo = URLGrabberFileObject(url, filename=None, opts=opts) + s = '' + try: + # this is an unfortunate thing. Some file-like objects + # have a default "limit" of None, while the built-in (real) + # file objects have -1. They each break the other, so for + # now, we just force the default if necessary. + if limit is None: s = fo.read() + else: s = fo.read(limit) + + if not opts.checkfunc is None: + cb_func, cb_args, cb_kwargs = \ + self._make_callback(opts.checkfunc) + obj = CallbackObject() + obj.data = s + obj.url = url + apply(cb_func, (obj, )+cb_args, cb_kwargs) + finally: + fo.close() + return s + + s = self._retry(opts, retryfunc, url, limit) + if limit and len(s) > limit: + raise URLGrabError(8, + _('Exceeded limit (%i): %s') % (limit, url)) + return s + + def _make_callback(self, callback_obj): + if callable(callback_obj): + return callback_obj, (), {} + else: + return callback_obj + +# create the default URLGrabber used by urlXXX functions. +# NOTE: actual defaults are set in URLGrabberOptions +default_grabber = URLGrabber() + +class URLGrabberFileObject: + """This is a file-object wrapper that supports progress objects + and throttling. + + This exists to solve the following problem: lets say you want to + drop-in replace a normal open with urlopen. You want to use a + progress meter and/or throttling, but how do you do that without + rewriting your code? Answer: urlopen will return a wrapped file + object that does the progress meter and-or throttling internally. + """ + + def __init__(self, url, filename, opts): + self.url = url + self.filename = filename + self.opts = opts + self.fo = None + self._rbuf = '' + self._rbufsize = 1024*8 + self._ttime = time.time() + self._tsize = 0 + self._amount_read = 0 + self._opener = None + self._do_open() + + def __getattr__(self, name): + """This effectively allows us to wrap at the instance level. + Any attribute not found in _this_ object will be searched for + in self.fo. This includes methods.""" + if hasattr(self.fo, name): + return getattr(self.fo, name) + raise AttributeError, name + + def _get_opener(self): + """Build a urllib2 OpenerDirector based on request options.""" + if self.opts.opener: + return self.opts.opener + elif self._opener is None: + handlers = [] + need_keepalive_handler = (have_keepalive and self.opts.keepalive) + need_range_handler = (range_handlers and \ + (self.opts.range or self.opts.reget)) + # if you specify a ProxyHandler when creating the opener + # it _must_ come before all other handlers in the list or urllib2 + # chokes. + if self.opts.proxies: + handlers.append( CachedProxyHandler(self.opts.proxies) ) + + # ------------------------------------------------------- + # OK, these next few lines are a serious kludge to get + # around what I think is a bug in python 2.2's + # urllib2. The basic idea is that default handlers + # get applied first. If you override one (like a + # proxy handler), then the default gets pulled, but + # the replacement goes on the end. In the case of + # proxies, this means the normal handler picks it up + # first and the proxy isn't used. Now, this probably + # only happened with ftp or non-keepalive http, so not + # many folks saw it. The simple approach to fixing it + # is just to make sure you override the other + # conflicting defaults as well. I would LOVE to see + # these go way or be dealt with more elegantly. The + # problem isn't there after 2.2. -MDS 2005/02/24 + if not need_keepalive_handler: + handlers.append( urllib2.HTTPHandler() ) + if not need_range_handler: + handlers.append( urllib2.FTPHandler() ) + # ------------------------------------------------------- + + ssl_factory = sslfactory.get_factory(self.opts.ssl_ca_cert, + self.opts.ssl_context) + + if need_keepalive_handler: + handlers.append(HTTPHandler()) + handlers.append(HTTPSHandler(ssl_factory)) + if need_range_handler: + handlers.extend( range_handlers ) + handlers.append( auth_handler ) + if self.opts.cache_openers: + self._opener = CachedOpenerDirector(ssl_factory, *handlers) + else: + self._opener = ssl_factory.create_opener(*handlers) + # OK, I don't like to do this, but otherwise, we end up with + # TWO user-agent headers. + self._opener.addheaders = [] + return self._opener + + def _do_open(self): + opener = self._get_opener() + + req = urllib2.Request(self.url, self.opts.data) # build request object + self._add_headers(req) # add misc headers that we need + self._build_range(req) # take care of reget and byterange stuff + + fo, hdr = self._make_request(req, opener) + if self.reget_time and self.opts.reget == 'check_timestamp': + # do this if we have a local file with known timestamp AND + # we're in check_timestamp reget mode. + fetch_again = 0 + try: + modified_tuple = hdr.getdate_tz('last-modified') + modified_stamp = rfc822.mktime_tz(modified_tuple) + if modified_stamp > self.reget_time: fetch_again = 1 + except (TypeError,): + fetch_again = 1 + + if fetch_again: + # the server version is newer than the (incomplete) local + # version, so we should abandon the version we're getting + # and fetch the whole thing again. + fo.close() + self.opts.reget = None + del req.headers['Range'] + self._build_range(req) + fo, hdr = self._make_request(req, opener) + + (scheme, host, path, parm, query, frag) = urlparse.urlparse(self.url) + path = urllib.unquote(path) + if not (self.opts.progress_obj or self.opts.raw_throttle() \ + or self.opts.timeout): + # if we're not using the progress_obj, throttling, or timeout + # we can get a performance boost by going directly to + # the underlying fileobject for reads. + self.read = fo.read + if hasattr(fo, 'readline'): + self.readline = fo.readline + elif self.opts.progress_obj: + try: + length = int(hdr['Content-Length']) + length = length + self._amount_read # Account for regets + except (KeyError, ValueError, TypeError): + length = None + + self.opts.progress_obj.start(str(self.filename), + urllib.unquote(self.url), + os.path.basename(path), + length, text=self.opts.text) + self.opts.progress_obj.update(0) + (self.fo, self.hdr) = (fo, hdr) + + def _add_headers(self, req): + if self.opts.user_agent: + req.add_header('User-agent', self.opts.user_agent) + try: req_type = req.get_type() + except ValueError: req_type = None + if self.opts.http_headers and req_type in ('http', 'https'): + for h, v in self.opts.http_headers: + req.add_header(h, v) + if self.opts.ftp_headers and req_type == 'ftp': + for h, v in self.opts.ftp_headers: + req.add_header(h, v) + + def _build_range(self, req): + self.reget_time = None + self.append = 0 + reget_length = 0 + rt = None + if have_range and self.opts.reget and type(self.filename) == type(''): + # we have reget turned on and we're dumping to a file + try: + s = os.stat(self.filename) + except OSError: + pass + else: + self.reget_time = s[ST_MTIME] + reget_length = s[ST_SIZE] + + # Set initial length when regetting + self._amount_read = reget_length + + rt = reget_length, '' + self.append = 1 + + if self.opts.range: + if not have_range: + raise URLGrabError(10, _('Byte range requested but range '\ + 'support unavailable')) + rt = self.opts.range + if rt[0]: rt = (rt[0] + reget_length, rt[1]) + + if rt: + header = range_tuple_to_header(rt) + if header: req.add_header('Range', header) + + def _make_request(self, req, opener): + try: + if have_socket_timeout and self.opts.timeout: + old_to = socket.getdefaulttimeout() + socket.setdefaulttimeout(self.opts.timeout) + try: + fo = opener.open(req) + finally: + socket.setdefaulttimeout(old_to) + else: + fo = opener.open(req) + hdr = fo.info() + except ValueError, e: + raise URLGrabError(1, _('Bad URL: %s') % (e, )) + except RangeError, e: + raise URLGrabError(9, str(e)) + except urllib2.HTTPError, e: + new_e = URLGrabError(14, str(e)) + new_e.code = e.code + new_e.exception = e + raise new_e + except IOError, e: + if hasattr(e, 'reason') and have_socket_timeout and \ + isinstance(e.reason, TimeoutError): + raise URLGrabError(12, _('Timeout: %s') % (e, )) + else: + raise URLGrabError(4, _('IOError: %s') % (e, )) + except OSError, e: + raise URLGrabError(5, _('OSError: %s') % (e, )) + except HTTPException, e: + raise URLGrabError(7, _('HTTP Exception (%s): %s') % \ + (e.__class__.__name__, e)) + else: + return (fo, hdr) + + def _do_grab(self): + """dump the file to self.filename.""" + if self.append: new_fo = open(self.filename, 'ab') + else: new_fo = open(self.filename, 'wb') + bs = 1024*8 + size = 0 + + block = self.read(bs) + size = size + len(block) + while block: + new_fo.write(block) + block = self.read(bs) + size = size + len(block) + + new_fo.close() + try: + modified_tuple = self.hdr.getdate_tz('last-modified') + modified_stamp = rfc822.mktime_tz(modified_tuple) + os.utime(self.filename, (modified_stamp, modified_stamp)) + except (TypeError,), e: pass + + return size + + def _fill_buffer(self, amt=None): + """fill the buffer to contain at least 'amt' bytes by reading + from the underlying file object. If amt is None, then it will + read until it gets nothing more. It updates the progress meter + and throttles after every self._rbufsize bytes.""" + # the _rbuf test is only in this first 'if' for speed. It's not + # logically necessary + if self._rbuf and not amt is None: + L = len(self._rbuf) + if amt > L: + amt = amt - L + else: + return + + # if we've made it here, then we don't have enough in the buffer + # and we need to read more. + + buf = [self._rbuf] + bufsize = len(self._rbuf) + while amt is None or amt: + # first, delay if necessary for throttling reasons + if self.opts.raw_throttle(): + diff = self._tsize/self.opts.raw_throttle() - \ + (time.time() - self._ttime) + if diff > 0: time.sleep(diff) + self._ttime = time.time() + + # now read some data, up to self._rbufsize + if amt is None: readamount = self._rbufsize + else: readamount = min(amt, self._rbufsize) + try: + new = self.fo.read(readamount) + except socket.error, e: + raise URLGrabError(4, _('Socket Error: %s') % (e, )) + except TimeoutError, e: + raise URLGrabError(12, _('Timeout: %s') % (e, )) + except IOError, e: + raise URLGrabError(4, _('IOError: %s') %(e,)) + newsize = len(new) + if not newsize: break # no more to read + + if amt: amt = amt - newsize + buf.append(new) + bufsize = bufsize + newsize + self._tsize = newsize + self._amount_read = self._amount_read + newsize + if self.opts.progress_obj: + self.opts.progress_obj.update(self._amount_read) + + self._rbuf = string.join(buf, '') + return + + def read(self, amt=None): + self._fill_buffer(amt) + if amt is None: + s, self._rbuf = self._rbuf, '' + else: + s, self._rbuf = self._rbuf[:amt], self._rbuf[amt:] + return s + + def readline(self, limit=-1): + i = string.find(self._rbuf, '\n') + while i < 0 and not (0 < limit <= len(self._rbuf)): + L = len(self._rbuf) + self._fill_buffer(L + self._rbufsize) + if not len(self._rbuf) > L: break + i = string.find(self._rbuf, '\n', L) + + if i < 0: i = len(self._rbuf) + else: i = i+1 + if 0 <= limit < len(self._rbuf): i = limit + + s, self._rbuf = self._rbuf[:i], self._rbuf[i:] + return s + + def close(self): + if self.opts.progress_obj: + self.opts.progress_obj.end(self._amount_read) + self.fo.close() + if self.opts.close_connection: + try: self.fo.close_connection() + except: pass + +_handler_cache = [] +def CachedOpenerDirector(ssl_factory = None, *handlers): + for (cached_handlers, opener) in _handler_cache: + if cached_handlers == handlers: + for handler in opener.handlers: + handler.add_parent(opener) + return opener + if not ssl_factory: + ssl_factory = sslfactory.get_factory() + opener = ssl_factory.create_opener(*handlers) + _handler_cache.append( (handlers, opener) ) + return opener + +_proxy_cache = [] +def CachedProxyHandler(proxies): + for (pdict, handler) in _proxy_cache: + if pdict == proxies: + if DEBUG: DEBUG.debug('re-using proxy settings: %s', proxies) + break + else: + for k, v in proxies.items(): + utype, url = urllib.splittype(v) + host, other = urllib.splithost(url) + if (utype is None) or (host is None): + raise URLGrabError(13, _('Bad proxy URL: %s') % v) + + if DEBUG: DEBUG.info('creating new proxy handler: %s', proxies) + handler = urllib2.ProxyHandler(proxies) + _proxy_cache.append( (proxies, handler) ) + return handler + +##################################################################### +# DEPRECATED FUNCTIONS +def set_throttle(new_throttle): + """Deprecated. Use: default_grabber.throttle = new_throttle""" + default_grabber.throttle = new_throttle + +def set_bandwidth(new_bandwidth): + """Deprecated. Use: default_grabber.bandwidth = new_bandwidth""" + default_grabber.bandwidth = new_bandwidth + +def set_progress_obj(new_progress_obj): + """Deprecated. Use: default_grabber.progress_obj = new_progress_obj""" + default_grabber.progress_obj = new_progress_obj + +def set_user_agent(new_user_agent): + """Deprecated. Use: default_grabber.user_agent = new_user_agent""" + default_grabber.user_agent = new_user_agent + +def retrygrab(url, filename=None, copy_local=0, close_connection=0, + progress_obj=None, throttle=None, bandwidth=None, + numtries=3, retrycodes=[-1,2,4,5,6,7], checkfunc=None): + """Deprecated. Use: urlgrab() with the retry arg instead""" + kwargs = {'copy_local' : copy_local, + 'close_connection' : close_connection, + 'progress_obj' : progress_obj, + 'throttle' : throttle, + 'bandwidth' : bandwidth, + 'retry' : numtries, + 'retrycodes' : retrycodes, + 'checkfunc' : checkfunc + } + return urlgrab(url, filename, **kwargs) + + +##################################################################### +# TESTING +def _main_test(): + import sys + try: url, filename = sys.argv[1:3] + except ValueError: + print 'usage:', sys.argv[0], \ + '<url> <filename> [copy_local=0|1] [close_connection=0|1]' + sys.exit() + + kwargs = {} + for a in sys.argv[3:]: + k, v = string.split(a, '=', 1) + kwargs[k] = int(v) + + set_throttle(1.0) + set_bandwidth(32 * 1024) + print "throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle, + default_grabber.bandwidth) + + try: from progress import text_progress_meter + except ImportError, e: pass + else: kwargs['progress_obj'] = text_progress_meter() + + try: name = apply(urlgrab, (url, filename), kwargs) + except URLGrabError, e: print e + else: print 'LOCAL FILE:', name + + +def _retry_test(): + import sys + try: url, filename = sys.argv[1:3] + except ValueError: + print 'usage:', sys.argv[0], \ + '<url> <filename> [copy_local=0|1] [close_connection=0|1]' + sys.exit() + + kwargs = {} + for a in sys.argv[3:]: + k, v = string.split(a, '=', 1) + kwargs[k] = int(v) + + try: from progress import text_progress_meter + except ImportError, e: pass + else: kwargs['progress_obj'] = text_progress_meter() + + def cfunc(filename, hello, there='foo'): + print hello, there + import random + rnum = random.random() + if rnum < .5: + print 'forcing retry' + raise URLGrabError(-1, 'forcing retry') + if rnum < .75: + print 'forcing failure' + raise URLGrabError(-2, 'forcing immediate failure') + print 'success' + return + + kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'}) + try: name = apply(retrygrab, (url, filename), kwargs) + except URLGrabError, e: print e + else: print 'LOCAL FILE:', name + +def _file_object_test(filename=None): + import random, cStringIO, sys + if filename is None: + filename = __file__ + print 'using file "%s" for comparisons' % filename + fo = open(filename) + s_input = fo.read() + fo.close() + + for testfunc in [_test_file_object_smallread, + _test_file_object_readall, + _test_file_object_readline, + _test_file_object_readlines]: + fo_input = cStringIO.StringIO(s_input) + fo_output = cStringIO.StringIO() + wrapper = URLGrabberFileObject(fo_input, None, 0) + print 'testing %-30s ' % testfunc.__name__, + testfunc(wrapper, fo_output) + s_output = fo_output.getvalue() + if s_output == s_input: print 'passed' + else: print 'FAILED' + +def _test_file_object_smallread(wrapper, fo_output): + while 1: + s = wrapper.read(23) + fo_output.write(s) + if not s: return + +def _test_file_object_readall(wrapper, fo_output): + s = wrapper.read() + fo_output.write(s) + +def _test_file_object_readline(wrapper, fo_output): + while 1: + s = wrapper.readline() + fo_output.write(s) + if not s: return + +def _test_file_object_readlines(wrapper, fo_output): + li = wrapper.readlines() + fo_output.write(string.join(li, '')) + +if __name__ == '__main__': + _main_test() + _retry_test() + _file_object_test('test') |