#!/usr/bin/env python """Interact with files.wowace.com.""" import logging import os import re import urlparse import zipfile import cPickle import sys import platform import urllib2 import gzip import time from StringIO import StringIO from optparse import OptionParser from xml.dom.minidom import parse __author__ = 'David Lynch (kemayo at gmail dot com)' __version__ = '2.0.8' __revision__ = '$Rev: 75 $' __date__ = '$Date: 2007-11-01 12:15:12 -0700 (Thu, 01 Nov 2007) $' __copyright__ = 'Copyright (c) 2007 David Lynch' __license__ = 'New BSD License' default_wowdir = False #Change this to, e.g. "F://World of Warcraft//Interface//Addons' if your wow directory is in a nonstandard location. default_wowace = 'http://files.wowace.com/' default_externals = False USER_AGENT = 'wowacepy/%s +http://code.google.com/p/wowacepy/' % __version__ class wowace: """Interacts with files.wowace.com.""" def __init__(self, wowdir = default_wowdir, wowace = default_wowace, externals = default_externals, logfile = False): if not wowdir: wowdir = get_wowdir() if not os.path.exists(wowdir): raise IOError, "World of Warcraft directory (%s) not found" % wowdir #set up logging if logfile: # Logging approach grabbed from: http://docs.python.org/lib/multiple-destinations.html logging.basicConfig(filename = logfile, filemode = 'a', level=logging.DEBUG, format='%(asctime)s %(levelname)-8s %(message)s',) console = logging.StreamHandler() # define a Handler which writes INFO messages or higher to the sys.stderr console.setLevel(logging.INFO) formatter = logging.Formatter('%(levelname)-8s %(message)s') # set a format which is simpler for console use console.setFormatter(formatter) # tell the handler to use this format logging.getLogger('').addHandler(console) # add the handler to the root logger else: # I might want to do something else here -- not sure. pass self.batchupdating = False self.wowdir = wowdir self.wowace = wowace self.externals = externals self.session = {} self.refresh() def refresh(self): """Refetch the local and remote addon listings.""" self.local_file = urlparse.urljoin(self.wowdir, self.externals and 'latest.xml' or 'latest-noext.xml') self.server_file = urlparse.urljoin(self.wowace, self.externals and 'latest.xml' or 'latest-noext.xml') self.latest_date = os.stat.st_size(self.local_file) # attempt at getting and comparing the date of the latest file self.current_date = time.localtime() if self.current_date - self.latest_date > 600: #if it's 10 minutes or older logging.info('Local index %s is older than 10 minutes. Fetching updated index %s to replace it.' % (self.local_file, self.server_file)) # don't know how to write it, but update local_file to be equivalent to server_file feed = self.local_file self.localaddons = self.get_local_addons() self.remoteaddons = self.get_remote_addons() self.session.clear() def update_addon(self, name, get_deps = True, unpackage = False, script = False, delete_old = True, force = False, optimize = False): """Update an addon. Required arguments: name -- name of the addon to update Keyword arguments: get_deps -- whether to fetch addon dependencies (default True) unpackage -- whether to unpackage addons that contain filelist.wau (default False) script -- whether to run any scripts found in the addon; this is much more potentially dangerous than unpackaging (default False) delete_old -- whether to delete the old addon directory before extracting the new one (default True) force -- whether to redownload the addon even if there isn't a new version (default False) If the addon directory contains '.svn' or '.ignore', nothing will be done. If the addon requested is not already installed, install it. save_local_addons will be called unless self.batchupdating == True. """ if self.addon_can_be_updated(name, force): old_version = self.localaddons.get(name, 'unknown') new_version = self.remoteaddons[name]['version'] if not os.path.exists(os.path.join(self.wowdir, name)): logging.info("Installing new addon: %s" % name) else: logging.info("Upgrading %s from %s to %s" % (name, old_version, new_version)) zip = _fetch(self.remoteaddons[name]['file'][0]) if zip: zipdata = zipfile.ZipFile(zip) if _diff_dir_zip(zipdata, self.wowdir): logging.info("You'll need to restart WoW, or this addon might act up") self.restart_required = True if delete_old: _removedir(os.path.join(self.wowdir, name)) _unzip(zipdata, self.wowdir) zipdata.close() self.localaddons[name] = new_version if get_deps: for dep in self.remoteaddons[name]['dependencies']: dep = self._real_dependency(dep) # This accounts for a few special cases. It's a pain. if not self.remoteaddons.has_key(dep) and not self.localaddons.has_key(dep): logging.warning("%s has a dependency (%s) which is not installed locally and cannot be found on the server" % (name, dep)) else: self.update_addon(dep, get_deps, unpackage, script, delete_old, force) if unpackage: self.unpackage_addon(name) elif script: os.chdir(os.path.join(self.wowdir, name)) for s in get_scripts('.'): os.system(os.path.join('.', s)) if optimize: # Comment out files that are missing in the .toc (speed reasons): toc = TocFile(os.path.join(self.wowdir, name, name+'.toc')) toc.comment_missing_files() toc.save() self.save_local_addons() self.session[name] = True else: logging.warning("Upgrade failed! Problem fetching/reading file.") def update_all(self, get_deps = True, unpackage = False, script = False, delete_old = True, force = False, optimize = False): """Update all addons""" self.batchupdating = True # Avoid pointlessly writing to disk on every update. for addon in self.addons_to_be_updated(force): self.update_addon(addon, get_deps, unpackage, script, delete_old, force, optimize) self.batchupdating = False self.save_local_addons() def addon_can_be_updated(self, name, force = False): """Checks whether an addon can be updated If force evaluates to True, this will return True regardless of the local and remote versions. This will ALWAYS return False if there's a .ignore or .svn path in the addon, or if the addon can't be found on the server. This will also always return False if an addon has already been updated this session. (This is to avoid repeatedly downloading dependencies when using force.) """ if self.localaddons.get(name) in ('svn', 'ignore') or not self.remoteaddons.has_key(name) or self.session.has_key(name): return False old_version = self.localaddons.get(name, 'unknown') new_version = self.remoteaddons[name]['version'] if force or old_version != new_version: return True def addons_to_be_updated(self, force = False): addons = self.localaddons.keys() addons.sort() # This is functionally unnecessary... but it looks nicer on the output. return [addon for addon in addons if self.addon_can_be_updated(addon, force)] def unpackage_addon(self, name): """Follow the instructions in filelist.wau to unpackage an addon. See http://wowace.com/wiki/WowAceUpdater#The_Package_System for details of the package system. """ if os.path.exists(os.path.join(self.wowdir, name+'.nounpack')): return filelist_path = os.path.join(self.wowdir, name, 'filelist.wau') if os.path.exists(filelist_path): self.batchupdating = True filelist_file = open(filelist_path) for package in filelist_file: package = package.strip() logging.info('Unpacking %s from %s' % (name, package)) if package.startswith('@'): package = package[1:] package_destination = '%s_%s' % (name, package) else: package_destination = package package_base = os.path.join(self.wowdir, name, package) package_destdir = os.path.join(self.wowdir, package_destination) if os.path.exists(package_base) and not os.path.exists(os.path.join(package_destdir, '.svn')) and not os.path.exists(os.path.join(package_destdir, '.ignore')): _removedir(package_destdir) os.rename(package_base, package_destdir) self.localaddons[package_destination] = 'package' # WAU behavior is to immediately update and unpackage the unpackaged folders. self.update_addon(package_destination, unpackage=True) filelist_file.close() self.batchupdating = False self.save_local_addons() def remove_addon(self, name): """Delete an addon, and remove its version information from localaddons.""" path = os.path.join(self.wowdir, name) if os.path.exists(path) and not os.path.exists(os.path.join(path, '.svn')) and not os.path.exists(os.path.join(path, '.ignore')): _removedir(path) del(self.localaddons[name]) self.save_local_addons() logging.info("Removing %s" % name) def get_local_addons(self): logging.info("Loading local addons") if os.path.exists(os.path.join(self.wowdir, 'addon_versions.pkl')): try: pickled_versions = open(os.path.join(self.wowdir, 'addon_versions.pkl'), 'rb') addons = cPickle.load(pickled_versions) pickled_versions.close() # We managed to load the list of addons. Now, let's check to see whether any have been uninstalled... for addon in addons.keys(): if (not os.path.isdir(os.path.join(self.wowdir, addon))): # Addon directory is gone. Drop version info. del(addons[addon]) logging.info("Couldn't find %s, removing from saved versions" % addon) except EOFError: addons = {} else: addons = {} for addon in [f for f in os.listdir(self.wowdir) if os.path.isdir(os.path.join(self.wowdir, f))]: version = addons.get(addon) or 'unknown' if os.path.exists(os.path.join(self.wowdir, addon, '.svn')): version = 'svn' elif os.path.exists(os.path.join(self.wowdir, addon, '.ignore')): version = 'ignore' elif (addons.get(addon) in ('ignore', 'svn') and not os.path.exists(os.path.join(self.wowdir, addon, addons.get(addon)))): version = 'unknown' elif not addons.has_key(addon): #Try to fall back to grabbing the changelog version. for f in os.listdir(os.path.join(self.wowdir, addon)): m = re.search(r"[Cc]hangelog.*-r([0-9]+\.?[0-9]*)\.txt", f) if m: version = m.groups()[0] break addons[addon] = version return addons def get_remote_addons(self): # feed = urlparse.urljoin(self.wowace, self.externals and 'latest.xml' or 'latest-noext.xml') logging.info('Checking %s for updated addons' % feed) addons = waFeed(_fetch(feed)).addons if len(addons)==0: logging.warning('No addons found at %s' % feed) return addons def save_local_addons(self): if not self.batchupdating: pickled_versions = open(os.path.join(self.wowdir, 'addon_versions.pkl'), 'wb') cPickle.dump(self.localaddons, pickled_versions) pickled_versions.close() def _real_dependency(self, name): # First, just return the addon name if the addon exists. if name in self.remoteaddons: return name # Then search for an addon that provides that dependency. for addon, data in self.remoteaddons.items(): if name in data['provides']: return addon # If we reach this point we haven't found the real dependency. So just return the name it gave us. return name class waFeed: def __init__(self, file): #file can be a local filename or an open file object. self.addons = {} d = parse(file) self.__handle_rss(d) d.unlink() def __handle_rss(self, rss): channel = rss.getElementsByTagName('channel')[0] self.__handle_items(channel.getElementsByTagName('item')) def __handle_items(self, items): for item in items: self.__handle_item(item) def __handle_item(self, item): self.addons[_get_text_from_single(item, 'title')] = { 'interface': _get_text_from_single(item, 'wowaddon:interface'), 'version': _get_text_from_single(item, 'wowaddon:version'), 'stable': _get_text_from_single(item, 'wowaddon:stable')=='true', 'updated': _get_text_from_single(item, 'pubDate'), 'description': _get_text_from_single(item, 'description'), 'author': _get_text_from_single(item, 'author'), 'category': _get_text_from_single(item, 'category'), 'link': _get_text_from_single(item, 'link'), 'comments': _get_text_from_single(item, 'comments'), 'file': (item.getElementsByTagName('enclosure')[0].getAttribute('url'), item.getElementsByTagName('enclosure')[0].getAttribute('length')), 'dependencies': tuple([_get_text(dep.childNodes) for dep in item.getElementsByTagName('wowaddon:dependencies')]), 'optionaldeps': tuple([_get_text(optdep.childNodes) for optdep in item.getElementsByTagName('wowaddon:optionaldeps')]), 'provides': tuple([_get_text(provides.childNodes) for provides in item.getElementsByTagName('wowaddon:provides')]), } class TocFile: """Read and write World of Warcraft .toc files. """ remeta = re.compile(r'^.*##\s*([^:\s]+)\s*:\s*(.+)\s*$') def __init__(self, filename): self.filename = filename self.meta = {} self.files = [] self.parse() self.dirty = False def parse(self): f = open(self.filename, 'r') for line in f.readlines(): line = line.decode('utf8').replace(u'\ufeff', u'') # kill the BOM if present if len(line.strip()) > 0: m = self.remeta.match(line) if m: #k,v = line.replace('##', '').strip().split(':') self.meta[m.group(1)] = m.group(2) else: self.files.append(line.strip()) f.close() def save(self): if self.dirty: f = open(self.filename, 'w') for i in self.meta.items(): f.write("## %s: %s\n" % i) for i in self.files: f.write(i+'\n') f.close() def comment_missing_files(self): d = os.path.dirname(self.filename) for f in self.files: if not os.path.exists(os.path.join(d, f.replace('\\', os.path.sep))): self.dirty = True self.files[self.files.index(f)] = '#'+f def fix_optdeps(self): pass #Stole this function wholesale from the python.org minidom example. #The necessity of this function helps explain why I hate the DOM. def _get_text(nodelist): rc = "" for node in nodelist: if node.nodeType == node.TEXT_NODE: rc = rc + node.data return rc def _get_text_from_single(dom, tag): l = dom.getElementsByTagName(tag) if l: return _get_text(l[0].childNodes) else: return '' #This isn't a very thorough diff. But it'll catch the main source of wow-restart-requiredness. #TODO: Make this better. def _diff_dir_zip(zip, path): for f in [f for f in zip.namelist() if not f.endswith('/') and not f.endswith('.txt')]: root, name = os.path.split(f) #The changelog filename, by virtue of containing the revision number, will alawys differ. if not name.startswith('Changelog'): #check for existance if not os.path.exists(os.path.join(path, f)): return True return False def _permissions_from_external_attr(l): """Creates a permission mask from the external_attr field of a zipfile.ZipInfo object, suitable for passing to os.chmod From my own somewhat limited investigation, bits 17-25 of the external_attr field are a *reversed* permissions bitmask e.g. bit 17 is the group execute bit, bit 18 is the group write bit, etc. """ p = [] for i in range(24,15,-1): # I'm awful at remembering how bitwise operations work. So, for my own reference in the future: # Shifts the value of l 'i' bits to the right (i.e. divides it by 2**i), and checks whether the first bit is 1 or 0. p.append((l >> i) & 1) # This would produce the standard octal string for permissions (e.g. 0755, which is rwxr-wr-w) #return str((p[0]+p[1]*2+p[2]*4))+str((p[3]+p[4]*2+p[5]*4))+str((p[6]+p[7]*2+p[8]*4)) # This produces an integer, suitable for passing to os.chmod (i.e., for 0755: 493) return int(''.join([str(i) for i in p]), 2) def _unzip(zip, path): for f in zip.namelist(): if not f.endswith('/'): root, name = os.path.split(f) directory = os.path.normpath(os.path.join(path, root)) if not os.path.isdir(directory): os.makedirs(directory) dest = os.path.join(directory, name) nf = file(dest, 'wb') nf.write(zip.read(f)) nf.close() permissions = _permissions_from_external_attr(zip.getinfo(f).external_attr) if permissions == 0: permissions = 0644 os.chmod(dest, permissions) def _fetch(url): request = urllib2.Request(url) request.add_header('Accept-encoding', 'gzip') request.add_header('User-agent', USER_AGENT) f = urllib2.urlopen(request) data = StringIO(f.read()) f.close() if f.headers.get('content-encoding', '') == 'gzip': data = gzip.GzipFile(fileobj=data) return data def _rmgeneric(path, __func__): try: __func__(path) except OSError, (errno, strerror): print "Error removing %(path)s, %(error)s " % {'path' : path, 'error': strerror } def _removedir(path): if not os.path.isdir(path): return for x in os.listdir(path): fullpath=os.path.join(path, x) if os.path.isfile(fullpath): _rmgeneric(fullpath, os.remove) elif os.path.isdir(fullpath): _removedir(fullpath) _rmgeneric(path, os.rmdir) def get_wowdir(): #Try to guess the wow directory, based on platform. s = platform.system() if s == 'Windows': return os.path.join(os.environ['PROGRAMFILES'], 'World of Warcraft\\Interface\\Addons') elif s == 'Darwin': # mac os x userdir = os.path.expanduser('~/Applications/World of Warcraft/Interface/Addons') if os.path.exists(userdir): return userdir else: return '/Applications/World of Warcraft/Interface/Addons' else: # We're screwed. Return a best guess, which might, maybe, work for linux. Though probably not. # If installed from wine, running from the same user's directory, it will likely be the following: userdir = os.path.expanduser('~/.wine/drive_c/World of Warcraft/Interface/AddOns') if os.path.exists(userdir): return userdir else: return '/usr/share/games/World of Warcraft' # If using an installation on a MS Windows partition, it of course would be under that mount location: #return '/mnt/win/World of Warcraft/Interface/AddOns' # or some such -- it'll be different for every system def get_scripts(path): s = platform.system() if s == 'Windows': scriptext = '.bat' else: scriptext = '.sh' for f in os.listdir(path): if f.endswith(scriptext): yield f # And now some functions that make this a standalone program, more or less. def _dispatch(): # get_deps = True, unpackage = False, delete_old = True, force = False parser = OptionParser(version="%%prog %s (%s)" % (__version__, __revision__), usage = "usage: %prog [options] [addon1] ... [addon99]") parser.add_option('-e', '--externals', action='store_true', dest='externals', default = False, help="Download addons with externals") parser.add_option('-n', '--nodeps', action='store_false', dest='get_deps', default=True, help="Don't fetch dependencies") parser.add_option('-u', '--unpackage', action='store_true', dest='unpackage', default=False, help="Unpackage downloaded addons") parser.add_option('-s', '--script', action='store_true', dest='script', default=False, help="Run scripts within an addon") parser.add_option('-k', '--keepold', action='store_false', dest='delete_old', default=True, help="Don't delete directories before replacing them") parser.add_option('--optimize', action='store_true', dest='optimize', default=False, help="Attempt to tweak TOCs to improve load times") parser.add_option('-f', '--force', action='store_true', dest='force', default=False, help="Redownload all addons, even if current") parser.add_option('-r', '--remove', action='store_true', dest='remove', default=False, help="Remove addons passed as arguments") parser.add_option('--wowdir', dest='wowdir', default = default_wowdir and default_wowdir or get_wowdir(), help="Set the WoW addon directory [default: %default]", metavar="DIR") parser.add_option('--wowace', dest='wowace', default = default_wowace, help="Set the wowace file repository location [default: %default]", metavar="URL") options, args = parser.parse_args(sys.argv[1:]) if options.script and options.unpackage: parser.error("Options -s and -u are mutually exclusive.") updater = wowace(wowdir=options.wowdir, wowace=options.wowace, externals=options.externals, logfile=os.path.join(options.wowdir, 'wowace.log')) if args: if options.remove: for arg in args: if updater.localaddons.has_key(arg): updater.remove_addon(arg) else: print "%s cannot be removed, as it cannot be found." % arg else: for arg in args: if updater.remoteaddons.has_key(arg): updater.update_addon(arg, get_deps=options.get_deps, unpackage=options.unpackage, delete_old=options.delete_old, force=options.force, script=options.script, optimize=options.optimize) else: print "%s cannot be found on the server." % arg else: updater.update_all(get_deps=options.get_deps, unpackage=options.unpackage, delete_old=options.delete_old, force=options.force, script=options.script, optimize=options.optimize) print "Like the updater? Consider donating to WowAce at: http://wowace.com/index.php/Donations" if __name__ == "__main__": _dispatch()