llmanifest.py
上传用户:king477883
上传日期:2021-03-01
资源大小:9553k
文件大小:25k
- """
- @file llmanifest.py
- @author Ryan Williams
- @brief Library for specifying operations on a set of files.
- $LicenseInfo:firstyear=2007&license=mit$
- Copyright (c) 2007-2010, Linden Research, Inc.
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
- $/LicenseInfo$
- """
- import commands
- import errno
- import filecmp
- import fnmatch
- import getopt
- import glob
- import os
- import re
- import shutil
- import sys
- import tarfile
- import errno
- def path_ancestors(path):
- drive, path = os.path.splitdrive(os.path.normpath(path))
- result = []
- while len(path) > 0 and path != os.path.sep:
- result.append(drive+path)
- path, sub = os.path.split(path)
- return result
- def proper_windows_path(path, current_platform = sys.platform):
- """ This function takes an absolute Windows or Cygwin path and
- returns a path appropriately formatted for the platform it's
- running on (as determined by sys.platform)"""
- path = path.strip()
- drive_letter = None
- rel = None
- match = re.match("/cygdrive/([a-z])/(.*)", path)
- if not match:
- match = re.match('([a-zA-Z]):\(.*)', path)
- if not match:
- return None # not an absolute path
- drive_letter = match.group(1)
- rel = match.group(2)
- if current_platform == "cygwin":
- return "/cygdrive/" + drive_letter.lower() + '/' + rel.replace('\', '/')
- else:
- return drive_letter.upper() + ':\' + rel.replace('/', '\')
- def get_default_platform(dummy):
- return {'linux2':'linux',
- 'linux1':'linux',
- 'cygwin':'windows',
- 'win32':'windows',
- 'darwin':'darwin'
- }[sys.platform]
- def get_default_version(srctree):
- # look up llversion.h and parse out the version info
- paths = [os.path.join(srctree, x, 'llversionviewer.h') for x in ['llcommon', '../llcommon', '../../indra/llcommon.h']]
- for p in paths:
- if os.path.exists(p):
- contents = open(p, 'r').read()
- major = re.search("LL_VERSION_MAJORs=s([0-9]+)", contents).group(1)
- minor = re.search("LL_VERSION_MINORs=s([0-9]+)", contents).group(1)
- patch = re.search("LL_VERSION_PATCHs=s([0-9]+)", contents).group(1)
- build = re.search("LL_VERSION_BUILDs=s([0-9]+)", contents).group(1)
- return major, minor, patch, build
- def get_channel(srctree):
- # look up llversionserver.h and parse out the version info
- paths = [os.path.join(srctree, x, 'llversionviewer.h') for x in ['llcommon', '../llcommon', '../../indra/llcommon.h']]
- for p in paths:
- if os.path.exists(p):
- contents = open(p, 'r').read()
- channel = re.search("LL_CHANNELs=s"(.+)";s*$", contents, flags = re.M).group(1)
- return channel
-
- DEFAULT_SRCTREE = os.path.dirname(sys.argv[0])
- DEFAULT_CHANNEL = 'Second Life Release'
- DEFAULT_CHANNEL_SNOWGLOBE = 'Snowglobe Release'
- ARGUMENTS=[
- dict(name='actions',
- description="""This argument specifies the actions that are to be taken when the
- script is run. The meaningful actions are currently:
- copy - copies the files specified by the manifest into the
- destination directory.
- package - bundles up the files in the destination directory into
- an installer for the current platform
- unpacked - bundles up the files in the destination directory into
- a simple tarball
- Example use: %(name)s --actions="copy unpacked" """,
- default="copy package"),
- dict(name='arch',
- description="""This argument is appended to the platform string for
- determining which manifest class to run.
- Example use: %(name)s --arch=i686
- On Linux this would try to use Linux_i686Manifest.""",
- default=""),
- dict(name='build', description='Build directory.', default=DEFAULT_SRCTREE),
- dict(name='buildtype', description='Build type (i.e. Debug, Release, RelWithDebInfo).', default=None),
- dict(name='branding_id', description="""Identifier for the branding set to
- use. Currently, 'secondlife' or 'snowglobe')""",
- default='secondlife'),
- dict(name='configuration',
- description="""The build configuration used.""",
- default="Release"),
- dict(name='dest', description='Destination directory.', default=DEFAULT_SRCTREE),
- dict(name='grid',
- description="""Which grid the client will try to connect to. Even
- though it's not strictly a grid, 'firstlook' is also an acceptable
- value for this parameter.""",
- default=""),
- dict(name='channel',
- description="""The channel to use for updates, packaging, settings name, etc.""",
- default=get_channel),
- dict(name='login_channel',
- description="""The channel to use for login handshake/updates only.""",
- default=None),
- dict(name='installer_name',
- description=""" The name of the file that the installer should be
- packaged up into. Only used on Linux at the moment.""",
- default=None),
- dict(name='login_url',
- description="""The url that the login screen displays in the client.""",
- default=None),
- dict(name='platform',
- description="""The current platform, to be used for looking up which
- manifest class to run.""",
- default=get_default_platform),
- dict(name='source',
- description='Source directory.',
- default=DEFAULT_SRCTREE),
- dict(name='artwork', description='Artwork directory.', default=DEFAULT_SRCTREE),
- dict(name='touch',
- description="""File to touch when action is finished. Touch file will
- contain the name of the final package in a form suitable
- for use by a .bat file.""",
- default=None),
- dict(name='version',
- description="""This specifies the version of Second Life that is
- being packaged up.""",
- default=get_default_version)
- ]
- def usage(srctree=""):
- nd = {'name':sys.argv[0]}
- print """Usage:
- %(name)s [options] [destdir]
- Options:
- """ % nd
- for arg in ARGUMENTS:
- default = arg['default']
- if hasattr(default, '__call__'):
- default = "(computed value) "" + str(default(srctree)) + '"'
- elif default is not None:
- default = '"' + default + '"'
- print "t--%s Default: %snt%sn" % (
- arg['name'],
- default,
- arg['description'] % nd)
- def main():
- option_names = [arg['name'] + '=' for arg in ARGUMENTS]
- option_names.append('help')
- options, remainder = getopt.getopt(sys.argv[1:], "", option_names)
- # convert options to a hash
- args = {'source': DEFAULT_SRCTREE,
- 'artwork': DEFAULT_SRCTREE,
- 'build': DEFAULT_SRCTREE,
- 'dest': DEFAULT_SRCTREE }
- for opt in options:
- args[opt[0].replace("--", "")] = opt[1]
- for k in 'artwork build dest source'.split():
- args[k] = os.path.normpath(args[k])
- print "Source tree:", args['source']
- print "Artwork tree:", args['artwork']
- print "Build tree:", args['build']
- print "Destination tree:", args['dest']
- # early out for help
- if 'help' in args:
- # *TODO: it is a huge hack to pass around the srctree like this
- usage(args['source'])
- return
- # defaults
- for arg in ARGUMENTS:
- if arg['name'] not in args:
- default = arg['default']
- if hasattr(default, '__call__'):
- default = default(args['source'])
- if default is not None:
- args[arg['name']] = default
- # fix up version
- if isinstance(args.get('version'), str):
- args['version'] = args['version'].split('.')
-
- # default and agni are default
- if args['grid'] in ['default', 'agni']:
- args['grid'] = ''
- if 'actions' in args:
- args['actions'] = args['actions'].split()
- # debugging
- for opt in args:
- print "Option:", opt, "=", args[opt]
- wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
- wm.do(*args['actions'])
- # Write out the package file in this format, so that it can easily be called
- # and used in a .bat file - yeah, it sucks, but this is the simplest...
- touch = args.get('touch')
- if touch:
- fp = open(touch, 'w')
- fp.write('set package_file=%sn' % wm.package_file)
- fp.close()
- print 'touched', touch
- return 0
- class LLManifestRegistry(type):
- def __init__(cls, name, bases, dct):
- super(LLManifestRegistry, cls).__init__(name, bases, dct)
- match = re.match("(w+)Manifest", name)
- if match:
- cls.manifests[match.group(1).lower()] = cls
- class LLManifest(object):
- __metaclass__ = LLManifestRegistry
- manifests = {}
- def for_platform(self, platform, arch = None):
- if arch:
- platform = platform + '_' + arch
- return self.manifests[platform.lower()]
- for_platform = classmethod(for_platform)
- def __init__(self, args):
- super(LLManifest, self).__init__()
- self.args = args
- self.file_list = []
- self.excludes = []
- self.actions = []
- self.src_prefix = [args['source']]
- self.artwork_prefix = [args['artwork']]
- self.build_prefix = [args['build']]
- self.dst_prefix = [args['dest']]
- self.created_paths = []
- self.package_name = "Unknown"
-
- def default_grid(self):
- return self.args.get('grid', None) == ''
- def default_channel(self):
- return self.args.get('channel', None) == DEFAULT_CHANNEL
-
- def default_channel_for_brand(self):
- if self.viewer_branding_id()=='secondlife':
- return self.args.get('channel', None) == DEFAULT_CHANNEL
- elif self.viewer_branding_id()=="snowglobe":
- return self.args.get('channel', None) == DEFAULT_CHANNEL_SNOWGLOBE
- raise ValueError, "Invalid branding id: " + self.viewer_branding_id()
- def construct(self):
- """ Meant to be overriden by LLManifest implementors with code that
- constructs the complete destination hierarchy."""
- pass # override this method
- def exclude(self, glob):
- """ Excludes all files that match the glob from being included
- in the file list by path()."""
- self.excludes.append(glob)
- def prefix(self, src='', build=None, dst=None):
- """ Pushes a prefix onto the stack. Until end_prefix is
- called, all relevant method calls (esp. to path()) will prefix
- paths with the entire prefix stack. Source and destination
- prefixes can be different, though if only one is provided they
- are both equal. To specify a no-op, use an empty string, not
- None."""
- if dst is None:
- dst = src
- if build is None:
- build = src
- self.src_prefix.append(src)
- self.artwork_prefix.append(src)
- self.build_prefix.append(build)
- self.dst_prefix.append(dst)
- return True # so that you can wrap it in an if to get indentation
- def end_prefix(self, descr=None):
- """Pops a prefix off the stack. If given an argument, checks
- the argument against the top of the stack. If the argument
- matches neither the source or destination prefixes at the top
- of the stack, then misnesting must have occurred and an
- exception is raised."""
- # as an error-prevention mechanism, check the prefix and see if it matches the source or destination prefix. If not, improper nesting may have occurred.
- src = self.src_prefix.pop()
- artwork = self.artwork_prefix.pop()
- build = self.build_prefix.pop()
- dst = self.dst_prefix.pop()
- if descr and not(src == descr or build == descr or dst == descr):
- raise ValueError, "End prefix '" + descr + "' didn't match '" +src+ "' or '" +dst + "'"
- def get_src_prefix(self):
- """ Returns the current source prefix."""
- return os.path.join(*self.src_prefix)
- def get_artwork_prefix(self):
- """ Returns the current artwork prefix."""
- return os.path.join(*self.artwork_prefix)
- def get_build_prefix(self):
- """ Returns the current build prefix."""
- return os.path.join(*self.build_prefix)
- def get_dst_prefix(self):
- """ Returns the current destination prefix."""
- return os.path.join(*self.dst_prefix)
- def src_path_of(self, relpath):
- """Returns the full path to a file or directory specified
- relative to the source directory."""
- return os.path.join(self.get_src_prefix(), relpath)
- def build_path_of(self, relpath):
- """Returns the full path to a file or directory specified
- relative to the build directory."""
- return os.path.join(self.get_build_prefix(), relpath)
- def dst_path_of(self, relpath):
- """Returns the full path to a file or directory specified
- relative to the destination directory."""
- return os.path.join(self.get_dst_prefix(), relpath)
- def ensure_src_dir(self, reldir):
- """Construct the path for a directory relative to the
- source path, and ensures that it exists. Returns the
- full path."""
- path = os.path.join(self.get_src_prefix(), reldir)
- self.cmakedirs(path)
- return path
- def ensure_dst_dir(self, reldir):
- """Construct the path for a directory relative to the
- destination path, and ensures that it exists. Returns the
- full path."""
- path = os.path.join(self.get_dst_prefix(), reldir)
- self.cmakedirs(path)
- return path
- def run_command(self, command):
- """ Runs an external command, and returns the output. Raises
- an exception if the command reurns a nonzero status code. For
- debugging/informational purpoases, prints out the command's
- output as it is received."""
- print "Running command:", command
- fd = os.popen(command, 'r')
- lines = []
- while True:
- lines.append(fd.readline())
- if lines[-1] == '':
- break
- else:
- print lines[-1],
- output = ''.join(lines)
- status = fd.close()
- if status:
- raise RuntimeError(
- "Command %s returned non-zero status (%s) noutput:n%s"
- % (command, status, output) )
- return output
- def created_path(self, path):
- """ Declare that you've created a path in order to
- a) verify that you really have created it
- b) schedule it for cleanup"""
- if not os.path.exists(path):
- raise RuntimeError, "Should be something at path " + path
- self.created_paths.append(path)
- def put_in_file(self, contents, dst):
- # write contents as dst
- f = open(self.dst_path_of(dst), "wb")
- f.write(contents)
- f.close()
- def replace_in(self, src, dst=None, searchdict={}):
- if dst == None:
- dst = src
- # read src
- f = open(self.src_path_of(src), "rbU")
- contents = f.read()
- f.close()
- # apply dict replacements
- for old, new in searchdict.iteritems():
- contents = contents.replace(old, new)
- self.put_in_file(contents, dst)
- self.created_paths.append(dst)
- def copy_action(self, src, dst):
- if src and (os.path.exists(src) or os.path.islink(src)):
- # ensure that destination path exists
- self.cmakedirs(os.path.dirname(dst))
- self.created_paths.append(dst)
- if not os.path.isdir(src):
- self.ccopy(src,dst)
- else:
- # src is a dir
- self.ccopytree(src,dst)
- else:
- print "Doesn't exist:", src
- def package_action(self, src, dst):
- pass
- def copy_finish(self):
- pass
- def package_finish(self):
- pass
- def unpacked_finish(self):
- unpacked_file_name = "unpacked_%(plat)s_%(vers)s.tar" % {
- 'plat':self.args['platform'],
- 'vers':'_'.join(self.args['version'])}
- print "Creating unpacked file:", unpacked_file_name
- # could add a gz here but that doubles the time it takes to do this step
- tf = tarfile.open(self.src_path_of(unpacked_file_name), 'w:')
- # add the entire installation package, at the very top level
- tf.add(self.get_dst_prefix(), "")
- tf.close()
- def cleanup_finish(self):
- """ Delete paths that were specified to have been created by this script"""
- for c in self.created_paths:
- # *TODO is this gonna be useful?
- print "Cleaning up " + c
- def process_file(self, src, dst):
- if self.includes(src, dst):
- # print src, "=>", dst
- for action in self.actions:
- methodname = action + "_action"
- method = getattr(self, methodname, None)
- if method is not None:
- method(src, dst)
- self.file_list.append([src, dst])
- return 1
- else:
- sys.stdout.write(" (excluding %r, %r)" % (src, dst))
- sys.stdout.flush()
- return 0
- def process_directory(self, src, dst):
- if not self.includes(src, dst):
- sys.stdout.write(" (excluding %r, %r)" % (src, dst))
- sys.stdout.flush()
- return 0
- names = os.listdir(src)
- self.cmakedirs(dst)
- errors = []
- count = 0
- for name in names:
- srcname = os.path.join(src, name)
- dstname = os.path.join(dst, name)
- if os.path.isdir(srcname):
- count += self.process_directory(srcname, dstname)
- else:
- count += self.process_file(srcname, dstname)
- return count
- def includes(self, src, dst):
- if src:
- for excl in self.excludes:
- if fnmatch.fnmatch(src, excl):
- return False
- return True
- def remove(self, *paths):
- for path in paths:
- if os.path.exists(path):
- print "Removing path", path
- if os.path.isdir(path):
- shutil.rmtree(path)
- else:
- os.remove(path)
- def ccopy(self, src, dst):
- """ Copy a single file or symlink. Uses filecmp to skip copying for existing files."""
- if os.path.islink(src):
- linkto = os.readlink(src)
- if os.path.islink(dst) or os.path.exists(dst):
- os.remove(dst) # because symlinking over an existing link fails
- os.symlink(linkto, dst)
- else:
- # Don't recopy file if it's up-to-date.
- # If we seem to be not not overwriting files that have been
- # updated, set the last arg to False, but it will take longer.
- if os.path.exists(dst) and filecmp.cmp(src, dst, True):
- return
- # only copy if it's not excluded
- if self.includes(src, dst):
- try:
- os.unlink(dst)
- except OSError, err:
- if err.errno != errno.ENOENT:
- raise
- shutil.copy2(src, dst)
- def ccopytree(self, src, dst):
- """Direct copy of shutil.copytree with the additional
- feature that the destination directory can exist. It
- is so dumb that Python doesn't come with this. Also it
- implements the excludes functionality."""
- if not self.includes(src, dst):
- return
- names = os.listdir(src)
- self.cmakedirs(dst)
- errors = []
- for name in names:
- srcname = os.path.join(src, name)
- dstname = os.path.join(dst, name)
- try:
- if os.path.isdir(srcname):
- self.ccopytree(srcname, dstname)
- else:
- self.ccopy(srcname, dstname)
- # XXX What about devices, sockets etc.?
- except (IOError, os.error), why:
- errors.append((srcname, dstname, why))
- if errors:
- raise RuntimeError, errors
- def cmakedirs(self, path):
- """Ensures that a directory exists, and doesn't throw an exception
- if you call it on an existing directory."""
- # print "making path: ", path
- path = os.path.normpath(path)
- self.created_paths.append(path)
- if not os.path.exists(path):
- os.makedirs(path)
- def find_existing_file(self, *list):
- for f in list:
- if os.path.exists(f):
- return f
- # didn't find it, return last item in list
- if len(list) > 0:
- return list[-1]
- else:
- return None
- def contents_of_tar(self, src_tar, dst_dir):
- """ Extracts the contents of the tarfile (specified
- relative to the source prefix) into the directory
- specified relative to the destination directory."""
- self.check_file_exists(src_tar)
- tf = tarfile.open(self.src_path_of(src_tar), 'r')
- for member in tf.getmembers():
- tf.extract(member, self.ensure_dst_dir(dst_dir))
- # TODO get actions working on these dudes, perhaps we should extract to a temporary directory and then process_directory on it?
- self.file_list.append([src_tar,
- self.dst_path_of(os.path.join(dst_dir,member.name))])
- tf.close()
- def wildcard_regex(self, src_glob, dst_glob):
- src_re = re.escape(src_glob)
- src_re = src_re.replace('*', '([-a-zA-Z0-9._ ]*)')
- dst_temp = dst_glob
- i = 1
- while dst_temp.count("*") > 0:
- dst_temp = dst_temp.replace('*', 'g<' + str(i) + '>', 1)
- i = i+1
- return re.compile(src_re), dst_temp
- def check_file_exists(self, path):
- if not os.path.exists(path) and not os.path.islink(path):
- raise RuntimeError("Path %s doesn't exist" % (
- os.path.normpath(os.path.join(os.getcwd(), path)),))
- wildcard_pattern = re.compile('*')
- def expand_globs(self, src, dst):
- src_list = glob.glob(src)
- src_re, d_template = self.wildcard_regex(src.replace('\', '/'),
- dst.replace('\', '/'))
- for s in src_list:
- d = src_re.sub(d_template, s.replace('\', '/'))
- yield os.path.normpath(s), os.path.normpath(d)
- def path(self, src, dst=None):
- sys.stdout.write("Processing %s => %s ... " % (src, dst))
- sys.stdout.flush()
- if src == None:
- raise RuntimeError("No source file, dst is " + dst)
- if dst == None:
- dst = src
- dst = os.path.join(self.get_dst_prefix(), dst)
- def try_path(src):
- # expand globs
- count = 0
- if self.wildcard_pattern.search(src):
- for s,d in self.expand_globs(src, dst):
- assert(s != d)
- count += self.process_file(s, d)
- else:
- # if we're specifying a single path (not a glob),
- # we should error out if it doesn't exist
- self.check_file_exists(src)
- # if it's a directory, recurse through it
- if os.path.isdir(src):
- count += self.process_directory(src, dst)
- else:
- count += self.process_file(src, dst)
- return count
- try:
- count = try_path(os.path.join(self.get_src_prefix(), src))
- except RuntimeError:
- try:
- count = try_path(os.path.join(self.get_artwork_prefix(), src))
- except RuntimeError:
- count = try_path(os.path.join(self.get_build_prefix(), src))
- print "%d files" % count
- def do(self, *actions):
- self.actions = actions
- self.construct()
- # perform finish actions
- for action in self.actions:
- methodname = action + "_finish"
- method = getattr(self, methodname, None)
- if method is not None:
- method()
- return self.file_list