''' Useful extensions and utilities for yabs. ''' from __future__ import generators import yabs, re, os, string, sys, yabserrors, inspect, traceback separator = ',' '''Used to separate items embedded in generated filenames.''' def _make_env(): '''returns dictionary with basic environment information - OS name, OS version and CPU type.''' try: uname = os.uname() except AttributeError, e: # Windows doesn't have os.uname return { 'os': 'unknown', 'osv': 'unknown', 'cpu': 'unknown'} osv = uname[2] osv = string.replace( osv, '/', '-') osv = string.replace( osv, '(', '{') osv = string.replace( osv, ')', '}') return { 'os': uname[0], 'osv': osv, 'cpu': uname[4]} def _env_to_string( env): ''' Converts an environment into a string. The returned string is formed by concatenating separator with each item in env. e.g.: ,os=CYGWIN_NT-5.1,cpu=i686,osv=1.5.9{0.112-4-2} ''' ret = '' for name,value in env.iteritems(): ret += separator + name + '=' + value return ret #yabs.default_state.env = _make_env() #yabs.default_state.envstring = _env_to_string( yabs.default_state.env) #print 'have set yabs.default_state.envstring:', yabs.default_state, yabs.default_state.envstring def make_space_list( items): '''Returns string consisting of each element in items preceded by a space.''' return reduce( lambda a, b: a + ' ' + b, items, '') def names_in_file( filename, regex): ''' Returns iterator for contents of file split first by line and then split by using the second param as a regex. ''' class internal: def __init__( self, filename, regex): #print 'internal__init__, filename=' + filename self.file = open( filename, 'r') self.regex = re.compile( regex) self.items = [] def __del__( self): #print 'names_in_file.__del__(): self=', self.__dict__ self.file.close() def __getitem__( self, i): while True: if self.items == []: line = self.file.readline() if line == '': raise IndexError, 'end of input' self.items = re.split( self.regex, line) ret = self.items[0] self.items = self.items[1:] if ret != '': return ret return internal( filename, regex) def make_version_numbers( text): '''Returns list of integers, from text assumed to be like "3.4.5". `-' is also treated as a separator - e.g. gcc version strings can be like `3.3.5-20050130'. ''' #print 'make_version_numbers: text=', text assert type( text)==str text = text.replace( '-', '.') substrings=string.split( text, '.') ret = [] for i in substrings: ret.append( int( i)) return ret def compare_version_numbers( a, b): ''' Compares version tuples returned from make_version_numbers(). Returns +1, 0 or -1 depending on whether a is greater, the same or less than b. a and b should be tuples of the form returned by make_version_number(). ''' #print 'compare_version_numbers', a, b assert type(a)==list or type(a)==tuple assert type(b)==list or type(b)==tuple for i in range( 0, max( len(a), len(b))): if i>=len(a): aa = 0 else: aa = a[i] if i>=len(a): bb = 0 else: bb = b[i] assert type( aa)==int assert type( bb)==int if aa > bb: #print 'compare_version_numbers returning 1' return 1 elif aa < bb: #print 'compare_version_numbers returning -1' return -1 return 0 def insert_leafsubdir( filename, subdir): '''Takes filename like foo/bar/pqr and returns foo/bar//pqr .''' head, tail = os.path.split( filename) ret = os.path.join( head, subdir) ret = os.path.join( ret, tail) return ret def split_dir( filename, directory): ''' Searches for element in called . Returns tuple consisting of two lists, containing the directory elements before and after in . If isn't in , first item in tuple contains all elements of , second item is []. Examples: search_for_dir_and_split( '/foo/bar/output/qwerty/abc.doc', 'output') -> ( ['foo', 'bar'], [ 'qwerty', 'abc.doc']) search_for_dir_and_split( '/foo/bar/output/qwerty/abc.doc', 'xyzqwerty') -> ( [ 'foo', 'bar', 'qwerty', 'abc.doc'], []) ''' elements = filename.split( os.sep) try: i = elements.index( directory) except ValueError: i = len( elements) return elements[:i], elements[i+1:] def add_patternrule( target_pattern, pre_pattern, command_pattern, phony = False, root = None, autocmds= None, internal= 0, always = False, categories = [], lockfilename=None, state = None, ): ''' Adds a rule similar to GNU make's % rules. uses % to match any number of characters. In both and , %N is repaced by what the Nth % matched in (N=1,2,...); $@ is replaced by the whole target. In only, $^ is replaced by the prerequisites, $< by the first prerequisite. After % substitution, is split by spaces, and used as the list of prerequisites. Pattern replacement is crude at the moment - there is no escaping of $ or % characters for example. Instead of being a string, can also be a function, taking parameters ( target, prerequisites, match, context). is the regex match object for the target, .out is where to send output text. If is true, the command is always run. This is the equivalent to a raw yabs rule returning a prerequisite of None. If is None, it is set to the directory containing the caller's module. Any non-absolute paths implied by , and are prefixed with . ''' if state is None: state= yabs.default_state if root is None: root = yabs.caller + 1 assert type( target_pattern)==str assert type( pre_pattern)==str or callable( pre_pattern), 'prepattern incorrect:' + repr( pre_pattern) assert type( command_pattern)==str or callable( command_pattern) assert phony is False or phony is True assert isinstance( state, yabs.State) target_pattern = '^' + target_pattern + '$' target_pattern = string.replace( target_pattern, '\\', '\\\\') target_pattern = string.replace( target_pattern, '.', '\.') target_pattern = string.replace( target_pattern, '(', '\(') target_pattern = string.replace( target_pattern, ')', '\)') target_pattern = string.replace( target_pattern, '[', '\[') target_pattern = string.replace( target_pattern, ']', '\]') target_pattern = string.replace( target_pattern, '%', '(.*)') #print 'yabs2.add_patternrule(): target_pattern=', target_pattern, 'root=', root try: regex = re.compile( target_pattern) except Exception, e: print 'yabs2.add_patternrule(): can\'t compile target_pattern regex:', target_pattern raise e def add_patternrule_rulefunction( target, state2): match = re.match( regex, target) if match is None: if state.debug>=4: print 'yabs2.add_patternrule: regex doesn\'t match target:' print target_pattern print target return None #print 'match=', match, dir(match), match.groupdict(), match.groups() # why-oh-why does ''.split(' ') return [''] ? means we have to # special-case an empty pre_pattern: if callable( pre_pattern): prerequisites = pre_pattern() elif pre_pattern=='': prerequisites = [] else: prerequisites = pre_pattern.split( ' ') if isinstance( prerequisites, str): prerequisites = [ prerequisites] pre_list = [] for i in prerequisites: i = i.replace( '\\', '\\\\') i = i.replace( '\n', ' ') i = i.replace( '$@', target) i = i.replace( '%', '\\') i = match.expand( i) pre_list.append( i) if callable( command_pattern): def command_helper( context): text = command_pattern( target, pre_list, match, context) if text!=None and type( text)!=str: raise Exception( 'yabs2 patternrule command function ' + str( command_pattern.func_code) + ' should have returned a None or a string, but has' + ' returned: ' + str( text)) return text command = command_helper else: command = command_pattern command = command.replace( '$@', target) #os.path.join( root, target)) command = command.replace( '\\', '\\\\') command = command.replace( '%%', '$@') # temporarily hde `%%'. command = command.replace( '%', '\\') command = command.replace( '$@', '%') # restore hidden `%%'. if len( pre_list)>0: command = command.replace( '$<', pre_list[0]) command = command.replace( '$^', string.join( pre_list)) command = match.expand( command) # prefix command with a mkdir -p, to ensure that the parent # directory of the target is present. yabs actually ensures that # this directory exists directly prior to running commands, but # adding an explicit mkdir -p means that the output from -n can # be used outside of yabs. # this doesn't work on windows - no mkdir command. if not phony and os.name!='nt': if type( command) == str: d = os.path.split( os.path.abspath( os.path.join( rule.root, target)))[0] d = yabs.relativepath( rule.root, d) if d!='': command = 'mkdir -p "' + d + '"\n'\ + command if always: semi_prereq = [ None] else: semi_prereq = [] return command, pre_list, semi_prereq rule = yabs.add_rule( add_patternrule_rulefunction, phony=phony, root=root, autocmds=autocmds, internal=internal, state=state, targetinfo=target_pattern, categories=categories, lockfilename=lockfilename) return rule def add_patternrule_phony( target_pattern, pre_pattern, command_pattern='', root = None, autocmds= None, internal= 0, categories = [], lockfilename=None, state = None): ''' Like add_patternrule(), but rule is always registered as phony. ''' if state is None: state=yabs.default_state # we use numerical value for to make yabs use our caller's # directory. we could do root=yabs.get_caller(2), but this involve calling # traceback.extract_stack() which is already called inside yabs.add_rule(), # and is a little slow, so we avoid calling it twice this way. if root is None: root = yabs.caller+2 rule = add_patternrule( target_pattern, pre_pattern, command_pattern, phony=True, autocmds=autocmds, root=root, internal=internal, state=state, categories=categories, lockfilename=lockfilename) return rule def add_rules( regex, phony=False, root=None, autocmds=None, autodeps=None, state=None, module=None): ''' Looks for all functions in whose names match (a regex string or a compiled regex). If is not specified, the caller's module is used. Matching functions that take two parameters are passed to yabs.add_rule(), along with the optional , , , and parameters. ''' if module is None: try: f = inspect.currentframe().f_back module = inspect.getmodule( f) finally: del f # to avoid reference cycles #print 'yabs2.add_rules: module=', module if type( regex)==str: regex = re.compile( regex) for fnname, fn in inspect.getmembers( module, inspect.isfunction): if regex.match( fnname): args = inspect.getargspec( fn)[0] if len( args)==2: #print 'yabs2.add_rules: passing to yabs.add_rule: ' + fnname yabs.add_rule( fn, phony=phony, root=root, autocmds=autocmds, autodeps=autodeps, state=state) appmake_help =\ '''Yabs options are: -b Sets default build string. -B Prints default build strings. --changed Assume has been changed by a yabs rule. --cwd Print filenames relative to current directory. -d Increment debugging level. --d Set debug level. -e Detailed control of diagnostics when -s and -S are not enough. The five sub-params are: T: target R: rule C: command O: output E: exception Each of the sub-params should be 0, 1, 2 or 3: 0: never show 1: show with: --summary auto 2: show if rule fails 3: show when running rule. Thus `-e 31222' will always display targets when they are made; if an error occurs, the command, output and any exception will be shown. If `--summary auto' is specified, the rule is displayed. Using E=3 is not useful because exceptions are errors. --echo-prefix Sets prefix used for all command output. -h Show this help. --help Show this help. -j Control concurrency. If is 0, no concurrency. Otherwise build targets as allowed by the resources returned by rules. also governs the maximum concurrency of the default resource. -k Keep going as much as possible after errors. -K Equivalent to: -k -e 30222. Useful when one wants to carry on after errors, but see information only about errors. -l Only used with -j. Only start making new targets if the system's load average is less than (interpreted as a floating-point number). -n Don't run commands. --new Assume is infinitely new. Same as -W. -o Assume is infinitely old. -O Assume doesn't exist. --oldprefix Assume that all filesnames starting with are infinitely old. --periodic-debug Output brief status information every seconds. --prefix Sets prefix used for all yabs diagnostics. --prefix-id Sets prefix to @: --psyco Attempt to use psyco.full() to optimise Yabs execution. --pty Use pty.fork and os.exec to run commands. This appears to work fine on linux, working with commands that read from stdin, while still allowing control of echoing and capture of the output. -s Quiet operation: don't show commands unless they fail. Equivalent to `-e 30233'. -S Very quiet operation: don't show commands or output unless the command fails. Equivalent to `-e 30222'. --setbuf Sets buffering to use for stdout and stderr. -ve sets to system default, 0 is no buffering, 1 is line buffering, other values are buffer size. --statefile Writes current blocking operation to . --summary --summaryf [+] Adds a summary of failures. is comma-separated list of items: 'rule': show the rule that failed. 'command': show command that failed. 'output': show output from rule that failed. 'except': show exception from failed rule. '': ignored '.': ignored 'auto': show just the information that was not output while running the commands - (e.g. if -s is specified, the summary shows the output from commands). This is the default if --summary is not specified. --summaryf writes the summary information to the specified file. If is prefixed with a `+', the data is appended to the file. --system Use os.system to run commands. This works better than the default when commands use stdin, but doesn't support non-echoing or capture of the output. --targets List info about available targets. Usually this is a list of regexes for targets that are defined in terms of a regex. --test-flock Runs a test that Yabs' flock abstraction works. --unchanged Assume has been left unchanged by a yabs rule. -v Print version. -W Assume is infinitely new. Same as --new. --xtermtitle Writes @: into xterm titlebar. Multiple single-hypen options can be grouped together, e.g. `-dds' is equivalent to `-d -d -s'. Option values can also be specified with '=', e.g. '-j=7' is equivalent to '-j 7'. Anything that doesn't start with `-' is taken as a target. ''' def readparams( argv=None, default_targets=None, default_params=None, default_build=None, state=None): ''' Reads GNU-style command-line parameters into state. The requested targets are put into the list state.targets. Also, non-target parameters are put into nontarget_args[]; this allows the original command params to be reused with different targets. ''' if state is None: state=yabs.default_state assert isinstance( state, yabs.State) if state.targets is None: state.targets = list() if argv is None: argv = sys.argv argv_has_flags = False if default_build is None: default_build = 'gcc,debug' state.defaultbuild = default_build prefix = None # some things to allow us to calculate state.nontarget_args: arg_i = [0] # use array to allow gnuargs() to modify this. target_args = [] # indexes of args that are actual targets. concurrent_jobs = None concurrent_max_load_average = None #print 'yabs2.readparams(): argv=', argv def gnuargs( args): ''' Simple generator that expands gnu-style single-hypen parameters into individual parameters, making for easier parsing. ''' for arg in args: if arg.startswith( '-'): # split things like '-j=8'. eq = arg.find( '=') if eq >= 0: for a in arg[:eq], arg[eq+1:]: #print 'yielding:', a yield a continue if arg.startswith( '--'): yield arg elif arg=='-': yield arg elif arg.startswith( '-'): for a in arg[1:]: yield '-' + a else: yield arg arg_i[0] += 1 args = gnuargs( argv[1:]) while True: try: arg = args.next() except StopIteration: break try: #print 'looking at param', arg #print 'debug=', state.debug, 'echo=', state.echo if arg[0]!='-': state.targets.append( os.path.abspath( arg)) target_args.append( arg_i[0]) else: argv_has_flags = True if arg=='-': pass elif arg=='-b': state.defaultbuild = args.next() state.defaultbuild2 = '' if state.debug>1: print 'yabs: default build is ' + state.defaultbuild elif arg=='-B': import yabs3 print 'yabs: default build is: ' + state.defaultbuild print 'yabs: expanded build is: ' + yabs3._get_defaultbuild( state) elif arg=='--changed': state.targetcache[ os.path.abspath( args.next())] = None, yabs.changed elif arg=='--cwd': state.show_cwd_paths = True elif arg=='-d': state.debug += 1 elif arg[:3]=='--d': state.debug = int( arg[3:]) elif arg=='-e': x = args.next() if len( x)!=5: raise RuntimeError( 'expected 5-digit param after `-e\', not: ' + str( x)) state.echo_target = int( x[0]) state.echo_rule = int( x[1]) state.echo_command = int( x[2]) state.echo_output = int( x[3]) state.echo_exception = int( x[4]) elif arg=='--echo-prefix': state.echo_prefix = args.next() elif arg=='-h': print appmake_help, elif arg=='--help': print appmake_help, #elif arg=='-j': state.mt_init( int( args.next())) elif arg=='-j': concurrent_jobs = int( args.next()) elif arg=='-k': state.keepgoing = True elif arg=='-l': concurrent_max_load_average = float( args.next()) elif arg=='-K': state.echo_target = 3 state.echo_rule = 0 state.echo_command = 2 state.echo_output = 2 state.echo_exception = 2 state.echo_timing = 0 state.keepgoing = True state.summary.append( ( 'auto', None)) state.summary.append( ( '.', None)) elif arg=='-n': state.dryrun = True elif arg=='-o': yabs.mtime_markold( args.next(), state.mtimecache) elif arg=='-O': yabs.mtime_mark_nonexistent( args.next(), state.mtimecache) elif arg=='--oldprefix': yabs.mtime_add_oldprefix( args.next(), state.mtimecache) elif arg=='--periodic-debug': state.periodic_debug_interval = int( args.next()) elif arg=='--prefix': state.prefix = args.next() elif arg=='--prefix-id': state.prefix = os.environ[ 'USER'] + '@' + os.uname()[1] + ': ' elif arg=='--psyco': try: import psyco except ImportError: print yabs.place(), 'failed to import psyco, continuing.' else: psyco.full() elif arg=='--pty': state.subprocessfn = yabs._subprocess_pty elif arg=='-s': state.echo_target = 3 state.echo_rule = 0 state.echo_command = 2 state.echo_output = 3 state.echo_exception = 2 elif arg=='-S': state.echo_target = 3 state.echo_rule = 0 state.echo_command = 2 state.echo_output = 2 state.echo_exception = 2 elif arg=='--setbuf': sys.stdout = os.fdopen( os.dup( sys.stdout.fileno()), 'w', int( args.next())) sys.stderr = os.fdopen( os.dup( sys.stderr.fileno()), 'w', int( args.next())) elif arg=='--statefile': state.statefile = args.next() elif arg=='--summary': state.summary.append( ( args.next(), None)) elif arg=='--summaryf': summary, filename = args.next(), args.next() os.system( 'echo -n > "' + filename + '"') state.summary.append( ( summary, filename)) elif arg=='--system': state.subprocessfn = yabs._subprocess_ossystem elif arg=='--targets': for root, pattern in sorted( yabs.known_targets()): if root: print os.path.join( root, pattern) else: print pattern elif arg=='--test-flock': print 'testing flock...' yabs.test_flock() elif arg=='--unchanged': state.targetcache[ os.path.abspath( args.next())] = None, yabs.unchanged elif arg=='-W' or arg=='--new': yabs.mtime_marknew( args.next(), state.mtimecache) elif arg=='--xtermtitle': state.premake_fn = yabs.xtermtitle_usertarget else: raise RuntimeError( 'Unrecognised parameter ' + arg) except StopIteration: raise Exception( 'Expected more parameters at end of command line') if concurrent_jobs is not None or concurrent_max_load_average is not None: state.mt_init( concurrent_jobs, concurrent_max_load_average) if state.summary == []: state.summary = [ ( 'auto', None)] if not argv_has_flags and default_params: # repeat, this time with default params. #print 'using default params:', default_params if type( default_params)==str: default_params = default_params.split() new_params = default_params new_params += argv[1:] state.targets = [] default_params.insert( 0, argv[0]) readparams( new_params, default_targets, None, default_build, state) if state.targets==[] and default_targets is not None: if type( default_targets)==str: default_targets = [default_targets] state.targets = default_targets # copy all non-target arguments into state.nontarget_args: #print 'target_args=', target_args state.nontarget_args = [] # don't use enumerate - not available on python-2.2. i = 0 for arg in argv[1:]: if i not in target_args: state.nontarget_args.append( arg) i += 1 def appmake2( state=None): ''' Expected to be called after readparams. Makes whatever targets are have been specified. Calls yabs.handle_sighup(state) so that SIGHUP will cause a diagnostic to be generated showing current state. ''' if state is None: state=yabs.default_state assert isinstance( state, yabs.State) yabs.handle_sighup( state) if len( state.targets) == 0: return elif len( state.targets) > 1 or state.mt: # make dummy top-level target, so that -k and the error reporting works # identically for top-level and intermediate targets. # # The `... or state.mt' is a hack to overcome a bug in multi-threaded # mode when there is one target - in this case, yabs.make() doesn't # return, due to loack of a notification when waiting for results. See # test32. toplevel_target = '/_yabs_multiple_targets' def toplevel_rule( target, state): if target==toplevel_target: return '', state.targets yabs.add_rule( toplevel_rule, phony=True, root=None) else: toplevel_target = state.targets[0] #print yabs.place(), toplevel_target ret = yabs.make( toplevel_target, state=state) if ret is yabs.changed or ret is yabs.unchanged: pass else: def prefix(): ''' todo: this is a copy of the prefix() fn in yabs.start_make(); should move code into separate fn. ''' p = state.prefix if p is None: p = state.prefix if callable( p): p = p() return p + 'yabs summary: ' for summary, whereto in state.summary: if whereto is None: whereto = sys.stdout elif whereto.startswith( '+'): whereto = open( whereto[1:], 'a') else: whereto = open( whereto, 'w') #whereto.write( prefix() + 'summary=' + summary + ':\n') yabs.print_failures( whereto=whereto, prefixes=prefix(), target=toplevel_target, rule=None, e=ret, depth=0, state=state, format=summary, recurse=True) if whereto != sys.stdout: whereto.close() yabs.mt_exit( state) return ret def appmake2exit( state=None): ''' Expects to be called after readparams. Makes whatever targets are have been specified, then calls sys.exit(0) or sys.exit(1). ''' ret = appmake2( state=state) if ret==yabs.changed or ret==yabs.unchanged: sys.exit(0) sys.exit(1) def appmake( argv=None, default_targets=None, default_params=None, default_build=None, state=None): ''' Behaves like GNU make. Reads flags and targets from , and then makes each target. Accepts yabs exceptions and converts them into an error message and a non-zero return code. ?. If everything succeeds, returns 0. can be a single target, or list/tuple of targets. It is used as the target(s) if no target is specified in argv. Similarly, is either a command-line style space-separated list of parameters or a list of parameters, and is used if doesn't contain any parameters. ''' readparams( argv, default_targets, default_params, default_build, state) return appmake2( state) def appmakeexit( argv=None, default_targets=None, default_params=None, default_build=None, state=None): ''' Intended to be called from a Yabs build script. As appmake(), but also calls sys.exit() with an appropriate exit code. ''' ret = appmake( argv, default_targets, default_params, default_build, state) #print yabs.place(), 'ret=', repr(ret) if ret==yabs.changed or ret==yabs.unchanged: sys.exit(0) sys.exit( 1)