#!/usr/bin/env python ''' Yabs - Yet Another Build System. Yabs is a simple build system, in the form of a Python library. Rules are expressed as python functions that are registered with Yabs using the add_rule() function. Targets are built by calling the make() function. Yabs requires python-2.2 or later. ''' from __future__ import generators import commands, exceptions, os, signal, string, sys, time, traceback, thread, threading, yabserrors # First check that we are running under a recent-enough # version of Python. This python-version trick was lifted from # http://tinyurl.com/3yhry. def _initial_init( s): ''' like int(), but ignores any trailing non-digits, instead of raising `ValueError: invalid literal for int()' ''' for i in range( len( s)): if not s[i].isdigit(): return int( s[:i]) return int( s) python_version_text = string.split( sys.version)[ 0] python_version_list = map( _initial_init, string.split( python_version_text, '.')) if python_version_list < [ 2, 2]: raise Exception( 'yabs: python version appears to be ' + python_version_text + '; yabs requires python-2.2 or later.') def _trace( state, info): ''' Writes `info' to .statefile, if it exists. Used to maintain information about the command/function that Yabs is running, useful when building large projects with echoing turned off. ''' if state.statefile: f = open( state.statefile, 'w') try: f.write( str( info).strip() + '\n') finally: f.close() def place( n=1): ''' Debugging fn; returns representation of source position of caller. ''' tb = traceback.extract_stack( None, 1+n) filename, line, function, text = tb[0] ret = os.path.split( filename)[1] + ':' + str( line) + ':' + function + ':' if 0: tid = str( threading.currentThread()) ret = '[' + tid + '] ' + ret return ret def callers(): tb = traceback.extract_stack( None, None) ret = 'thread=' + str( threading.currentThread()) i = 0 for filename, line, function, text in tb: ret += ' ' + filename + ':' + str( line) + ':' + function + '\n' return ret def subprocess( command, echo=None, state=None, fn=None, endtime=None, prefix=None): ''' Runs using , which defaults to .subprocessfn, which in turn defaults to _subprocess_popen. If specified, should be an absolute time in seconds since epoc, as returned by time.time(), and the command is terminated at this time. This is not supported by all _subprocess_* functions. can be a string, or a function that returns a string; it will be used to prefix all lines returned in (and echoed to screen if is set). If None, is used. Returns (e, text), where is the integer return code, and is the text output by the command. ''' if state is None: state=default_state if echo is None: echo=state.echo_output if echo<0: echo = 0 if prefix is None: prefix = state.echo_prefix if not fn: fn = state.subprocessfn if state.debug>=5: print place(), 'command=', command print place(), 'echo=', echo print place(), 'prefix=', prefix print place(), 'fn=', fn print place(), 'state.subprocessfn=', state.subprocessfn print place(), 'caller=', callers() _trace( state, command) e, text = fn( command, echo=echo, state=state, endtime=endtime, prefix=prefix) return e, text def subprocess_text( command, echo=None, state=None, fn=None, endtime=None, prefix=None): ''' Returns just the text part of the return from subprocess() function. Raises exception if the subprocess() function indicates an error. ''' e, text = subprocess( command, echo, state, fn, endtime, prefix) if e: raise yabserrors.command_error( command, e, text) return text def _subprocess_pty( command, echo=False, state=None, endtime=None, prefix=None): ''' Implementation of subprocess() that uses pty. ''' import pty, select, threading if state is None: state = default_state if prefix == None: prefix = state.echo_prefix #print 'endtime=', endtime if state.debug>1: print '_subprocess_pty: ', command pid, fd = pty.fork() #print '_subprocess_pty: pid,fd=', pid, fd if pid==0: #print 'child:' #args = command.split( ' ') # should handle \ escapes args = [ '/bin/sh', '-c', command] try: os.execvp( '/bin/sh', args) #try: os.execvp( args[0], args) except OSError, e: print e sys.exit( e.errno) # should only get OSError from os.execvp, but just in case: except Exception, e: print 'os.execvp failed:', e sys.exit( 1) else: #print 'parent' if echo: out = _PrefixOutput( prefix, sys.stdout) else: out = _PrefixOutput( prefix, None) def threadfn(): pollblock = select.poll() pollblock.register( fd, select.POLLIN) pollblock.register( sys.stdin, select.POLLIN) while 1: if endtime: timeout = 1000 * (endtime-time.time()) if timeout<1: timeout = 1 pollresult = pollblock.poll( timeout) else: pollresult = pollblock.poll() #print 'pollresult=', pollresult if len( pollresult)==0: # timeout c = 'yabs._subprocess_pty(): pid=' + str(pid) + ', timeout exceeded.\n' print c, out.buffer += c try: # give it some time to exit gracefully, then # kill it brutally. hope the pid isn't reused # while we sleep... os.kill( pid, signal.SIGTERM) time.sleep( 5) os.kill( pid, signal.SIGKILL) except OSError: pass for f, r in pollresult: assert f==fd or f==sys.stdin.fileno() if r & select.POLLERR or r & select.POLLHUP: return assert r & select.POLLIN if f==fd: c = os.read( f, 1) out.write( c) elif f==sys.stdin.fileno(): l = sys.stdin.readline() os.write( fd, l) t = threading.Thread( None, threadfn) t.start() dummypid, status = os.waitpid( pid, 0) if ( os.WIFSIGNALED( status)): print 'subprocess_pty: signal ', os.WTERMSIG( status) e = 1 elif ( os.WIFEXITED( status)): e = os.WEXITSTATUS( status) else: raise Exception( 'subprocess_ptr: child has stopped?') t.join() os.close( fd) #print '_subprocess_pty: returning ', e, text return e, out.buffer def _subprocess_ossystem( command, echo=False, state=None, endtime=None, prefix=None): ''' Implementation of subprocess() that uses os.system. Doesn't support , and returns empty output string. ''' #print place(), command if endtime != None: raise Exception( 'yabs._subprocess_ossystem() doesn\'t support ') return os.system( command), '' def _subprocess_popen( command, echo=False, state=None, endtime=None, prefix=None): ''' Implementation of subprocess() that uses os.popen. Doesn't support . Runs specified command, returning (e, t), where is return code and is the text from command's stdout and stderr. if is True, also outputs line-by-line using print. ''' #print place(), command #print '_subprocess_popen: command=', command, 'echo=', echo assert type( command)==str if state is None: state = default_state if prefix is None: prefix = state.echo_prefix if endtime != None: raise Exception( 'yabs._subprocess_popen() doesn\'t support ') if os.name=='nt': # the following works on windows/python-24 - see test12 stdin, out = os.popen4( command) text = '' while True: l = out.readline() if l=='': break text += l if echo: print l, if echo and l!='' and not l.endswith( '\n'): print stdin.close() e = out.close() if e is None: e = 0 else: # this is based on commands.getstatusoutput(). differences # are that we read a line at a time, so that we can echo # to the screen while the command executes. # there should be a better way of capturing both stdout and # stderr, than using shell redirection as here, but i can't # find one - tried popen2 for example. # maybe python2.4's subcommand will provide a nice way. if echo: out = _PrefixOutput( prefix, sys.stdout) else: out = _PrefixOutput( prefix, None) pipe = os.popen('{ ' + command + '; } 2>&1', 'r') #print place(), pipe e = None while True: try: line = pipe.readline() #print place(), repr( line) except exceptions.KeyboardInterrupt, e: print 'yabs: pipe_readline() returned exception:', e, e.__class__ line='' e = 0x0002 # fake a SIGINT if line=='': break # eof out.write( line) if e is None: e = pipe.close() else: pipe.close() if e is None: e = 0 if 0: # popen2.popen4 doesn't seem to send out stdin to the child. import popen2 # popen2.popen4 doesn't seem to capture stderr, so we do it by hand. p = popen2.Popen4( command + ' 2>&1') text = '' while True: l = p.fromchild.readline() if l=='': break text += l if echo: print l, if echo and l!='' and not l.endswith( '\n'): print e = p.wait() return e, out.buffer # Things for concurrency. Yabs has a single lock. Most of Yabs runs with # this lock held. The lock is released only while running commands to # build targets. There is also a single condition, which is signaled # whenever things have changed, allowing blocked threads to attempt to # claim resources or detect that targets have been built. def mt_exit( state): if not state.mt: return state.lock() try: state.mt_exited = True state.mt_condition.notifyAll() finally: state.unlock() def mt_target_fn( ruleresult, target, state): ''' run in separate thread by mt_spawntasks(). builds , then releases the 's resource and calls state.mt_condition.notifyAll(). ''' state.lock() try: try: if state.debug>=2: print state.prefix(), 'building:', ruleresult.target run_ruleresult( ruleresult, state) #state.mt_condition.notifyAll() except Exception, e: traceback.print_exc() os.abort() finally: if state.debug>=2: print prefix(), 'built: ', ruleresult.target # release the resource and tell everyone # that they may be able to claim a new # resource: ruleresult.resource.release() state.mt_condition.notifyAll() del state.mt_threads[ threading.currentThread()] state.unlock() def mt_spawntasks( state): ''' Repeatedly attempts to claim resources required to build targets in .mt_targets. When a target's resource is acquired, we spawn a new thread to build that target. We wait on state.mt_condition. We always scan targets from the first target, so if more than target share a resource, the earliest target in .mt_targets will always be built first. Currently there is no attempt to reuse threads. We return when .mt_exited is set. ''' state.lock() try: try: while 1: if state.mt_exited: if state.debug>=2: print place(), 'mt_exited set, exitting' break i = 0 #print place(), 'mt_targets=', state.mt_targets while i < len( state.mt_targets): target, ruleresult, prefix = state.mt_targets[i] if ruleresult.resource.tryacquire(): # we can spawn a new thread to run the rule result: del state.mt_targets[i] t = threading.Thread( target=mt_target_fn, name=ruleresult.target, args=( ruleresult, target, state)) state.mt_threads[ t] = ruleresult.target, len( state.mt_threads) if state.debug>=4: print 'starting new thread: ' + str( len( build.state.mt_threads))\ + ': ' + t.getName() if state.debug>=5: print 'new thread is for:', ruleresult.target t.start() i = 0 else: #print place(), 'couldn\'t acquire resource:', ruleresult.resource i += 1 state.wait() #print place() + 'state.wait() returned' finally: state.unlock() except Exception, e: traceback.print_exc() assert 0, str( e) class mt_Resource: ''' Simple default resource for use by concurrent yabs. The resource can have up to users. We are protected by state's mutex. Note that resources are not owned by specific threads. ''' def __init__( self, n=1): self.n = n self.count = 0 def tryacquire( self): ''' Returns True if resource was acquired, else returns False. ''' try: assert self.count <= self.n if self.count < self.n: self.count += 1 #print 'giving access to default resource, count=', self.count return True else: #print 'refuxing access to default resource, count=', self.count return False finally: pass def release( self, n=0): try: assert self.count > 0 self.count -= 1 #print 'releasing resource, count=', self.count finally: pass def __str__( self): return 'n=%i, count=%i' % ( self.n, self.count) return 'n=%i, count=%i, owners=%s' % ( self.n, self.count, str( self.owners)) # tdb_* - thread debugging. # # is a mapping of thread id to the mutex/condition that # the thread is blocked on. # # is mapping of mutex to thread that owns the mutex. # # protects and . # tdb_threads = {} tdb_mutexes = {} tdb_lock = threading.Lock() def tdb_str(): ret = '' ret += '\nblocked threads:\n' for t, ( m, c) in tdb_threads.items(): ret += ' t=' + str( t) + ', m=' + str( m) + '\n' ret += c #for cc in c: # ret += str( cc) + '\n' ret += '\nowned mutexes:\n' for m, t in tdb_mutexes.items(): ret += ' m=' + str( m) + ', t=' + str( t) + '\n' return ret return '\nblocked threads:' + str( tdb_threads) + '\nowned mutexes:' + str( tdb_mutexes) def tdb_willdeadlock( m): ''' if attempting to acquire mutex will deadlock, returns list of ,,,... that describe the deadlock. Otherwise returns False. ''' ret = [] ret.append( m) while 1: #print 'ret=', ret t = tdb_mutexes.get( m, None) if t is None: return False ret.append( t) if t == threading.currentThread(): #return True print 'deadlock detected:', ret return ret m, c = tdb_threads.get( t, ( None, None)) ret.append( ( m, c)) def tdb_acquire( mutex): ''' acquires , asserting if there would be a deadlock. ''' tdb_lock.acquire() try: assert tdb_mutexes.get( mutex, None) != threading.currentThread() assert not tdb_threads.has_key( threading.currentThread()) assert not tdb_willdeadlock( mutex) tdb_threads[ threading.currentThread()] = mutex, callers finally: tdb_lock.release() mutex.acquire() tdb_lock.acquire() try: del tdb_threads[ threading.currentThread()] tdb_mutexes[ mutex] = threading.currentThread() finally: tdb_lock.release() def tdb_release( mutex): ''' releases . ''' tdb_lock.acquire() try: assert not tdb_threads.has_key( threading.currentThread()) assert tdb_mutexes[ mutex] == threading.currentThread() del tdb_mutexes[ mutex] finally: tdb_lock.release() mutex.release() def tdb_wait( mutex, condition): ''' does a wait on ,. ''' tdb_lock.acquire() try: assert tdb_mutexes.get( mutex, None) == threading.currentThread() assert not tdb_threads.has_key( threading.currentThread()) del tdb_mutexes[ mutex] tdb_threads[ threading.currentThread()] = condition, callers() finally: tdb_lock.release() condition.wait() tdb_lock.acquire() try: del tdb_threads[ threading.currentThread()] tdb_mutexes[ mutex] = threading.currentThread() finally: tdb_lock.release() if 0: # test deadlock-detection: l = threading.Lock() tdb_acquire( l) assert tdb_willdeadlock( l) if 0: l1 = threading.Lock() l2 = threading.Lock() l3 = threading.Lock() def a(): tdb_acquire( l1) time.sleep(1) tdb_acquire( l2) def b(): tdb_acquire( l2) time.sleep(1) tdb_acquire( l3) def c(): tdb_acquire( l3) time.sleep(1) tdb_acquire( l1) print 'running deliberate deadlock' t1 = threading.Thread( target=a) t2 = threading.Thread( target=b) t3 = threading.Thread( target=c) t1.start() t2.start() t3.start() t1.join() t2.join() t3.join() class MtimeCache: ''' Caches modification times for files. Also maintains statistics about the number of times we have called os.stat(). ''' def __init__( self): self.cache = dict() self.oldprefixes = [] self.num_nonstat = 0 self.num_stat = 0 class NotifyingDict( dict): ''' A dictionary that calls a specified fn when it is modified. ''' def __init__( self, fn): ''' Modifications will cause () to be called. ''' self.fn = fn def __setitem__( self, i, v): dict.__setitem__( self, i, v) self.fn() class State: ''' Container for all state used by yabs functions. self.premake_fn is called before each target is run, passing and . ''' def __init__( self): self.rules = list() self.targetcache = NotifyingDict( lambda self=self: self.notifyAll()) self.mtimecache = MtimeCache() self.rulecache = dict() self.defaultbuild = '' # echo defaults: self.echo_target = 3 self.echo_rule = 1 self.echo_command = 3 self.echo_output = 3 self.echo_exception = 3 self.dryrun = False self.debug = 0 self.targets = list() self.pending_targets= [] self.autocmds_rules = dict() self.show_cwd_paths = False self.echo_prefix = '' # prefix for output from commands self.subprocessfn = _subprocess_popen self.premake_fn = None self.keepgoing = False self.statefile = None self.summary = [] self.prefix = '' # prefix for yabs diagnostics. self.xtermtitle_target = None # concurrency: self.mt = 0 # default to no concurrency. self.mt_exited = False self.mt_targets = [] # all additions to this must be followed by # state.mt_condition.notifyAll(). Unlike self.targetcache(), # self.mt_targets doesn't do this automatically. Really # need a NotifyingList class. def notifyAll( self): if self.mt: self.mt_condition.notifyAll() def mt_init( self, mt): ''' If is not zero, activates concurrency. Also sets the default resource so that we use up to separate threads for default targets. ''' if self.mt: raise Exception( 'mt_init() called twice') assert mt>=0 if mt>0: self.mt = mt self.mt_mutex = threading.Lock() self.mt_condition = threading.Condition( self.mt_mutex) self.mt_targets = [] self.mt_threads = { threading.currentThread(): ( '', 0)} self.mt_default_resource = mt_Resource( mt) self.mt_spawnthread = threading.Thread( target=mt_spawntasks, args=( self,), name='mt_spawntasks') self.mt_spawnthread.start() def lock( self): if self.mt: tdb_acquire( self.mt_mutex) def unlock( self): if self.mt: tdb_release( self.mt_mutex) def wait( self): if self.mt: tdb_wait( self.mt_mutex, self.mt_condition) default_state = State() ''' A global state object for Yabs. Many of the Yabs functions default to using this global if the user doesn't specify the state. There is usually little need to use a different state object. ''' class changed: pass class pending: pass class unchanged: pass class Rule: ''' Represents a rule. Used only by add_rule(). We store a complete traceback so that if a rule fails, we can display who created the rule. We require our caller to pass us this backtrace. ''' def __init__( self, rule_fn, phony, root, autocmds, autodeps, internal, backtrace): ''' Only called from yabs.add_rule(). must be either None or a string ending with os.sep. should be from traceback.extract_stack(), and should exclude the yabs.add_rule() fn. we use it to calculate if is an integer, and it is also used in diagnostics if a rule fails. thus the design only requires one call to the relatively slow traceback.extract_stack() function. ''' assert callable( rule_fn) or type( rule_fn)==str assert phony is False or phony is True assert root is None or type( root)==str if type(root)==str: assert root.endswith( os.sep) # have disabled this because it makes things difficult to # debug. we prefer out caller to use yabs.fn() explicitly. #if type( rule_fn)==str: # #print 'calling fn, rule_fn=', rule_fn # rule_fn = fn( rule_fn, up=1) assert callable( rule_fn) self.rule_fn = rule_fn self.phony = phony self.root = root self.autocmds = autocmds self.autodeps = autodeps self.backtrace = backtrace self.internal = internal def __repr__( self): return 'fn=' + str( self.rule_fn.func_code) caller = 1 ''' A constant for use with get_caller_directory() and the parameter in add_rule(). ''' def get_caller_directory( n=caller): ''' Returns absolute path of the directory containing the python module corresponding to the function levels up the call stack. Directory's trailing os.sep is retained. E.g. yabs.get_caller_directory(0) returns the directory containing the yabs.py module (which contains the get_caller_directory() function itself). get_caller_directory(1) returns the directory containing the caller's module. get_caller_directory(2) returns the directory containing the caller's caller's module. Usually get_caller_directory(1) is the desired call. yabs.caller is a constant that gives the same effect, e.g.: yabs.get_caller_directory( yabs.caller). ''' assert n>=0 tb = traceback.extract_stack( None, 1+n) # traceback.extract_stack() puts leaf fns at end of the array. we've # deliberately asked for the n+1 items most recent fns, so the first item # is the one we want. filename, line, function, text = tb[0] root = os.path.split( os.path.abspath( filename))[0] + os.sep # very occasionally, we seem to end up with an incorrect directory. # touching yabs.py stops this error, so this is just to enable a # non-invasive way of turning checking on. if os.environ.has_key( 'YABS_SANITY') and 'home/home' in root: print '***' print 'possible problem in yabs.get_caller_directory' print 'tb=', tb print 'filename=', filename print 'os.path.abspath( filename)=', os.path.abspath( filename) print 'root=', root print '***' return root # The following fns find the globals()/locals() dict's of caller # function levels up the stack. based on code in: # http://groups.google.co.uk/group/comp.lang.python/browse_thread/thread/c8750a0adefe0c30/cb738b32e45a3bd0?lnk=st&q=%22Steven+D.+Majewski%22+uplocals&rnum=1&hl=en#cb738b32e45a3bd0 def _linearize( head, link_attr ): #print 'linearize: head = ', head list = [] obj = head while obj: list.append( obj ) obj = getattr( obj, link_attr ) #print 'linearize: returning ', list return list def _callers(): # traceback doesn't seem to give a way of directly # getting complete info about the current stack. # so we use the lower-level `sys.exc_info()' which # gives info about the current stack from within # an exception handler. try: raise Exception() except: # The original code used: # tb = sys.exc_traceback.tb_frame # but we try to be thread-safe by using sys.exc_info(). tb = sys.exc_info()[2] try: #print 'tb=', tb traces = _linearize( tb.tb_frame, 'f_back' ) list = [] for frame in traces : list.append( frame ) return list[1:] # return my caller's info finally: # avoid cycles to help the garbage collector del tb def _upglobals( n ): return _callers()[n+1].f_globals def _uplocals( n ): return _callers()[n+1].f_locals def _upfname( n ): return _callers()[n+1].f_code.co_name def fn( text, globals_=None, locals_=None, up=None): ''' Returns an anonymous function created by calling exec on . should be a string containing a Python function definition, ommiting the initial `def ' (actually, leading `def' can be retained, for clarity). Any leading/trailing white space is removed using str.strip(). Leading white space on all lines will be handled automatically by exec, so you can use an indented python triple-quoted string. In addition, newlines and backslashes are re-escaped inside single/double-quoted strings. this enables nested use of fn(). e.g.: yabs.fn( """ def ( target, state): if target != 'foo': return None return [], None, 'touch foo' """) If globals_/locals_ are not specified, fn() will find its caller's globals/locals using yabs._upglobals()/yabs._uplocals(). If is specified, yabs.fn() will look at the stack frame levels up. Unfortunately the use of globals_/locals_ doesn't work in exactly the way one would hope - the resultant function doesn't seem to be able to access locals variables. E.g.: import yabs def a(): x = 42 f = yabs.fn( """ def (): print x # fails def g(): print x # succeeds f() g() a() See make.py:test22(). The created function will have a .func_code attribute like conventional functions. Unfortunately this attribute is read-only, and the only element that we can control is .func_code.co_name, which we set to encode our caller's filename, function and line number (using '_' for non-alphanumberic characters). An example of a .func_code passed through str() is: import yabs f = yabs.fn( """ def(): pass """) print f.func_code - which outputs: ", line 1> ''' if up is None: if globals_ is None: globals_ = _upglobals(1) #print 'yabs.fn(): have set globals_ to:', globals_ if locals_ is None: locals_ = _uplocals(1) #print 'yabs.fn(): have set locals_ to:', locals_ else: assert globals_ is None assert locals_ is None globals_ = _upglobals( 1+up) locals_ = _uplocals( 1+up) #print #print 'yabs.fn: globals_=', globals_ #print 'yabs.fn: locals_=', locals_ text = text.strip() if text.startswith( 'def'): text = text[ 3:].strip() if not text.startswith( '('): raise Exception( 'fn string must start with `(\': ', repr( text)) def _escape_quotes( text): ''' escape newlines and backslashes that are inside single/double-quoted strings. should probably do something about backslashes inside triple-quoted strings, but we don't bother for now. ''' quote = None ret = '' for c in text: if quote: if c=='\n': ret += '\\n' elif c=='\\': ret += '\\\\' else: if c==quote: quote = None ret += c else: if c=='\'' or c=='"': quote = c ret += c return ret text = _escape_quotes( text) #print 'execing:', text # the exec will put the fn in the locals_ dict. we return it after # removing it from this dict. we construct a function name that # roughly encodes our caller's file, function and line. if up is None: tb = traceback.extract_stack( None, 2) else: tb = traceback.extract_stack( None, 2+up) #print 'tb[0]=', tb[0] tb_filename, tb_line, tb_function, tb_text = tb[0] fnname = '_yabs_fn_temp_' for i in tb_filename + '_' + tb_function: if i in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_': fnname += i else: fnname += '_' fnname += '_' + str( tb_line) # now create the function using exec. exec ( 'def ' + fnname + text) in globals_, locals_ ret = locals_[ fnname] del locals_[ fnname] if 0: # make the new function have some properties of our caller's # function. unfortunately these attributes are read-only, # so this doesn't work. tb = traceback.extract_stack( None, 1) filename, line, function, text = tb[0] ret.func_code.co_filename = filename ret.func_code.co_firstlineno = line return ret def add_rule( rule_fn, phony=False, root=None, autocmds=None, autodeps=None, internal=0, state=None): ''' Adds a new Yabs rule. should be a function that takes a target filename and a context (an object with a .state member that is the Yabs state, and a .out member to which text output should be sent). If the rule cannot create the target, it should return None. Otherwise, it should return a tuple with an initial portion of the following list: A string consisting of individual commands separated by newlines, or an empty string if no command is necessary. A list of prerequisites (files that are required to build the target). : A list of files that don't have to exist, but will cause a remake of if they can't be built or are newer than . A function that will be called after all prerequisites and semi-prerequisites have been created, which should return any additional prereqistes that can only be specified by examining the existing prerequisites. This function takes a single State parameter, and should return a list of additional prerequisites. It is repeatedly called until it returns an empty list. A number that is added to the rule's value. This is useful if a rule applies to different types of targets which require different levels of diagnostics. Resource required by the rule when it runs. Only used if Yabs is concurrent. must provide two methods: tryacquire(): Returns True if resource is acquired, or False. release(): Releases resource. *Disabled*: if is a string, it will be compiled into an anonymous function using yabs.fn(). If a command is prefixed with a '-', any error it returns is ignored. Rules are inserted at front of 's rules, so rules added later will be found first by the yabs.make() function. The returned prerequisites and semi-prerequisites can instead be functions, which take a single state parameter, which return a list of filenames when called. Alternatively, they can be a string, which is converted into a single (semi-)prerequisite. Similarly, the returned command can be a function taking a single state parameter that returns a command-string or None, or raises an exception. This function can generate the target itself directly, but it is usually better to return a command-string to allow autocmds and autodeps to work. A prerequisite or semi-prerequisite that is None is treated specially - it ensures that a target is always remade. If is True, Yabs never looks at 's datestamp, and doesn't require that exists after the rule's commands are run. The target is still treated as a real filename though - it will be converted to an absolute filename using (if specified) or the current directory. If is None, the rule is always passed absolute filenames, and must return absolute filenames for both prerequisites and semi-prerequisites. Otherwise, can be an integer, which identifies a function in the backtrace, whose module's directory is taken as the root for the rule. E.g. root=yabs.caller will use the module of the function that called yabs.add_rule(). root=yabs.caller+1 will use this function's caller's module etc. Otherwise, must be an absolute path, (optionally ending with os.sep) and all targets will be given relative to . Any prerequisite or semi-prerequisite returned from that are not an absolute path will have prepended. The command returned from the rule will be run with the current directory set to . Alternatively, can be yabs.caller, in which case yabs will replace it with the directory containing the file that calls yabs.add_rule(). If not None, is treated as a filename suffix that is appended to a target filename to form a dependency filename. If the dependency file exists, its contents are treated like extra semi-prerequisites. When the rule's commands are run, all files that are opened for reading by these commands are logged to the dependency file. Note that this system has only been implemented for Unix systems, using the $LD_PRELOAD environmental variable with a special shared library, yabs.autodeps_so, that is built automatically. This shared library has only been built for OpenBSD and Linux at the time of writing; other Unix systems may require slightly different build parameters. Also, files that the command attempts, but fails, to open, are also logged. They are treated as a third form of prerequisite, where inability to remake the prerequisite doesn't force a remake of the target; this is the correct behaviour - if such a prerequisites exists, then the command will always be re-run, but if it still doesn't exist, it is safe to not re-run the command. [At the moment, this is done using a hack - the semi-prerequisite is prefixed with a `-'; ultimately, it would make more sense to recognise the three forms of prerequisites that seem to be relevent to Yabs, perhaps using simple `', `-' and `--' prefixes.] This is necessary in situations involving multiple include-paths, such as the following: compiler include path is: /usr/local/include:/usr/include /usr/local/include/foo.h does not exist. /usr/include/foo.h exists. source file main.c contains #include "foo.h" Yabs is asked to build main.o, so compiles main.c /usr/local/include/foo.h is created. Yabs is asked to build main.o again. The creation of the file /usr/local/include/foo.h should force a recompilation of main.c, even though the attempt to open that file will have failed previously, and the filename will not be listed in tradional auto-dependency information such as the output from gcc -M. Note that gcc doesn't seem to attempt to open the header file when the parent directory doesn't exist, so this system isn't bomb-proof. If not None, is treated as a filename suffix that is appended to target filenames, and used to store the command(s) that returns when asked how to create a target. If the commands are changed (e.g. is modified), Yabs will detect this and force a remake of the target, even if it is newer than all of its prerequisites. The system works in the following way: yabs.add_rule() makes a call to yabs.add_autocmds_rule(), so that Yabs will have a rule available that will write the commands for a target to a file . Also, Yabs ensures that is always added to the prerequisites returned from . Note that this mechanism results in an extra file per target; if there are many targets, it might be better to use the technique in yabs3's compilation rules where command files are per-directory. is used to reduce the diagnostics when this rule is used. For example, setting internal=1 will reduce the effective state.debug level by 1 when outputing diagnostics in connection with this rule. [this is currently broken]. If is None, yabs.default_state is used. We return the created Rule object. This is useful because it contains the rule's root if applicable. Example usage: def myrule( target, state): if target!='myprog.exe': return None prereqs = [ 'foo.o', 'bar.o'] semiprereqs = [ 'foo.h', 'bar.h'] command = 'link -o ' + target + ' ' + string.join( prereqs, ' ') return prereqs, semiprereqs, command yabs.add_rule( myrule) ''' if state is None: state = default_state # get our caller's backtrace, as required by class Rule. backtrace = traceback.extract_stack( None, None)[:-1] if type( root)==int: # has leaf fns at the end of the array, so we count from # the end. e.g. if root=yabs.caller, this will be backtrace[-1], i.e. # the last item in the array i.e. our immediate caller. r = root filename, line, function, text = backtrace[-root] root = os.path.split( os.path.abspath( filename))[0] + os.sep #assert root == get_caller_directory( r+1) # the above line slows things down a bit. but it should be ok to # uncomment it. if type(root)==str and not root.endswith( os.sep): root = root + os.sep rule = Rule( rule_fn, phony, root, autocmds, autodeps, internal, backtrace) # insert new rule at head of list of rules: state.rules.insert( 0, rule) if autocmds: add_autocmds_rule( autocmds, state=state) return rule def _file_rule( filename, state): ''' Default rule, for human-written files. This is automatically registered as the lowest-priority rule. Note that this rule is never used if a previous rule has matched the target, but has failed due to failure to create prerequisites. This avoids most problems caused by old generated files existing when a rule is modified incorrectly. Maybe this rule should not be registered. Instead, a rule such as yabs3._source_rule() that ignores files whose name implies that they are generated files is better. This would avoid problems with Yabs seeming to work because a generated file exists even when there is no rule that will regenerate this file, which can happen when a rule is changed. ''' assert isinstance( state, State) if mtime( filename, state.mtimecache) == 0: if state.debug>4: print 'yabs: file_rule failed for ' + filename return None else: if state.debug>4: print 'yabs: file_rule succeeded for ' + filename return '', add_rule( _file_rule, internal=1) def updatediff( filename, new_contents): ''' Writes to if doesn't exist, or 's existing contents differ from . File's parent directory is always made if not already present. Returns True if file has been changed, or False if file has been left unchanged. ''' try: f = open( filename, 'r') contents = f.read() f.close() #print 'updatediff: contents=', contents #print 'updatediff: new_contents=', new_contents if contents==new_contents: #print 'contents are unchanged' return False # spot the inefficiency... except IOError: pass #print 'contents are changed' f = open( filename, 'w') f.write( new_contents) f.close() return True def add_autocmds_rule( autocmds, state=None): ''' This is normally called only internally by Yabs (when add_rule() is called with a parameter). Adds a rule for filenames ending with . Given a target foo, this rule looks in Yabs's rule cache for the command for target foo, and writes it to foo using yabs.updatediff(). This function can be called multiple times with the same - it checks for multiple registration. ''' if state is None: state=default_state if type( autocmds)!=str: raise Exception( 'autocmds should be a string, but is type ', type( autocmds), ':', autocmds) if state.autocmds_rules.has_key( autocmds): return #print 'add_autocmds_rule: autocmds=', autocmds def autocmds_rule( target, state): #print 'yabs.add_autocmds_rule.autocmds_rule: target=', target, 'autocmds=', autocmds if target.endswith( autocmds): #print 'yabs.add_autocmds_rule.autocmds_rule: target=', target, 'autocmds=', autocmds target0 = target[:-len(autocmds)] rrp = findrule_fromcache( target0, state) if rrp is None or len( rrp)!=2: if state.debug>=3: print place(), 'autocmds failure, suffix=', autocmds, ', target', target, ', rrp=', rrp print place(), 'rulecache=' for i in state.rulecache: print place(), ' ', i return None #print 'rrp=', rrp, 'len(rrp)=', len(rrp) rule, ruleresult = rrp if rule is None: return None command = command_gettext( ruleresult.command, state) def autocmds_rule_command( context): if context.state.debug>3: print 'yabs: updating commandfile ' + target if updatediff( target, command): if context.state.debug>2: print 'yabs: command file changed: ' + target else: if context.state.debug>2: print 'yabs: command file unchanged: ' + target return '' return autocmds_rule_command, [], [ None] add_rule( autocmds_rule, state=state, internal=1) state.autocmds_rules[ autocmds] = autocmds_rule class _PrefixOutput: ''' Behaves like an output stream object. Stores all output in self.buffer. If is not None, also sends all output to , prefixing all lines with . We take care not to output the prefix as the last line when the last input ends with '\n' - the prefix is only output when we know that there are following characters. Also, if prefix is callable, it is expected to take no parameters and return a prefix, and is called each time it is used ''' def __init__( self, prefix='', stream=None): self.buffer = '' self.startofline = True self.stream = stream #print '_PrefixOutput: prefix=', prefix if callable( prefix): self.prefix = prefix else: self.prefix = lambda : prefix def write( self, text): if text=='': return def write_both( text): if self.stream: self.stream.write( text) self.buffer += text if self.startofline: write_both( self.prefix()) lines = text.split( '\n') write_both( lines[0]) self.startofline = False for line in lines[1:]: if line=='': self.startofline = True write_both( '\n') else: self.startofline = False write_both( '\n' + self.prefix() + line) def flush( self): pass def fileno( self): if self.stream: return self.stream.fileno() else: return -1 def _print_withprefix( whereto, prefix, text): _PrefixOutput( prefix, whereto).write( text) def command_gettext( command, state, prefix=None): ''' This is normally called only internally by Yabs. Calls if it is callable, to get the text of the command. Return text command, or raises exception containing any output from the callable command. If .echo is False, (the default is True), the callable command will be given dummy sys.stdout and sys.stderr which write to an internal buffer. This buffer is included in any exception, but is otherwise discarded. See yabs2's -e option. Note that this facility is turned off when yabs is concurrent. ''' assert isinstance( state, State) if prefix is None: prefix = state.echo_prefix if callable( command): #print 'command_gettext: command is', command.func_code # use single-item list to hold buffer, so that instance of our # nested Buffer class can modify it. if state.echo_output == 3: prefixoutput = _PrefixOutput( prefix, sys.stdout) else: prefixoutput = _PrefixOutput( prefix, None) if not state.mt: old_stdout = sys.stdout old_stderr = sys.stderr sys.stdout = prefixoutput sys.stderr = prefixoutput class Context: pass context = Context() context.state = state context.out = prefixoutput try: try: _trace( state, command.func_code) command2 = command( context) finally: if not state.mt: sys.stdout = old_stdout sys.stderr = old_stderr except: raise yabserrors.commandfn_error( command, prefixoutput.buffer) # maybe we should translate to a yabserrors.command_error? if command2 is None: command2='' if type( command2)!=str: raise Exception( 'rule-command ' + str( command.func_code) + ' has returned type ' + str( type( command2)) + ', instead of a string: ' + str( command2)) command = command2 else: if type( command) != str: raise Exception( 'rule-command ' + str( command) + ' is not a string') return command # Things for autodeps # autodeps doesn't work on Windows - I don't know of any way to detect # what files are opened by child tasks on Windows. if os.name!='nt': uname = os.uname() uname_os = uname[0] uname_osv = uname[2] uname_cpu = uname[4] string.replace( uname_osv, '/', '-') string.replace( uname_osv, '(', '{') string.replace( uname_osv, ')', '}') uname_env = 'os=' + uname_os + ',osv=' + uname_osv + ',cpu=' + uname_cpu yabs_root = get_caller_directory() autodeps_so = os.path.join( yabs_root, 'autodeps-' + uname_env + '.so') #print 'yabs_root=', yabs_root # is a shared library that we use with $LD_PRELOAD # when running sub-commands. We use the suffix so that # different systems can share a Yabs directory. Naturally, we use a # Yabs rule to build . def _autodeps_rule( target, state): if target!=autodeps_so: return None #print place(), 'target=', target src = os.path.join( yabs_root, 'autodeps.c') obj = src + '.' + uname_env + '.o' if uname_os=='OpenBSD': return ( 'gcc -W -Wall -c -o ' + obj + ' ' + src + '\n' + 'gcc -shared -fpic -o ' + target + ' ' + obj, [ src], ) # could probably avoid the intermediate file, like # the linux version below. elif uname_os=='Linux': r = ( 'gcc -W -Wall -fPIC -shared -ldl -o ' + target + ' ' + src, [ src], ) #print place(), 'returning:', r return r else: raise Exception( 'Don\'t know how to build autodeps shared object for ' + uname_os) return None add_rule( _autodeps_rule, autocmds='.cmds', internal=1) def run_ruleresult( ruleresult, state): ''' Runs the command/fn returned from a rule, by calling command_run(). Unlocks Yabs's lock while doing so. ''' if state.premake_fn: state.premake_fn( ruleresult.target, state) if ruleresult.rule.autodeps: autodeps_filename = ruleresult.target + ruleresult.rule.autodeps try: os.remove( autodeps_filename) except OSError: pass else: autodeps_filename = None old_mtime = mtime( ruleresult.target, state.mtimecache) if not ruleresult.rule.phony: _ensure_directory( os.path.dirname( ruleresult.target), state) # run command with state unlocked, so that concurrency works. state.unlock() try: e, text = command_run( ruleresult.command, state, ruleresult.rule.root, autodeps_filename) finally: state.lock() mtime_flush( ruleresult.target, state.mtimecache) new_mtime = mtime( ruleresult.target, state.mtimecache) if e: pass else: if ruleresult.rule.phony: e = changed elif new_mtime == 0: e = yabserrors.file_command_error( ruleresult.target, ruleresult.command) if state.echo_exception==3: prefix = state.echo_prefix if callable( prefix): prefix = prefix() sys.stdout.write( prefix + '*** ' + str( e) + '\n') elif new_mtime <= old_mtime: e = unchanged else: e = changed state.targetcache[ ruleresult.target] = ruleresult.rule, e def command_run_text( command, state=None, cwd=None, autodeps_filename=None, fn=None, endtime=None): ''' As command_run, but returns text, or raises exception. ''' e, text = command_run( command, state, cwd, autodeps_filename, fn, endtime) if e: raise e return text def command_run( command, state=None, cwd=None, autodeps_filename=None, fn=None, endtime=None): ''' Runs newline-separated commands in , prefixing each with `cd && ' if is specified. Also prefixes with settings for $LD_PRELOAD and $YABS_AUTODEPS_FILENAME if is set, so that auto dependencies are written to . If .silent is False, commands are written to stdout before being run. If .dryrun is True, command are not actually run. If .echo is True (the default), output from commands is written to stdout as the command executes; see yabs2's -e option. Returns ( e, outputtext). e is 0 if no error occurred, else a yabserrors.command_error, which will also contain the output text from the last sub-command. If state.use_os_system is True, always calls commands using os.system. This is to overcome problems with child tasks not responding properly to stdin etc. When python 2.4 is widespred, will hopefully be able to rely on the subprocess module. Absorbs exceptions. ''' if state is None: state = default_state assert isinstance( state, State) try: commands = command_gettext( command, state) except Exception, e: if state.echo_exception==3: # use _print_failed_prerequisites2() to output just the # exception. fixme: very crude, we pass None for lots of # params that we know won't be used. _print_failed_prerequisites2( sys.stdout, state.echo_prefix, None, None, e, 0, state, False, False, False, False, True, False) return e, '' out_all = '' for i in string.split( commands, '\n'): command = string.strip( i) ignore_error = False if len(command)>0 and command[0]=='-': ignore_error = True command = command[1:] if len(command)==0: continue if cwd: command = 'cd ' + cwd + ' && ' + command if autodeps_filename: if os.name=='nt': raise Exception( 'rules with auto-dependency files are ' 'not supported under win32') # trying to use multiple chain LD_PRELOAD libraries # sometimes hangs, so we don't attempt to do it. if 0: ld_preload = os.getenv( 'LD_PRELOAD') else: ld_preload = None if ld_preload: ld_preload = autodeps_so + ':' + ld_preload else: ld_preload = autodeps_so command = 'export LD_PRELOAD=' + ld_preload\ + ' YABS_AUTODEPS_FILENAME=' + autodeps_filename + ' && '\ + command if state.echo_command==3: #if state.dryrun: print 'yabs: would run:' if 0 and state.mt: print '[' + str( threading.currentThread()) + '] ' + command else: print command if not state.dryrun: e, out = subprocess( command, echo=state.echo_output==3, state=state, fn=fn, endtime=endtime) out_all += out #print place(), 'yabs: e=', e, ', out=', out if e: if ignore_error: if state.debug >= 2: print 'yabs: ignored error from last command' else: # prob better to use _print_failed_prerequisites2() # to avoid duplication of the `*** return value # was...' code. if state.echo_exception==3: prefix = state.echo_prefix if callable( prefix): prefix = prefix() sys.stdout.write( prefix + ( '*** return value was: 0x%x' % e) + ': ' + yabserrors.exit_description( e) + ':\n') return yabserrors.command_error( command, e, out), out_all return 0, out_all def _check_rule_command( rule, command): ''' Checks that command is either a string or is callable. Commands are eventually run with command_gettext()/command_run(), which will check that if command is callable, it returns a string. ''' if not callable( command) and type( command)!=type(''): raise Exception( 'command returned from rule [' + str( rule.rule_fn.func_code) + '] is type ' + str( type( command)) + '; should be string or function.') def mtime_raw( filename, mtimecache=None): ''' Returns modification time of , or 0 if it doesn't exist. ''' assert type( filename)==str if mtimecache==None: mtimecache=default_state.mtimecache for prefix in mtimecache.oldprefixes: if filename.startswith( prefix): #print 'mtime_raw: not stat-ing: ' + filename mtimecache.num_nonstat += 1 return 0 try: mtimecache.num_stat += 1 t = os.path.getmtime( filename) except os.error: t = 0 return t def mtime( filename, mtimecache=None): ''' Returns modification time of , or 0 if doesn't exist, using caching. ''' if mtimecache==None: mtimecache=default_state.mtimecache assert type( filename)==str, str( filename) filename = os.path.abspath( filename) #print 'yabs: mtime: filename0=' + filename0 + ', filename=' + filename if not mtimecache.cache.has_key( filename): mtimecache.cache[ filename] = mtime_raw( filename) ret = mtimecache.cache[ filename] #print 'mtime, returning ' + str( ret) + ', real=' + str( mtime_raw( filename))\ # + ': ' + filename return ret def mtime_flush( filename, mtimecache=None): ''' Removes any cached modification time for , including entries from calls to mtime_markold() and mtime_marknew(). For example, this is called after a command is run that remakes a target. ''' if mtimecache==None: mtimecache=default_state.mtimecache assert type( filename)==str filename = os.path.abspath( filename) if mtimecache.cache.has_key( filename): del mtimecache.cache[ filename] def mtime_markold( filename, mtimecache=None): ''' Future calls to mtime( , ) will return 1. ''' if mtimecache==None: mtimecache= default_state.mtimecache assert type( filename)==str filename = os.path.abspath( filename) mtimecache.cache[ filename] = 1 def mtime_add_oldprefix( tree, mtimecache=None): if mtimecache==None: mtimecache=default_state.mtimecache mtimecache.oldprefixes.append( tree) def mtime_marknew( filename, mtimecache=None): ''' Future calls to mtime( , ) will return fixed value that is a long time in the future. ''' if mtimecache==None: mtimecache=default_state.mtimecache assert type( filename)==str filename = os.path.abspath( filename) #print 'yabs: mtime_marknew: filename=' + filename mtimecache.cache[ filename] = 3600*24*365*10*1000 # yuk - simply set to 10,000 years after epoc. class Prerequisite: ''' A normal prerequisite. ''' pass class SemiPrerequisite: ''' A prerequisite, such as a header file, that is allowed to fail. ''' pass class FailedSemiPrerequisite: ''' prerequisite that was tried last time but didn't exist last time. ''' pass class Ruleresult: ''' An easy-to-use representation of the tuples returned from rules. ''' def __init__( self, ruleresult, rule, target, state): ''' is the value returned from a rule, i.e. initial portion of [ , , , ...]. ''' #print place(), ruleresult, rule, target self.extraprereq_fn = None self.internal = 0 try: assert len( ruleresult) >= 1 except: raise Exception( 'rule result from [' + str( rule.rule_fn.func_code) + '] is not sequence of at least one item: ' + str( ruleresult)) def set( i, default): if len( ruleresult) >= i+1: return ruleresult[i] else: return default self.rule = rule self.target = target self.command = set( 0, '') self.raw_prereqs = set( 1, []) self.raw_semiprereqs = set( 2, []) self.extraprereq_fn = set( 3, None) self.internal = set( 4, 0) if state.mt: self.resource = set( 5, state.mt_default_resource) else: self.resource = None if self.extraprereq_fn is None or callable( self.extraprereq_fn): pass else: raise Exception( 'rule result from [' + str( rule.rule_fn.func_code) + '] has invalid extraprereq_fn, should be None or callable, but is: ' + repr( self.extraprereq_fn)) def prerequisites( self, state): ''' Generator, returning ( ptype, prerequisite), where is Prerequisite, SemiPrerequisite or FailedSemiPrerequisite. ''' def recurse( p, state): if type(p)==str or p==None: yield p elif callable( p): for pp in recurse( p( state), state): yield pp else: # should assert that x is iterable. for pp in p: for ppp in recurse( pp, state): yield ppp for p in recurse( self.raw_prereqs, state): yield Prerequisite, p for p in recurse( self.raw_semiprereqs, state): yield SemiPrerequisite, p if self.rule.autodeps: for ptype, p in autodeps_get( self.target + self.rule.autodeps, state): yield ptype, p yield SemiPrerequisite, autodeps_so return def __str__( self): return ( 'target=' + str( self.target) + ', command=' + str( self.command) + ', prerequisites=' + str( self.prerequisites) + ', resource=' + str( self.resource) + ', internal=' + str( self.internal) ) def _output_ruleinfo( prefix, rule, ruleresult, state): ''' Outputs debug information about a rule and its result (if not None). ''' assert isinstance( state, State) if 1 or state.debug>3: if rule.phony: print prefix + ' rule is (phony) ' + str( rule.rule_fn.func_code) else: print prefix + ' rule is ' + str( rule.rule_fn.func_code) print prefix + ' ruleroot is: ' + str( rule.root) if state.debug>3: print prefix + ' rule registration: ' + str( rule.backtrace) print prefix + ' rule autocmds: ' + str( rule.autocmds) print prefix + ' rule autodeps: ' + str( rule.autodeps) if ruleresult is not None: print prefix + ' prerequisites are: ' + str( ruleresult.prerequisites) print prefix + ' command is ' + str( ruleresult.command) def findrule_fromcache( target, state=None): ''' Finds matching rule for target, from 's rule cache. This assumes that the rule has already been found by make(). Returns a tuple: ( rule, ruleresult, phony), where is True/False, and is the tuple returned from the rule function. If rule not found, raises exception. ''' if state is None: state=default_state if not state.rulecache.has_key( target): if state.debug>2: print 'No previously found rule for target ' + target return None return state.rulecache[ target] def _ensure_directory( directory, state): ''' Ensure exists. Uses mtime cache, so can be called repeatedly without being slow. ''' if mtime( directory, state.mtimecache)==0: os.makedirs( directory) mtime_flush( directory) def _str_cwd( filename, state): ''' Returns filename relative to current directory, if .show_cwd_paths is True. Otherwise returns filename. ''' if state.show_cwd_paths: cwd = os.getcwd() + os.sep if filename.startswith( cwd): return '.' + os.sep + filename[ len( cwd):] return filename def _print_failed_prerequisites( whereto, prefix, target, rule, e, depth, state, format, recurse): ''' Outputs human-readable information about a failed call to yabs.make(). should be an error returned from make() - see yabs.make() for a description of the format. ''' show_target = False show_rule = False show_command = False show_output = False show_exception = False if 0: print place(), 'state.echo_target=', state.echo_target print place(), 'state.echo_rule=', state.echo_rule print place(), 'state.echo_command=', state.echo_command print place(), 'state.echo_output=', state.echo_output print place(), 'state.echo_exception=', state.echo_exception if format=='auto': # show all info that hasn't already been displayed. show_target = 1# == state.echo_target show_rule = 1 == state.echo_rule show_command = 1 == state.echo_command show_output = 1 == state.echo_output show_exception = 1 == state.echo_exception else: assert type( format) == str for f in format.split( ','): if f=='' or f=='.': pass elif f=='target': show_target = True elif f=='rule': show_rule = True elif f=='command': show_command = True elif f=='output': show_output = True elif f=='except': show_exception = True else: whereto.write( prefix() + 'ignoring unrecognised summary item: ' + f + '\n') if 0 and depth==0: print ( place() + 'show rule=', show_rule, 'command=', show_command, 'output=', show_output, 'exception=', show_exception) #print callers() _print_failed_prerequisites2( whereto, prefix, target, rule, e, depth, state, show_target, show_rule, show_command, show_output, show_exception, recurse) def _print_failed_prerequisites2( whereto, prefix, target, rule, e, depth, state, show_target, show_rule, show_command, show_output, show_exception, recurse): #print callers() #print 'recurse=', recurse #print 'depth=', repr(depth) #print 'prefix=', repr(prefix) #print 'prefix()=', repr(prefix()) #print 'e=', repr(e) #print 'show_target, show_rule, show_command, show_output, show_exception:',\ # show_target, show_rule, show_command, show_output, show_exception if callable( prefix): prefix = prefix() prefix2 = prefix + str( depth) + ': ' + ' '*depth if show_target and target: if 0: whereto.write( prefix2 + '*** target failed:\n') whereto.write( prefix2 + ' ' + _str_cwd( target, state) + '\n') else: whereto.write( prefix2 + '*** target failed: ' + _str_cwd( target, state) + '\n') if show_rule and rule: whereto.write( prefix2 + '*** rule:\n') whereto.write( prefix2 + ' ' + str( rule.rule_fn.func_code) + '\n') if type(e)==list: for rule, prereq, ee in e: if show_rule and rule: whereto.write( prefix2 + '*** rule:\n') whereto.write( prefix2 + ' ' + str( rule.rule_fn.func_code) + '\n') if recurse: _print_failed_prerequisites2( whereto, prefix, prereq, None, ee, depth+1, state, show_target, show_rule, show_command, show_output, show_exception, recurse) elif isinstance( e, yabserrors.command_error): if show_command: whereto.write( prefix2 + '*** command was:\n') if callable( e.command): whereto.write( prefix2 + ' ' + str( e.command.func_code) + '\n') else: whereto.write( prefix2 + ' ' + str( e.command).strip() + '\n') if show_output: whereto.write( prefix2 + '*** output was:\n') _print_withprefix( whereto, prefix2 + ' ' , str( e.text)) if show_exception: if os.name=='nt': whereto.write( prefix2 + '*** return value was: 0x%x:\n' % e.ret) else: whereto.write( prefix2 + '*** return value was: 0x%x' % e.ret\ + ': '\ + yabserrors.exit_description( e.ret) + ':\n') elif isinstance( e, yabserrors.commandfn_error): if show_command: whereto.write( prefix2 + '*** command-fn failed:\n') whereto.write( prefix2 + ' ' + str( e.commandfn.func_code) + '\n') if show_output: whereto.write( prefix2 + '*** output was:\n') _print_withprefix( whereto, prefix2 + ' ', str( e.text)) if show_exception: whereto.write( prefix2 + '*** error was:\n') traceback.print_exception( e.exc_info[0], e.exc_info[1], e.exc_info[2], file=_PrefixOutput( prefix2 + ' ', whereto) ) elif isinstance( e, yabserrors.rulefn_error): if show_command: whereto.write( prefix2 + '*** rule failed:\n') traceback.print_exception( e.exc_info[0], e.exc_info[1], e.exc_info[2], file=_PrefixOutput( prefix2 + ' ', whereto) ) elif isinstance( e, Exception): if show_exception: whereto.write( prefix2 + '*** exception was:\n') _print_withprefix( whereto, prefix2, str(e).strip() + '\n') elif e==changed: pass elif e==unchanged: pass elif e==pending: pass else: print e.__class__, e def relativepath( from0, to_, sep=os.sep, up='..'+os.sep): ''' constructs relative path, from directory to path . this has be optimised for speed. ''' if len( from0)==0: return to_ elif from0[-1]==sep: from1 = from0 else: from1 = from0 + sep last_sep = 0 for i in xrange( min( len( from1), len( to_))): if from1[i] != to_[i]: break if from1[i]==sep: last_sep = i num_up = from1.count( sep, last_sep) - 1 #print 'from=', from0, 'to=', to_, ', last_sep=', last_sep, ', num_up=', num_up return num_up * up + to_[last_sep+1:] if 0: relative_path_tests = [ ( '/home/foo/bar', '/home/qwerty/abcd/pqr', '../../qwerty/abcd/pqr'), ( '/home/qwerty/abcd/pqr', '/home/foo/bar', '../../../foo/bar'), ( '/home/foo/abcd/pqr/', '/home/foo/bar', '../../bar'), ( '/foo', '/home/qwerty/abcd/pqr/', '../home/qwerty/abcd/pqr/'), ( '/foo/bar/qw', '/foo/qw', '../../qw'), ( '/foo/bar/qw', '/foo/qw/', '../../qw/'), ( '/foo', '/foo/qw/', 'qw/'), ( '', '/foo/qw/', '/foo/qw/'), ] # from, to, correct-result. for f, t, r in relative_path_tests: assert relativepath( f, t) == r if 1: # alternative implementations of relativepath. are up to 2x slower. def relativepath0( from_, to_, sep=os.sep, up='..'+os.sep): if from_=='' or from_ is None: return to_ # would be nice to not special case, but string.join is not exactly right. while from_.endswith( sep): from_=from_[:-1] #while to_.endswith( sep): to_=to_[:-1] f = from_.split( sep) t = to_.split( sep) i = 0 while 1: if i>=len( t) or i>=len( f) or t[i] != f[i]: break i += 1 #print 'i=', i # first i directories are the same relative_path = (len(f)-i) * up relative_path += string.join( t[ i:], sep) return relative_path def relativepath1( from_0, to_, sep=os.sep, up='..'+os.sep): if len(from_0)==0: return to_ elif from_0[-1]==sep: from_ = from_0 else: from_ = from_0 + sep p = os.path.commonprefix( ( from_, to_)) s = p.rfind( sep) + 1 num_up = from_.count( sep, len(p)) if from_[-1]!=sep: num_up += 1 #print 'from=', from_0, 'to=', to_, ', p=', p, ', s=', s, ', num_up=', num_up return num_up * up + to_[s:] for k in relativepath1, relativepath0, relativepath: t = time.time() for i in range( 10*1000): for j in relative_path_tests: s = k( j[0], j[1]) assert s==j[2], ( j[2], s) t = time.time() - t print k, t def relativepath_user( user, to, sep=os.sep, up='..'+os.sep): ''' If is within ~, returns representation of starting with `~'. Otherwise returns absolute path of . ''' user_path = os.path.expanduser( '~' + user) to_abs = os.path.abspath( to) if to_abs.startswith( user_path): return '~' + user + to_abs[ len( user_path):] else: return to_abs def xtermtitle_usertarget( target, state): ''' writes @: into titlebar of xterm. intended for use as .premake_fn. ''' import socket if os.isatty(0): sys.stdout.write( '\033]2;' + os.environ['USER'] + '@' + socket.gethostname() + ':' + relativepath_user( os.environ['USER'], target) + '\007') def call_rule( rule, target, state, prefix): #print 'target=', target if rule.root: target2 = relativepath( rule.root, target) if state.debug>=4: print prefix() + 'converted target: ' + target print prefix() + ' to relative path: ' + target2 print prefix() + ' ruleroot=: ' + rule.root else: target2 = target try: state.debug -= rule.internal try: ruleresult_raw = rule.rule_fn( target2, state) finally: state.debug += rule.internal except Exception, e: e = yabserrors.rulefn_error( e) if state.echo_command>=2: _print_failed_prerequisites2( sys.stdout, state.echo_prefix, None, None, e, 0, state, False, False, True, False, False, False) return e if ruleresult_raw is None: if state.debug>=5: print prefix() + 'rule doesn\'t match' return return Ruleresult( ruleresult_raw, rule, target, state) def cancel_targets( targets, state): for target in targets: for i in range( len( state.mt_targets)): if state.mt_targets[i][0]==target: t, rr, p = state.mt_targets[i] if state.debug>=2: print p(), 'cancelling:', t del state.mt_targets[i] def show_result( target, rule, result, prefix, state, nesting): if result!=changed and result!=unchanged: if 0: print 'state.echo_target=', state.echo_target print 'state.echo_rule=', state.echo_rule print 'state.echo_command=', state.echo_command print 'state.echo_output=', state.echo_output print 'state.echo_exception=', state.echo_exception _print_failed_prerequisites2( sys.stdout, prefix, target, rule, result, nesting, state, 2 == state.echo_target, 2 == state.echo_rule, 2 == state.echo_command, 2 == state.echo_output, 2 == state.echo_exception, recurse=False) #print prefix() + '0: target failed: ' + _str_cwd( target, state) def try_get_result( targets, state, prefix, nesting): ''' Looks in .targetcache for result for any target in . if found, removes from and returns ( , result), else returns None. ''' # don't use enumerate - not available on python-2.2. for i in range( len( targets)): target = targets[i] rule, result = state.targetcache[ target] if result != pending: del targets[i] show_result( target, rule, result, prefix, state, nesting) return target, result def wait_result( targets, state, prefix, nesting): ''' Waits for any item in to be made. Removes the item from , and returns ( , ,). Relies on state.mt_condition.notifyAll() being called whenever .targetcache is changed. This is done automatically - .targetcache is a NotifyingDict. ''' while 1: i = 0 while 1: if i==len( targets): break target = targets[i] assert state.targetcache.has_key( target), target rule, result = state.targetcache[ target] if result!=pending: del targets[i] show_result( target, rule, result, prefix, state, nesting) return target, result else: i += 1 if len( targets)==0: break state.wait() def autodeps_get( autodeps_name, state): ''' A generator that yields prerequisites from an autodeps file. ''' try: f = file( autodeps_name, 'r') except IOError: return while True: line = f.readline() if line=='': break c = line[0] filename = line[1:-1] # remove initial +/- and trailing newline. if c=='+': yield SemiPrerequisite, filename elif c=='-': # this file failed to open last time. so if it # now exists, we should force a rebuild. we # cheat a little by also pretending the file # is very new, in case someone has created # it as an old file. if mtime( filename): if state.debug>=1: print 'file failed to open previously, but now exists:'\ + filename mtime_marknew( filename) yield FailedSemiPrerequisite, filename f.close() def prefixfn( prefix0, nesting, state): def fn(): p = prefix0 if p is None: p = state.prefix if callable( p): p = p() return p + 'yabs ' + str( nesting) + ': ' + nesting*' ' return fn def start_make( target, prefix0, state, nesting=0): ''' This is the heart of Yabs. Builds , returning same as make(). Can also return if .mt is set, in which case one can use wait_result() to wait for to be built. Calls itself recursively to build prerequisites. The returning of is very useful when building prerequisites recursively - we start building all of a target's prerequisites, then repeatedly wait for any of them to finish using wait_result(). ''' prefix = prefixfn( prefix0, nesting, state) if state.targetcache.has_key( target): if state.debug>=3: print prefix() + 'have already dealt with ' + _str_cwd( target, state) + ': ' + repr( state.targetcache[ target]) rule, result = state.targetcache[ target] return result # while we are attempting to build , mark it as pending, so # that recursive/concurrent builds work. state.targetcache[ target] = None, pending failed_prereqs = [] # info about prereqs that have failed to make. newer_prereqs = [] # prereqs that are phony or newer than . ruleresult = None if state.debug >= 2: print prefix(), 'target: ' + _str_cwd( target, state) rule_ok = None # set to True if rule's prerequistes have built succesfully. Set to # False if one or more of the rule's prerequisites have failed. # We try each rule in turn. If a rule matches, we attempt to build # all of its prerequisites. If any of the prerequisites fail to # build, we carry on looking at other rules. for rule in state.rules: if rule.rule_fn==_file_rule and len( failed_prereqs)>0: if state.debug>=3: print prefix() + 'ignoring file_rule, because previous rules have '\ 'failed due to failed prerequisites' #_print_failed_prerequisites( sys.stdout, prefix, failed_prereqs, 0, state, 'rule') continue ruleresult = call_rule( rule, target, state, prefix) if state.debug>=4: print prefix(), 'ruleresult=', ruleresult if ruleresult is None: continue if isinstance( ruleresult, Exception): #print 'rule exception:', ruleresult.__class__, ruleresult failed_prereqs.append( ( rule, None, ruleresult)) #failed_prereqs.append( ruleresult) rule_ok = None ruleresult = None continue rule_ok = None if state.debug>=2: print prefix(), 'target:', target print prefix(), 'rule matches:', rule print prefix(), 'ruleresult=', ruleresult #print prefix(), ' prereqs:', ruleresult.prerequisites # We provisionally set rulecache[ target] to this rule, for use by # yabs2.command_rule(). state.rulecache[ target] = rule, ruleresult if rule.autocmds: # add autocmds file as a semi-prerequisites. ruleresult.raw_semiprereqs.append( target + rule.autocmds) pending_prereqs = [] # prereqs that we have started to make. prereqs = [] # absolute prerequisite filenames. for ptype, p in ruleresult.prerequisites( state): if p is None: newer_prereqs.append( p) else: if not os.path.isabs( p): if rule.root: pp = os.path.join( rule.root, p) else: print prefix() + 'warning: rule returned a relative prerequisite, but rule has no root-dir.' print prefix() + 'rule is:' _output_ruleinfo( prefix() + ' ', rule, None, state) print prefix() + 'target is: ' + target print prefix() + 'prerequiste is: ' + p print prefix() + 'ignoring rule' preq_make_result = [] rule_ok = False break else: pp = p if pp==target: if ptype==Prerequisite: print prefix() + 'recursion detected for target ' + _str_cwd( target, state) print prefix() + 'rule is:' _output_ruleinfo( prefix() + ' ', rule, None, state) print prefix() + 'target is: ' + target print prefix() + 'prerequiste is: ' + p rule_ok = False break else: # ignore recursion for semi-prerequistes, # because they often include the target. pass else: prereqs.append( ( ptype, pp)) if rule_ok==False: continue # Loop continually, starting to make prerequisites and checking # for prerequisites that have been made. This technique works # whether Yabs is concurrent or not. i = 0 while 1: if i < len( prereqs): # start making another prerequiste: prereq_type, prereq = prereqs[i] if prereq is None: # forces us to rebuild the target. newer_prereqs.append( prereq) else: start_make( prereq, prefix0, state, nesting+1) pending_prereqs.append( prereq) # do a non-blocking check of whether any prerequisites # have been made. # # If we are concurrent, start_make (prereq) will not # have built anything completely, so we will only get a # result for prerequisites that were already made. # # If we are not concurrent, start_make( prereq) will # have completed building , so there will always # be a result available. targetresult = try_get_result( pending_prereqs, state, prefix, nesting+1) else: # we have no more prerequisites to start to make, so do # a blocking wait - we have nothing else to do. If we # are not concurrent, will be empty. targetresult = wait_result( pending_prereqs, state, prefix, nesting+1) if targetresult is not None: prereq, result = targetresult # start_make() has no knowledge of whether a target # is a prerequisite or semi-prerequisite. But we need # to know, so recover the prerequisite-type from our # original list of prerequisites: prereq_type = None for p in prereqs: if p[1]==prereq: prereq_type = p[0] break assert prereq_type != None if result is changed: # Note that all phony prereqs will be marked as # changed. newer_prereqs.append( ( prereq, prereq_type)) elif result is unchanged: #if mtime( prereq, state.mtimecache) > mtime( target): # newer_prereqs.append( ( prereq, prereq_type)) pass else: # failed to build . if prereq_type is Prerequisite: failed_prereqs.append( ( rule, prereq, result)) rule_ok = False if state.keepgoing: pass else: # Stop building all the other prerequisites. cancel_targets( pending_prereqs, state) pending_prereqs = [] break else: # failed to build semi-prerequisite. if state.debug >= 2: print prefix(), 'ignoring failure to build semi-prerequisite:', prereq pass if len( pending_prereqs)==0 and i>=len( prereqs): # All prereqs have been made and we have handled their # results. if rule_ok is None: rule_ok = True break i += 1 if rule_ok is True: break # We've processed all the prerequisites. Now decide whether to run the # rule's command. if rule_ok is True: # Prerequisites were created successfully. Decide whether to # build ; if so, set to text describing why. remake = None if ruleresult.rule.phony: remake = 'target is phony' else: target_mtime = mtime( target, state.mtimecache) if target_mtime==0: remake = 'target doesn\'t exist' else: if newer_prereqs: if newer_prereqs[0]==None: remake = 'prerequisite is None' else: remake = 'prerequisite changed: ' + newer_prereqs[0][0] remake = 'prerequisite changed: ' + str( newer_prereqs) else: for ptype, p in prereqs: if mtime( p, state.mtimecache) > target_mtime: remake = 'prerequisite newer than target: ' + p break target_text = _str_cwd( target, state) if ruleresult.rule.phony: target_text = 'phony ' + target_text if remake: #print place(), 'ruleresult=', ruleresult if state.echo_target==3: #print 'ruleresult.rule.internal=', ruleresult.rule.internal #print 'ruleresult.internal=', ruleresult.internal if ruleresult.rule.internal + ruleresult.internal > 0 and state.debug<=1: #print prefix(), 'remaking (internal), debug=', state.debug, ',' + target_text pass else: print prefix(), 'remaking: ' + target_text if state.debug>=1: print prefix(), '- because ' + remake if state.echo_rule==3: print prefix(), 'rule: ' + str( ruleresult.rule.rule_fn.func_code) if state.mt: state.mt_targets.append( ( target, ruleresult, prefix)) state.mt_condition.notifyAll() else: run_ruleresult( ruleresult, state) else: if state.debug>=2: print prefix(), 'not remaking: ' + target_text rule, result = state.targetcache[ target] state.targetcache[ target] = rule, unchanged else: #print place(), 'rule_ok=', rule_ok, failed_prereqs if ruleresult is None: # no matching rule if state.debug>=2: print prefix(), 'no matching rule for:', target show_result( target, None, failed_prereqs, prefix, state, nesting) if state.echo_target==3: print prefix() + 'couldn\'t make: ' + _str_cwd( target, state) assert state.targetcache.has_key( target), 'target=' + target + ', targetcache keys=' + str( state.targetcache.keys()) #rule, result = state.targetcache[ target] state.targetcache[ target] = rule, failed_prereqs rule, ret = state.targetcache[ target] #print place(), ret return ret def make( target, prefix=None, state=None): ''' If is a relative path, is converted to an absolute path using os.path.abspath(). Uses the registered rules in to try to remake 's prerequisites, semi-prerequisites and itself, returning one of the following: yabs.unchanged - 's rule's command was not run because was found to be up-to-date, or the command was run but the timestamp of was unchanged. yabs.changed - 's rule's command was run and it updated , or it was run and was a phony rule. A list containing information about why could not be made. Each item in the list is a 3-tuple ( , , ), where is an instance of the yabs.Rule class, describing a rule that claimed to be able to remake , but which required a prerequisite that could not be made. The item is the error returned from make() when it was recursively called to remake , so the list returned from make() is actually a tree-like structure. Note that does not appear in the list. The function yabs._print_failed_prerequisites() can be used to display the list in a human readable format. All diagnostics are prefixed with . can be a function taking no params, in which case it should return a prefix string. Note that make() does not first check that rules exist for all of a rule's prerequisites before attempting to create the first prerequisite. This has the disadvantage that it means that rules could be run even when they ultimately don't work because some other prerequisites cannot be made. It has the advantage that the rules can depend on files created by earlier rules; for example a rule could untar a .tar file, and subsequent rules will be able to respond to the extracted files. It might even be possible to import extra rules from these files into the build programme itself. See http://tinyurl.com/yqk6l for a usenet posting that mentions this issue. ''' if state is None: state = default_state if not os.path.isabs( target): target = os.path.abspath( target) state.lock() try: r = start_make( target, prefix, state) target, r = wait_result( [target], state, prefixfn( prefix, 0, state), 0) #print place(), 'target=', target, ', r=', r return r finally: state.unlock() def tdb_status( state): while 1: time.sleep(5) tdb_lock.acquire() try: print 'tdb_status:', tdb_str().replace( '\n', '\n\r') print 'tdb_status:', 'all threads:', str( threading.enumerate()).replace( '\n', '\n\r') print 'threading.enumerate():', threading.enumerate() print 'state.m=', state.mt print 'state.mt_exited=', state.mt_exited finally: tdb_lock.release() if state.mt_exited: print place(), 'state.mt_exited is set' break if 0: # set up a background thread that periodically prints status of # other threads. useful when debugging concurrent useage. tdb_dummy = threading.Thread( target=tdb_status, name="tdb dummy", args=(default_state,)) tdb_dummy.start() #thread.start_new_thread( tdb_status, (None,))