SCons.Taskmaster

Package SCons :: Module Taskmaster |
[hide private]
[frames] | no frames]

# Source Code for Module SCons.Taskmaster

      1  #
      2  # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 The SCons Foundation
      3  #
      4  # Permission is hereby granted, free of charge, to any person obtaining
      5  # a copy of this software and associated documentation files (the
      6  # "Software"), to deal in the Software without restriction, including
      7  # without limitation the rights to use, copy, modify, merge, publish,
      8  # distribute, sublicense, and/or sell copies of the Software, and to
      9  # permit persons to whom the Software is furnished to do so, subject to
     10  # the following conditions:
     11  #
     12  # The above copyright notice and this permission notice shall be included
     13  # in all copies or substantial portions of the Software.
     14  #
     15  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
     16  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
     17  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
     18  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
     19  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
     20  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
     21  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     22  #
     23
     24  __doc__ = """
     25  Generic Taskmaster module for the SCons build engine.
     26
     27  This module contains the primary interface(s) between a wrapping user
     28  interface and the SCons build engine.  There are two key classes here:
     29
     30      Taskmaster
     31          This is the main engine for walking the dependency graph and
     32          calling things to decide what does or doesn't need to be built.
     33
     34      Task
     35          This is the base class for allowing a wrapping interface to
     36          decide what does or doesn't actually need to be done.  The
     37          intention is for a wrapping interface to subclass this as
     38          appropriate for different types of behavior it may need.
     39
     40          The canonical example is the SCons native Python interface,
     41          which has Task subclasses that handle its specific behavior,
     42          like printing "`foo' is up to date" when a top-level target
     43          doesn't need to be built, and handling the -c option by removing
     44          targets as its "build" action.  There is also a separate subclass
     45          for suppressing this output when the -q option is used.
     46
     47          The Taskmaster instantiates a Task object for each (set of)
     48          target(s) that it decides need to be evaluated and/or built.
     49  """
     50
     51  __revision__ = "src/engine/SCons/Taskmaster.py 2725 2008/03/31 12:52:02 knight"
     52
     53  import SCons.compat
     54
     55  import operator
     56  import string
     57  import sys
     58  import traceback
     59
     60  import SCons.Node
     61  import SCons.Errors
     62
     63  StateString = SCons.Node.StateString
     64
     65
     66
     67  # A subsystem for recording stats about how different Nodes are handled by
     68  # the main Taskmaster loop.  There's no external control here (no need for
     69  # a --debug= option); enable it by changing the value of CollectStats.
     70
     71  CollectStats = None
     72
     73 -class Stats:
     74      """
     75      A simple class for holding statistics about the disposition of a
     76      Node by the Taskmaster.  If we're collecting statistics, each Node
     77      processed by the Taskmaster gets one of these attached, in which case
     78      the Taskmaster records its decision each time it processes the Node.
     79      (Ideally, that's just once per Node.)
     80      """
     81 -    def __init__(self):
     82          """
     83          Instantiates a Taskmaster.Stats object, initializing all
     84          appropriate counters to zero.
     85          """
     86          self.considered  = 0
     87          self.already_handled  = 0
     88          self.problem  = 0
     89          self.child_failed  = 0
     90          self.not_built  = 0
     91          self.side_effects  = 0
     92          self.build  = 0
     93
     94  StatsNodes = []
     95
     96  fmt = "%(considered)3d "\
     97        "%(already_handled)3d " \
     98        "%(problem)3d " \
     99        "%(child_failed)3d " \
    100        "%(not_built)3d " \
    101        "%(side_effects)3d " \
    102        "%(build)3d "
    103
    104 -def dump_stats():
    105      StatsNodes.sort(lambda a, b: cmp(str(a), str(b)))
    106      for n in StatsNodes:
    107          print (fmt % n.stats.__dict__) + str(n)
    108
    109
    110
    111 -class Task:
    112      """
    113      Default SCons build engine task.
    114
    115      This controls the interaction of the actual building of node
    116      and the rest of the engine.
    117
    118      This is expected to handle all of the normally-customizable
    119      aspects of controlling a build, so any given application
    120      *should* be able to do what it wants by sub-classing this
    121      class and overriding methods as appropriate.  If an application
    122      needs to customze something by sub-classing Taskmaster (or
    123      some other build engine class), we should first try to migrate
    124      that functionality into this class.
    125
    126      Note that it's generally a good idea for sub-classes to call
    127      these methods explicitly to update state, etc., rather than
    128      roll their own interaction with Taskmaster from scratch.
    129      """
    130 -    def __init__(self, tm, targets, top, node):
    131          self.tm = tm
    132          self.targets = targets
    133          self.top = top
    134          self.node = node
    135          self.exc_clear()
    136
    137 -    def display(self, message):
    138          """
    139          Hook to allow the calling interface to display a message.
    140
    141          This hook gets called as part of preparing a task for execution
    142          (that is, a Node to be built).  As part of figuring out what Node
    143          should be built next, the actually target list may be altered,
    144          along with a message describing the alteration.  The calling
    145          interface can subclass Task and provide a concrete implementation
    146          of this method to see those messages.
    147          """
    148          pass
    149
    150 -    def prepare(self):
    151          """
    152          Called just before the task is executed.
    153
    154          This is mainly intended to give the target Nodes a chance to
    155          unlink underlying files and make all necessary directories before
    156          the Action is actually called to build the targets.
    157          """
    158
    159          # Now that it's the appropriate time, give the TaskMaster a
    160          # chance to raise any exceptions it encountered while preparing
    161          # this task.
    162          self.exception_raise()
    163
    164          if self.tm.message:
    165              self.display(self.tm.message)
    166              self.tm.message = None
    167
    168          for t in self.targets:
    169              t.prepare()
    170              for s in t.side_effects:
    171                  s.prepare()
    172
    173 -    def get_target(self):
    174          """Fetch the target being built or updated by this task.
    175          """
    176          return self.node
    177
    178 -    def execute(self):
    179          """
    180          Called to execute the task.
    181
    182          This method is called from multiple threads in a parallel build,
    183          so only do thread safe stuff here.  Do thread unsafe stuff in
    184          prepare(), executed() or failed().
    185          """
    186
    187          try:
    188              everything_was_cached = 1
    189              for t in self.targets:
    190                  if not t.retrieve_from_cache():
    191                      everything_was_cached = 0
    192                      break
    193              if not everything_was_cached:
    194                  self.targets[0].build()
    195          except KeyboardInterrupt:
    196              raise
    197          except SystemExit:
    198              exc_value = sys.exc_info()[1]
    199              raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code)
    200          except SCons.Errors.UserError:
    201              raise
    202          except SCons.Errors.BuildError:
    203              raise
    204          except:
    205              raise SCons.Errors.TaskmasterException(self.targets[0],
    206                                                     sys.exc_info())
    207
    208 -    def executed_without_callbacks(self):
    209          """
    210          Called when the task has been successfully executed
    211          and the Taskmaster instance doesn't want to call
    212          the Node's callback methods.
    213          """
    214          for t in self.targets:
    215              if t.get_state() == SCons.Node.executing:
    216                  for side_effect in t.side_effects:
    217                      side_effect.set_state(SCons.Node.no_state)
    218                  t.set_state(SCons.Node.executed)
    219
    220 -    def executed_with_callbacks(self):
    221          """
    222          Called when the task has been successfully executed and
    223          the Taskmaster instance wants to call the Node's callback
    224          methods.
    225
    226          This may have been a do-nothing operation (to preserve build
    227          order), so we must check the node's state before deciding whether
    228          it was "built", in which case we call the appropriate Node method.
    229          In any event, we always call "visited()", which will handle any
    230          post-visit actions that must take place regardless of whether
    231          or not the target was an actual built target or a source Node.
    232          """
    233          for t in self.targets:
    234              if t.get_state() == SCons.Node.executing:
    235                  for side_effect in t.side_effects:
    236                      side_effect.set_state(SCons.Node.no_state)
    237                  t.set_state(SCons.Node.executed)
    238                  t.built()
    239              t.visited()
    240
    241      executed = executed_with_callbacks
    242
    243 -    def failed(self):
    244          """
    245          Default action when a task fails:  stop the build.
    246          """
    247          self.fail_stop()
    248
    249 -    def fail_stop(self):
    250          """
    251          Explicit stop-the-build failure.
    252          """
    253          for t in self.targets:
    254              t.set_state(SCons.Node.failed)
    255          self.tm.stop()
    256
    257          # We're stopping because of a build failure, but give the
    258          # calling Task class a chance to postprocess() the top-level
    259          # target under which the build failure occurred.
    260          self.targets = [self.tm.current_top]
    261          self.top = 1
    262
    263 -    def fail_continue(self):
    264          """
    265          Explicit continue-the-build failure.
    266
    267          This sets failure status on the target nodes and all of
    268          their dependent parent nodes.
    269          """
    270          for t in self.targets:
    271              # Set failure state on all of the parents that were dependent
    272              # on this failed build.
    273              def set_state(node): node.set_state(SCons.Node.failed)
    274              t.call_for_all_waiting_parents(set_state)
    275
    276 -    def make_ready_all(self):
    277          """
    278          Marks all targets in a task ready for execution.
    279
    280          This is used when the interface needs every target Node to be
    281          visited--the canonical example being the "scons -c" option.
    282          """
    283          self.out_of_date = self.targets[:]
    284          for t in self.targets:
    285              t.disambiguate().set_state(SCons.Node.executing)
    286              for s in t.side_effects:
    287                  s.set_state(SCons.Node.executing)
    288
    289 -    def make_ready_current(self):
    290          """
    291          Marks all targets in a task ready for execution if any target
    292          is not current.
    293
    294          This is the default behavior for building only what's necessary.
    295          """
    296          self.out_of_date = []
    297          for t in self.targets:
    298              try:
    299                  t.disambiguate().make_ready()
    300                  is_up_to_date = not t.has_builder() or \
    301                                  (not t.always_build and t.is_up_to_date())
    302              except EnvironmentError, e:
    303                  raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filename=e.filename)
    304              if is_up_to_date:
    305                  t.set_state(SCons.Node.up_to_date)
    306              else:
    307                  self.out_of_date.append(t)
    308                  t.set_state(SCons.Node.executing)
    309                  for s in t.side_effects:
    310                      s.set_state(SCons.Node.executing)
    311
    312      make_ready = make_ready_current
    313
    314 -    def postprocess(self):
    315          """
    316          Post-processes a task after it's been executed.
    317
    318          This examines all the targets just built (or not, we don't care
    319          if the build was successful, or even if there was no build
    320          because everything was up-to-date) to see if they have any
    321          waiting parent Nodes, or Nodes waiting on a common side effect,
    322          that can be put back on the candidates list.
    323          """
    324
    325          # We may have built multiple targets, some of which may have
    326          # common parents waiting for this build.  Count up how many
    327          # targets each parent was waiting for so we can subtract the
    328          # values later, and so we *don't* put waiting side-effect Nodes
    329          # back on the candidates list if the Node is also a waiting
    330          # parent.
    331
    332          targets = set(self.targets)
    333
    334          parents = {}
    335          for t in targets:
    336              for p in t.waiting_parents.keys():
    337                  parents[p] = parents.get(p, 0) + 1
    338
    339          for t in targets:
    340              for s in t.side_effects:
    341                  if s.get_state() == SCons.Node.executing:
    342                      s.set_state(SCons.Node.no_state)
    343                      for p in s.waiting_parents.keys():
    344                          if not parents.has_key(p):
    345                              parents[p] = 1
    346                  for p in s.waiting_s_e.keys():
    347                      if p.ref_count == 0:
    348                          self.tm.candidates.append(p)
    349
    350          for p, subtract in parents.items():
    351              p.ref_count = p.ref_count - subtract
    352              if p.ref_count == 0:
    353                  self.tm.candidates.append(p)
    354
    355          for t in targets:
    356              t.postprocess()
    357
    358      # Exception handling subsystem.
    359      #
    360      # Exceptions that occur while walking the DAG or examining Nodes
    361      # must be raised, but must be raised at an appropriate time and in
    362      # a controlled manner so we can, if necessary, recover gracefully,
    363      # possibly write out signature information for Nodes we've updated,
    364      # etc.  This is done by having the Taskmaster tell us about the
    365      # exception, and letting
    366
    367 -    def exc_info(self):
    368          """
    369          Returns info about a recorded exception.
    370          """
    371          return self.exception
    372
    373 -    def exc_clear(self):
    374          """
    375          Clears any recorded exception.
    376
    377          This also changes the "exception_raise" attribute to point
    378          to the appropriate do-nothing method.
    379          """
    380          self.exception = (None, None, None)
    381          self.exception_raise = self._no_exception_to_raise
    382
    383 -    def exception_set(self, exception=None):
    384          """
    385          Records an exception to be raised at the appropriate time.
    386
    387          This also changes the "exception_raise" attribute to point
    388          to the method that will, in fact
    389          """
    390          if not exception:
    391              exception = sys.exc_info()
    392          self.exception = exception
    393          self.exception_raise = self._exception_raise
    394
    395 -    def _no_exception_to_raise(self):
    396          pass
    397
    398 -    def _exception_raise(self):
    399          """
    400          Raises a pending exception that was recorded while getting a
    401          Task ready for execution.
    402          """
    403          exc = self.exc_info()[:]
    404          try:
    405              exc_type, exc_value, exc_traceback = exc
    406          except ValueError:
    407              exc_type, exc_value = exc
    408              exc_traceback = None
    409          raise exc_type, exc_value, exc_traceback
    410
    411
    412 -def find_cycle(stack):
    413      if stack[0] == stack[-1]:
    414          return stack
    415      for n in stack[-1].waiting_parents.keys():
    416          stack.append(n)
    417          if find_cycle(stack):
    418              return stack
    419          stack.pop()
    420      return None
    421
    422
    423 -class Taskmaster:
    424      """
    425      The Taskmaster for walking the dependency DAG.
    426      """
    427
    428 -    def __init__(self, targets=[], tasker=Task, order=None, trace=None):
    429          self.original_top = targets
    430          self.top_targets_left = targets[:]
    431          self.top_targets_left.reverse()
    432          self.candidates = []
    433          self.tasker = tasker
    434          if not order:
    435              order = lambda l: l
    436          self.order = order
    437          self.message = None
    438          self.trace = trace
    439          self.next_candidate = self.find_next_candidate
    440
    441 -    def find_next_candidate(self):
    442          """
    443          Returns the next candidate Node for (potential) evaluation.
    444
    445          The candidate list (really a stack) initially consists of all of
    446          the top-level (command line) targets provided when the Taskmaster
    447          was initialized.  While we walk the DAG, visiting Nodes, all the
    448          children that haven't finished processing get pushed on to the
    449          candidate list.  Each child can then be popped and examined in
    450          turn for whether *their* children are all up-to-date, in which
    451          case a Task will be created for their actual evaluation and
    452          potential building.
    453
    454          Here is where we also allow candidate Nodes to alter the list of
    455          Nodes that should be examined.  This is used, for example, when
    456          invoking SCons in a source directory.  A source directory Node can
    457          return its corresponding build directory Node, essentially saying,
    458          "Hey, you really need to build this thing over here instead."
    459          """
    460          try:
    461              return self.candidates.pop()
    462          except IndexError:
    463              pass
    464          try:
    465              node = self.top_targets_left.pop()
    466          except IndexError:
    467              return None
    468          self.current_top = node
    469          alt, message = node.alter_targets()
    470          if alt:
    471              self.message = message
    472              self.candidates.append(node)
    473              self.candidates.extend(self.order(alt))
    474              node = self.candidates.pop()
    475          return node
    476
    477 -    def no_next_candidate(self):
    478          """
    479          Stops Taskmaster processing by not returning a next candidate.
    480          """
    481          return None
    482
    483 -    def _find_next_ready_node(self):
    484          """
    485          Finds the next node that is ready to be built.
    486
    487          This is *the* main guts of the DAG walk.  We loop through the
    488          list of candidates, looking for something that has no un-built
    489          children (i.e., that is a leaf Node or has dependencies that are
    490          all leaf Nodes or up-to-date).  Candidate Nodes are re-scanned
    491          (both the target Node itself and its sources, which are always
    492          scanned in the context of a given target) to discover implicit
    493          dependencies.  A Node that must wait for some children to be
    494          built will be put back on the candidates list after the children
    495          have finished building.  A Node that has been put back on the
    496          candidates list in this way may have itself (or its sources)
    497          re-scanned, in order to handle generated header files (e.g.) and
    498          the implicit dependencies therein.
    499
    500          Note that this method does not do any signature calculation or
    501          up-to-date check itself.  All of that is handled by the Task
    502          class.  This is purely concerned with the dependency graph walk.
    503          """
    504
    505          self.ready_exc = None
    506
    507          T = self.trace
    508
    509          while 1:
    510              node = self.next_candidate()
    511              if node is None:
    512                  return None
    513
    514              node = node.disambiguate()
    515              state = node.get_state()
    516
    517              if CollectStats:
    518                  if not hasattr(node, 'stats'):
    519                      node.stats = Stats()
    520                      StatsNodes.append(node)
    521                  S = node.stats
    522                  S.considered = S.considered + 1
    523              else:
    524                  S = None
    525
    526              if T: T.write('Taskmaster: %s:' % repr(str(node)))
    527
    528              # Skip this node if it has already been evaluated:
    529              if state > SCons.Node.pending:
    530                  if S: S.already_handled = S.already_handled + 1
    531                  if T: T.write(' already handled (%s)\n' % StateString[state])
    532                  continue
    533
    534              # Mark this node as being on the execution stack:
    535              node.set_state(SCons.Node.pending)
    536
    537              try:
    538                  children = node.children() + node.prerequisites
    539              except SystemExit:
    540                  exc_value = sys.exc_info()[1]
    541                  e = SCons.Errors.ExplicitExit(node, exc_value.code)
    542                  self.ready_exc = (SCons.Errors.ExplicitExit, e)
    543                  if T: T.write(' SystemExit\n')
    544                  return node
    545              except KeyboardInterrupt:
    546                  if T: T.write(' KeyboardInterrupt\n')
    547                  raise
    548              except:
    549                  # We had a problem just trying to figure out the
    550                  # children (like a child couldn't be linked in to a
    551                  # VariantDir, or a Scanner threw something).  Arrange to
    552                  # raise the exception when the Task is "executed."
    553                  self.ready_exc = sys.exc_info()
    554                  if S: S.problem = S.problem + 1
    555                  if T: T.write(' exception\n')
    556                  return node
    557
    558              if T and children:
    559                  c = map(str, children)
    560                  c.sort()
    561                  T.write(' children:\n    %s\n   ' % c)
    562
    563              childstate = map(lambda N: (N, N.get_state()), children)
    564
    565              # Detect dependency cycles:
    566              pending_nodes = filter(lambda I: I[1] == SCons.Node.pending, childstate)
    567              if pending_nodes:
    568                  for p in pending_nodes:
    569                      cycle = find_cycle([p[0], node])
    570                      if cycle:
    571                          desc = "Dependency cycle: " + string.join(map(str, cycle), " -> ")
    572                          if T: T.write(' dependency cycle\n')
    573                          raise SCons.Errors.UserError, desc
    574
    575              not_built = filter(lambda I: I[1] <= SCons.Node.executing, childstate)
    576              if not_built:
    577                  # We're waiting on one or more derived targets that have
    578                  # not yet finished building.
    579
    580                  not_visited = filter(lambda I: not I[1], not_built)
    581                  if not_visited:
    582                      # Some of them haven't even been visited yet.
    583                      # Add them to the list so that on some next pass
    584                      # we can take a stab at evaluating them (or
    585                      # their children).
    586                      not_visited = map(lambda I: I[0], not_visited)
    587                      not_visited.reverse()
    588                      self.candidates.extend(self.order(not_visited))
    589
    590                  n_b_nodes = map(lambda I: I[0], not_built)
    591
    592                  # Add this node to the waiting parents lists of anything
    593                  # we're waiting on, with a reference count so we can be
    594                  # put back on the list for re-evaluation when they've
    595                  # all finished.
    596                  map(lambda n, P=node: n.add_to_waiting_parents(P), n_b_nodes)
    597                  node.ref_count = len(set(n_b_nodes))
    598
    599                  if S: S.not_built = S.not_built + 1
    600                  if T:
    601                      c = map(str, n_b_nodes)
    602                      c.sort()
    603                      T.write(' waiting on unfinished children:\n    %s\n' % c)
    604                  continue
    605
    606              # Skip this node if it has side-effects that are
    607              # currently being built:
    608              side_effects = filter(lambda N:
    609                                    N.get_state() == SCons.Node.executing,
    610                                    node.side_effects)
    611              if side_effects:
    612                  map(lambda n, P=node: n.add_to_waiting_s_e(P), side_effects)
    613                  if S: S.side_effects = S.side_effects + 1
    614                  if T:
    615                      c = map(str, side_effects)
    616                      c.sort()
    617                      T.write(' waiting on side effects:\n    %s\n' % c)
    618                  continue
    619
    620              # Skip this node if any of its children have failed.
    621              #
    622              # This catches the case where we're descending a top-level
    623              # target and one of our children failed while trying to be
    624              # built by a *previous* descent of an earlier top-level
    625              # target.
    626              #
    627              # It can also occur if a node is reused in multiple
    628              # targets. One first descends though the one of the
    629              # target, the next time occurs through the other target.
    630              #
    631              # Note that we can only have failed_children if the
    632              # --keep-going flag was used, because without it the build
    633              # will stop before diving in the other branch.
    634              #
    635              # Note that even if one of the children fails, we still
    636              # added the other children to the list of candidate nodes
    637              # to keep on building (--keep-going).
    638              failed_children = filter(lambda I: I[1] == SCons.Node.failed,
    639                                       childstate)
    640              if failed_children:
    641                  node.set_state(SCons.Node.failed)
    642                  if S: S.child_failed = S.child_failed + 1
    643                  if T:
    644                      c = map(lambda I: str(I[0]), failed_children)
    645                      c.sort()
    646                      T.write(' children failed:\n    %s\n' % c)
    647                  continue
    648
    649              # The default when we've gotten through all of the checks above:
    650              # this node is ready to be built.
    651              if S: S.build = S.build + 1
    652              if T: T.write(' evaluating %s\n' % node)
    653              return node
    654
    655          return None
    656
    657 -    def next_task(self):
    658          """
    659          Returns the next task to be executed.
    660
    661          This simply asks for the next Node to be evaluated, and then wraps
    662          it in the specific Task subclass with which we were initialized.
    663          """
    664          node = self._find_next_ready_node()
    665
    666          if node is None:
    667              return None
    668
    669          tlist = node.get_executor().targets
    670
    671          task = self.tasker(self, tlist, node in self.original_top, node)
    672          try:
    673              task.make_ready()
    674          except KeyboardInterrupt:
    675              raise
    676          except:
    677              # We had a problem just trying to get this task ready (like
    678              # a child couldn't be linked in to a VariantDir when deciding
    679              # whether this node is current).  Arrange to raise the
    680              # exception when the Task is "executed."
    681              self.ready_exc = sys.exc_info()
    682
    683          if self.ready_exc:
    684              task.exception_set(self.ready_exc)
    685
    686          self.ready_exc = None
    687
    688          return task
    689
    690 -    def stop(self):
    691          """
    692          Stops the current build completely.
    693          """
    694          self.next_candidate = self.no_next_candidate
    695