SCons.Taskmaster

1 # 2 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 The SCons Foundation 3 # 4 # Permission is hereby granted, free of charge, to any person obtaining 5 # a copy of this software and associated documentation files (the 6 # "Software"), to deal in the Software without restriction, including 7 # without limitation the rights to use, copy, modify, merge, publish, 8 # distribute, sublicense, and/or sell copies of the Software, and to 9 # permit persons to whom the Software is furnished to do so, subject to 10 # the following conditions: 11 # 12 # The above copyright notice and this permission notice shall be included 13 # in all copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 # 23 24 __doc__ = """ 25 Generic Taskmaster module for the SCons build engine. 26 27 This module contains the primary interface(s) between a wrapping user 28 interface and the SCons build engine. There are two key classes here: 29 30 Taskmaster 31 This is the main engine for walking the dependency graph and 32 calling things to decide what does or doesn't need to be built. 33 34 Task 35 This is the base class for allowing a wrapping interface to 36 decide what does or doesn't actually need to be done. The 37 intention is for a wrapping interface to subclass this as 38 appropriate for different types of behavior it may need. 39 40 The canonical example is the SCons native Python interface, 41 which has Task subclasses that handle its specific behavior, 42 like printing "`foo' is up to date" when a top-level target 43 doesn't need to be built, and handling the -c option by removing 44 targets as its "build" action. There is also a separate subclass 45 for suppressing this output when the -q option is used. 46 47 The Taskmaster instantiates a Task object for each (set of) 48 target(s) that it decides need to be evaluated and/or built. 49 """ 50 51 __revision__ = "src/engine/SCons/Taskmaster.py 2725 2008/03/31 12:52:02 knight" 52 53 import SCons.compat 54 55 import operator 56 import string 57 import sys 58 import traceback 59 60 import SCons.Node 61 import SCons.Errors 62 63 StateString = SCons.Node.StateString 64 65 66 67 # A subsystem for recording stats about how different Nodes are handled by 68 # the main Taskmaster loop. There's no external control here (no need for 69 # a --debug= option); enable it by changing the value of CollectStats. 70 71 CollectStats = None 72

73 -class Stats:

74 """ 75 A simple class for holding statistics about the disposition of a 76 Node by the Taskmaster. If we're collecting statistics, each Node 77 processed by the Taskmaster gets one of these attached, in which case 78 the Taskmaster records its decision each time it processes the Node. 79 (Ideally, that's just once per Node.) 80 """

81 - def __init__(self):

82 """ 83 Instantiates a Taskmaster.Stats object, initializing all 84 appropriate counters to zero. 85 """ 86 self.considered = 0 87 self.already_handled = 0 88 self.problem = 0 89 self.child_failed = 0 90 self.not_built = 0 91 self.side_effects = 0 92 self.build = 0

93 94 StatsNodes = [] 95 96 fmt = "%(considered)3d "\ 97 "%(already_handled)3d " \ 98 "%(problem)3d " \ 99 "%(child_failed)3d " \ 100 "%(not_built)3d " \ 101 "%(side_effects)3d " \ 102 "%(build)3d " 103

104 -def dump_stats():

105 StatsNodes.sort(lambda a, b: cmp(str(a), str(b))) 106 for n in StatsNodes: 107 print (fmt % n.stats.__dict__) + str(n)

108 109 110

111 -class Task:

112 """ 113 Default SCons build engine task. 114 115 This controls the interaction of the actual building of node 116 and the rest of the engine. 117 118 This is expected to handle all of the normally-customizable 119 aspects of controlling a build, so any given application 120 *should* be able to do what it wants by sub-classing this 121 class and overriding methods as appropriate. If an application 122 needs to customze something by sub-classing Taskmaster (or 123 some other build engine class), we should first try to migrate 124 that functionality into this class. 125 126 Note that it's generally a good idea for sub-classes to call 127 these methods explicitly to update state, etc., rather than 128 roll their own interaction with Taskmaster from scratch. 129 """

130 - def __init__(self, tm, targets, top, node):

131 self.tm = tm 132 self.targets = targets 133 self.top = top 134 self.node = node 135 self.exc_clear()

136

137 - def display(self, message):

138 """ 139 Hook to allow the calling interface to display a message. 140 141 This hook gets called as part of preparing a task for execution 142 (that is, a Node to be built). As part of figuring out what Node 143 should be built next, the actually target list may be altered, 144 along with a message describing the alteration. The calling 145 interface can subclass Task and provide a concrete implementation 146 of this method to see those messages. 147 """ 148 pass

149

150 - def prepare(self):

151 """ 152 Called just before the task is executed. 153 154 This is mainly intended to give the target Nodes a chance to 155 unlink underlying files and make all necessary directories before 156 the Action is actually called to build the targets. 157 """ 158 159 # Now that it's the appropriate time, give the TaskMaster a 160 # chance to raise any exceptions it encountered while preparing 161 # this task. 162 self.exception_raise() 163 164 if self.tm.message: 165 self.display(self.tm.message) 166 self.tm.message = None 167 168 for t in self.targets: 169 t.prepare() 170 for s in t.side_effects: 171 s.prepare()

172

173 - def get_target(self):

174 """Fetch the target being built or updated by this task. 175 """ 176 return self.node

177

178 - def execute(self):

179 """ 180 Called to execute the task. 181 182 This method is called from multiple threads in a parallel build, 183 so only do thread safe stuff here. Do thread unsafe stuff in 184 prepare(), executed() or failed(). 185 """ 186 187 try: 188 everything_was_cached = 1 189 for t in self.targets: 190 if not t.retrieve_from_cache(): 191 everything_was_cached = 0 192 break 193 if not everything_was_cached: 194 self.targets[0].build() 195 except KeyboardInterrupt: 196 raise 197 except SystemExit: 198 exc_value = sys.exc_info()[1] 199 raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) 200 except SCons.Errors.UserError: 201 raise 202 except SCons.Errors.BuildError: 203 raise 204 except: 205 raise SCons.Errors.TaskmasterException(self.targets[0], 206 sys.exc_info())

207

208 - def executed_without_callbacks(self):

209 """ 210 Called when the task has been successfully executed 211 and the Taskmaster instance doesn't want to call 212 the Node's callback methods. 213 """ 214 for t in self.targets: 215 if t.get_state() == SCons.Node.executing: 216 for side_effect in t.side_effects: 217 side_effect.set_state(SCons.Node.no_state) 218 t.set_state(SCons.Node.executed)

219

220 - def executed_with_callbacks(self):

221 """ 222 Called when the task has been successfully executed and 223 the Taskmaster instance wants to call the Node's callback 224 methods. 225 226 This may have been a do-nothing operation (to preserve build 227 order), so we must check the node's state before deciding whether 228 it was "built", in which case we call the appropriate Node method. 229 In any event, we always call "visited()", which will handle any 230 post-visit actions that must take place regardless of whether 231 or not the target was an actual built target or a source Node. 232 """ 233 for t in self.targets: 234 if t.get_state() == SCons.Node.executing: 235 for side_effect in t.side_effects: 236 side_effect.set_state(SCons.Node.no_state) 237 t.set_state(SCons.Node.executed) 238 t.built() 239 t.visited()

240 241 executed = executed_with_callbacks 242

243 - def failed(self):

244 """ 245 Default action when a task fails: stop the build. 246 """ 247 self.fail_stop()

248

249 - def fail_stop(self):

250 """ 251 Explicit stop-the-build failure. 252 """ 253 for t in self.targets: 254 t.set_state(SCons.Node.failed) 255 self.tm.stop() 256 257 # We're stopping because of a build failure, but give the 258 # calling Task class a chance to postprocess() the top-level 259 # target under which the build failure occurred. 260 self.targets = [self.tm.current_top] 261 self.top = 1

262

263 - def fail_continue(self):

264 """ 265 Explicit continue-the-build failure. 266 267 This sets failure status on the target nodes and all of 268 their dependent parent nodes. 269 """ 270 for t in self.targets: 271 # Set failure state on all of the parents that were dependent 272 # on this failed build. 273 def set_state(node): node.set_state(SCons.Node.failed) 274 t.call_for_all_waiting_parents(set_state)

275

276 - def make_ready_all(self):

277 """ 278 Marks all targets in a task ready for execution. 279 280 This is used when the interface needs every target Node to be 281 visited--the canonical example being the "scons -c" option. 282 """ 283 self.out_of_date = self.targets[:] 284 for t in self.targets: 285 t.disambiguate().set_state(SCons.Node.executing) 286 for s in t.side_effects: 287 s.set_state(SCons.Node.executing)

288

289 - def make_ready_current(self):

290 """ 291 Marks all targets in a task ready for execution if any target 292 is not current. 293 294 This is the default behavior for building only what's necessary. 295 """ 296 self.out_of_date = [] 297 for t in self.targets: 298 try: 299 t.disambiguate().make_ready() 300 is_up_to_date = not t.has_builder() or \ 301 (not t.always_build and t.is_up_to_date()) 302 except EnvironmentError, e: 303 raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filename=e.filename) 304 if is_up_to_date: 305 t.set_state(SCons.Node.up_to_date) 306 else: 307 self.out_of_date.append(t) 308 t.set_state(SCons.Node.executing) 309 for s in t.side_effects: 310 s.set_state(SCons.Node.executing)

311 312 make_ready = make_ready_current 313

314 - def postprocess(self):

315 """ 316 Post-processes a task after it's been executed. 317 318 This examines all the targets just built (or not, we don't care 319 if the build was successful, or even if there was no build 320 because everything was up-to-date) to see if they have any 321 waiting parent Nodes, or Nodes waiting on a common side effect, 322 that can be put back on the candidates list. 323 """ 324 325 # We may have built multiple targets, some of which may have 326 # common parents waiting for this build. Count up how many 327 # targets each parent was waiting for so we can subtract the 328 # values later, and so we *don't* put waiting side-effect Nodes 329 # back on the candidates list if the Node is also a waiting 330 # parent. 331 332 targets = set(self.targets) 333 334 parents = {} 335 for t in targets: 336 for p in t.waiting_parents.keys(): 337 parents[p] = parents.get(p, 0) + 1 338 339 for t in targets: 340 for s in t.side_effects: 341 if s.get_state() == SCons.Node.executing: 342 s.set_state(SCons.Node.no_state) 343 for p in s.waiting_parents.keys(): 344 if not parents.has_key(p): 345 parents[p] = 1 346 for p in s.waiting_s_e.keys(): 347 if p.ref_count == 0: 348 self.tm.candidates.append(p) 349 350 for p, subtract in parents.items(): 351 p.ref_count = p.ref_count - subtract 352 if p.ref_count == 0: 353 self.tm.candidates.append(p) 354 355 for t in targets: 356 t.postprocess()

357 358 # Exception handling subsystem. 359 # 360 # Exceptions that occur while walking the DAG or examining Nodes 361 # must be raised, but must be raised at an appropriate time and in 362 # a controlled manner so we can, if necessary, recover gracefully, 363 # possibly write out signature information for Nodes we've updated, 364 # etc. This is done by having the Taskmaster tell us about the 365 # exception, and letting 366

367 - def exc_info(self):

368 """ 369 Returns info about a recorded exception. 370 """ 371 return self.exception

372

373 - def exc_clear(self):

374 """ 375 Clears any recorded exception. 376 377 This also changes the "exception_raise" attribute to point 378 to the appropriate do-nothing method. 379 """ 380 self.exception = (None, None, None) 381 self.exception_raise = self._no_exception_to_raise

382

383 - def exception_set(self, exception=None):

384 """ 385 Records an exception to be raised at the appropriate time. 386 387 This also changes the "exception_raise" attribute to point 388 to the method that will, in fact 389 """ 390 if not exception: 391 exception = sys.exc_info() 392 self.exception = exception 393 self.exception_raise = self._exception_raise

394

395 - def _no_exception_to_raise(self):

396 pass

397

398 - def _exception_raise(self):

399 """ 400 Raises a pending exception that was recorded while getting a 401 Task ready for execution. 402 """ 403 exc = self.exc_info()[:] 404 try: 405 exc_type, exc_value, exc_traceback = exc 406 except ValueError: 407 exc_type, exc_value = exc 408 exc_traceback = None 409 raise exc_type, exc_value, exc_traceback

410 411

412 -def find_cycle(stack):

413 if stack[0] == stack[-1]: 414 return stack 415 for n in stack[-1].waiting_parents.keys(): 416 stack.append(n) 417 if find_cycle(stack): 418 return stack 419 stack.pop() 420 return None

421 422

423 -class Taskmaster:

424 """ 425 The Taskmaster for walking the dependency DAG. 426 """ 427

428 - def __init__(self, targets=[], tasker=Task, order=None, trace=None):

429 self.original_top = targets 430 self.top_targets_left = targets[:] 431 self.top_targets_left.reverse() 432 self.candidates = [] 433 self.tasker = tasker 434 if not order: 435 order = lambda l: l 436 self.order = order 437 self.message = None 438 self.trace = trace 439 self.next_candidate = self.find_next_candidate

440

441 - def find_next_candidate(self):

442 """ 443 Returns the next candidate Node for (potential) evaluation. 444 445 The candidate list (really a stack) initially consists of all of 446 the top-level (command line) targets provided when the Taskmaster 447 was initialized. While we walk the DAG, visiting Nodes, all the 448 children that haven't finished processing get pushed on to the 449 candidate list. Each child can then be popped and examined in 450 turn for whether *their* children are all up-to-date, in which 451 case a Task will be created for their actual evaluation and 452 potential building. 453 454 Here is where we also allow candidate Nodes to alter the list of 455 Nodes that should be examined. This is used, for example, when 456 invoking SCons in a source directory. A source directory Node can 457 return its corresponding build directory Node, essentially saying, 458 "Hey, you really need to build this thing over here instead." 459 """ 460 try: 461 return self.candidates.pop() 462 except IndexError: 463 pass 464 try: 465 node = self.top_targets_left.pop() 466 except IndexError: 467 return None 468 self.current_top = node 469 alt, message = node.alter_targets() 470 if alt: 471 self.message = message 472 self.candidates.append(node) 473 self.candidates.extend(self.order(alt)) 474 node = self.candidates.pop() 475 return node

476

477 - def no_next_candidate(self):

478 """ 479 Stops Taskmaster processing by not returning a next candidate. 480 """ 481 return None

482

483 - def _find_next_ready_node(self):

484 """ 485 Finds the next node that is ready to be built. 486 487 This is *the* main guts of the DAG walk. We loop through the 488 list of candidates, looking for something that has no un-built 489 children (i.e., that is a leaf Node or has dependencies that are 490 all leaf Nodes or up-to-date). Candidate Nodes are re-scanned 491 (both the target Node itself and its sources, which are always 492 scanned in the context of a given target) to discover implicit 493 dependencies. A Node that must wait for some children to be 494 built will be put back on the candidates list after the children 495 have finished building. A Node that has been put back on the 496 candidates list in this way may have itself (or its sources) 497 re-scanned, in order to handle generated header files (e.g.) and 498 the implicit dependencies therein. 499 500 Note that this method does not do any signature calculation or 501 up-to-date check itself. All of that is handled by the Task 502 class. This is purely concerned with the dependency graph walk. 503 """ 504 505 self.ready_exc = None 506 507 T = self.trace 508 509 while 1: 510 node = self.next_candidate() 511 if node is None: 512 return None 513 514 node = node.disambiguate() 515 state = node.get_state() 516 517 if CollectStats: 518 if not hasattr(node, 'stats'): 519 node.stats = Stats() 520 StatsNodes.append(node) 521 S = node.stats 522 S.considered = S.considered + 1 523 else: 524 S = None 525 526 if T: T.write('Taskmaster: %s:' % repr(str(node))) 527 528 # Skip this node if it has already been evaluated: 529 if state > SCons.Node.pending: 530 if S: S.already_handled = S.already_handled + 1 531 if T: T.write(' already handled (%s)\n' % StateString[state]) 532 continue 533 534 # Mark this node as being on the execution stack: 535 node.set_state(SCons.Node.pending) 536 537 try: 538 children = node.children() + node.prerequisites 539 except SystemExit: 540 exc_value = sys.exc_info()[1] 541 e = SCons.Errors.ExplicitExit(node, exc_value.code) 542 self.ready_exc = (SCons.Errors.ExplicitExit, e) 543 if T: T.write(' SystemExit\n') 544 return node 545 except KeyboardInterrupt: 546 if T: T.write(' KeyboardInterrupt\n') 547 raise 548 except: 549 # We had a problem just trying to figure out the 550 # children (like a child couldn't be linked in to a 551 # VariantDir, or a Scanner threw something). Arrange to 552 # raise the exception when the Task is "executed." 553 self.ready_exc = sys.exc_info() 554 if S: S.problem = S.problem + 1 555 if T: T.write(' exception\n') 556 return node 557 558 if T and children: 559 c = map(str, children) 560 c.sort() 561 T.write(' children:\n %s\n ' % c) 562 563 childstate = map(lambda N: (N, N.get_state()), children) 564 565 # Detect dependency cycles: 566 pending_nodes = filter(lambda I: I[1] == SCons.Node.pending, childstate) 567 if pending_nodes: 568 for p in pending_nodes: 569 cycle = find_cycle([p[0], node]) 570 if cycle: 571 desc = "Dependency cycle: " + string.join(map(str, cycle), " -> ") 572 if T: T.write(' dependency cycle\n') 573 raise SCons.Errors.UserError, desc 574 575 not_built = filter(lambda I: I[1] <= SCons.Node.executing, childstate) 576 if not_built: 577 # We're waiting on one or more derived targets that have 578 # not yet finished building. 579 580 not_visited = filter(lambda I: not I[1], not_built) 581 if not_visited: 582 # Some of them haven't even been visited yet. 583 # Add them to the list so that on some next pass 584 # we can take a stab at evaluating them (or 585 # their children). 586 not_visited = map(lambda I: I[0], not_visited) 587 not_visited.reverse() 588 self.candidates.extend(self.order(not_visited)) 589 590 n_b_nodes = map(lambda I: I[0], not_built) 591 592 # Add this node to the waiting parents lists of anything 593 # we're waiting on, with a reference count so we can be 594 # put back on the list for re-evaluation when they've 595 # all finished. 596 map(lambda n, P=node: n.add_to_waiting_parents(P), n_b_nodes) 597 node.ref_count = len(set(n_b_nodes)) 598 599 if S: S.not_built = S.not_built + 1 600 if T: 601 c = map(str, n_b_nodes) 602 c.sort() 603 T.write(' waiting on unfinished children:\n %s\n' % c) 604 continue 605 606 # Skip this node if it has side-effects that are 607 # currently being built: 608 side_effects = filter(lambda N: 609 N.get_state() == SCons.Node.executing, 610 node.side_effects) 611 if side_effects: 612 map(lambda n, P=node: n.add_to_waiting_s_e(P), side_effects) 613 if S: S.side_effects = S.side_effects + 1 614 if T: 615 c = map(str, side_effects) 616 c.sort() 617 T.write(' waiting on side effects:\n %s\n' % c) 618 continue 619 620 # Skip this node if any of its children have failed. 621 # 622 # This catches the case where we're descending a top-level 623 # target and one of our children failed while trying to be 624 # built by a *previous* descent of an earlier top-level 625 # target. 626 # 627 # It can also occur if a node is reused in multiple 628 # targets. One first descends though the one of the 629 # target, the next time occurs through the other target. 630 # 631 # Note that we can only have failed_children if the 632 # --keep-going flag was used, because without it the build 633 # will stop before diving in the other branch. 634 # 635 # Note that even if one of the children fails, we still 636 # added the other children to the list of candidate nodes 637 # to keep on building (--keep-going). 638 failed_children = filter(lambda I: I[1] == SCons.Node.failed, 639 childstate) 640 if failed_children: 641 node.set_state(SCons.Node.failed) 642 if S: S.child_failed = S.child_failed + 1 643 if T: 644 c = map(lambda I: str(I[0]), failed_children) 645 c.sort() 646 T.write(' children failed:\n %s\n' % c) 647 continue 648 649 # The default when we've gotten through all of the checks above: 650 # this node is ready to be built. 651 if S: S.build = S.build + 1 652 if T: T.write(' evaluating %s\n' % node) 653 return node 654 655 return None

656

657 - def next_task(self):

658 """ 659 Returns the next task to be executed. 660 661 This simply asks for the next Node to be evaluated, and then wraps 662 it in the specific Task subclass with which we were initialized. 663 """ 664 node = self._find_next_ready_node() 665 666 if node is None: 667 return None 668 669 tlist = node.get_executor().targets 670 671 task = self.tasker(self, tlist, node in self.original_top, node) 672 try: 673 task.make_ready() 674 except KeyboardInterrupt: 675 raise 676 except: 677 # We had a problem just trying to get this task ready (like 678 # a child couldn't be linked in to a VariantDir when deciding 679 # whether this node is current). Arrange to raise the 680 # exception when the Task is "executed." 681 self.ready_exc = sys.exc_info() 682 683 if self.ready_exc: 684 task.exception_set(self.ready_exc) 685 686 self.ready_exc = None 687 688 return task

689

690 - def stop(self):

691 """ 692 Stops the current build completely. 693 """ 694 self.next_candidate = self.no_next_candidate

695

Source Code for Module SCons.Taskmaster