SCons.CacheDir

Package SCons :: Module CacheDir |
[hide private]
[frames] | no frames]

# Source Code for Module SCons.CacheDir

      1  #
      2  # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 The SCons Foundation
      3  #
      4  # Permission is hereby granted, free of charge, to any person obtaining
      5  # a copy of this software and associated documentation files (the
      6  # "Software"), to deal in the Software without restriction, including
      7  # without limitation the rights to use, copy, modify, merge, publish,
      8  # distribute, sublicense, and/or sell copies of the Software, and to
      9  # permit persons to whom the Software is furnished to do so, subject to
     10  # the following conditions:
     11  #
     12  # The above copyright notice and this permission notice shall be included
     13  # in all copies or substantial portions of the Software.
     14  #
     15  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
     16  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
     17  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
     18  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
     19  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
     20  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
     21  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     22  #
     23
     24  __revision__ = "src/engine/SCons/CacheDir.py 2725 2008/03/31 12:52:02 knight"
     25
     26  __doc__ = """
     27  CacheDir support
     28  """
     29
     30  import os.path
     31  import stat
     32  import string
     33  import sys
     34
     35  import SCons.Action
     36
     37  cache_enabled = True
     38  cache_debug = False
     39  cache_force = False
     40  cache_show = False
     41
     42 -def CacheRetrieveFunc(target, source, env):
     43      t = target[0]
     44      fs = t.fs
     45      cd = env.get_CacheDir()
     46      cachedir, cachefile = cd.cachepath(t)
     47      if not fs.exists(cachefile):
     48          cd.CacheDebug('CacheRetrieve(%s):  %s not in cache\n', t, cachefile)
     49          return 1
     50      cd.CacheDebug('CacheRetrieve(%s):  retrieving from %s\n', t, cachefile)
     51      if SCons.Action.execute_actions:
     52          if fs.islink(cachefile):
     53              fs.symlink(fs.readlink(cachefile), t.path)
     54          else:
     55              env.copy_from_cache(cachefile, t.path)
     56          st = fs.stat(cachefile)
     57          fs.chmod(t.path, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
     58      return 0
     59
     60 -def CacheRetrieveString(target, source, env):
     61      t = target[0]
     62      fs = t.fs
     63      cd = env.get_CacheDir()
     64      cachedir, cachefile = cd.cachepath(t)
     65      if t.fs.exists(cachefile):
     66          return "Retrieved `%s' from cache" % t.path
     67      return None
     68
     69  CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
     70
     71  CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
     72
     73 -def CachePushFunc(target, source, env):
     74      t = target[0]
     75      if t.nocache:
     76          return
     77      fs = t.fs
     78      cd = env.get_CacheDir()
     79      cachedir, cachefile = cd.cachepath(t)
     80      if fs.exists(cachefile):
     81          # Don't bother copying it if it's already there.  Note that
     82          # usually this "shouldn't happen" because if the file already
     83          # existed in cache, we'd have retrieved the file from there,
     84          # not built it.  This can happen, though, in a race, if some
     85          # other person running the same build pushes their copy to
     86          # the cache after we decide we need to build it but before our
     87          # build completes.
     88          cd.CacheDebug('CachePush(%s):  %s already exists in cache\n', t, cachefile)
     89          return
     90
     91      cd.CacheDebug('CachePush(%s):  pushing to %s\n', t, cachefile)
     92
     93      tempfile = cachefile+'.tmp'+str(os.getpid())
     94      errfmt = "Unable to copy %s to cache. Cache file is %s"
     95
     96      if not fs.isdir(cachedir):
     97          try:
     98              fs.makedirs(cachedir)
     99          except EnvironmentError:
    100              # We may have received an exception because another process
    101              # has beaten us creating the directory.
    102              if not fs.isdir(cachedir):
    103                  msg = errfmt % (str(target), cachefile)
    104                  raise SCons.Errors.EnvironmentError, msg
    105
    106      try:
    107          if fs.islink(t.path):
    108              fs.symlink(fs.readlink(t.path), tempfile)
    109          else:
    110              fs.copy2(t.path, tempfile)
    111          fs.rename(tempfile, cachefile)
    112          st = fs.stat(t.path)
    113          fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
    114      except EnvironmentError:
    115          # It's possible someone else tried writing the file at the
    116          # same time we did, or else that there was some problem like
    117          # the CacheDir being on a separate file system that's full.
    118          # In any case, inability to push a file to cache doesn't affect
    119          # the correctness of the build, so just print a warning.
    120          msg = errfmt % (str(target), cachefile)
    121          SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
    122
    123  CachePush = SCons.Action.Action(CachePushFunc, None)
    124
    125 -class CacheDir:
    126
    127 -    def __init__(self, path):
    128          try:
    129              import hashlib
    130          except ImportError:
    131              msg = "No hashlib or MD5 module available, CacheDir() not supported"
    132              SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg)
    133              self.path = None
    134          else:
    135              self.path = path
    136          self.current_cache_debug = None
    137          self.debugFP = None
    138
    139 -    def CacheDebug(self, fmt, target, cachefile):
    140          if cache_debug != self.current_cache_debug:
    141              if cache_debug == '-':
    142                  self.debugFP = sys.stdout
    143              elif cache_debug:
    144                  self.debugFP = open(cache_debug, 'w')
    145              else:
    146                  self.debugFP = None
    147              self.current_cache_debug = cache_debug
    148          if self.debugFP:
    149              self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
    150
    151 -    def is_enabled(self):
    152          return (cache_enabled and not self.path is None)
    153
    154 -    def cachepath(self, node):
    155          """
    156          """
    157          if not self.is_enabled():
    158              return None, None
    159
    160          sig = node.get_cachedir_bsig()
    161          subdir = string.upper(sig[0])
    162          dir = os.path.join(self.path, subdir)
    163          return dir, os.path.join(dir, sig)
    164
    165 -    def retrieve(self, node):
    166          """
    167          This method is called from multiple threads in a parallel build,
    168          so only do thread safe stuff here. Do thread unsafe stuff in
    169          built().
    170
    171          Note that there's a special trick here with the execute flag
    172          (one that's not normally done for other actions).  Basically
    173          if the user requested a no_exec (-n) build, then
    174          SCons.Action.execute_actions is set to 0 and when any action
    175          is called, it does its showing but then just returns zero
    176          instead of actually calling the action execution operation.
    177          The problem for caching is that if the file does NOT exist in
    178          cache then the CacheRetrieveString won't return anything to
    179          show for the task, but the Action.__call__ won't call
    180          CacheRetrieveFunc; instead it just returns zero, which makes
    181          the code below think that the file *was* successfully
    182          retrieved from the cache, therefore it doesn't do any
    183          subsequent building.  However, the CacheRetrieveString didn't
    184          print anything because it didn't actually exist in the cache,
    185          and no more build actions will be performed, so the user just
    186          sees nothing.  The fix is to tell Action.__call__ to always
    187          execute the CacheRetrieveFunc and then have the latter
    188          explicitly check SCons.Action.execute_actions itself.
    189          """
    190          if not self.is_enabled():
    191              return False
    192
    193          retrieved = False
    194
    195          if cache_show:
    196              if CacheRetrieveSilent(node, [], node.get_build_env(), execute=1) == 0:
    197                  node.build(presub=0, execute=0)
    198                  retrieved = 1
    199          else:
    200              if CacheRetrieve(node, [], node.get_build_env(), execute=1) == 0:
    201                  retrieved = 1
    202          if retrieved:
    203              # Record build signature information, but don't
    204              # push it out to cache.  (We just got it from there!)
    205              node.set_state(SCons.Node.executed)
    206              SCons.Node.Node.built(node)
    207
    208          return retrieved
    209
    210 -    def push(self, node):
    211          if not self.is_enabled():
    212              return
    213          return CachePush(node, [], node.get_build_env())
    214
    215 -    def push_if_forced(self, node):
    216          if cache_force:
    217              return self.push(node)
    218