Halberd.crew

Package Halberd :: Module crew |
[hide private]
[frames] | no frames]

# Source Code for Module Halberd.crew

      1  # -*- coding: iso-8859-1 -*-
      2
      3  """\
      4  Work crew pattern of parallel scanners
      5  ======================================
      6
      7  Overview
      8  --------
      9
     10  A work crew is instantiated passing a ScanTask object as a parameter, thus
     11  defining the target and the way the scanning should be done. After the
     12  initialization of the work crew it can be used to scan the target and get the
     13  obtained clues back.
     14
     15      >>> crew = WorkCrew(scantask)
     16      >>> clues = crew.scan()
     17
     18  Requirements
     19  ------------
     20
     21  These are the features that the WorkCrew must provide:
     22
     23      1. There are 3 different types of consumers:
     24          - Controller thread (Performs timing + error-checking).
     25          - Local scanning thread.
     26          - Remote scanning thread.
     27
     28      2. We need a way to signal:
     29          - When a fatal error has happened.
     30          - When the user has pressed Control-C
     31
     32  Types of scanning threads
     33  -------------------------
     34
     35  The WorkCrew object spawns different kinds of threads. Here's a brief summary
     36  of what they do:
     37
     38      - Manager: Detects when the time for performing the scan has expired
     39      and notifies the rest of the threads. This code is executed in the main
     40      thread in order to be able to appropriately catch signals, etc.
     41
     42      - Scanner: Performs a load-balancer scan from the current machine.
     43
     44  The following is a diagram showing the way it works::
     45
     46                                       .--> Manager --.
     47                                       |              |
     48                                       +--> Scanner --+
     49          .----------.   .----------.  |              |   .-------.
     50   IN --> | ScanTask |->-| WorkCrew |--+--> Scanner --+->-| Clues |--> OUT
     51          `----------'   `----------'  |              |   `-------'
     52                                       +--> Scanner --+
     53                                       |              |
     54                                       `--> Scanner --'
     55  """
     56
     57  # Copyright (C) 2004, 2005, 2006, 2010  Juan M. Bello Rivas <jmbr@superadditive.com>
     58  #
     59  # This program is free software; you can redistribute it and/or modify
     60  # it under the terms of the GNU General Public License as published by
     61  # the Free Software Foundation; either version 2 of the License, or
     62  # (at your option) any later version.
     63  #
     64  # This program is distributed in the hope that it will be useful,
     65  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     66  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     67  # GNU General Public License for more details.
     68  #
     69  # You should have received a copy of the GNU General Public License
     70  # along with this program; if not, write to the Free Software
     71  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
     72
     73
     74  import sys
     75  import time
     76  import math
     77  import copy
     78  import signal
     79  import threading
     80
     81  import Halberd.logger
     82  import Halberd.clues.Clue
     83  import Halberd.clientlib as clientlib
     84
     85
     86  __all__ = ['WorkCrew']
     87
     88
     89 -class ScanState:
     90      """Shared state among scanner threads.
     91
     92      @ivar shouldstop: Signals when the threads should stop scanning.
     93      @type shouldstop: C{threading.Event}
     94
     95      caught with an exception).
     96      """
     97 -    def __init__(self):
     98          """Initializes shared state among scanning threads.
     99          """
    100          self.__mutex = threading.Lock()
    101          self.shouldstop = threading.Event()
    102          self.__error = None
    103          self.__clues = []
    104
    105          self.__missed = 0
    106          self.__replies = 0
    107
    108 -    def getStats(self):
    109          """Provides statistics about the scanning process.
    110
    111          @return: Number of clues gathered so far, number of successful requests
    112          and number of unsuccessful ones (missed replies).
    113          @rtype: C{tuple}
    114          """
    115          # xxx - I badly need read/write locks.
    116          self.__mutex.acquire()
    117          nclues = len(self.__clues)
    118          replies = self.__replies
    119          missed = self.__missed
    120          self.__mutex.release()
    121
    122          return (nclues, replies, missed)
    123
    124 -    def insertClue(self, clue):
    125          """Inserts a clue in the list if it is new.
    126          """
    127          self.__mutex.acquire()
    128
    129          count = clue.getCount()
    130          self.__replies += count
    131          try:
    132              idx = self.__clues.index(clue)
    133              self.__clues[idx].incCount(count)
    134          except ValueError:
    135              self.__clues.append(clue)
    136
    137          self.__mutex.release()
    138
    139 -    def getClues(self):
    140          """Clue accessor.
    141
    142          @return: A copy of all obtained clues.
    143          @rtype: C{list}
    144          """
    145          self.__mutex.acquire()
    146          clues = self.__clues[:]
    147          self.__mutex.release()
    148
    149          return clues
    150
    151 -    def incMissed(self):
    152          """Increase the counter of missed replies.
    153          """
    154          self.__mutex.acquire()
    155          self.__missed += 1
    156          self.__mutex.release()
    157
    158 -    def setError(self, err):
    159          """Signal an error condition.
    160          """
    161          self.__mutex.acquire()
    162          if self.__error is not None:
    163              # An error has already been signalled.
    164              self.__mutex.release()
    165              return
    166          self.__error = err
    167          self.shouldstop.set()
    168          self.__mutex.release()
    169
    170 -    def getError(self):
    171          """Returns the reason of the error condition.
    172          """
    173          self.__mutex.acquire()
    174          # Since we don't know what the nature of __error will be, we need to
    175          # provide a clean copy of it to the caller so that no possible
    176          # references or changes to __error can affect the object we return.
    177          err = copy.deepcopy(self.__error)
    178          self.__mutex.release()
    179
    180          return err
    181
    182
    183 -class WorkCrew:
    184      """Pool of scanners working in parallel.
    185
    186      @ivar task: A reference to scantask.
    187      @type task: L{ScanTask}
    188
    189      @ivar working: Indicates whether the crew is working or idle.
    190      @type working: C{bool}
    191
    192      @ivar prev: Previous SIGINT handler.
    193      """
    194 -    def __init__(self, scantask):
    195          self.workers = []
    196          self.task = scantask
    197
    198          self.state = ScanState()
    199
    200          self.working = False
    201
    202          self.prev = None
    203
    204 -    def _setupSigHandler(self):
    205          """Performs what's needed to catch SIGINT.
    206          """
    207          def interrupt(signum, frame):
    208              """SIGINT handler
    209              """
    210              self.state.setError('received SIGINT')
    211
    212          self.prev = signal.signal(signal.SIGINT, interrupt)
    213
    214 -    def _restoreSigHandler(self):
    215          """Restore previous SIGINT handler.
    216          """
    217          signal.signal(signal.SIGINT, self.prev)
    218
    219 -    def _initLocal(self):
    220          """Initializes conventional (local) scanner threads.
    221          """
    222          for i in xrange(self.task.parallelism):
    223              worker = Scanner(self.state, self.task)
    224              self.workers.append(worker)
    225
    226 -    def scan(self):
    227          """Perform a parallel load-balancer scan.
    228          """
    229          self.working = True
    230          self._setupSigHandler()
    231
    232          self._initLocal()
    233
    234          for worker in self.workers:
    235              worker.start()
    236
    237          # The Manager executes in the main thread WHILE the others are working
    238          # so that signals are correctly caught.
    239          manager = Manager(self.state, self.task)
    240          manager.run()
    241
    242          for worker in self.workers:
    243              worker.join()
    244
    245          # Display status information for the last time.
    246          manager.showStats()
    247          sys.stdout.write('\n\n')
    248
    249          self._restoreSigHandler()
    250          self.working = False
    251
    252          err = self.state.getError()
    253          if err is not None:
    254              sys.stderr.write('*** finished (%s) ***\n\n' % err)
    255
    256          return self._getClues()
    257
    258 -    def _getClues(self):
    259          """Returns a sequence of clues obtained during the scan.
    260          """
    261          assert not self.working
    262
    263          return self.state.getClues()
    264
    265
    266 -class BaseScanner(threading.Thread):
    267      """Base class for load balancer scanning threads.
    268
    269      @ivar timeout: Time (in seconds since the UNIX Epoch) when the scan will be
    270      stopped.
    271      @type timeout: C{float}
    272      """
    273 -    def __init__(self, state, scantask):
    274          """Initializes the scanning thread.
    275
    276          @param state: Container to store the results of the scan (shared among
    277          scanning threads).
    278          @type state: C{instanceof(ScanState)}
    279
    280          @param scantask: Object providing information needed to perform the
    281          scan.
    282          @type scantask: C{instanceof(ScanTask)}
    283          """
    284          threading.Thread.__init__(self)
    285          self.state = state
    286          self.task = scantask
    287          self.timeout = 0
    288          self.logger = Halberd.logger.getLogger()
    289
    290 -    def remaining(self, end=None):
    291          """Seconds left until a given point in time.
    292
    293          @param end: Ending time.
    294          @type end: C{float}
    295
    296          @return: Remaining time until L{self.timeout}
    297          @rtype: C{int}
    298          """
    299          if not end:
    300              end = self.timeout
    301          return int(end - time.time())
    302
    303 -    def hasExpired(self):
    304          """Expiration predicate.
    305
    306          @return: True if the timeout has expired, False otherwise.
    307          @rtype: C{bool}
    308          """
    309          return (self.remaining() <= 0)
    310
    311 -    def setTimeout(self, secs):
    312          """Compute an expiration time.
    313
    314          @param secs: Amount of seconds to spend scanning the target.
    315          @type secs: C{int}
    316
    317          @return: The moment in time when the task expires.
    318          @rtype: C{float}
    319          """
    320          self.timeout = time.time() + secs
    321
    322 -    def run(self):
    323          """Perform the scan.
    324          """
    325          self.setTimeout(self.task.scantime)
    326
    327          while not self.state.shouldstop.isSet():
    328              self.process()
    329
    330 -    def process(self):
    331          """Perform a scanning task.
    332
    333          This method should be overriden to do actual work.
    334          """
    335          pass
    336
    337 -class Scanner(BaseScanner):
    338      """Scans the target host from the local machine.
    339      """
    340 -    def process(self):
    341          """Gathers clues connecting directly to the target web server.
    342          """
    343          client = clientlib.clientFactory(self.task)
    344
    345          fatal_exceptions = (
    346              clientlib.ConnectionRefused,
    347              clientlib.UnknownReply,
    348              clientlib.HTTPSError,
    349          )
    350
    351          try:
    352              ts, hdrs = client.getHeaders(self.task.addr, self.task.url)
    353          except fatal_exceptions, msg:
    354              self.state.setError(msg)
    355          except clientlib.TimedOut, msg:
    356              self.state.incMissed()
    357          else:
    358              self.state.insertClue(self.makeClue(ts, hdrs))
    359
    360 -    def makeClue(self, timestamp, headers):
    361          """Compose a clue object.
    362
    363          @param timestamp: Time when the reply was received.
    364          @type timestamp: C{float}
    365
    366          @param headers: MIME headers coming from an HTTP response.
    367          @type headers: C{str}
    368
    369          @return: A valid clue
    370          @rtype: C{Clue}
    371          """
    372          clue = Halberd.clues.Clue.Clue()
    373          clue.setTimestamp(timestamp)
    374          clue.parse(headers)
    375
    376          return clue
    377
    378
    379 -class Manager(BaseScanner):
    380      """Performs management tasks during the scan.
    381      """
    382      # Indicates how often the state must be refreshed (in seconds).
    383      refresh_interval = 0.25
    384
    385 -    def process(self):
    386          """Controls the whole scanning process.
    387
    388          This method checks when the timeout has expired and notifies the rest
    389          of the scanning threads that they should stop. It also displays (in
    390          case the user asked for it) detailed information regarding the process.
    391          """
    392          self.showStats()
    393
    394          if self.hasExpired():
    395              self.state.shouldstop.set()
    396          try:
    397              time.sleep(self.refresh_interval)
    398          except IOError:
    399              # Catch interrupted system call exception (it happens when
    400              # CONTROL-C is pressed on win32 systems).
    401              self.state.shouldstop.set()
    402
    403 -    def showStats(self):
    404          """Displays certain statistics while the scan is happening.
    405          """
    406          if not self.task.verbose:
    407              return
    408
    409          def statbar(elapsed, total):
    410              """Compose a status bar string showing progress.
    411              """
    412              done = int(math.floor(float(total - elapsed)/total * 10))
    413              notdone = int(math.ceil(float(elapsed)/total * 10))
    414              return '[' + '#' * done + ' ' * notdone + ']'
    415
    416          nclues, replies, missed = self.state.getStats()
    417
    418          # We put a lower bound on the remaining time.
    419          if self.remaining() < 0:
    420              remaining = 0
    421          else:
    422              remaining = self.remaining()
    423
    424          statusline = '\r' + self.task.addr.ljust(15) + \
    425                      '  %s  clues: %3d | replies: %3d | missed: %3d' \
    426                      % (statbar(remaining, self.task.scantime),
    427                         nclues, replies, missed)
    428          sys.stdout.write(statusline)
    429          sys.stdout.flush()
    430
    431
    432  # vim: ts=4 sw=4 et
    433