paramiko.file

Package paramiko :: Module file |
[frames] | no frames]

# Source Code for Module paramiko.file

      1  # Copyright (C) 2003-2007  Robey Pointer <robey@lag.net>
      2  #
      3  # This file is part of paramiko.
      4  #
      5  # Paramiko is free software; you can redistribute it and/or modify it under the
      6  # terms of the GNU Lesser General Public License as published by the Free
      7  # Software Foundation; either version 2.1 of the License, or (at your option)
      8  # any later version.
      9  #
     10  # Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY
     11  # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
     12  # A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
     13  # details.
     14  #
     15  # You should have received a copy of the GNU Lesser General Public License
     16  # along with Paramiko; if not, write to the Free Software Foundation, Inc.,
     17  # 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
     18
     19  """
     20  BufferedFile.
     21  """
     22
     23  from cStringIO import StringIO
     24
     25
     26 -class BufferedFile (object):
     27      """
     28      Reusable base class to implement python-style file buffering around a
     29      simpler stream.
     30      """
     31
     32      _DEFAULT_BUFSIZE = 8192
     33
     34      SEEK_SET = 0
     35      SEEK_CUR = 1
     36      SEEK_END = 2
     37
     38      FLAG_READ = 0x1
     39      FLAG_WRITE = 0x2
     40      FLAG_APPEND = 0x4
     41      FLAG_BINARY = 0x10
     42      FLAG_BUFFERED = 0x20
     43      FLAG_LINE_BUFFERED = 0x40
     44      FLAG_UNIVERSAL_NEWLINE = 0x80
     45
     46 -    def __init__(self):
     47          self.newlines = None
     48          self._flags = 0
     49          self._bufsize = self._DEFAULT_BUFSIZE
     50          self._wbuffer = StringIO()
     51          self._rbuffer = ''
     52          self._at_trailing_cr = False
     53          self._closed = False
     54          # pos - position within the file, according to the user
     55          # realpos - position according the OS
     56          # (these may be different because we buffer for line reading)
     57          self._pos = self._realpos = 0
     58          # size only matters for seekable files
     59          self._size = 0
     60
     61 -    def __del__(self):
     62          self.close()
     63
     64 -    def __iter__(self):
     65          """
     66          Returns an iterator that can be used to iterate over the lines in this
     67          file.  This iterator happens to return the file itself, since a file is
     68          its own iterator.
     69
     70          @raise ValueError: if the file is closed.
     71
     72          @return: an interator.
     73          @rtype: iterator
     74          """
     75          if self._closed:
     76              raise ValueError('I/O operation on closed file')
     77          return self
     78
     79 -    def close(self):
     80          """
     81          Close the file.  Future read and write operations will fail.
     82          """
     83          self.flush()
     84          self._closed = True
     85
     86 -    def flush(self):
     87          """
     88          Write out any data in the write buffer.  This may do nothing if write
     89          buffering is not turned on.
     90          """
     91          self._write_all(self._wbuffer.getvalue())
     92          self._wbuffer = StringIO()
     93          return
     94
     95 -    def next(self):
     96          """
     97          Returns the next line from the input, or raises L{StopIteration} when
     98          EOF is hit.  Unlike python file objects, it's okay to mix calls to
     99          C{next} and L{readline}.
    100
    101          @raise StopIteration: when the end of the file is reached.
    102
    103          @return: a line read from the file.
    104          @rtype: str
    105          """
    106          line = self.readline()
    107          if not line:
    108              raise StopIteration
    109          return line
    110
    111 -    def read(self, size=None):
    112          """
    113          Read at most C{size} bytes from the file (less if we hit the end of the
    114          file first).  If the C{size} argument is negative or omitted, read all
    115          the remaining data in the file.
    116
    117          @param size: maximum number of bytes to read
    118          @type size: int
    119          @return: data read from the file, or an empty string if EOF was
    120              encountered immediately
    121          @rtype: str
    122          """
    123          if self._closed:
    124              raise IOError('File is closed')
    125          if not (self._flags & self.FLAG_READ):
    126              raise IOError('File is not open for reading')
    127          if (size is None) or (size < 0):
    128              # go for broke
    129              result = self._rbuffer
    130              self._rbuffer = ''
    131              self._pos += len(result)
    132              while True:
    133                  try:
    134                      new_data = self._read(self._DEFAULT_BUFSIZE)
    135                  except EOFError:
    136                      new_data = None
    137                  if (new_data is None) or (len(new_data) == 0):
    138                      break
    139                  result += new_data
    140                  self._realpos += len(new_data)
    141                  self._pos += len(new_data)
    142              return result
    143          if size <= len(self._rbuffer):
    144              result = self._rbuffer[:size]
    145              self._rbuffer = self._rbuffer[size:]
    146              self._pos += len(result)
    147              return result
    148          while len(self._rbuffer) < size:
    149              read_size = size - len(self._rbuffer)
    150              if self._flags & self.FLAG_BUFFERED:
    151                  read_size = max(self._bufsize, read_size)
    152              try:
    153                  new_data = self._read(read_size)
    154              except EOFError:
    155                  new_data = None
    156              if (new_data is None) or (len(new_data) == 0):
    157                  break
    158              self._rbuffer += new_data
    159              self._realpos += len(new_data)
    160          result = self._rbuffer[:size]
    161          self._rbuffer = self._rbuffer[size:]
    162          self._pos += len(result)
    163          return result
    164
    165 -    def readline(self, size=None):
    166          """
    167          Read one entire line from the file.  A trailing newline character is
    168          kept in the string (but may be absent when a file ends with an
    169          incomplete line).  If the size argument is present and non-negative, it
    170          is a maximum byte count (including the trailing newline) and an
    171          incomplete line may be returned.  An empty string is returned only when
    172          EOF is encountered immediately.
    173
    174          @note: Unlike stdio's C{fgets()}, the returned string contains null
    175          characters (C{'\\0'}) if they occurred in the input.
    176
    177          @param size: maximum length of returned string.
    178          @type size: int
    179          @return: next line of the file, or an empty string if the end of the
    180              file has been reached.
    181          @rtype: str
    182          """
    183          # it's almost silly how complex this function is.
    184          if self._closed:
    185              raise IOError('File is closed')
    186          if not (self._flags & self.FLAG_READ):
    187              raise IOError('File not open for reading')
    188          line = self._rbuffer
    189          while True:
    190              if self._at_trailing_cr and (self._flags & self.FLAG_UNIVERSAL_NEWLINE) and (len(line) > 0):
    191                  # edge case: the newline may be '\r\n' and we may have read
    192                  # only the first '\r' last time.
    193                  if line[0] == '\n':
    194                      line = line[1:]
    195                      self._record_newline('\r\n')
    196                  else:
    197                      self._record_newline('\r')
    198                  self._at_trailing_cr = False
    199              # check size before looking for a linefeed, in case we already have
    200              # enough.
    201              if (size is not None) and (size >= 0):
    202                  if len(line) >= size:
    203                      # truncate line and return
    204                      self._rbuffer = line[size:]
    205                      line = line[:size]
    206                      self._pos += len(line)
    207                      return line
    208                  n = size - len(line)
    209              else:
    210                  n = self._bufsize
    211              if ('\n' in line) or ((self._flags & self.FLAG_UNIVERSAL_NEWLINE) and ('\r' in line)):
    212                  break
    213              try:
    214                  new_data = self._read(n)
    215              except EOFError:
    216                  new_data = None
    217              if (new_data is None) or (len(new_data) == 0):
    218                  self._rbuffer = ''
    219                  self._pos += len(line)
    220                  return line
    221              line += new_data
    222              self._realpos += len(new_data)
    223          # find the newline
    224          pos = line.find('\n')
    225          if self._flags & self.FLAG_UNIVERSAL_NEWLINE:
    226              rpos = line.find('\r')
    227              if (rpos >= 0) and ((rpos < pos) or (pos < 0)):
    228                  pos = rpos
    229          xpos = pos + 1
    230          if (line[pos] == '\r') and (xpos < len(line)) and (line[xpos] == '\n'):
    231              xpos += 1
    232          self._rbuffer = line[xpos:]
    233          lf = line[pos:xpos]
    234          line = line[:pos] + '\n'
    235          if (len(self._rbuffer) == 0) and (lf == '\r'):
    236              # we could read the line up to a '\r' and there could still be a
    237              # '\n' following that we read next time.  note that and eat it.
    238              self._at_trailing_cr = True
    239          else:
    240              self._record_newline(lf)
    241          self._pos += len(line)
    242          return line
    243
    244 -    def readlines(self, sizehint=None):
    245          """
    246          Read all remaining lines using L{readline} and return them as a list.
    247          If the optional C{sizehint} argument is present, instead of reading up
    248          to EOF, whole lines totalling approximately sizehint bytes (possibly
    249          after rounding up to an internal buffer size) are read.
    250
    251          @param sizehint: desired maximum number of bytes to read.
    252          @type sizehint: int
    253          @return: list of lines read from the file.
    254          @rtype: list
    255          """
    256          lines = []
    257          bytes = 0
    258          while True:
    259              line = self.readline()
    260              if len(line) == 0:
    261                  break
    262              lines.append(line)
    263              bytes += len(line)
    264              if (sizehint is not None) and (bytes >= sizehint):
    265                  break
    266          return lines
    267
    268 -    def seek(self, offset, whence=0):
    269          """
    270          Set the file's current position, like stdio's C{fseek}.  Not all file
    271          objects support seeking.
    272
    273          @note: If a file is opened in append mode (C{'a'} or C{'a+'}), any seek
    274              operations will be undone at the next write (as the file position
    275              will move back to the end of the file).
    276
    277          @param offset: position to move to within the file, relative to
    278              C{whence}.
    279          @type offset: int
    280          @param whence: type of movement: 0 = absolute; 1 = relative to the
    281              current position; 2 = relative to the end of the file.
    282          @type whence: int
    283
    284          @raise IOError: if the file doesn't support random access.
    285          """
    286          raise IOError('File does not support seeking.')
    287
    288 -    def tell(self):
    289          """
    290          Return the file's current position.  This may not be accurate or
    291          useful if the underlying file doesn't support random access, or was
    292          opened in append mode.
    293
    294          @return: file position (in bytes).
    295          @rtype: int
    296          """
    297          return self._pos
    298
    299 -    def write(self, data):
    300          """
    301          Write data to the file.  If write buffering is on (C{bufsize} was
    302          specified and non-zero), some or all of the data may not actually be
    303          written yet.  (Use L{flush} or L{close} to force buffered data to be
    304          written out.)
    305
    306          @param data: data to write.
    307          @type data: str
    308          """
    309          if self._closed:
    310              raise IOError('File is closed')
    311          if not (self._flags & self.FLAG_WRITE):
    312              raise IOError('File not open for writing')
    313          if not (self._flags & self.FLAG_BUFFERED):
    314              self._write_all(data)
    315              return
    316          self._wbuffer.write(data)
    317          if self._flags & self.FLAG_LINE_BUFFERED:
    318              # only scan the new data for linefeed, to avoid wasting time.
    319              last_newline_pos = data.rfind('\n')
    320              if last_newline_pos >= 0:
    321                  wbuf = self._wbuffer.getvalue()
    322                  last_newline_pos += len(wbuf) - len(data)
    323                  self._write_all(wbuf[:last_newline_pos + 1])
    324                  self._wbuffer = StringIO()
    325                  self._wbuffer.write(wbuf[last_newline_pos + 1:])
    326              return
    327          # even if we're line buffering, if the buffer has grown past the
    328          # buffer size, force a flush.
    329          if self._wbuffer.tell() >= self._bufsize:
    330              self.flush()
    331          return
    332
    333 -    def writelines(self, sequence):
    334          """
    335          Write a sequence of strings to the file.  The sequence can be any
    336          iterable object producing strings, typically a list of strings.  (The
    337          name is intended to match L{readlines}; C{writelines} does not add line
    338          separators.)
    339
    340          @param sequence: an iterable sequence of strings.
    341          @type sequence: sequence
    342          """
    343          for line in sequence:
    344              self.write(line)
    345          return
    346
    347 -    def xreadlines(self):
    348          """
    349          Identical to C{iter(f)}.  This is a deprecated file interface that
    350          predates python iterator support.
    351
    352          @return: an iterator.
    353          @rtype: iterator
    354          """
    355          return self
    356
    357
    358      ###  overrides...
    359
    360
    361 -    def _read(self, size):
    362          """
    363          I{(subclass override)}
    364          Read data from the stream.  Return C{None} or raise C{EOFError} to
    365          indicate EOF.
    366          """
    367          raise EOFError()
    368
    369 -    def _write(self, data):
    370          """
    371          I{(subclass override)}
    372          Write data into the stream.
    373          """
    374          raise IOError('write not implemented')
    375
    376 -    def _get_size(self):
    377          """
    378          I{(subclass override)}
    379          Return the size of the file.  This is called from within L{_set_mode}
    380          if the file is opened in append mode, so the file position can be
    381          tracked and L{seek} and L{tell} will work correctly.  If the file is
    382          a stream that can't be randomly accessed, you don't need to override
    383          this method,
    384          """
    385          return 0
    386
    387
    388      ###  internals...
    389
    390
    391 -    def _set_mode(self, mode='r', bufsize=-1):
    392          """
    393          Subclasses call this method to initialize the BufferedFile.
    394          """
    395          # set bufsize in any event, because it's used for readline().
    396          self._bufsize = self._DEFAULT_BUFSIZE
    397          if bufsize < 0:
    398              # do no buffering by default, because otherwise writes will get
    399              # buffered in a way that will probably confuse people.
    400              bufsize = 0
    401          if bufsize == 1:
    402              # apparently, line buffering only affects writes.  reads are only
    403              # buffered if you call readline (directly or indirectly: iterating
    404              # over a file will indirectly call readline).
    405              self._flags |= self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED
    406          elif bufsize > 1:
    407              self._bufsize = bufsize
    408              self._flags |= self.FLAG_BUFFERED
    409              self._flags &= ~self.FLAG_LINE_BUFFERED
    410          elif bufsize == 0:
    411              # unbuffered
    412              self._flags &= ~(self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED)
    413
    414          if ('r' in mode) or ('+' in mode):
    415              self._flags |= self.FLAG_READ
    416          if ('w' in mode) or ('+' in mode):
    417              self._flags |= self.FLAG_WRITE
    418          if ('a' in mode):
    419              self._flags |= self.FLAG_WRITE | self.FLAG_APPEND
    420              self._size = self._get_size()
    421              self._pos = self._realpos = self._size
    422          if ('b' in mode):
    423              self._flags |= self.FLAG_BINARY
    424          if ('U' in mode):
    425              self._flags |= self.FLAG_UNIVERSAL_NEWLINE
    426              # built-in file objects have this attribute to store which kinds of
    427              # line terminations they've seen:
    428              # <http://www.python.org/doc/current/lib/built-in-funcs.html>
    429              self.newlines = None
    430
    431 -    def _write_all(self, data):
    432          # the underlying stream may be something that does partial writes (like
    433          # a socket).
    434          while len(data) > 0:
    435              count = self._write(data)
    436              data = data[count:]
    437              if self._flags & self.FLAG_APPEND:
    438                  self._size += count
    439                  self._pos = self._realpos = self._size
    440              else:
    441                  self._pos += count
    442                  self._realpos += count
    443          return None
    444
    445 -    def _record_newline(self, newline):
    446          # silliness about tracking what kinds of newlines we've seen.
    447          # i don't understand why it can be None, a string, or a tuple, instead
    448          # of just always being a tuple, but we'll emulate that behavior anyway.
    449          if not (self._flags & self.FLAG_UNIVERSAL_NEWLINE):
    450              return
    451          if self.newlines is None:
    452              self.newlines = newline
    453          elif (type(self.newlines) is str) and (self.newlines != newline):
    454              self.newlines = (self.newlines, newline)
    455          elif newline not in self.newlines:
    456              self.newlines += (newline,)
    457