/*
 * E57FormatImpl.cpp - implementation of private functions of E57 format.
 *
 * Copyright 2009 - 2010 Kevin Ackley (kackley@gwi.net)
 *
 * Permission is hereby granted, free of charge, to any person or organization
 * obtaining a copy of the software and accompanying documentation covered by
 * this license (the "Software") to use, reproduce, display, distribute,
 * execute, and transmit the Software, and to prepare derivative works of the
 * Software, and to permit third-parties to whom the Software is furnished to
 * do so, all subject to the following:
 *
 * The copyright notices in the Software and this entire statement, including
 * the above license grant, this restriction and the following disclaimer,
 * must be included in all copies of the Software, in whole or in part, and
 * all derivative works of the Software, unless such copies or derivative
 * works are solely in the form of machine-executable object code generated by
 * a source language processor.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
 * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
 * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <cmath>
#include <cstring>

#include "E57FormatImpl.h"

#include "CheckedFile.h"
#include "Decoder.h"
#include "Encoder.h"
#include "ImageFileImpl.h"
#include "SourceDestBufferImpl.h"


using namespace e57;
using namespace std;

/// Section types
enum
{
   BLOB_SECTION = 0,
   COMPRESSED_VECTOR_SECTION,
};

struct BlobSectionHeader
{
    const uint8_t     sectionId = BLOB_SECTION;

    uint8_t     reserved1[7] = {};           // must be zero
    uint64_t    sectionLogicalLength = 0;    // byte length of whole section

#ifdef E57_DEBUG
    void        dump(int indent = 0, std::ostream& os = std::cout);
#endif
};

#ifdef E57_DEBUG
void BlobSectionHeader::dump(int indent, std::ostream& os)
{
    os << space(indent) << "sectionId:            " << sectionId << endl;
    os << space(indent) << "sectionLogicalLength: " << sectionLogicalLength << endl;
}
#endif


struct CompressedVectorSectionHeader
{
    const uint8_t     sectionId = COMPRESSED_VECTOR_SECTION;

    uint8_t     reserved1[7] = {};          // must be zero
    uint64_t    sectionLogicalLength = 0;   // byte length of whole section
    uint64_t    dataPhysicalOffset = 0;     // offset of first data packet
    uint64_t    indexPhysicalOffset = 0;    // offset of first index packet

                CompressedVectorSectionHeader();
    void        verify(uint64_t filePhysicalSize=0);

#ifdef E57_DEBUG
    void        dump(int indent = 0, std::ostream& os = std::cout) const;
#endif
};

//================================================================================================

VectorNodeImpl::VectorNodeImpl(ImageFileImplWeakPtr destImageFile, bool allowHeteroChildren)
: StructureNodeImpl(destImageFile),
  allowHeteroChildren_(allowHeteroChildren)
{
    /// don't checkImageFileOpen, StructNodeImpl() will do it
}

bool VectorNodeImpl::isTypeEquivalent(NodeImplSharedPtr ni)
{
    /// don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_VECTOR)
        return(false);

    /// Downcast to shared_ptr<VectorNodeImpl>
    shared_ptr<VectorNodeImpl> ai(dynamic_pointer_cast<VectorNodeImpl>(ni));
    if (!ai)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// allowHeteroChildren must match
    if (allowHeteroChildren_ != ai->allowHeteroChildren_)
        return(false);

    /// Same number of children?
    if (childCount() != ai->childCount())
        return(false);

    /// Check each child, must be in same order
    for (unsigned i = 0; i < childCount(); i++) {
        if (!children_.at(i)->isTypeEquivalent(ai->children_.at(i)))
            return(false);
    }

    /// Types match
    return(true);
}

bool VectorNodeImpl::allowHeteroChildren() const
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return allowHeteroChildren_;
}

void VectorNodeImpl::set(int64_t index64, NodeImplSharedPtr ni)
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    if (!allowHeteroChildren_) {
        /// New node type must match all existing children
        for ( auto &child : children_ )
        {
            if (!child->isTypeEquivalent(ni))
            {
                throw E57_EXCEPTION2(E57_ERROR_HOMOGENEOUS_VIOLATION, "this->pathName=" + this->pathName());
            }
        }
    }

    ///??? for now, use base implementation
    StructureNodeImpl::set(index64, ni);
}

void VectorNodeImpl::writeXml(ImageFileImplSharedPtr imf, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    /// don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"Vector\" allowHeterogeneousChildren=\"" << static_cast<int64_t>(allowHeteroChildren_) << "\">\n";
    for ( auto &child : children_ )
    {
       child->writeXml(imf, cf, indent+2, "vectorChild");
    }
    cf << space(indent) << "</"<< fieldName << ">\n";
}

#ifdef E57_DEBUG
void VectorNodeImpl::dump(int indent, ostream& os) const
{
    /// don't checkImageFileOpen
    os << space(indent) << "type:        Vector" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "allowHeteroChildren: " << allowHeteroChildren() << endl;
    for (unsigned i = 0; i < children_.size(); i++) {
        os << space(indent) << "child[" << i << "]:" << endl;
        children_.at(i)->dump(indent+2, os);
    }
}
#endif

//=============================================================================
CompressedVectorNodeImpl::CompressedVectorNodeImpl(ImageFileImplWeakPtr destImageFile)
: NodeImpl(destImageFile)
{
    // don't checkImageFileOpen, NodeImpl() will do it
}

void CompressedVectorNodeImpl::setPrototype(const NodeImplSharedPtr &prototype)
{
    // don't checkImageFileOpen, ctor did it

    //??? check ok for proto, no Blob CompressedVector, empty?
    //??? throw E57_EXCEPTION2(E57_ERROR_BAD_PROTOTYPE)

    /// Can't set prototype twice.
    if (prototype_)
        throw E57_EXCEPTION2(E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName());

    /// prototype can't have a parent (must be a root node)
    if (!prototype->isRoot()) {
        throw E57_EXCEPTION2(E57_ERROR_ALREADY_HAS_PARENT,
                             "this->pathName=" + this->pathName() +
                             " prototype->pathName=" + prototype->pathName());
    }

    /// Verify that prototype is destined for same ImageFile as this is
    ImageFileImplSharedPtr thisDest(destImageFile());
    ImageFileImplSharedPtr prototypeDest(prototype->destImageFile());
    if (thisDest != prototypeDest) {
        throw E57_EXCEPTION2(E57_ERROR_DIFFERENT_DEST_IMAGEFILE,
                             "this->destImageFile" + thisDest->fileName()
                             + " prototype->destImageFile" + prototypeDest->fileName());
    }

    //!!! check for incomplete CompressedVectors when closing file
    prototype_ = prototype;

    /// Note that prototype is not attached to CompressedVector in a parent/child relationship.
    /// This means that prototype is a root node (has no parent).
}

NodeImplSharedPtr CompressedVectorNodeImpl::getPrototype()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(prototype_);  //??? check defined
}

void CompressedVectorNodeImpl::setCodecs(const shared_ptr<VectorNodeImpl> &codecs)
{
    // don't checkImageFileOpen, ctor did it

    //??? check ok for codecs, empty vector, or each element has "inputs" vector of strings, codec substruct

    /// Can't set codecs twice.
    if (codecs_)
        throw E57_EXCEPTION2(E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName());

    /// codecs can't have a parent (must be a root node)
    if (!codecs->isRoot()) {
        throw E57_EXCEPTION2(E57_ERROR_ALREADY_HAS_PARENT,
                             "this->pathName=" + this->pathName() +
                             " codecs->pathName=" + codecs->pathName());
    }

    /// Verify that codecs is destined for same ImageFile as this is
    ImageFileImplSharedPtr thisDest(destImageFile());
    ImageFileImplSharedPtr codecsDest(codecs->destImageFile());
    if (thisDest != codecsDest) {
        throw E57_EXCEPTION2(E57_ERROR_DIFFERENT_DEST_IMAGEFILE,
                             "this->destImageFile" + thisDest->fileName()
                             + " codecs->destImageFile" + codecsDest->fileName());
    }

    codecs_ = codecs;

    /// Note that codecs is not attached to CompressedVector in a parent/child relationship.
    /// This means that codecs is a root node (has no parent).
}

shared_ptr<VectorNodeImpl> CompressedVectorNodeImpl::getCodecs()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(codecs_);  //??? check defined
}

bool CompressedVectorNodeImpl::isTypeEquivalent(NodeImplSharedPtr ni)
{
    // don't checkImageFileOpen

    //??? is this test a good idea?

    /// Same node type?
    if (ni->type() != E57_COMPRESSED_VECTOR)
        return(false);

    /// Downcast to shared_ptr<CompressedVectorNodeImpl>
    shared_ptr<CompressedVectorNodeImpl> cvi(dynamic_pointer_cast<CompressedVectorNodeImpl>(ni));
    if (!cvi)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// recordCount must match
    if (recordCount_ != cvi->recordCount_)
        return(false);

    /// Prototypes and codecs must match ???
    if (!prototype_->isTypeEquivalent(cvi->prototype_))
        return(false);
    if (!codecs_->isTypeEquivalent(cvi->codecs_))
        return(false);

    return(true);
}

bool CompressedVectorNodeImpl::isDefined(const ustring& pathName)
{
    throw E57_EXCEPTION2(E57_ERROR_NOT_IMPLEMENTED, "this->pathName=" + this->pathName() + " pathName=" + pathName);
}

void CompressedVectorNodeImpl::setAttachedRecursive()
{
    /// Mark this node as attached to an ImageFile
    isAttached_ = true;

    /// Mark nodes in prototype tree, if defined
    if (prototype_)
        prototype_->setAttachedRecursive();

    /// Mark nodes in codecs tree if defined
    if (codecs_)
        codecs_->setAttachedRecursive();
}

int64_t CompressedVectorNodeImpl::childCount()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(recordCount_);
}

void CompressedVectorNodeImpl::checkLeavesInSet(const StringSet& /*pathNames*/, NodeImplSharedPtr /*origin*/)
{
    // don't checkImageFileOpen

    /// Since only called for prototype nodes, should't be able to get here since CompressedVectors can't be in prototypes
    throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->pathName=" + this->pathName());
}

void CompressedVectorNodeImpl::writeXml(ImageFileImplSharedPtr imf, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    uint64_t physicalStart = cf.logicalToPhysical(binarySectionLogicalStart_);

    cf << space(indent) << "<" << fieldName << " type=\"CompressedVector\"";
    cf << " fileOffset=\"" << physicalStart;
    cf << "\" recordCount=\"" << recordCount_ << "\">\n";

    if (prototype_)
        prototype_->writeXml(imf, cf, indent+2, "prototype");
    if (codecs_)
        codecs_->writeXml(imf, cf, indent+2, "codecs");
    cf << space(indent) << "</"<< fieldName << ">\n";
}

#ifdef E57_DEBUG
void CompressedVectorNodeImpl::dump(int indent, ostream& os) const
{
    os << space(indent) << "type:        CompressedVector" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    if (prototype_) {
        os << space(indent) << "prototype:" << endl;
        prototype_->dump(indent+2, os);
    } else
        os << space(indent) << "prototype: <empty>" << endl;
    if (codecs_) {
        os << space(indent) << "codecs:" << endl;
        codecs_->dump(indent+2, os);
    } else
        os << space(indent) << "codecs: <empty>" << endl;
    os << space(indent) << "recordCount:                " << recordCount_ << endl;
    os << space(indent) << "binarySectionLogicalStart:  " << binarySectionLogicalStart_ << endl;
}
#endif

shared_ptr<CompressedVectorWriterImpl> CompressedVectorNodeImpl::writer(vector<SourceDestBuffer> sbufs)
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    ImageFileImplSharedPtr destImageFile(destImageFile_);

    /// Check don't have any writers/readers open for this ImageFile
    if (destImageFile->writerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_WRITERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }
    if (destImageFile->readerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_READERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }

    /// sbufs can't be empty
    if (sbufs.empty())
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT, "fileName=" + destImageFile->fileName());

    if (!destImageFile->isWriter())
        throw E57_EXCEPTION2(E57_ERROR_FILE_IS_READ_ONLY, "fileName=" + destImageFile->fileName());

    if (!isAttached())
        throw E57_EXCEPTION2(E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName());

    /// Get pointer to me (really shared_ptr<CompressedVectorNodeImpl>)
    NodeImplSharedPtr ni(shared_from_this());

    /// Downcast pointer to right type
    shared_ptr<CompressedVectorNodeImpl> cai(dynamic_pointer_cast<CompressedVectorNodeImpl>(ni));
    if (!cai)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// Return a shared_ptr to new object
    shared_ptr<CompressedVectorWriterImpl> cvwi(new CompressedVectorWriterImpl(cai, sbufs));
    return(cvwi);
}

shared_ptr<CompressedVectorReaderImpl> CompressedVectorNodeImpl::reader(vector<SourceDestBuffer> dbufs)
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    ImageFileImplSharedPtr destImageFile(destImageFile_);

    /// Check don't have any writers/readers open for this ImageFile
    if (destImageFile->writerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_WRITERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }
    if (destImageFile->readerCount() > 0) {
        throw E57_EXCEPTION2(E57_ERROR_TOO_MANY_READERS,
                             "fileName=" + destImageFile->fileName()
                             + " writerCount=" + toString(destImageFile->writerCount())
                             + " readerCount=" + toString(destImageFile->readerCount()));
    }

    /// dbufs can't be empty
    if (dbufs.empty())
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT, "fileName=" + destImageFile->fileName());

    /// Can be read or write mode, but must be attached
    if (!isAttached())
        throw E57_EXCEPTION2(E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName());

    /// Get pointer to me (really shared_ptr<CompressedVectorNodeImpl>)
    NodeImplSharedPtr ni(shared_from_this());
#ifdef E57_MAX_VERBOSE
    //cout << "constructing CAReader, ni:" << endl;
    //ni->dump(4);
#endif

    /// Downcast pointer to right type
    shared_ptr<CompressedVectorNodeImpl> cai(dynamic_pointer_cast<CompressedVectorNodeImpl>(ni));
    if (!cai)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());
#ifdef E57_MAX_VERBOSE
    //cout<<"constructing CAReader, cai:"<<endl;
    //cai->dump(4);
#endif
    /// Return a shared_ptr to new object
    shared_ptr<CompressedVectorReaderImpl> cvri(new CompressedVectorReaderImpl(cai, dbufs));
    return(cvri);
}

//=====================================================================
IntegerNodeImpl::IntegerNodeImpl(ImageFileImplWeakPtr destImageFile, int64_t value, int64_t minimum, int64_t maximum)
: NodeImpl(destImageFile),
  value_(value),
  minimum_(minimum),
  maximum_(maximum)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Enforce the given bounds
    if (value < minimum || maximum < value)
    {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " value=" + toString(value)
                             + " minimum=" + toString(minimum)
                             + " maximum=" + toString(maximum));
    }
}

bool IntegerNodeImpl::isTypeEquivalent(NodeImplSharedPtr ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_INTEGER)
        return(false);

    /// Downcast to shared_ptr<IntegerNodeImpl>
    shared_ptr<IntegerNodeImpl> ii(dynamic_pointer_cast<IntegerNodeImpl>(ni));
    if (!ii)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// minimum must match
    if (minimum_ != ii->minimum_)
        return(false);

    /// maximum must match
    if (maximum_ != ii->maximum_)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool IntegerNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return pathName.empty();
}

int64_t IntegerNodeImpl::value()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(value_);
}

int64_t IntegerNodeImpl::minimum()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(minimum_);
}

int64_t IntegerNodeImpl::maximum()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(maximum_);
}

void IntegerNodeImpl::checkLeavesInSet(const StringSet &pathNames, NodeImplSharedPtr origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set.
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void IntegerNodeImpl::writeXml(ImageFileImplSharedPtr /*imf???*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"Integer\"";

    /// Don't need to write if are default values
    if (minimum_ != E57_INT64_MIN)
        cf << " minimum=\"" << minimum_ << "\"";
    if (maximum_ != E57_INT64_MAX)
        cf << " maximum=\"" << maximum_ << "\"";

    /// Write value as child text, unless it is the default value
    if (value_ != 0)
        cf << ">" << value_ << "</" << fieldName << ">\n";
    else
        cf << "/>\n";
}

#ifdef E57_DEBUG
void IntegerNodeImpl::dump(int indent, ostream& os) const
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        Integer" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "value:       " << value_ << endl;
    os << space(indent) << "minimum:     " << minimum_ << endl;
    os << space(indent) << "maximum:     " << maximum_ << endl;
}
#endif

//=============================================================================
ScaledIntegerNodeImpl::ScaledIntegerNodeImpl(ImageFileImplWeakPtr destImageFile, int64_t rawValue, int64_t minimum, int64_t maximum, double scale, double offset)
: NodeImpl(destImageFile),
  value_(rawValue),
  minimum_(minimum),
  maximum_(maximum),
  scale_(scale),
  offset_(offset)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Enforce the given bounds on raw value
    if (rawValue < minimum || maximum < rawValue) {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " rawValue=" + toString(rawValue)
                             + " minimum=" + toString(minimum)
                             + " maximum=" + toString(maximum));
    }
}
//=============================================================================
ScaledIntegerNodeImpl::ScaledIntegerNodeImpl(ImageFileImplWeakPtr destImageFile, double scaledValue, double scaledMinimum, double scaledMaximum, double scale, double offset)
: NodeImpl(destImageFile),
  value_(static_cast<int64_t>(floor((scaledValue - offset)/scale +.5))),
  minimum_(static_cast<int64_t>(floor((scaledMinimum - offset)/scale +.5))),
  maximum_(static_cast<int64_t>(floor((scaledMaximum - offset)/scale +.5))),
  scale_(scale),
  offset_(offset)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Enforce the given bounds on raw value
    if (scaledValue < scaledMinimum || scaledMaximum < scaledValue) {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " scaledValue=" + toString(scaledValue)
                             + " scaledMinimum=" + toString(scaledMinimum)
                             + " scaledMaximum=" + toString(scaledMaximum));
    }
}

bool ScaledIntegerNodeImpl::isTypeEquivalent(NodeImplSharedPtr ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_SCALED_INTEGER)
        return(false);

    /// Downcast to shared_ptr<ScaledIntegerNodeImpl>
    shared_ptr<ScaledIntegerNodeImpl> ii(dynamic_pointer_cast<ScaledIntegerNodeImpl>(ni));
    if (!ii)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// minimum must match
    if (minimum_ != ii->minimum_)
        return(false);

    /// maximum must match
    if (maximum_ != ii->maximum_)
        return(false);

    /// scale must match
    if (scale_ != ii->scale_)
        return(false);

    /// offset must match
    if (offset_ != ii->offset_)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool ScaledIntegerNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return pathName.empty();
}

int64_t ScaledIntegerNodeImpl::rawValue()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(value_);
}

double ScaledIntegerNodeImpl::scaledValue()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(value_ * scale_ + offset_);
}

int64_t ScaledIntegerNodeImpl::minimum()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(minimum_);
}
double ScaledIntegerNodeImpl::scaledMinimum()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(minimum_ * scale_ + offset_);
}

int64_t ScaledIntegerNodeImpl::maximum()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(maximum_);
}
double ScaledIntegerNodeImpl::scaledMaximum()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(maximum_ * scale_ + offset_);
}

double ScaledIntegerNodeImpl::scale()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(scale_);
}

double ScaledIntegerNodeImpl::offset()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(offset_);
}

void ScaledIntegerNodeImpl::checkLeavesInSet(const StringSet &pathNames, NodeImplSharedPtr origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set.
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void ScaledIntegerNodeImpl::writeXml(ImageFileImplSharedPtr /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"ScaledInteger\"";

    /// Don't need to write if are default values
    if (minimum_ != E57_INT64_MIN)
        cf << " minimum=\"" << minimum_ << "\"";
    if (maximum_ != E57_INT64_MAX)
        cf << " maximum=\"" << maximum_ << "\"";
    if (scale_ != 1.0)
        cf << " scale=\""  << scale_  << "\"";
    if (offset_ != 0.0)
        cf << " offset=\"" << offset_ << "\"";

    /// Write value as child text, unless it is the default value
    if (value_ != 0)
        cf << ">" << value_ << "</" << fieldName << ">\n";
    else
        cf << "/>\n";
}

#ifdef E57_DEBUG
void ScaledIntegerNodeImpl::dump(int indent, ostream& os) const
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        ScaledInteger" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "rawValue:    " << value_ << endl;
    os << space(indent) << "minimum:     " << minimum_ << endl;
    os << space(indent) << "maximum:     " << maximum_ << endl;
    os << space(indent) << "scale:       " << scale_ << endl;
    os << space(indent) << "offset:      " << offset_ << endl;
}
#endif

//=============================================================================

FloatNodeImpl::FloatNodeImpl(ImageFileImplWeakPtr destImageFile, double value, FloatPrecision precision, double minimum, double maximum)
: NodeImpl(destImageFile),
  value_(value),
  precision_(precision),
  minimum_(minimum),
  maximum_(maximum)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Since this ctor also used to construct single precision, and defaults for minimum/maximum are for double precision,
    /// adjust bounds smaller if single.
    if (precision_ == E57_SINGLE)
    {
        if (minimum_ < E57_FLOAT_MIN)
            minimum_ = E57_FLOAT_MIN;
        if (maximum_ > E57_FLOAT_MAX)
            maximum_ = E57_FLOAT_MAX;
    }

    /// Enforce the given bounds on raw value
    if (value < minimum || maximum < value)
    {
        throw E57_EXCEPTION2(E57_ERROR_VALUE_OUT_OF_BOUNDS,
                             "this->pathName=" + this->pathName()
                             + " value=" + toString(value)
                             + " minimum=" + toString(minimum)
                             + " maximum=" + toString(maximum));
    }
}

bool FloatNodeImpl::isTypeEquivalent(NodeImplSharedPtr ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_FLOAT)
        return(false);

    /// Downcast to shared_ptr<FloatNodeImpl>
    shared_ptr<FloatNodeImpl> fi(dynamic_pointer_cast<FloatNodeImpl>(ni));
    if (!fi)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// precision must match
    if (precision_ != fi->precision_)
        return(false);

    /// minimum must match
    if (minimum_ != fi->minimum_)
        return(false);

    /// maximum must match
    if (maximum_ != fi->maximum_)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool FloatNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return pathName.empty();
}

double FloatNodeImpl::value() const
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return value_;
}

FloatPrecision FloatNodeImpl::precision() const
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return precision_;
}

double FloatNodeImpl::minimum() const
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return minimum_;
}

double FloatNodeImpl::maximum() const
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return maximum_;
}

void FloatNodeImpl::checkLeavesInSet(const StringSet &pathNames, NodeImplSharedPtr origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set (either relative or absolute form)
    if (pathNames.find(relativePathName(origin)) == pathNames.end() && pathNames.find(pathName()) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void FloatNodeImpl::writeXml(ImageFileImplSharedPtr /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"Float\"";
    if (precision_ == E57_SINGLE) {
        cf << " precision=\"single\"";

        /// Don't need to write if are default values
        if (minimum_ > E57_FLOAT_MIN)
            cf << " minimum=\"" << static_cast<float>(minimum_) << "\"";
        if (maximum_ < E57_FLOAT_MAX)
            cf << " maximum=\"" << static_cast<float>(maximum_) << "\"";

        /// Write value as child text, unless it is the default value
        if (value_ != 0.0)
            cf << ">" << static_cast<float>(value_) << "</" << fieldName << ">\n";
        else
            cf << "/>\n";
    } else {
        /// Don't need to write precision="double", because that's the default

        /// Don't need to write if are default values
        if (minimum_ > E57_DOUBLE_MIN)
            cf << " minimum=\"" << minimum_ << "\"";
        if (maximum_ < E57_DOUBLE_MAX)
            cf << " maximum=\"" << maximum_ << "\"";

        /// Write value as child text, unless it is the default value
        if (value_ != 0.0)
            cf << ">" << value_ << "</" << fieldName << ">\n";
        else
            cf << "/>\n";
    }
}

#ifdef E57_DEBUG
void FloatNodeImpl::dump(int indent, ostream& os) const
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        Float" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "precision:   ";
    if (precision() == E57_SINGLE)
        os << "single" << endl;
    else
        os << "double" << endl;

    /// Save old stream config
    const streamsize oldPrecision = os.precision();
    const ios_base::fmtflags oldFlags = os.flags();

    os << space(indent) << scientific << setprecision(17) << "value:       " << value_ << endl;
    os << space(indent) << "minimum:     " << minimum_ << endl;
    os << space(indent) << "maximum:     " << maximum_ << endl;

    /// Restore old stream config
    os.precision(oldPrecision);
    os.flags(oldFlags);
}
#endif

//=============================================================================

StringNodeImpl::StringNodeImpl(ImageFileImplWeakPtr destImageFile, const ustring &value)
: NodeImpl(destImageFile),
  value_(value)
{
    // don't checkImageFileOpen, NodeImpl() will do it
}

bool StringNodeImpl::isTypeEquivalent(NodeImplSharedPtr ni)
{
    // don't checkImageFileOpen

    /// Same node type?
    if (ni->type() != E57_STRING)
        return(false);

    /// ignore value_, doesn't have to match

    /// Types match
    return(true);
}

bool StringNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen

    /// We have no sub-structure, so if path not empty return false
    return pathName.empty();
}

ustring StringNodeImpl::value()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(value_);
}

void StringNodeImpl::checkLeavesInSet(const StringSet &pathNames, NodeImplSharedPtr origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set.
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void StringNodeImpl::writeXml(ImageFileImplSharedPtr /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    cf << space(indent) << "<" << fieldName << " type=\"String\"";

    /// Write value as child text, unless it is the default value
    if (value_.empty()) {
        cf << "/>\n";
    } else {
        cf << "><![CDATA[";

        size_t currentPosition = 0;
        size_t len = value_.length();

        /// Loop, searching for occurences of "]]>", which will be split across two CDATA directives
        while (currentPosition < len)
        {
            size_t found = value_.find("]]>", currentPosition);

            if (found == string::npos)
            {
                /// Didn't find any more "]]>", so can send the rest.
                cf << value_.substr(currentPosition);
                break;
            }

             /// Must output in two pieces, first send upto end of "]]"  (don't send the following ">").
             cf << value_.substr(currentPosition, found-currentPosition+2);

             /// Then start a new CDATA
             cf << "]]><![CDATA[";

             /// Keep looping to send the ">" plus the remaining part of the string
             currentPosition = found+2;
        }
        cf << "]]></" << fieldName << ">\n";
    }
}

#ifdef E57_DEBUG
void StringNodeImpl::dump(int indent, ostream& os) const
{
    os << space(indent) << "type:        String" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "value:       '" << value_ << "'" << endl;
}
#endif

//=============================================================================

BlobNodeImpl::BlobNodeImpl(ImageFileImplWeakPtr destImageFile, int64_t byteCount)
: NodeImpl(destImageFile)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    ImageFileImplSharedPtr imf(destImageFile);

    /// This what caller thinks blob length is
    blobLogicalLength_ = byteCount;

    /// Round segment length up to multiple of 4 bytes
    binarySectionLogicalLength_ = sizeof(BlobSectionHeader) + blobLogicalLength_;
    unsigned remainder = binarySectionLogicalLength_ % 4;
    if (remainder > 0)
        binarySectionLogicalLength_ += 4 - remainder;

    /// Reserve space for blob in file, extend with zeros since writes will happen at later time by caller
    binarySectionLogicalStart_ = imf->allocateSpace(binarySectionLogicalLength_, true);

    /// Prepare BlobSectionHeader
    BlobSectionHeader header;
    header.sectionLogicalLength = binarySectionLogicalLength_;
#ifdef E57_MAX_VERBOSE
    header.dump(); //???
#endif

    /// Write header at beginning of section
    imf->file_->seek(binarySectionLogicalStart_);
    imf->file_->write(reinterpret_cast<char*>(&header), sizeof(header));
}

BlobNodeImpl::BlobNodeImpl(ImageFileImplWeakPtr destImageFile, int64_t fileOffset, int64_t length)
: NodeImpl(destImageFile)
{
    /// Init blob object that already exists in E57 file currently reading.

    // don't checkImageFileOpen, NodeImpl() will do it

    ImageFileImplSharedPtr imf(destImageFile);

    /// Init state from values read from XML
    blobLogicalLength_ = length;
    binarySectionLogicalStart_ = imf->file_->physicalToLogical(fileOffset);
    binarySectionLogicalLength_ = sizeof(BlobSectionHeader) + blobLogicalLength_;
}

bool BlobNodeImpl::isTypeEquivalent(NodeImplSharedPtr ni)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// Same node type?
    if (ni->type() != E57_BLOB)
        return(false);

    /// Downcast to shared_ptr<BlobNodeImpl>
    shared_ptr<BlobNodeImpl> bi(dynamic_pointer_cast<BlobNodeImpl>(ni));
    if (!bi)  // check if failed
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "this->elementName=" + this->elementName() + " elementName=" + ni->elementName());

    /// blob lengths must match
    if (blobLogicalLength_ != bi->blobLogicalLength_)
        return(false);

    /// ignore blob contents, doesn't have to match

    /// Types match
    return(true);
}

bool BlobNodeImpl::isDefined(const ustring& pathName)
{
    // don't checkImageFileOpen, NodeImpl() will do it

    /// We have no sub-structure, so if path not empty return false
    return pathName.empty();
}

int64_t BlobNodeImpl::byteCount()
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    return(blobLogicalLength_);
}

void BlobNodeImpl::read(uint8_t* buf, int64_t start, size_t count)
{
    //??? check start not negative

    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    if (static_cast<uint64_t>(start)+count > blobLogicalLength_) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "this->pathName=" + this->pathName()
                             + " start=" + toString(start)
                             + " count=" + toString(count)
                             + " length=" + toString(blobLogicalLength_));
    }

    ImageFileImplSharedPtr imf(destImageFile_);
    imf->file_->seek(binarySectionLogicalStart_ + sizeof(BlobSectionHeader) + start);
    imf->file_->read(reinterpret_cast<char*>(buf), static_cast<size_t>(count));  //??? arg1 void* ?
}

void BlobNodeImpl::write(uint8_t* buf, int64_t start, size_t count)
{
    //??? check start not negative
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    ImageFileImplSharedPtr destImageFile(destImageFile_);

    if (!destImageFile->isWriter())
        throw E57_EXCEPTION2(E57_ERROR_FILE_IS_READ_ONLY, "fileName=" + destImageFile->fileName());
    if (!isAttached())
        throw E57_EXCEPTION2(E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName());

    if (static_cast<uint64_t>(start)+count > blobLogicalLength_) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "this->pathName=" + this->pathName()
                             + " start=" + toString(start)
                             + " count=" + toString(count)
                             + " length=" + toString(blobLogicalLength_));
    }

    ImageFileImplSharedPtr imf(destImageFile_);
    imf->file_->seek(binarySectionLogicalStart_ + sizeof(BlobSectionHeader) + start);
    imf->file_->write(reinterpret_cast<char*>(buf), static_cast<size_t>(count));  //??? arg1 void* ?
}

void BlobNodeImpl::checkLeavesInSet(const StringSet &pathNames, NodeImplSharedPtr origin)
{
    // don't checkImageFileOpen

    /// We are a leaf node, so verify that we are listed in set. ???true for blobs? what exception get if try blob in compressedvector?
    if (pathNames.find(relativePathName(origin)) == pathNames.end())
        throw E57_EXCEPTION2(E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName());
}

void BlobNodeImpl::writeXml(ImageFileImplSharedPtr /*imf*/, CheckedFile& cf, int indent, const char* forcedFieldName)
{
    // don't checkImageFileOpen

    ustring fieldName;
    if (forcedFieldName != nullptr)
        fieldName = forcedFieldName;
    else
        fieldName = elementName_;

    //??? need to implement
    //??? Type --> type
    //??? need to have length?, check same as in section header?
    uint64_t physicalOffset = cf.logicalToPhysical(binarySectionLogicalStart_);
    cf << space(indent) << "<" << fieldName << " type=\"Blob\" fileOffset=\"" << physicalOffset << "\" length=\"" << blobLogicalLength_ << "\"/>\n";
}

#ifdef E57_DEBUG
void BlobNodeImpl::dump(int indent, ostream& os) const
{
    // don't checkImageFileOpen
    os << space(indent) << "type:        Blob" << " (" << type() << ")" << endl;
    NodeImpl::dump(indent, os);
    os << space(indent) << "blobLogicalLength_:           " << blobLogicalLength_ << endl;
    os << space(indent) << "binarySectionLogicalStart:    " << binarySectionLogicalStart_ << endl;
    os << space(indent) << "binarySectionLogicalLength:   " << binarySectionLogicalLength_ << endl;
//    size_t i;
//    for (i = 0; i < blobLogicalLength_ && i < 10; i++) {
//        uint8_t b;
//        read(&b, i, 1);
//        os << space(indent) << "data[" << i << "]: "<< static_cast<int>(b) << endl;
//    }
//    if (i < blobLogicalLength_)
//        os << space(indent) << "more data unprinted..." << endl;
}
#endif

//================================================================

CompressedVectorSectionHeader::CompressedVectorSectionHeader()
{
   /// Double check that header is correct length.  Watch out for RTTI increasing the size.
   static_assert( sizeof( CompressedVectorSectionHeader ) == 32, "Unexpected size of CompressedVectorSectionHeader" );
}

void CompressedVectorSectionHeader::verify(uint64_t filePhysicalSize)
{
    /// Verify that section is correct type
    if (sectionId != COMPRESSED_VECTOR_SECTION)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER, "sectionId=" + toString(sectionId));

    /// Verify reserved fields are zero. ???  if fileversion==1.0 ???
    for (unsigned i=0; i < sizeof(reserved1); i++)
    {
        if (reserved1[i] != 0)
        {
           throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER, "i=" + toString(i) + " reserved=" + toString(reserved1[i]));
        }
    }

    /// Check section length is multiple of 4
    if (sectionLogicalLength % 4)
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER, "sectionLogicalLength=" + toString(sectionLogicalLength));

    /// Check sectionLogicalLength is in bounds
    if (filePhysicalSize > 0 && sectionLogicalLength >= filePhysicalSize) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER,
                             "sectionLogicalLength=" + toString(sectionLogicalLength)
                             + " filePhysicalSize=" + toString(filePhysicalSize));
    }

    /// Check dataPhysicalOffset is in bounds
    if (filePhysicalSize > 0 && dataPhysicalOffset >= filePhysicalSize) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER,
                             "dataPhysicalOffset=" + toString(dataPhysicalOffset)
                             + " filePhysicalSize=" + toString(filePhysicalSize));
    }

    /// Check indexPhysicalOffset is in bounds
    if (filePhysicalSize > 0 && indexPhysicalOffset >= filePhysicalSize) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_CV_HEADER,
                             "indexPhysicalOffset=" + toString(indexPhysicalOffset)
                             + " filePhysicalSize=" + toString(filePhysicalSize));
    }
}

#ifdef E57_DEBUG
void CompressedVectorSectionHeader::dump(int indent, std::ostream& os) const
{
    os << space(indent) << "sectionId:            " << static_cast<unsigned>(sectionId) << endl;
    os << space(indent) << "sectionLogicalLength: " << sectionLogicalLength << endl;
    os << space(indent) << "dataPhysicalOffset:   " << dataPhysicalOffset << endl;
    os << space(indent) << "indexPhysicalOffset:  " << indexPhysicalOffset << endl;
}
#endif

///================================================================

struct SortByBytestreamNumber
{
    bool operator () (const shared_ptr<Encoder> &lhs , const shared_ptr<Encoder> &rhs) const {
        return(lhs->bytestreamNumber() < rhs->bytestreamNumber());
    }
};

CompressedVectorWriterImpl::CompressedVectorWriterImpl(shared_ptr<CompressedVectorNodeImpl> ni, vector<SourceDestBuffer>& sbufs)
: cVector_(ni),
  isOpen_(false)  // set to true when succeed below
{
    //???  check if cvector already been written (can't write twice)

    /// Empty sbufs is an error
    if (sbufs.empty()) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }

    /// Get CompressedArray's prototype node (all array elements must match this type)
    proto_ = cVector_->getPrototype();

    /// Check sbufs well formed (matches proto exactly)
    setBuffers(sbufs); //??? copy code here?

    /// For each individual sbuf, create an appropriate Encoder based on the cVector_ attributes
    for (unsigned i=0; i < sbufs_.size(); i++) {
        /// Create vector of single sbuf  ??? for now, may have groups later
        vector<SourceDestBuffer> vTemp;
        vTemp.push_back(sbufs_.at(i));

        ustring codecPath = sbufs_.at(i).pathName();

        /// Calc which stream the given path belongs to.  This depends on position of the node in the proto tree.
        NodeImplSharedPtr readNode = proto_->get(sbufs.at(i).pathName());
        uint64_t bytestreamNumber = 0;
        if (!proto_->findTerminalPosition(readNode, bytestreamNumber))
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "sbufIndex=" + toString(i));

        /// EncoderFactory picks the appropriate encoder to match type declared in prototype
        bytestreams_.push_back(Encoder::EncoderFactory(static_cast<unsigned>(bytestreamNumber), cVector_, vTemp, codecPath));
    }

    /// The bytestreams_ vector must be ordered by bytestreamNumber, not by order called specified sbufs, so sort it.
    sort(bytestreams_.begin(), bytestreams_.end(), SortByBytestreamNumber());
#ifdef E57_MAX_DEBUG
    /// Double check that all bytestreams are specified
    for (unsigned i=0; i < bytestreams_.size(); i++) {
        if (bytestreams_.at(i)->bytestreamNumber() != i){
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                                 "bytestreamIndex=" + toString(i)
                                 + " bytestreamNumber=" + toString(bytestreams_.at(i)->bytestreamNumber()));
        }
    }
#endif

    ImageFileImplSharedPtr imf(ni->destImageFile_);

    /// Reserve space for CompressedVector binary section header, record location so can save to when writer closes.
    /// Request that file be extended with zeros since we will write to it at a later time (when writer closes).
    sectionHeaderLogicalStart_ = imf->allocateSpace(sizeof(CompressedVectorSectionHeader), true);

    sectionLogicalLength_   = 0;
    dataPhysicalOffset_     = 0;
    topIndexPhysicalOffset_ = 0;
    recordCount_            = 0;
    dataPacketsCount_       = 0;
    indexPacketsCount_      = 0;

    /// Just before return (and can't throw) increment writer count  ??? safer way to assure don't miss close?
    imf->incrWriterCount();

    /// If get here, the writer is open
    isOpen_ = true;
}

CompressedVectorWriterImpl::~CompressedVectorWriterImpl()
{
#ifdef E57_MAX_VERBOSE
    cout << "~CompressedVectorWriterImpl() called" << endl; //???
#endif

    try {
        if (isOpen_)
            close();
    } catch (...) {
        //??? report?
    }
}

void CompressedVectorWriterImpl::close()
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorWriterImpl::close() called" << endl; //???
#endif
    ImageFileImplSharedPtr imf(cVector_->destImageFile_);

    /// Before anything that can throw, decrement writer count
    imf->decrWriterCount();

    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    /// don't call checkWriterOpen();

    if (!isOpen_)
        return;

    /// Set closed before do anything, so if get fault and start unwinding, don't try to close again.
    isOpen_ = false;

    /// If have any data, write packet
    /// Write all remaining ioBuffers and internal encoder register cache into file.
    /// Know we are done when totalOutputAvailable() returns 0 after a flush().
    flush();
    while (totalOutputAvailable() > 0) {
        packetWrite();
        flush();
    }

    /// Compute length of whole section we just wrote (from section start to current start of free space).
    sectionLogicalLength_ = imf->unusedLogicalStart_ - sectionHeaderLogicalStart_;
#ifdef E57_MAX_VERBOSE
    cout << "  sectionLogicalLength_=" << sectionLogicalLength_ << endl; //???
#endif

    /// Prepare CompressedVectorSectionHeader
    CompressedVectorSectionHeader header;
    header.sectionLogicalLength = sectionLogicalLength_;
    header.dataPhysicalOffset   = dataPhysicalOffset_;   ///??? can be zero, if no data written ???not set yet
    header.indexPhysicalOffset  = topIndexPhysicalOffset_;  ///??? can be zero, if no data written ???not set yet
#ifdef E57_MAX_VERBOSE
    cout << "  CompressedVectorSectionHeader:" << endl;
    header.dump(4); //???
#endif
#ifdef E57_DEBUG
    /// Verify OK before write it.
    header.verify(imf->file_->length(CheckedFile::Physical));
#endif

    /// Write header at beginning of section, previously allocated
    imf->file_->seek(sectionHeaderLogicalStart_);
    imf->file_->write(reinterpret_cast<char*>(&header), sizeof(header));

    /// Set address and size of associated CompressedVector
    cVector_->setRecordCount(recordCount_);
    cVector_->setBinarySectionLogicalStart(sectionHeaderLogicalStart_);

    /// Free channels
    bytestreams_.clear();

#ifdef E57_MAX_VERBOSE
    cout << "  CompressedVectorWriter:" << endl;
    dump(4);
#endif
}

bool CompressedVectorWriterImpl::isOpen() const
{
    /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), or checkWriterOpen()
    return isOpen_;
}

std::shared_ptr<CompressedVectorNodeImpl> CompressedVectorWriterImpl::compressedVectorNode() const
{
    return cVector_;
}

void CompressedVectorWriterImpl::setBuffers(vector<SourceDestBuffer>& sbufs)
{
    /// don't checkImageFileOpen

    /// If had previous sbufs_, check to see if new ones have changed in incompatible way
    if ( !sbufs_.empty() )
    {
        if (sbufs_.size() != sbufs.size())
        {
            throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                                 "oldSize=" + toString(sbufs_.size())
                                 + " newSize=" + toString(sbufs.size()));
        }

        for (size_t i = 0; i < sbufs_.size(); ++i)
        {
            shared_ptr<SourceDestBufferImpl> oldbuf = sbufs_[i].impl();
            shared_ptr<SourceDestBufferImpl> newBuf = sbufs[i].impl();

            /// Throw exception if old and new not compatible
            oldbuf->checkCompatible(newBuf);
        }
    }

    /// Check sbufs well formed: no dups, no missing, no extra
    /// For writing, all data fields in prototype must be presented for writing at same time.
    proto_->checkBuffers(sbufs, false);

    sbufs_ = sbufs;
}

void CompressedVectorWriterImpl::write(vector<SourceDestBuffer>& sbufs, const size_t requestedRecordCount)
{
    /// don't checkImageFileOpen, write(unsigned) will do it
    /// don't checkWriterOpen(), write(unsigned) will do it

    setBuffers(sbufs);
    write(requestedRecordCount);
}

void CompressedVectorWriterImpl::write(const size_t requestedRecordCount)
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorWriterImpl::write() called" << endl; //???
#endif
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    checkWriterOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    /// Check that requestedRecordCount is not larger than the sbufs
    if (requestedRecordCount > sbufs_.at(0).impl()->capacity()) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "requested=" + toString(requestedRecordCount)
                             + " capacity=" + toString(sbufs_.at(0).impl()->capacity())
                             + " imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }

    /// Rewind all sbufs so start reading from beginning
    for ( auto &sbuf : sbufs_ )
    {
       sbuf.impl()->rewind();
    }

    /// Loop until all channels have completed requestedRecordCount transfers
    uint64_t endRecordIndex = recordCount_ + requestedRecordCount;
    for (;;) {
        /// Calc remaining record counts for all channels
        uint64_t totalRecordCount = 0;
        for ( auto &bytestream : bytestreams_ )
        {
            totalRecordCount += endRecordIndex - bytestream->currentRecordIndex();
        }
#ifdef E57_MAX_VERBOSE
        cout << "  totalRecordCount=" << totalRecordCount << endl; //???
#endif

        /// We are done if have no more work, break out of loop
        if (totalRecordCount == 0)
            break;

        /// Estimate how many records can write before have enough data to fill data packet to efficient length
        /// Efficient packet length is >= 75% of maximum packet length.
        /// It is OK if get too much data (more than one packet) in an iteration.
        /// Reader will be able to handle packets whose streams are not exactly synchronized to the record boundaries.
        /// But try to do a good job of keeping the stream synchronization "close enough" (so a reader that can cache only two packets is efficient).

#ifdef E57_MAX_VERBOSE
        cout << "  currentPacketSize()=" << currentPacketSize() << endl; //???
#endif

#ifdef E57_WRITE_CRAZY_PACKET_MODE
///??? depends on number of streams
#  define E57_TARGET_PACKET_SIZE    500
#else
#  define E57_TARGET_PACKET_SIZE    (DATA_PACKET_MAX*3/4)
#endif
        /// If have more than target fraction of packet, send it now
        if (currentPacketSize() >= E57_TARGET_PACKET_SIZE) {  //???
            packetWrite();
            continue;  /// restart loop so recalc statistics (packet size may not be zero after write, if have too much data)
        }

        ///??? useful?
        /// Get approximation of number of bytes per record of CompressedVector and total of bytes used
        float totalBitsPerRecord = 0;  // an estimate of future performance
        for ( auto &bytestream : bytestreams_ )
        {
            totalBitsPerRecord += bytestream->bitsPerRecord();
        }

#ifdef E57_MAX_VERBOSE
        float totalBytesPerRecord = max(totalBitsPerRecord/8, 0.1F); //??? trust

        cout << "  totalBytesPerRecord=" << totalBytesPerRecord << endl; //???
#endif

        /// Don't allow straggler to get too far behind. ???
        /// Don't allow a single channel to get too far ahead ???
        /// Process channels that are furthest behind first. ???

        ///!!!! For now just process one record per loop until packet is full enough, or completed request
        for ( auto &bytestream : bytestreams_ )
        {
             if (bytestream->currentRecordIndex() < endRecordIndex)
             {
                //!!! For now, process up to 50 records at a time
                uint64_t recordCount = endRecordIndex - bytestream->currentRecordIndex();
                recordCount = (recordCount<50ULL)?recordCount:50ULL; //min(recordCount, 50ULL);
                bytestream->processRecords(static_cast<unsigned>(recordCount));
            }
        }
    }

    recordCount_ += requestedRecordCount;

    /// When we leave this function, will likely still have data in channel ioBuffers as well as partial words in Encoder registers.
}

size_t CompressedVectorWriterImpl::totalOutputAvailable() const
{
    size_t total = 0;

    for ( const auto & bytestream : bytestreams_ )
    {
        total += bytestream->outputAvailable();
    }

    return total;
}

size_t CompressedVectorWriterImpl::currentPacketSize() const
{
    /// Calc current packet size
    return(sizeof(DataPacketHeader) + bytestreams_.size()*sizeof(uint16_t) + totalOutputAvailable());
}

uint64_t CompressedVectorWriterImpl::packetWrite()
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorWriterImpl::packetWrite() called" << endl; //???
#endif

    /// Double check that we have work to do
    size_t totalOutput = totalOutputAvailable();
    if (totalOutput == 0)
        return(0);
#ifdef E57_MAX_VERBOSE
    cout << "  totalOutput=" << totalOutput << endl; //???
#endif

    /// Calc maximum number of bytestream values can put in data packet.
    size_t packetMaxPayloadBytes = DATA_PACKET_MAX - sizeof(DataPacketHeader) - bytestreams_.size()*sizeof(uint16_t);
#ifdef E57_MAX_VERBOSE
    cout << "  packetMaxPayloadBytes=" << packetMaxPayloadBytes << endl; //???
#endif

    /// Allocate vector for number of bytes that each bytestream will write to file.
    vector<size_t> count(bytestreams_.size());

    /// See if we can fit into a single data packet
    if (totalOutput < packetMaxPayloadBytes) {
        /// We can fit everything in one packet
        for (unsigned i=0; i < bytestreams_.size(); i++)
            count.at(i) = bytestreams_.at(i)->outputAvailable();
    } else {
        /// We have too much data for one packet.  Send proportional amounts from each bytestream.
        /// Adjust packetMaxPayloadBytes down by one so have a little slack for floating point weirdness.
        float fractionToSend =  (packetMaxPayloadBytes-1) / static_cast<float>(totalOutput);
        for (unsigned i=0; i < bytestreams_.size(); i++) {
            /// Round down here so sum <= packetMaxPayloadBytes
            count.at(i) = static_cast<unsigned>(floor(fractionToSend * bytestreams_.at(i)->outputAvailable()));
        }
    }
#ifdef E57_MAX_VERBOSE
    for (unsigned i=0; i < bytestreams_.size(); i++)
        cout << "  count[" << i << "]=" << count.at(i) << endl; //???
#endif

#ifdef E57_DEBUG
    /// Double check sum of count is <= packetMaxPayloadBytes
    size_t totalByteCount = 0;
    for (size_t i : count)
        totalByteCount += i;
    if (totalByteCount > packetMaxPayloadBytes) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "totalByteCount=" + toString(totalByteCount)
                             + " packetMaxPayloadBytes=" + toString(packetMaxPayloadBytes));
    }
#endif

    /// Get smart pointer to ImageFileImpl from associated CompressedVector
    ImageFileImplSharedPtr imf(cVector_->destImageFile_);

    /// Use temp buf in object (is 64KBytes long) instead of allocating each time here
    char* packet = reinterpret_cast<char*>(&dataPacket_);
#ifdef E57_MAX_VERBOSE
    cout << "  packet=" << packet << endl; //???
#endif

    /// To be safe, clear header part of packet
    dataPacket_.header.reset();

    /// Write bytestreamBufferLength[bytestreamCount] after header, in dataPacket_
    auto bsbLength = reinterpret_cast<uint16_t*>(&packet[sizeof(DataPacketHeader)]);
#ifdef E57_MAX_VERBOSE
    cout << "  bsbLength=" << bsbLength << endl; //???
#endif
    for (unsigned i=0; i < bytestreams_.size(); i++) {
        bsbLength[i] = static_cast<uint16_t>(count.at(i));      // %%% Truncation
#ifdef E57_MAX_VERBOSE
        cout << "  Writing " << bsbLength[i] << " bytes into bytestream " << i << endl; //???
#endif
    }

    /// Get pointer to end of data so far
    char* p = reinterpret_cast<char*>(&bsbLength[bytestreams_.size()]);
#ifdef E57_MAX_VERBOSE
    cout << "  after bsbLength, p=" << p << endl; //???
#endif

    /// Write contents of each bytestream in dataPacket_
    for (size_t i=0; i < bytestreams_.size(); i++) {
        size_t n = count.at(i);

#ifdef E57_DEBUG
        /// Double check we aren't accidentally going to write off end of vector<char>
        if (&p[n] > &packet[DATA_PACKET_MAX])
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "n=" + toString(n));
#endif

        /// Read from encoder output into packet
        bytestreams_.at(i)->outputRead(p, n);

        /// Move pointer to end of current data
        p += n;
    }

    /// Length of packet is difference in beginning pointer and ending pointer
    auto packetLength = static_cast<unsigned>(p - packet);  ///??? pointer diff portable?
#ifdef E57_MAX_VERBOSE
    cout << "  packetLength=" << packetLength << endl; //???
#endif

#ifdef E57_DEBUG
    /// Double check that packetLength is what we expect
    if (packetLength != sizeof(DataPacketHeader) + bytestreams_.size()*sizeof(uint16_t) + totalByteCount) {
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "packetLength=" + toString(packetLength)
                             + " bytestreamSize=" + toString(bytestreams_.size()*sizeof(uint16_t))
                             + " totalByteCount=" + toString(totalByteCount));
    }
#endif

    /// packetLength must be multiple of 4, if not, add some zero padding
    while (packetLength % 4) {
        /// Double check we aren't accidentally going to write off end of vector<char>
        if (p >= &packet[DATA_PACKET_MAX-1])
            throw E57_EXCEPTION1(E57_ERROR_INTERNAL);
        *p++ = 0;
        packetLength++;
#ifdef E57_MAX_VERBOSE
        cout << "  padding with zero byte, new packetLength=" << packetLength << endl; //???
#endif
    }

    /// Prepare header in dataPacket_, now that we are sure of packetLength
    dataPacket_.header.packetLogicalLengthMinus1 = static_cast<uint16_t>(packetLength-1);          // %%% Truncation
    dataPacket_.header.bytestreamCount = static_cast<uint16_t>(bytestreams_.size());       // %%% Truncation

    /// Double check that data packet is well formed
    dataPacket_.verify(packetLength);

    /// Write whole data packet at beginning of free space in file
    uint64_t packetLogicalOffset = imf->allocateSpace(packetLength, false);
    uint64_t packetPhysicalOffset = imf->file_->logicalToPhysical(packetLogicalOffset);
    imf->file_->seek(packetLogicalOffset);  //??? have seekLogical and seekPhysical instead? more explicit
    imf->file_->write(packet, packetLength);

#ifdef E57_MAX_VERBOSE
//  cout << "data packet:" << endl;
//  dataPacket_.dump(4);
#endif

    /// If first data packet written for this CompressedVector binary section, save address to put in section header
    ///??? what if no data packets?
    ///??? what if have exceptions while write, what is state of file?  will close report file good/bad?
    if (dataPacketsCount_ == 0)
        dataPhysicalOffset_ = packetPhysicalOffset;
    dataPacketsCount_++;

    ///!!! update seekIndex here? if started new chunk?

    /// Return physical offset of data packet for potential use in seekIndex
    return(packetPhysicalOffset); //??? needed
}

void CompressedVectorWriterImpl::flush()
{
    for ( auto &bytestream : bytestreams_ )
    {
       bytestream->registerFlushToOutput();
    }
}

void CompressedVectorWriterImpl::checkImageFileOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName) const
{
   // unimplemented...
}

void CompressedVectorWriterImpl::checkWriterOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName) const
{
    if (!isOpen_)
    {
        throw E57Exception(E57_ERROR_WRITER_NOT_OPEN,
                           "imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName(),
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
}

void CompressedVectorWriterImpl::dump(int indent, std::ostream& os)
{
    os << space(indent) << "isOpen:" << isOpen_ << endl;

    for (unsigned i = 0; i < sbufs_.size(); i++) {
        os << space(indent) << "sbufs[" << i << "]:" << endl;
        sbufs_.at(i).dump(indent+4, os);
    }

    os << space(indent) << "cVector:" << endl;
    cVector_->dump(indent+4, os);

    os << space(indent) << "proto:" << endl;
    proto_->dump(indent+4, os);

    for (unsigned i = 0; i < bytestreams_.size(); i++) {
        os << space(indent) << "bytestreams[" << i << "]:" << endl;
        bytestreams_.at(i)->dump(indent+4, os);
    }

    /// Don't call dump() for DataPacket, since it may contain junk when debugging.  Just print a few byte values.
    os << space(indent) << "dataPacket:" << endl;
    auto p = reinterpret_cast<uint8_t*>(&dataPacket_);

    for (unsigned i = 0; i < 40; ++i)
    {
        os << space(indent+4) << "dataPacket[" << i << "]: " << static_cast<unsigned>(p[i]) << endl;
    }
    os << space(indent+4) << "more unprinted..." << endl;

    os << space(indent) << "sectionHeaderLogicalStart: " << sectionHeaderLogicalStart_ << endl;
    os << space(indent) << "sectionLogicalLength:      " << sectionLogicalLength_ << endl;
    os << space(indent) << "dataPhysicalOffset:        " << dataPhysicalOffset_ << endl;
    os << space(indent) << "topIndexPhysicalOffset:    " << topIndexPhysicalOffset_ << endl;
    os << space(indent) << "recordCount:               " << recordCount_ << endl;
    os << space(indent) << "dataPacketsCount:          " << dataPacketsCount_ << endl;
    os << space(indent) << "indexPacketsCount:         " << indexPacketsCount_ << endl;
}

///================================================================
///================================================================
///================================================================

CompressedVectorReaderImpl::CompressedVectorReaderImpl(shared_ptr<CompressedVectorNodeImpl> cvi, vector<SourceDestBuffer>& dbufs)
: isOpen_(false),  // set to true when succeed below
  cVector_(cvi)
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorReaderImpl() called" << endl; //???
#endif
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    /// Allow reading of a completed CompressedVector, whether file is being read or currently being written.
    ///??? what other situations need checking for?
    ///??? check if CV not yet written to?
    ///??? file in error state?

    /// Empty dbufs is an error
    if (dbufs.empty()) {
        throw E57_EXCEPTION2(E57_ERROR_BAD_API_ARGUMENT,
                             "imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }

    /// Get CompressedArray's prototype node (all array elements must match this type)
    proto_ = cVector_->getPrototype();

    /// Check dbufs well formed (matches proto exactly)
    setBuffers(dbufs);

    /// For each dbuf, create an appropriate Decoder based on the cVector_ attributes
    for (unsigned i=0; i < dbufs_.size(); i++) {
        vector<SourceDestBuffer> theDbuf;
        theDbuf.push_back(dbufs.at(i));

        shared_ptr<Decoder> decoder =  Decoder::DecoderFactory(i, cVector_, theDbuf, ustring());

        /// Calc which stream the given path belongs to.  This depends on position of the node in the proto tree.
        NodeImplSharedPtr readNode = proto_->get(dbufs.at(i).pathName());
        uint64_t bytestreamNumber = 0;
        if (!proto_->findTerminalPosition(readNode, bytestreamNumber))
            throw E57_EXCEPTION2(E57_ERROR_INTERNAL, "dbufIndex=" + toString(i));

        channels_.emplace_back(dbufs.at(i), decoder, static_cast<unsigned>(bytestreamNumber), cVector_->childCount());
    }

    recordCount_ = 0;

    /// Get how many records are actually defined
    maxRecordCount_ = cvi->childCount();

    ImageFileImplSharedPtr imf(cVector_->destImageFile_);

    //??? what if fault in this constructor?
    cache_ = new PacketReadCache(imf->file_, 32);

    /// Read CompressedVector section header
    CompressedVectorSectionHeader sectionHeader;
    uint64_t sectionLogicalStart = cVector_->getBinarySectionLogicalStart();
    if (sectionLogicalStart == 0) {
        //??? should have caught this before got here, in XML read, get this if CV wasn't written to by writer.
        throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                             "imageFileName=" + cVector_->imageFileName()
                             + " cvPathName=" + cVector_->pathName());
    }
    imf->file_->seek(sectionLogicalStart, CheckedFile::Logical);
    imf->file_->read(reinterpret_cast<char*>(&sectionHeader), sizeof(sectionHeader));

#ifdef E57_DEBUG
    sectionHeader.verify(imf->file_->length(CheckedFile::Physical));
#endif

    /// Pre-calc end of section, so can tell when we are out of packets.
    sectionEndLogicalOffset_ = sectionLogicalStart + sectionHeader.sectionLogicalLength;

    /// Convert physical offset to first data packet to logical
    uint64_t dataLogicalOffset = imf->file_->physicalToLogical(sectionHeader.dataPhysicalOffset);

    /// Verify that packet given by dataPhysicalOffset is actually a data packet, init channels
    {
        char* anyPacket = nullptr;
        unique_ptr<PacketLock> packetLock = cache_->lock(dataLogicalOffset, anyPacket);

        auto dpkt = reinterpret_cast<DataPacket*>(anyPacket);

        /// Double check that have a data packet
        if (dpkt->header.packetType != DATA_PACKET)
        {
           throw E57_EXCEPTION2(E57_ERROR_BAD_CV_PACKET, "packetType=" + toString(dpkt->header.packetType));
        }

        /// Have good packet, initialize channels
        for ( auto &channel : channels_ )
        {
            channel.currentPacketLogicalOffset    = dataLogicalOffset;
            channel.currentBytestreamBufferIndex  = 0;
            channel.currentBytestreamBufferLength = dpkt->getBytestreamBufferLength(channel.bytestreamNumber);
        }
    }

    /// Just before return (and can't throw) increment reader count  ??? safer way to assure don't miss close?
    imf->incrReaderCount();

    /// If get here, the reader is open
    isOpen_ = true;
}

CompressedVectorReaderImpl::~CompressedVectorReaderImpl()
{
#ifdef E57_MAX_VERBOSE
    cout << "~CompressedVectorReaderImpl() called" << endl; //???
    //dump(4);
#endif

    if (isOpen_) {
        try {
            close();  ///??? what if already closed?
        } catch (...) {
                //??? report?
        }
    }
}

void CompressedVectorReaderImpl::setBuffers(vector<SourceDestBuffer>& dbufs)
{
    /// don't checkImageFileOpen
    /// don't checkReaderOpen

    /// Check dbufs well formed: no dups, no extra, missing is ok
    proto_->checkBuffers(dbufs, true);

    /// If had previous dbufs_, check to see if new ones have changed in incompatible way
    if (!dbufs_.empty())
    {
        if (dbufs_.size() != dbufs.size())
        {
            throw E57_EXCEPTION2(E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                                 "oldSize=" + toString(dbufs_.size())
                                 + " newSize=" + toString(dbufs.size()));
        }
        for (size_t i = 0; i < dbufs_.size(); i++)
        {
            shared_ptr<SourceDestBufferImpl> oldBuf = dbufs_[i].impl();
            shared_ptr<SourceDestBufferImpl> newBuf = dbufs[i].impl();

            /// Throw exception if old and new not compatible
            oldBuf->checkCompatible(newBuf);
        }
    }

    dbufs_ = dbufs;
}

unsigned CompressedVectorReaderImpl::read(vector<SourceDestBuffer>& dbufs)
{
    /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), read() will do it

    checkReaderOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    /// Check compatible with current dbufs
    setBuffers(dbufs);

    return(read());
}

unsigned CompressedVectorReaderImpl::read()
{
#ifdef E57_MAX_VERBOSE
    cout << "CompressedVectorReaderImpl::read() called" << endl; //???
#endif
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));
    checkReaderOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    /// Rewind all dbufs so start writing to them at beginning
    for ( auto &dbuf : dbufs_ )
    {
       dbuf.impl()->rewind();
    }

    /// Allow decoders to use data they already have in their queue to fill newly empty dbufs
    /// This helps to keep decoder input queues smaller, which reduces backtracking in the packet cache.
    for ( auto &channel : channels_ )
    {
       channel.decoder->inputProcess( nullptr, 0 );
    }

    /// Loop until every dbuf is full or we have reached end of the binary section.
    while (true)
    {
        /// Find the earliest packet position for channels that are still hungry
        /// It's important to call inputProcess of the decoders before this call, so current hungriness level is reflected.
        uint64_t earliestPacketLogicalOffset = earliestPacketNeededForInput();

        /// If nobody's hungry, we are done with the read
        if (earliestPacketLogicalOffset == E57_UINT64_MAX)
            break;

        /// Feed packet to the hungry decoders
        feedPacketToDecoders(earliestPacketLogicalOffset);
    }

    /// Verify that each channel produced the same number of records
    unsigned outputCount = 0;
    for (unsigned i = 0; i < channels_.size(); i++)
    {
        DecodeChannel* chan = &channels_[i];
        if (i == 0)
        {
           outputCount = chan->dbuf.impl()->nextIndex();
        }
        else
        {
            if (outputCount != chan->dbuf.impl()->nextIndex())
            {
                throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                                     "outputCount=" + toString(outputCount)
                                     + " nextIndex=" + toString(chan->dbuf.impl()->nextIndex()));
            }
        }
    }

    /// Return number of records transferred to each dbuf.
    return outputCount;
}

uint64_t CompressedVectorReaderImpl::earliestPacketNeededForInput() const
{
    uint64_t earliestPacketLogicalOffset = E57_UINT64_MAX;
#ifdef E57_MAX_VERBOSE
    unsigned earliestChannel = 0;
#endif

    for (unsigned i = 0; i < channels_.size(); i++)
    {
        const DecodeChannel* chan = &channels_[i];

        /// Test if channel needs more input.
        /// Important to call inputProcess just before this, so these tests work.
        if (!chan->isOutputBlocked() && !chan->inputFinished)
        {
            /// Check if earliest so far
            if (chan->currentPacketLogicalOffset < earliestPacketLogicalOffset)
            {
                earliestPacketLogicalOffset = chan->currentPacketLogicalOffset;
#ifdef E57_MAX_VERBOSE
                earliestChannel = i;
#endif
            }
        }
    }
#ifdef E57_MAX_VERBOSE
    if (earliestPacketLogicalOffset == E57_UINT64_MAX)
        cout << "earliestPacketNeededForInput returning none found" << endl;
    else
        cout << "earliestPacketNeededForInput returning " << earliestPacketLogicalOffset << " for channel[" << earliestChannel << "]" << endl;
#endif
    return earliestPacketLogicalOffset;
}

DataPacket *CompressedVectorReaderImpl::dataPacket( uint64_t inLogicalOffset ) const
{
   char  *packet = nullptr;

   unique_ptr<PacketLock> packetLock = cache_->lock( inLogicalOffset, packet );

   return reinterpret_cast<DataPacket*>( packet );
}

void CompressedVectorReaderImpl::feedPacketToDecoders( uint64_t currentPacketLogicalOffset )
{
   /// Read earliest packet into cache and send data to decoders with unblocked output
   bool     channelHasExhaustedPacket = false;
   uint64_t nextPacketLogicalOffset = E57_UINT64_MAX;

   /// Get packet at currentPacketLogicalOffset into memory.
   auto dpkt = dataPacket( currentPacketLogicalOffset );

   /// Double check that have a data packet.  Should have already determined this.
   if ( dpkt->header.packetType != DATA_PACKET )
   {
     throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "packetType=" + toString( dpkt->header.packetType ) );
   }

   /// Feed bytestreams to channels with unblocked output that are reading from this packet
   for ( DecodeChannel &channel : channels_ )
   {
      /// Skip channels that have already read this packet.
      if (channel.currentPacketLogicalOffset != currentPacketLogicalOffset || channel.isOutputBlocked())
      {
         continue;
      }

      /// Get bytestream buffer for this channel from packet
      unsigned int   bsbLength = 0;
      const char     *bsbStart = dpkt->getBytestream(channel.bytestreamNumber, bsbLength);

      /// Calc where we are in the buffer
      const char     *uneatenStart = &bsbStart[channel.currentBytestreamBufferIndex];
      const size_t   uneatenLength = bsbLength - channel.currentBytestreamBufferIndex;

      /// Double check we are not off end of buffer
      if ( channel.currentBytestreamBufferIndex > bsbLength )
      {
          throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                               "currentBytestreamBufferIndex =" + toString(channel.currentBytestreamBufferIndex)
                               + " bsbLength=" + toString(bsbLength));
      }

      if ( &uneatenStart[uneatenLength] > &bsbStart[bsbLength] )
      {
          throw E57_EXCEPTION2(E57_ERROR_INTERNAL,
                               "uneatenLength=" + toString(uneatenLength)
                               + " bsbLength=" + toString(bsbLength));
      }

      /// Feed into decoder
      size_t bytesProcessed = channel.decoder->inputProcess(uneatenStart, uneatenLength);

   #ifdef E57_MAX_VERBOSE
      cout << "  stream[" << channel.bytestreamNumber << "]: feeding decoder " << uneatenLength << " bytes" << endl;
      if (uneatenLength == 0)
      {
         channel.dump(8);
      }

      cout << "  stream[" << channel.bytestreamNumber << "]: bytesProcessed=" << bytesProcessed << endl;
   #endif

      /// Adjust counts of bytestream location
      channel.currentBytestreamBufferIndex += bytesProcessed;

      /// Check if this channel has exhausted its bytestream buffer in this packet
      if ( channel.isInputBlocked() )
      {
   #ifdef E57_MAX_VERBOSE
          cout << "  stream[" << channel.bytestreamNumber << "] has exhausted its input in current packet" << endl;
   #endif
          channelHasExhaustedPacket = true;
          nextPacketLogicalOffset = currentPacketLogicalOffset + dpkt->header.packetLogicalLengthMinus1 + 1;
      }
   }

   /// Skip over any index or empty packets to next data packet.
   nextPacketLogicalOffset = findNextDataPacket( nextPacketLogicalOffset );

   /// If some channel has exhausted this packet, find next data packet and update currentPacketLogicalOffset for all interested channels.
   if ( channelHasExhaustedPacket )
   {
     if ( nextPacketLogicalOffset < E57_UINT64_MAX )
     { //??? huh?
         /// Get packet at nextPacketLogicalOffset into memory.
         dpkt = dataPacket( nextPacketLogicalOffset );

         /// Got a data packet, update the channels with exhausted input
         for ( DecodeChannel &channel : channels_ )
         {
            if ( (channel.currentPacketLogicalOffset == currentPacketLogicalOffset) && channel.isInputBlocked() )
            {
                 channel.currentPacketLogicalOffset = nextPacketLogicalOffset;
                 channel.currentBytestreamBufferIndex = 0;

                 /// It is OK if the next packet doesn't contain any data for this channel, will skip packet on next iter of loop
                 channel.currentBytestreamBufferLength = dpkt->getBytestreamBufferLength( channel.bytestreamNumber );

   #ifdef E57_MAX_VERBOSE
                 cout << "  set new stream buffer for channel[" << channel.bytestreamNumber << "], length=" << channel.currentBytestreamBufferLength << endl;
   #endif
                 /// ??? perform flush if new packet flag set?
             }
         }
     }
     else
     {
         /// Reached end without finding data packet, mark exhausted channels as finished
   #ifdef E57_MAX_VERBOSE
         cout << "  at end of data packets" << endl;
   #endif
         if ( nextPacketLogicalOffset >= sectionEndLogicalOffset_ )
         {
             for ( DecodeChannel &channel : channels_ )
             {
                 if ( (channel.currentPacketLogicalOffset == currentPacketLogicalOffset) && channel.isInputBlocked() )
                 {
   #ifdef E57_MAX_VERBOSE
                     cout << "  Marking channel[" << channel.bytestreamNumber << "] as finished" << endl;
   #endif
                     channel.inputFinished = true;
                 }
             }
         }
     }
   }
}

uint64_t CompressedVectorReaderImpl::findNextDataPacket(uint64_t nextPacketLogicalOffset)
{
#ifdef E57_MAX_VERBOSE
    cout << "  searching for next data packet, nextPacketLogicalOffset=" << nextPacketLogicalOffset
         << " sectionEndLogicalOffset=" << sectionEndLogicalOffset_ << endl;
#endif

    /// Starting at nextPacketLogicalOffset, search for next data packet until hit end of binary section.
    while (nextPacketLogicalOffset < sectionEndLogicalOffset_)
    {
        char* anyPacket = nullptr;

        unique_ptr<PacketLock> packetLock = cache_->lock(nextPacketLogicalOffset, anyPacket);

        /// Guess it's a data packet, if not continue to next packet
        auto dpkt = reinterpret_cast<const DataPacket*>(anyPacket);

        if (dpkt->header.packetType == DATA_PACKET)
        {
#ifdef E57_MAX_VERBOSE
            cout << "  Found next data packet at nextPacketLogicalOffset=" << nextPacketLogicalOffset << endl;
#endif
            return nextPacketLogicalOffset;
        }

        /// All packets have length in same place, so can use the field to skip to next packet.
        nextPacketLogicalOffset += dpkt->header.packetLogicalLengthMinus1 + 1;
    }

    /// Ran off end of section, so return failure code.
    return E57_UINT64_MAX;
}

void CompressedVectorReaderImpl::seek(uint64_t /*recordNumber*/)
{
    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    ///!!! implement
    throw E57_EXCEPTION1(E57_ERROR_NOT_IMPLEMENTED);
}

bool CompressedVectorReaderImpl::isOpen() const
{
    /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), or checkReaderOpen()
    return(isOpen_);
}

std::shared_ptr<CompressedVectorNodeImpl> CompressedVectorReaderImpl::compressedVectorNode() const
{
    return(cVector_);
}

void CompressedVectorReaderImpl::close()
{
    /// Before anything that can throw, decrement reader count
    ImageFileImplSharedPtr imf(cVector_->destImageFile_);
    imf->decrReaderCount();

    checkImageFileOpen(__FILE__, __LINE__, static_cast<const char *>(__FUNCTION__));

    /// No error if reader not open
    if (!isOpen_)
        return;

    /// Destroy decoders
    channels_.clear();

    delete cache_;
    cache_ = nullptr;

    isOpen_ = false;
}

void CompressedVectorReaderImpl::checkImageFileOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName) const
{
   // unimplemented...
}

void CompressedVectorReaderImpl::checkReaderOpen(const char* srcFileName, int srcLineNumber, const char* srcFunctionName) const
{
    if (!isOpen_) {
        throw E57Exception(E57_ERROR_READER_NOT_OPEN,
                           "imageFileName=" + cVector_->imageFileName()
                           + " cvPathName=" + cVector_->pathName(),
                           srcFileName,
                           srcLineNumber,
                           srcFunctionName);
    }
}

void CompressedVectorReaderImpl::dump(int indent, std::ostream& os)
{
    os << space(indent) << "isOpen:" << isOpen_ << endl;

    for (unsigned i = 0; i < dbufs_.size(); i++) {
        os << space(indent) << "dbufs[" << i << "]:" << endl;
        dbufs_[i].dump(indent+4, os);
    }

    os << space(indent) << "cVector:" << endl;
    cVector_->dump(indent+4, os);

    os << space(indent) << "proto:" << endl;
    proto_->dump(indent+4, os);

    for (unsigned i = 0; i < channels_.size(); i++) {
        os << space(indent) << "channels[" << i << "]:" << endl;
        channels_[i].dump(indent+4, os);
    }

    os << space(indent) << "recordCount:             " << recordCount_ << endl;
    os << space(indent) << "maxRecordCount:          " << maxRecordCount_ << endl;
    os << space(indent) << "sectionEndLogicalOffset: " << sectionEndLogicalOffset_ << endl;
}

//================================================================

DecodeChannel::DecodeChannel(SourceDestBuffer dbuf_arg, shared_ptr<Decoder> decoder_arg, unsigned bytestreamNumber_arg, uint64_t maxRecordCount_arg)
: dbuf(dbuf_arg),
  decoder(decoder_arg),
  bytestreamNumber(bytestreamNumber_arg)
{
    maxRecordCount = maxRecordCount_arg;
    currentPacketLogicalOffset = 0;
    currentBytestreamBufferIndex = 0;
    currentBytestreamBufferLength = 0;
    inputFinished = false;
}

bool DecodeChannel::isOutputBlocked() const
{
    /// If we have completed the entire vector, we are done
    if (decoder->totalRecordsCompleted() >= maxRecordCount)
        return(true);

    /// If we have filled the dest buffer, we are blocked
    return(dbuf.impl()->nextIndex() == dbuf.impl()->capacity());
}

bool DecodeChannel::isInputBlocked() const
{
    /// If have read until the section end, we are done
    if (inputFinished)
        return(true);

    /// If have eaten all the input in the current packet, we are blocked.
    return(currentBytestreamBufferIndex == currentBytestreamBufferLength);
}

void DecodeChannel::dump(int indent, std::ostream& os)
{
    os << space(indent) << "dbuf" << endl;
    dbuf.dump(indent+4, os);

    os << space(indent) << "decoder:" << endl;
    decoder->dump(indent+4, os);

    os << space(indent) << "bytestreamNumber:              " << bytestreamNumber << endl;
    os << space(indent) << "maxRecordCount:                " << maxRecordCount << endl;
    os << space(indent) << "currentPacketLogicalOffset:    " << currentPacketLogicalOffset << endl;
    os << space(indent) << "currentBytestreamBufferIndex:  " << currentBytestreamBufferIndex << endl;
    os << space(indent) << "currentBytestreamBufferLength: " << currentBytestreamBufferLength << endl;
    os << space(indent) << "inputFinished:                 " << inputFinished << endl;
    os << space(indent) << "isInputBlocked():              " << isInputBlocked() << endl;
    os << space(indent) << "isOutputBlocked():             " << isOutputBlocked() << endl;
}
