dlib C++ Library - input_abstract.h

    // Copyright (C) 2015  Davis E. King (davis@dlib.net)
    // License: Boost Software License   See LICENSE.txt for the full license.
    #undef DLIB_DNn_INPUT_ABSTRACT_H_
    #ifdef DLIB_DNn_INPUT_ABSTRACT_H_

    #include "../matrix.h"
    #include "../pixel.h"

    namespace dlib
    {

    // ----------------------------------------------------------------------------------------

        class EXAMPLE_INPUT_LAYER
        {
            /*!
                WHAT THIS OBJECT REPRESENTS
                    Each deep neural network model in dlib begins with an input layer. The job
                    of the input layer is to convert an input_type into a tensor.  Nothing more
                    and nothing less.

                    Note that there is no dlib::EXAMPLE_INPUT_LAYER type.  It is shown here
                    purely to document the interface that an input layer object must implement.
                    If you are using some kind of image or matrix object as your input_type
                    then you can use the provided dlib::input layer defined below.  Otherwise,
                    you need to define your own custom input layer.

                THREAD SAFETY
                    to_tensor() must be thread safe.  That is, multiple threads must be able to
                    make calls to to_tensor() on a single instance of this object at the same
                    time.
            !*/
        public:

            EXAMPLE_INPUT_LAYER(
            );
            /*!
                ensures
                    - Default constructs this object.  This function is not required to do
                      anything in particular but it must exist, that is, it is required that
                      layer objects be default constructable.
            !*/

            EXAMPLE_INPUT_LAYER (
                const EXAMPLE_INPUT_LAYER& item
            );
            /*!
                ensures
                    - EXAMPLE_INPUT_LAYER objects are copy constructable
            !*/

            EXAMPLE_INPUT_LAYER(
                const some_other_input_layer_type& item
            );
            /*!
                ensures
                    - Constructs this object from item.  This form of constructor is optional
                      but it allows you to provide a conversion from one input layer type to
                      another.  For example, the following code is valid only if my_input_layer2 can
                      be constructed from my_input_layer1:
                        relu<fc<relu<fc<my_input_layer1>>>> my_dnn1;
                        relu<fc<relu<fc<my_input_layer2>>>> my_dnn2(my_dnn1);
                      This kind of pattern is useful if you want to use one type of input layer
                      during training but a different type of layer during testing since it
                      allows you to easily convert between related deep neural network types.
            !*/

            typedef whatever_type_to_tensor_expects input_type;

            template <typename forward_iterator>
            void to_tensor (
                forward_iterator ibegin,
                forward_iterator iend,
                resizable_tensor& data
            ) const;
            /*!
                requires
                    - [ibegin, iend) is an iterator range over input_type objects.
                    - std::distance(ibegin,iend) > 0
                ensures
                    - Converts the iterator range into a tensor and stores it into #data.
                    - #data.num_samples()%distance(ibegin,iend) == 0.
                      Normally you would have #data.num_samples() == distance(ibegin,iend) but
                      you can also expand the output by some integer factor so long as the loss
                      you use can deal with it correctly.
                    - The data in the ith sample of #data corresponds to the input_type object
                      *(ibegin+i/sample_expansion_factor).
                      where sample_expansion_factor==#data.num_samples()/distance(ibegin,iend).
            !*/
        };

        std::ostream& operator<<(std::ostream& out, const EXAMPLE_INPUT_LAYER& item);
        /*!
            print a string describing this layer.
        !*/

        void to_xml(const EXAMPLE_INPUT_LAYER& item, std::ostream& out);
        /*!
            This function is optional, but required if you want to print your networks with
            net_to_xml().  Therefore, to_xml() prints a layer as XML.
        !*/

        void serialize(const EXAMPLE_INPUT_LAYER& item, std::ostream& out);
        void deserialize(EXAMPLE_INPUT_LAYER& item, std::istream& in);
        /*!
            provides serialization support
        !*/

    // ----------------------------------------------------------------------------------------

        template <
            typename T
            >
        class input
        {
            /*!
                REQUIREMENTS ON T
                    One of the following must be true:
                        - T is a matrix or array2d object and it must contain some kind of
                          pixel type.  I.e. pixel_traits<T::type> must be defined.
                        - T is a std::array<matrix<U>> where U is any built in scalar type like
                          float, double, or unsigned char.

                WHAT THIS OBJECT REPRESENTS
                    This is a basic input layer that simply copies images into a tensor.
            !*/

        public:
            typedef T input_type;

            template <typename forward_iterator>
            void to_tensor (
                forward_iterator ibegin,
                forward_iterator iend,
                resizable_tensor& data
            ) const;
            /*!
                requires
                    - [ibegin, iend) is an iterator range over input_type objects.
                    - std::distance(ibegin,iend) > 0
                    - The input range should contain image objects that all have the same
                      dimensions.
                ensures
                    - Converts the iterator range into a tensor and stores it into #data.  In
                      particular, if the input images have R rows, C columns, and K channels
                      (where K is given by pixel_traits::num or std::array::size() if
                      std::array inputs are used) then we will have:
                        - #data.num_samples() == std::distance(ibegin,iend)
                        - #data.nr() == R
                        - #data.nc() == C
                        - #data.k() == K
                      For example, a matrix<float,3,3> would turn into a tensor with 3 rows, 3
                      columns, and k()==1.  Or a matrix<rgb_pixel,4,5> would turn into a tensor
                      with 4 rows, 5 columns, and k()==3 (since rgb_pixels have 3 channels).
                      Or a std::array<matrix<float,3,3>,5> would turn into a tensor with 3 rows
                      and columns, and k()==5 channels.
                    - If the input data contains pixels of type unsigned char, rgb_pixel, or
                      other pixel types with a basic_pixel_type of unsigned char then each
                      value written to the output tensor is first divided by 256.0 so that the
                      resulting outputs are all in the range [0,1].
            !*/

            // Provided for compatibility with input_rgb_image_pyramid's interface
            bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
            drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
            drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
        };

    // ----------------------------------------------------------------------------------------

        class input_rgb_image
        {
            /*!
                WHAT THIS OBJECT REPRESENTS
                    This input layer works with RGB images of type matrix<rgb_pixel>.  It is
                    very similar to the dlib::input layer except that it allows you to subtract
                    the average color value from each color channel when converting an image to
                    a tensor.
            !*/
        public:
            typedef matrix<rgb_pixel> input_type;

            input_rgb_image (
            );
            /*!
                ensures
                    - #get_avg_red()   == 122.782
                    - #get_avg_green() == 117.001
                    - #get_avg_blue()  == 104.298
            !*/

            input_rgb_image (
                float avg_red,
                float avg_green,
                float avg_blue
            );
            /*!
                ensures
                    - #get_avg_red() == avg_red
                    - #get_avg_green() == avg_green
                    - #get_avg_blue() == avg_blue
            !*/

            float get_avg_red(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the red color channel.
            !*/

            float get_avg_green(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the green color channel.
            !*/

            float get_avg_blue(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the blue color channel.
            !*/

            template <typename forward_iterator>
            void to_tensor (
                forward_iterator ibegin,
                forward_iterator iend,
                resizable_tensor& data
            ) const;
            /*!
                requires
                    - [ibegin, iend) is an iterator range over input_type objects.
                    - std::distance(ibegin,iend) > 0
                    - The input range should contain images that all have the same
                      dimensions.
                ensures
                    - Converts the iterator range into a tensor and stores it into #data.  In
                      particular, if the input images have R rows, C columns then we will have:
                        - #data.num_samples() == std::distance(ibegin,iend)
                        - #data.nr() == R
                        - #data.nc() == C
                        - #data.k() == 3
                      Moreover, each color channel is normalized by having its average value
                      subtracted (according to get_avg_red(), get_avg_green(), or
                      get_avg_blue()) and then is divided by 256.0.
            !*/

            // Provided for compatibility with input_rgb_image_pyramid's interface
            bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
            drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
            drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }
        };

    // ----------------------------------------------------------------------------------------

        template <size_t NR, size_t NC=NR>
        class input_rgb_image_sized
        {
            /*!
                WHAT THIS OBJECT REPRESENTS
                    This layer has an interface and behavior identical to input_rgb_image
                    except that it requires input images to have NR rows and NC columns.  This
                    is checked by a DLIB_CASSERT inside to_tensor().

                    You can also convert between input_rgb_image and input_rgb_image_sized by
                    copy construction or assignment.
            !*/

        };

    // ----------------------------------------------------------------------------------------

        class input_rgb_image_pair
        {
            /*!
                WHAT THIS OBJECT REPRESENTS
                    This input layer works with std::pair of RGB images of type matrix<rgb_pixel>.
                    It is useful when you want to input image pairs that are related to each other,
                    for instance, they are different distorted views of the same original image.
                    It is mainly supposed to be used with unsupervised loss functions such as
                    loss_barlow_twins_. You can also convert between input_rgb_image and
                    input_rgb_image_pair by copy construction or assignment.
            !*/
        public:
            typedef std::pair<matrix<rgb_pixel>, matrix<rgb_pixel>> input_type;

            input_rgb_image_pair (
            );
            /*!
                ensures
                    - #get_avg_red()   == 122.782
                    - #get_avg_green() == 117.001
                    - #get_avg_blue()  == 104.298
            !*/

            input_rgb_image_pair (
                float avg_red,
                float avg_green,
                float avg_blue
            );
            /*!
                ensures
                    - #get_avg_red() == avg_red
                    - #get_avg_green() == avg_green
                    - #get_avg_blue() == avg_blue
            !*/

            inline input_rgb_image_pair (
                const input_rgb_image& item
            );
            /*!
                ensures
                    - #get_avg_red() == item.get_avg_red()
                    - #get_avg_green() == item.get_avg_green()
                    - #get_avg_blue() == item.get_avg_blue()
            !*/

            template <size_t NR, size_t NC>
            inline input_rgb_image_pair (
                const input_rgb_image_sized<NR, NC>& item
            );
            /*!
                ensures
                    - #get_avg_red() == item.get_avg_red()
                    - #get_avg_green() == item.get_avg_green()
                    - #get_avg_blue() == item.get_avg_blue()
            !*/

            float get_avg_red(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the red color channel.
            !*/

            float get_avg_green(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the green color channel.
            !*/

            float get_avg_blue(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the blue color channel.
            !*/

            void to_tensor (
                forward_iterator ibegin,
                forward_iterator iend,
                resizable_tensor& data
            ) const;
            /*!
                requires
                    - [ibegin, iend) is an iterator range over input_type objects.
                    - std::distance(ibegin,iend) > 0
                    - The input range should contain images that all have the same
                      dimensions.
                ensures
                    - Converts the iterator range into a tensor and stores it into #data.  In
                      particular, if the input images have R rows, C columns then we will have:
                        - #data.num_samples() == 2 * std::distance(ibegin,iend)
                        - #data.nr() == R
                        - #data.nc() == C
                        - #data.k() == 3
                      Moreover, each color channel is normalized by having its average value
                      subtracted (according to get_avg_red(), get_avg_green(), or
                      get_avg_blue()) and then is divided by 256.0.
                      Additionally, the first elements in each pair are placed in the first half
                      of the batch, and the second elements in the second half.
            !*/

            // Provided for compatibility with input_rgb_image_pyramid's interface
            bool image_contained_point ( const tensor& data, const point& p) const { return get_rect(data).contains(p); }
            drectangle tensor_space_to_image_space ( const tensor& /*data*/, drectangle r) const { return r; }
            drectangle image_space_to_tensor_space ( const tensor& /*data*/, double /*scale*/, drectangle r ) const { return r; }

    // ----------------------------------------------------------------------------------------

        template <
            typename PYRAMID_TYPE
            >
        class input_grayscale_image_pyramid
        {
            /*!
                REQUIREMENTS ON PYRAMID_TYPE
                    PYRAMID_TYPE must be an instance of the dlib::pyramid_down template.

                WHAT THIS OBJECT REPRESENTS
                    This input layer works with gray scale images of type matrix<unsigned char>.
                    It is identical to input layer except that it outputs a tensor containing a tiled
                    image pyramid of each input image rather than a simple copy of each image.
                    The tiled image pyramid is created using create_tiled_pyramid().
            !*/

        public:

            typedef matrix<unsigned char> input_type;
            typedef PYRAMID_TYPE pyramid_type;
            input_grayscale_image_pyramid (
            );
            /*!
                ensures
                    - #get_pyramid_padding() == 10
                    - #get_pyramid_outer_padding() == 11
            !*/

            unsigned long get_pyramid_padding (
            ) const;
            /*!
                ensures
                    - When this object creates a pyramid it will call create_tiled_pyramid() and
                      set create_tiled_pyramid's pyramid_padding parameter to get_pyramid_padding().
            !*/

            void set_pyramid_padding (
                unsigned long value
            );
            /*!
                ensures
                    - #get_pyramid_padding() == value
            !*/

            unsigned long get_pyramid_outer_padding (
            ) const;
            /*!
                ensures
                    - When this object creates a pyramid it will call create_tiled_pyramid()
                      and set create_tiled_pyramid's pyramid_outer_padding parameter to
                      get_pyramid_outer_padding().
            !*/

            void set_pyramid_outer_padding (
                unsigned long value
            );
            /*!
                ensures
                    - #get_pyramid_outer_padding() == value
            !*/

            template <typename forward_iterator>
            void to_tensor (
                forward_iterator ibegin,
                forward_iterator iend,
                resizable_tensor& data
            ) const;
            /*!
                requires
                    - [ibegin, iend) is an iterator range over input_type objects.
                    - std::distance(ibegin,iend) > 0
                    - The input range should contain images that all have the same
                      dimensions.
                ensures
                    - Converts the iterator range into a tensor and stores it into #data.  In
                      particular, we will have:
                        - #data.num_samples() == std::distance(ibegin,iend)
                        - #data.k() == 1
                        - Each sample in #data contains a tiled image pyramid of the
                          corresponding input image.  The tiled pyramid is created by
                          create_tiled_pyramid().
                      Moreover, each pixel is normalized, dividing them by 256.0.
            !*/

            bool image_contained_point (
                const tensor& data,
                const point& p
            ) const;
            /*!
                requires
                    - data is a tensor that was produced by this->to_tensor()
                ensures
                    - Since data is a tensor that is built from a bunch of identically sized
                      images, we can ask if those images were big enough to contain the point
                      p.  This function returns the answer to that question.
            !*/

            drectangle image_space_to_tensor_space (
                const tensor& data,
                double scale,
                drectangle r
            ) const;
            /*!
                requires
                    - data is a tensor that was produced by this->to_tensor()
                    - 0 < scale <= 1
                ensures
                    - This function maps from to_tensor()'s input image space to its output
                      tensor space.  Therefore, given that data is a tensor produced by
                      to_tensor(), image_space_to_tensor_space() allows you to ask for the
                      rectangle in data that corresponds to a rectangle in the original image
                      space.

                      Note that since the output tensor contains an image pyramid, there are
                      multiple points in the output tensor that correspond to any input
                      location.  So you must also specify a scale so we know what level of the
                      pyramid is needed.  So given a rectangle r in an input image, you can
                      ask, what rectangle in data corresponds to r when things are scale times
                      smaller?  That rectangle is returned by this function.
                    - A scale of 1 means we don't move anywhere in the pyramid scale space relative
                      to the input image while smaller values of scale mean we move down the
                      pyramid.
            !*/

            drectangle tensor_space_to_image_space (
                const tensor& data,
                drectangle r
            ) const;
            /*!
                requires
                    - data is a tensor that was produced by this->to_tensor()
                ensures
                    - This function maps from to_tensor()'s output tensor space to its input
                      image space.  Therefore, given that data is a tensor produced by
                      to_tensor(), tensor_space_to_image_space() allows you to ask for the
                      rectangle in the input image that corresponds to a rectangle in data.
                    - It should be noted that this function isn't always an inverse of
                      image_space_to_tensor_space().  This is because you can ask
                      image_space_to_tensor_space() for the coordinates of points outside the input
                      image and they will be mapped to somewhere that doesn't have an inverse.
                      But for points actually inside the input image this function performs an
                      approximate inverse mapping.  I.e. when image_contained_point(data,center(r))==true
                      there is an approximate inverse.
            !*/

        };

    // ----------------------------------------------------------------------------------------

        template <
            typename PYRAMID_TYPE
            >
        class input_rgb_image_pyramid
        {
            /*!
                REQUIREMENTS ON PYRAMID_TYPE
                    PYRAMID_TYPE must be an instance of the dlib::pyramid_down template.

                WHAT THIS OBJECT REPRESENTS
                    This input layer works with RGB images of type matrix<rgb_pixel>.  It is
                    identical to input_rgb_image except that it outputs a tensor containing a
                    tiled image pyramid of each input image rather than a simple copy of each
                    image.  The tiled image pyramid is created using create_tiled_pyramid().
            !*/

        public:

            typedef matrix<rgb_pixel> input_type;
            typedef PYRAMID_TYPE pyramid_type;

            input_rgb_image_pyramid (
            );
            /*!
                ensures
                    - #get_avg_red()   == 122.782
                    - #get_avg_green() == 117.001
                    - #get_avg_blue()  == 104.298
                    - #get_pyramid_padding() == 10
                    - #get_pyramid_outer_padding() == 11
            !*/

            input_rgb_image_pyramid (
                float avg_red,
                float avg_green,
                float avg_blue
            );
            /*!
                ensures
                    - #get_avg_red() == avg_red
                    - #get_avg_green() == avg_green
                    - #get_avg_blue() == avg_blue
                    - #get_pyramid_padding() == 10
                    - #get_pyramid_outer_padding() == 11
            !*/

            float get_avg_red(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the red color channel.
            !*/

            float get_avg_green(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the green color channel.
            !*/

            float get_avg_blue(
            ) const;
            /*!
                ensures
                    - returns the value subtracted from the blue color channel.
            !*/

            unsigned long get_pyramid_padding (
            ) const;
            /*!
                ensures
                    - When this object creates a pyramid it will call create_tiled_pyramid() and
                      set create_tiled_pyramid's pyramid_padding parameter to get_pyramid_padding().
            !*/
            void set_pyramid_padding (
                unsigned long value
            );
            /*!
                ensures
                    - #get_pyramid_padding() == value
            !*/

            unsigned long get_pyramid_outer_padding (
            ) const;
            /*!
                ensures
                    - When this object creates a pyramid it will call create_tiled_pyramid()
                      and set create_tiled_pyramid's pyramid_outer_padding parameter to
                      get_pyramid_outer_padding().
            !*/
            void set_pyramid_outer_padding (
                unsigned long value
            );
            /*!
                ensures
                    - #get_pyramid_outer_padding() == value
            !*/

            template <typename forward_iterator>
            void to_tensor (
                forward_iterator ibegin,
                forward_iterator iend,
                resizable_tensor& data
            ) const;
            /*!
                requires
                    - [ibegin, iend) is an iterator range over input_type objects.
                    - std::distance(ibegin,iend) > 0
                    - The input range should contain images that all have the same
                      dimensions.
                ensures
                    - Converts the iterator range into a tensor and stores it into #data.  In
                      particular, we will have:
                        - #data.num_samples() == std::distance(ibegin,iend)
                        - #data.k() == 3
                        - Each sample in #data contains a tiled image pyramid of the
                          corresponding input image.  The tiled pyramid is created by
                          create_tiled_pyramid().
                      Moreover, each color channel is normalized by having its average value
                      subtracted (according to get_avg_red(), get_avg_green(), or
                      get_avg_blue()) and then is divided by 256.0.
            !*/

            bool image_contained_point (
                const tensor& data,
                const point& p
            ) const;
            /*!
                requires
                    - data is a tensor that was produced by this->to_tensor()
                ensures
                    - Since data is a tensor that is built from a bunch of identically sized
                      images, we can ask if those images were big enough to contain the point
                      p.  This function returns the answer to that question.
            !*/

            drectangle image_space_to_tensor_space (
                const tensor& data,
                double scale,
                drectangle r
            ) const;
            /*!
                requires
                    - data is a tensor that was produced by this->to_tensor()
                    - 0 < scale <= 1
                ensures
                    - This function maps from to_tensor()'s input image space to its output
                      tensor space.  Therefore, given that data is a tensor produced by
                      to_tensor(), image_space_to_tensor_space() allows you to ask for the
                      rectangle in data that corresponds to a rectangle in the original image
                      space.

                      Note that since the output tensor contains an image pyramid, there are
                      multiple points in the output tensor that correspond to any input
                      location.  So you must also specify a scale so we know what level of the
                      pyramid is needed.  So given a rectangle r in an input image, you can
                      ask, what rectangle in data corresponds to r when things are scale times
                      smaller?  That rectangle is returned by this function.
                    - A scale of 1 means we don't move anywhere in the pyramid scale space relative
                      to the input image while smaller values of scale mean we move down the
                      pyramid.
            !*/

            drectangle tensor_space_to_image_space (
                const tensor& data,
                drectangle r
            ) const;
            /*!
                requires
                    - data is a tensor that was produced by this->to_tensor()
                ensures
                    - This function maps from to_tensor()'s output tensor space to its input
                      image space.  Therefore, given that data is a tensor produced by
                      to_tensor(), tensor_space_to_image_space() allows you to ask for the
                      rectangle in the input image that corresponds to a rectangle in data.
                    - It should be noted that this function isn't always an inverse of
                      image_space_to_tensor_space().  This is because you can ask
                      image_space_to_tensor_space() for the coordinates of points outside the input
                      image and they will be mapped to somewhere that doesn't have an inverse.
                      But for points actually inside the input image this function performs an
                      approximate inverse mapping.  I.e. when image_contained_point(data,center(r))==true
                      there is an approximate inverse.
            !*/

        };

    // ----------------------------------------------------------------------------------------

        class input_tensor
        {
            /*!
                WHAT THIS OBJECT REPRESENTS
                    This input layer works with dlib::tensor objects. It is very similar to
                    the dlib::input layer except that it allows for concatenating data that
                    already resides in GPU memory.
            !*/

        public:
            typedef tensor input_type;

            input_tensor(
            );
            /*!
                ensures
                    - input_tensor objects are default constructable
            !*/

            input_tensor(
                const input_tensor& item
            );
            /*!
                ensures
                    - input_tensor objects are copy constructable
            !*/

            template <typename forward_iterator>
            void to_tensor(
                forward_iterator ibegin,
                forward_iterator iend,
                resizable_tensor& data
            ) const;
            /*!
                requires
                    - [ibegin, iend) is an iterator range over input_type objects.
                    - std::distance(ibegin,iend) > 0
                    - The input range should contain tensor objects that all have the same
                      dimensions.
                ensures
                    - Copies the iterator range into #data.  In particular, if the input tensors
                      have R rows, C columns, and K channels then we will have:
                        - #data.num_samples() == count_samples(ibegin,iend)
                        - #data.nr() == R
                        - #data.nc() == C
                        - #data.k() == K
                      This results in a tensor concatenation along the sample dimension.
            !*/
        };

    // ----------------------------------------------------------------------------------------
    }

    #endif // DLIB_DNn_INPUT_ABSTRACT_H_