dlib C++ Library - lspi.h

    // Copyright (C) 2015  Davis E. King (davis@dlib.net)
    // License: Boost Software License   See LICENSE.txt for the full license.
    #ifndef DLIB_LSPI_Hh_
    #define DLIB_LSPI_Hh_

    #include "lspi_abstract.h"
    #include "approximate_linear_models.h"

    namespace dlib
    {

    // ----------------------------------------------------------------------------------------

        template <
            typename feature_extractor
            >
        class lspi
        {
        public:
            typedef feature_extractor feature_extractor_type;
            typedef typename feature_extractor::state_type state_type;
            typedef typename feature_extractor::action_type action_type;

            explicit lspi(
                const feature_extractor& fe_
            ) : fe(fe_)
            {
                init();
            }

            lspi(
            )
            {
                init();
            }

            double get_discount (
            ) const { return discount; }

            void set_discount (
                double value
            )
            {
                // make sure requires clause is not broken
                DLIB_ASSERT(0 < value && value <= 1,
                    "\t void lspi::set_discount(value)"
                    << "\n\t invalid inputs were given to this function"
                    << "\n\t value: " << value
                    );
                discount = value;
            }

            const feature_extractor& get_feature_extractor (
            ) const { return fe; }

            void be_verbose (
            )
            {
                verbose = true;
            }

            void be_quiet (
            )
            {
                verbose = false;
            }

            void set_epsilon (
                double eps_
            )
            {
                // make sure requires clause is not broken
                DLIB_ASSERT(eps_ > 0,
                    "\t void lspi::set_epsilon(eps_)"
                    << "\n\t invalid inputs were given to this function"
                    << "\n\t eps_: " << eps_
                    );
                eps = eps_;
            }

            double get_epsilon (
            ) const
            {
                return eps;
            }

            void set_lambda (
                double lambda_
            )
            {
                // make sure requires clause is not broken
                DLIB_ASSERT(lambda_ >= 0,
                    "\t void lspi::set_lambda(lambda_)"
                    << "\n\t invalid inputs were given to this function"
                    << "\n\t lambda_: " << lambda_
                    );
                lambda = lambda_;
            }

            double get_lambda (
            ) const
            {
                return lambda;
            }

            void set_max_iterations (
                unsigned long max_iter
            ) { max_iterations = max_iter; }

            unsigned long get_max_iterations (
            ) { return max_iterations; }

            template <typename vector_type>
            policy<feature_extractor> train (
                const vector_type& samples
            ) const
            {
                // make sure requires clause is not broken
                DLIB_ASSERT(samples.size() > 0,
                    "\t policy lspi::train(samples)"
                    << "\n\t invalid inputs were given to this function"
                    );

                matrix<double,0,1> w(fe.num_features());
                w = 0;
                matrix<double,0,1> prev_w, b, f1, f2;

                matrix<double> A;

                double change;
                unsigned long iter = 0;
                do
                {
                    A = identity_matrix<double>(fe.num_features())*lambda;
                    b = 0;
                    for (unsigned long i = 0; i < samples.size(); ++i)
                    {
                        fe.get_features(samples[i].state, samples[i].action, f1);
                        fe.get_features(samples[i].next_state,
                                        fe.find_best_action(samples[i].next_state,w),
                                        f2);
                        A += f1*trans(f1 - discount*f2);
                        b += f1*samples[i].reward;
                    }

                    prev_w = w;
                    if (feature_extractor::force_last_weight_to_1)
                        w = join_cols(pinv(colm(A,range(0,A.nc()-2)))*(b-colm(A,A.nc()-1)),mat(1.0));
                    else
                        w = pinv(A)*b;

                    change = length(w-prev_w);
                    ++iter;

                    if (verbose)
                        std::cout << "iteration: " << iter << "\tchange: " << change << std::endl;

                } while(change > eps && iter < max_iterations);

                return policy<feature_extractor>(w,fe);
            }

        private:

            void init()
            {
                lambda = 0.01;
                discount = 0.8;
                eps = 0.01;
                verbose = false;
                max_iterations = 100;
            }

            double lambda;
            double discount;
            double eps;
            bool verbose;
            unsigned long max_iterations;
            feature_extractor fe;
        };

    // ----------------------------------------------------------------------------------------

    }

    #endif // DLIB_LSPI_Hh_