dlib C++ Library - find_max_global.h

    // Copyright (C) 2017  Davis E. King (davis@dlib.net)
    // License: Boost Software License   See LICENSE.txt for the full license.
    #ifndef DLIB_FiND_GLOBAL_MAXIMUM_hH_
    #define DLIB_FiND_GLOBAL_MAXIMUM_hH_

    #include "find_max_global_abstract.h"
    #include "global_function_search.h"
    #include "../metaprogramming.h"
    #include <utility>
    #include <chrono>
    #include <memory>
    #include <thread>
    #include <functional>
    #include "../threads/thread_pool_extension.h"
    #include "../statistics/statistics.h"
    #include "../enable_if.h"

    namespace dlib
    {
        namespace gopt_impl
        {

        // ----------------------------------------------------------------------------------------

            class disable_decay_to_scalar
            {
                const matrix<double,0,1>& a;
            public:
                disable_decay_to_scalar(const matrix<double,0,1>& a) : a(a){}
                operator const matrix<double,0,1>&() const { return a;}
            };

            template <typename T, size_t... indices>
            auto _cwv (
                T&& f,
                const matrix<double,0,1>& a,
                std::index_sequence<indices...>
            ) -> decltype(f(a(indices)...))
            {
                DLIB_CASSERT(a.size() == sizeof...(indices),
                    "You invoked dlib::call_function_and_expand_args(f,a) but the number of arguments expected by f() doesn't match the size of 'a'. "
                    << "Expected " << sizeof...(indices) << " arguments but got " << a.size() << "."
                );
                return f(a(indices)...);
            }

            // Visual studio, as of November 2017, doesn't support C++11 and can't compile this code.
            // So we write the terrible garbage in the #else for visual studio.  When Visual Studio supports C++11 I'll update this #ifdef to use the C++11 code.
    #ifndef _MSC_VER
            template <size_t max_unpack>
            struct call_function_and_expand_args
            {
                template <typename T>
                static auto go(T&& f, const matrix<double,0,1>& a) -> decltype(_cwv(std::forward<T>(f),a,std::make_index_sequence<max_unpack>{}))
                {
                    return _cwv(std::forward<T>(f),a, std::make_index_sequence<max_unpack>{});
                }

                template <typename T>
                static auto go(T&& f, const matrix<double,0,1>& a) -> decltype(call_function_and_expand_args<max_unpack-1>::template go<T>(std::forward<T>(f),a))
                {
                    return call_function_and_expand_args<max_unpack-1>::go(std::forward<T>(f),a);
                }
            };

            template <>
            struct call_function_and_expand_args<0>
            {
                template <typename T>
                static auto go(T&& f, const matrix<double,0,1>& a) -> decltype(f(disable_decay_to_scalar(a)))
                {
                    return f(disable_decay_to_scalar(a));
                }
            };
    #else
            template <size_t max_unpack>
            struct call_function_and_expand_args
            {
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(disable_decay_to_scalar(a)))  {return f(disable_decay_to_scalar(a));   }
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(a(0))) { DLIB_CASSERT(a.size() == 1); return f(a(0)); }
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(a(0),a(1))) { DLIB_CASSERT(a.size() == 2); return f(a(0),a(1)); }
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(a(0), a(1), a(2))) { DLIB_CASSERT(a.size() == 3); return f(a(0), a(1),a(2)); }
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(a(0), a(1), a(2), a(3))) { DLIB_CASSERT(a.size() == 4); return f(a(0), a(1), a(2), a(3)); }
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(a(0), a(1), a(2), a(3), a(4))) { DLIB_CASSERT(a.size() == 5); return f(a(0), a(1), a(2), a(3), a(4)); }
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(a(0), a(1), a(2), a(3), a(4), a(5))) { DLIB_CASSERT(a.size() == 6); return f(a(0), a(1), a(2), a(3), a(4), a(5)); }
    template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> decltype(f(a(0), a(1), a(2), a(3), a(4), a(5), a(6))) { DLIB_CASSERT(a.size() == 7); return f(a(0), a(1), a(2), a(3), a(4), a(5), a(6)); }
            };
    #endif
        }

    // ----------------------------------------------------------------------------------------
    // ----------------------------------------------------------------------------------------

        template <typename T>
        auto call_function_and_expand_args(
            T&& f,
            const matrix<double,0,1>& a
        ) -> decltype(gopt_impl::call_function_and_expand_args<40>::go(f,a))
        {
            // unpack up to 40 parameters when calling f()
            return gopt_impl::call_function_and_expand_args<40>::go(std::forward<T>(f),a);
        }

    // ----------------------------------------------------------------------------------------
    // ----------------------------------------------------------------------------------------

        struct max_function_calls
        {
            max_function_calls() = default;
            explicit max_function_calls(size_t max_calls) : max_calls(max_calls) {}
            size_t max_calls = std::numeric_limits<size_t>::max();
        };

    // ----------------------------------------------------------------------------------------

        const auto FOREVER = std::chrono::hours(24*365*200); // 200 years
        using stop_condition = std::function<bool(double)>;
        const stop_condition never_stop_early = [](double) { return false; };

    // ----------------------------------------------------------------------------------------

        namespace impl
        {
            template <
                typename funct
                >
            std::pair<size_t,function_evaluation> find_max_global (
                double ymult,
                thread_pool& tp,
                std::vector<funct>& functions,
                std::vector<function_spec> specs,
                const max_function_calls num,
                const std::chrono::nanoseconds max_runtime = FOREVER,
                double solver_epsilon = 0,
                std::vector<std::vector<function_evaluation>> initial_function_evals = {},
                stop_condition should_stop = never_stop_early
            )
            {
                // Decide which parameters should be searched on a log scale.  Basically, it's
                // common for machine learning models to have parameters that should be searched on
                // a log scale (e.g. SVM C).  These parameters are usually identifiable because
                // they have bounds like [1e-5 1e10], that is, they span a very large range of
                // magnitudes from really small to really big.  So there we are going to check for
                // that and if we find parameters with that kind of bound constraints we will
                // transform them to a log scale automatically.
                std::vector<std::vector<bool>> log_scale(specs.size());
                for (size_t i = 0; i < specs.size(); ++i)
                {
                    for (long j = 0; j < specs[i].lower.size(); ++j)
                    {
                        if (!specs[i].is_integer_variable[j] && specs[i].lower(j) > 0 && specs[i].upper(j)/specs[i].lower(j) >= 1000)
                        {
                            log_scale[i].push_back(true);
                            specs[i].lower(j) = std::log(specs[i].lower(j));
                            specs[i].upper(j) = std::log(specs[i].upper(j));
                        }
                        else
                        {
                            log_scale[i].push_back(false);
                        }
                    }
                }

                if (initial_function_evals.empty())
                {
                    initial_function_evals.resize(specs.size());
                }
                for (auto& evals : initial_function_evals) {
                    for (auto& eval : evals) {
                        eval.y *= ymult;
                    }
                }

                global_function_search opt(specs, {initial_function_evals});
                opt.set_solver_epsilon(solver_epsilon);

                running_stats_decayed<double> objective_funct_eval_time(functions.size()*5);
                std::mutex eval_time_mutex;

                using std::chrono::steady_clock;
                using std::chrono::duration_cast;
                using std::chrono::nanoseconds;
                const auto time_to_stop = steady_clock::now() + max_runtime;
                //atomic<bool> doesn't support .fetch_or, use std::atomic<int> instead
                std::atomic<int> this_should_stop{false};

                double max_solver_overhead_time = 0;

                // Now run the main solver loop.
                for (size_t i = 0; i < num.max_calls && steady_clock::now() < time_to_stop && !this_should_stop.load(); ++i)
                {
                    const auto get_next_x_start_time = steady_clock::now();
                    auto next = std::make_shared<function_evaluation_request>(opt.get_next_x());
                    const auto get_next_x_runtime = steady_clock::now() - get_next_x_start_time;

                    auto execute_call = [&functions,&ymult,&log_scale,&eval_time_mutex,&objective_funct_eval_time,next,&should_stop,&this_should_stop]() {
                        matrix<double,0,1> x = next->x();
                        // Undo any log-scaling that was applied to the variables before we pass them
                        // to the functions being optimized.
                        for (long j = 0; j < x.size(); ++j)
                        {
                            if (log_scale[next->function_idx()][j])
                                x(j) = std::exp(x(j));
                        }
                        const auto funct_eval_start = steady_clock::now();
                        double y = ymult*call_function_and_expand_args(functions[next->function_idx()], x);
                        const double funct_eval_runtime = duration_cast<nanoseconds>(steady_clock::now() - funct_eval_start).count();
                        this_should_stop.fetch_or(should_stop(y*ymult));
                        next->set(y);

                        std::lock_guard<std::mutex> lock(eval_time_mutex);
                        objective_funct_eval_time.add(funct_eval_runtime);
                    };

                    tp.add_task_by_value(execute_call);

                    std::lock_guard<std::mutex> lock(eval_time_mutex);
                    const double obj_funct_time = objective_funct_eval_time.mean()/std::max(1ul,tp.num_threads_in_pool());
                    const double solver_overhead_time = duration_cast<nanoseconds>(get_next_x_runtime).count();
                    max_solver_overhead_time = std::max(max_solver_overhead_time, solver_overhead_time);
                    // Don't start thinking about the logic below until we have at least 5 objective
                    // function samples for each objective function.  This way we have a decent idea how
                    // fast these things are.  The solver overhead is really small initially so none of
                    // the stuff below really matters in the beginning anyway.
                    if (objective_funct_eval_time.current_n() > functions.size()*5)
                    {
                        // If calling opt.get_next_x() is taking a long time relative to how long it takes
                        // to evaluate the objective function then we should spend less time grinding on the
                        // internal details of the optimizer and more time running the actual objective
                        // function.  E.g. if we could just run 2x more objective function calls in the same
                        // amount of time then we should just do that.  The main slowness in the solver is
                        // from the Monte Carlo sampling, which we can turn down if the objective function
                        // is really fast to evaluate.  This is because the point of the Monte Carlo part is
                        // to try really hard to avoid calls to really expensive objective functions.  But
                        // if the objective function is not expensive then we should just call it.
                        if (obj_funct_time < solver_overhead_time)
                        {
                            // Reduce the amount of Monte Carlo sampling we do.  If it goes low enough
                            // we will disable it altogether.
                            const size_t new_val = static_cast<size_t>(std::floor(opt.get_monte_carlo_upper_bound_sample_num()*0.8));
                            opt.set_monte_carlo_upper_bound_sample_num(std::max<size_t>(1, new_val));
                            // At this point just disable the upper bounding Monte Carlo search stuff and
                            // use only pure random search since the objective function is super cheap to
                            // evaluate, making this more fancy search a waste of time.
                            if (opt.get_monte_carlo_upper_bound_sample_num() == 1)
                            {
                                opt.set_pure_random_search_probability(1);
                            }
                        } else if (obj_funct_time > 1.5*max_solver_overhead_time) // Consider reenabling
                        {
                            // The Monte Carlo overhead grows over time as the solver accumulates more
                            // information about the objective function.  So we only want to reenable it
                            // or make it bigger if the objective function really is more expensive.  So
                            // we compare to the max solver runtime we have seen so far. If the
                            // objective function has suddenly gotten more expensive then we start to
                            // turn the Monte Carlo modeling back on.
                            const size_t new_val = static_cast<size_t>(std::ceil(opt.get_monte_carlo_upper_bound_sample_num()*1.28));
                            opt.set_monte_carlo_upper_bound_sample_num(std::min<size_t>(5000, new_val));
                            // Set this back to its default value.
                            opt.set_pure_random_search_probability(0.02);
                        }
                    }
                }
                tp.wait_for_all_tasks();

                matrix<double,0,1> x;
                double y;
                size_t function_idx;
                opt.get_best_function_eval(x,y,function_idx);
                // Undo any log-scaling that was applied to the variables before we output them.
                for (long j = 0; j < x.size(); ++j)
                {
                    if (log_scale[function_idx][j])
                        x(j) = std::exp(x(j));
                }
                return std::make_pair(function_idx, function_evaluation(x,y/ymult));
            }

            // This overload allows the order of max_runtime and num to be reversed.
            template <
                typename funct,
                typename ...Args
                >
            std::pair<size_t,function_evaluation> find_max_global (
                double ymult,
                thread_pool& tp,
                std::vector<funct>& functions,
                std::vector<function_spec> specs,
                const std::chrono::nanoseconds max_runtime,
                const max_function_calls num,
                double solver_epsilon = 0,
                Args&& ...args
            )
            {
                return find_max_global(ymult, tp, functions, std::move(specs), num, max_runtime, solver_epsilon, std::forward<Args>(args)...);
            }

            // This overload allows the num argument to be skipped.
            template <
                typename funct,
                typename ...Args
                >
            std::pair<size_t,function_evaluation> find_max_global (
                double ymult,
                thread_pool& tp,
                std::vector<funct>& functions,
                std::vector<function_spec> specs,
                const std::chrono::nanoseconds max_runtime,
                double solver_epsilon = 0,
                Args&& ...args
            )
            {
                return find_max_global(ymult, tp, functions, std::move(specs), max_function_calls(), max_runtime, solver_epsilon, std::forward<Args>(args)...);
            }

            // This overload allows the max_runtime argument to be skipped.
            template <
                typename funct,
                typename ...Args
                >
            std::pair<size_t,function_evaluation> find_max_global (
                double ymult,
                thread_pool& tp,
                std::vector<funct>& functions,
                std::vector<function_spec> specs,
                const max_function_calls num,
                double solver_epsilon,
                Args&& ...args
            )
            {
                return find_max_global(ymult, tp, functions, std::move(specs), num, FOREVER, solver_epsilon, std::forward<Args>(args)...);
            }

            // This overload makes the thread_pool argument optional.
            template <
                typename funct,
                typename ...Args
                >
            std::pair<size_t,function_evaluation> find_max_global (
                double ymult,
                std::vector<funct>& functions,
                Args&& ...args
            )
            {
                // disabled, don't use any threads
                thread_pool tp(0);

                return find_max_global(ymult, tp, functions, std::forward<Args>(args)...);
            }

            // The point of normalize() is to handle some of the overloaded argument types in
            // find_max_global() instances below and turn them into the argument types expected by
            // find_max_global() above.
            template <typename T>
            const T& normalize(const T& item)
            {
                return item;
            }

            inline std::vector<std::vector<function_evaluation>> normalize(
                const std::vector<function_evaluation>& initial_function_evals
            )
            {
                return {initial_function_evals};
            }
        }

    // ----------------------------------------------------------------------------------------

        template <
            typename funct,
            typename ...Args
            >
        std::pair<size_t,function_evaluation> find_max_global (
            std::vector<funct>& functions,
            std::vector<function_spec> specs,
            Args&& ...args
        )
        {
            return impl::find_max_global(+1, functions, std::move(specs), std::forward<Args>(args)...);
        }

        template <
            typename funct,
            typename ...Args
            >
        std::pair<size_t,function_evaluation> find_min_global (
            std::vector<funct>& functions,
            std::vector<function_spec> specs,
            Args&& ...args
        )
        {
            return impl::find_max_global(-1, functions, std::move(specs), std::forward<Args>(args)...);
        }

        template <
            typename funct,
            typename ...Args
            >
        std::pair<size_t,function_evaluation> find_max_global (
            thread_pool& tp,
            std::vector<funct>& functions,
            std::vector<function_spec> specs,
            Args&& ...args
        )
        {
            return impl::find_max_global(+1, tp, functions, std::move(specs), std::forward<Args>(args)...);
        }

        template <
            typename funct,
            typename ...Args
            >
        std::pair<size_t,function_evaluation> find_min_global (
            thread_pool& tp,
            std::vector<funct>& functions,
            std::vector<function_spec> specs,
            Args&& ...args
        )
        {
            return impl::find_max_global(-1, tp, functions, std::move(specs), std::forward<Args>(args)...);
        }

    // ----------------------------------------------------------------------------------------

    // Overloads that take function objects and simple matrix bounds instead of function_specs.
        template <
            typename funct,
            typename ...Args
            >
        function_evaluation find_max_global (
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const std::vector<bool>& is_integer_variable,
            Args&& ...args
        )
        {
            std::vector<funct> functions(1,std::move(f));
            std::vector<function_spec> specs(1, function_spec(bound1, bound2, is_integer_variable));
            return find_max_global(functions, std::move(specs), impl::normalize(args)...).second;
        }

        template <
            typename funct,
            typename ...Args
            >
        function_evaluation find_min_global (
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const std::vector<bool>& is_integer_variable,
            Args&& ...args
        )
        {
            std::vector<funct> functions(1,std::move(f));
            std::vector<function_spec> specs(1, function_spec(bound1, bound2, is_integer_variable));
            return find_min_global(functions, std::move(specs), impl::normalize(args)...).second;
        }

        template <
            typename funct,
            typename ...Args
            >
        function_evaluation find_max_global (
            thread_pool& tp,
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const std::vector<bool>& is_integer_variable,
            Args&& ...args
        )
        {
            std::vector<funct> functions(1,std::move(f));
            std::vector<function_spec> specs(1, function_spec(bound1, bound2, is_integer_variable));
            return find_max_global(tp, functions, std::move(specs), impl::normalize(args)...).second;
        }

        template <
            typename funct,
            typename ...Args
            >
        function_evaluation find_min_global (
            thread_pool& tp,
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const std::vector<bool>& is_integer_variable,
            Args&& ...args
        )
        {
            std::vector<funct> functions(1,std::move(f));
            std::vector<function_spec> specs(1, function_spec(bound1, bound2, is_integer_variable));
            return find_min_global(tp, functions, std::move(specs), impl::normalize(args)...).second;
        }

    // ----------------------------------------------------------------------------------------

    // overloads that are the same as above, but is_integer_variable defaulted to false for all parameters.
        template <
            typename funct,
            typename T,
            typename ...Args
            >
        typename disable_if<std::is_same<T,std::vector<bool>>, function_evaluation>::type
        find_max_global (
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const T& arg,
            Args&& ...args
        )
        {
            const std::vector<bool> is_integer_variable(bound1.size(),false);
            return find_max_global(std::move(f), bound1, bound2, is_integer_variable, arg, impl::normalize(args)...);
        }

        template <
            typename funct,
            typename T,
            typename ...Args
            >
        typename disable_if<std::is_same<T,std::vector<bool>>, function_evaluation>::type
        find_min_global (
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const T& arg,
            Args&& ...args
        )
        {
            const std::vector<bool> is_integer_variable(bound1.size(),false);
            return find_min_global(std::move(f), bound1, bound2, is_integer_variable, arg, impl::normalize(args)...);
        }

        template <
            typename funct,
            typename T,
            typename ...Args
            >
        typename disable_if<std::is_same<T,std::vector<bool>>, function_evaluation>::type
        find_max_global (
            thread_pool& tp,
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const T& arg,
            Args&& ...args
        )
        {
            const std::vector<bool> is_integer_variable(bound1.size(),false);
            return find_max_global(tp, std::move(f), bound1, bound2, is_integer_variable, arg, impl::normalize(args)...);
        }

        template <
            typename funct,
            typename T,
            typename ...Args
            >
        typename disable_if<std::is_same<T,std::vector<bool>>, function_evaluation>::type
        find_min_global (
            thread_pool& tp,
            funct f,
            const matrix<double,0,1>& bound1,
            const matrix<double,0,1>& bound2,
            const T& arg,
            Args&& ...args
        )
        {
            const std::vector<bool> is_integer_variable(bound1.size(),false);
            return find_min_global(tp, std::move(f), bound1, bound2, is_integer_variable, arg, impl::normalize(args)...);
        }

    // ----------------------------------------------------------------------------------------

    // overloads for a function taking a single scalar.
        template <
            typename funct,
            typename T,
            typename ...Args
            >
        function_evaluation find_max_global (
            funct f,
            const double bound1,
            const double bound2,
            const T& arg,
            Args&& ...args
        )
        {
            return find_max_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), arg, impl::normalize(args)...);
        }

        template <
            typename funct,
            typename T,
            typename ...Args
            >
        function_evaluation find_min_global (
            funct f,
            const double bound1,
            const double bound2,
            const T& arg,
            Args&& ...args
        )
        {
            return find_min_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), arg, impl::normalize(args)...);
        }

        template <
            typename funct,
            typename T,
            typename ...Args
            >
        function_evaluation find_max_global (
            thread_pool& tp,
            funct f,
            const double bound1,
            const double bound2,
            const T& arg,
            Args&& ...args
        )
        {
            return find_max_global(tp, std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), arg, impl::normalize(args)...);
        }

        template <
            typename funct,
            typename T,
            typename ...Args
            >
        function_evaluation find_min_global (
            thread_pool& tp,
            funct f,
            const double bound1,
            const double bound2,
            const T& arg,
            Args&& ...args
        )
        {
            return find_min_global(tp, std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), arg, impl::normalize(args)...);
        }

    // ----------------------------------------------------------------------------------------

    }

    #endif // DLIB_FiND_GLOBAL_MAXIMUM_hH_