dlib C++ Library - http_client.cpp

    #include "../sockets.h"
    #include "../string.h"
    #include "../logger.h"
    #include "../sockstreambuf.h"
    #include "../timeout.h"
    #include "http_client.h"
    #include <time.h>
    #include <stdio.h>
    #include <fstream>
    #include <sstream>
    #include <iostream>

    namespace dlib
    {

        typedef std::shared_ptr<dlib::timeout> timeout_ptr;

    #ifdef _MSC_VER
    #define BR_CASECMP strnicmp
    #else
    #define BR_CASECMP strncasecmp
    #endif
    // Default timeout after 60 seconds
    #define DEFAULT_TIMEOUT 60000

    // ----------------------------------------------------------------------------------------

        inline bool isXdigit( char c )
        {
            return  (c >= '0' && c <= '9') ||
                    (c >= 'A' && c <= 'Z') ||
                    (c >= 'a' && c <= 'z');
        }

    // ----------------------------------------------------------------------------------------

        std::string http_client::urldecode( const std::string& s )
        {
            std::stringstream ss;

            for ( char const * p_read = s.c_str(), * p_end = (s.c_str() + s.size()); p_read < p_end; p_read++ )
            {
                if ( p_read[0] == '%' && p_read+1 != p_end && p_read+2 != p_end && isXdigit(p_read[1]) && isXdigit(p_read[2]) )
                {
                    ss << static_cast<char>((( (p_read[1] & 0xf) + ((p_read[1] >= 'A') ? 9 : 0) ) << 4 ) | ( (p_read[2] & 0xf) + ((p_read[2] >= 'A') ? 9 : 0) ));
                    p_read += 2;
                }
                else if ( p_read[0] == '+' )
                {
                    // Undo the encoding that replaces spaces with plus signs.
                    ss << ' ';
                }
                else
                {
                    ss << p_read[0];
                }
            }

            return ss.str();
        }

    // ----------------------------------------------------------------------------------------

    //! \return modified string ``s'' with spaces trimmed from left
        inline std::string& triml(std::string& s)
        {
            std::string::size_type pos(0);
            for ( ; s[pos] == ' ' || s[pos] == '\t' || s[pos] == '\r' || s[pos] == '\n' ; ++pos );
            s.erase(0, pos);
            return s;
        }

    // ----------------------------------------------------------------------------------------

    //! \return modified string ``s'' with spaces trimmed from right
        inline std::string& trimr(std::string& s)
        {
            std::string::size_type pos(s.size());
            for ( ; pos && (s[pos-1] == ' ' || s[pos-1] == '\t' || s[pos-1] == '\r' || s[pos-1] == '\n') ; --pos );
            s.erase(pos, s.size()-pos);
            return s;
        }

    // ----------------------------------------------------------------------------------------

    //! \return modified string ``s'' with spaces trimmed from edges
        inline std::string& trim(std::string& s)
        {
            return triml(trimr(s));
        }

    // ----------------------------------------------------------------------------------------

        http_client::
        http_client(
        ) :
            http_return(0),
            timeout(DEFAULT_TIMEOUT),
            OnDownload(0)
        {
        }

    // ----------------------------------------------------------------------------------------

        std::string http_client::get_header(const std::string& header_name) const
        {
            stringmap::const_iterator ci = headers.find(header_name);
            return ci != headers.end() ? ci->second : std::string();
        }

    // ----------------------------------------------------------------------------------------

        void http_client::set_header(const std::string& header_name, long header_value)
        {
            char buf[21] = { 0 };
    #ifdef __WXMSW__
            ::ltoa(header_value, buf, 10);
    #else
            snprintf(buf, sizeof(buf), "%ld", header_value);
    #endif
            set_header(header_name, buf);
        }

    // ----------------------------------------------------------------------------------------

        void http_client::set_header(const std::string& header_name, const std::string& header_value)
        {
            headers[header_name] = header_value;
        }

    // ----------------------------------------------------------------------------------------

        bool http_client::is_header_set(const std::string& header_name) const
        {
            stringmap::const_iterator ci = headers.find(header_name);
            return ci != headers.end() && !ci->second.empty();
        }

    // ----------------------------------------------------------------------------------------

        void http_client::remove_header(const std::string& header_name)
        {
            headers.erase(header_name);
        }

    // ----------------------------------------------------------------------------------------

        void http_client::set_cookie(const std::string& cookie_name, long cookie_value)
        {
            char buf[21] = { 0 };
    #ifdef __WXMSW__
            ::ltoa(cookie_value, buf, 10);
    #else
            snprintf(buf, sizeof(buf), "%ld", cookie_value);
    #endif
            set_cookie(cookie_name, buf);
        }

    // ----------------------------------------------------------------------------------------

        void http_client::set_cookie(const std::string& cookie_name, const std::string& cookie_value)
        {
            cookies[cookie_name] = cookie_value;
        }

    // ----------------------------------------------------------------------------------------

        void http_client::remove_cookie(const std::string& cookie_name)
        {
            cookies.erase(cookie_name);
        }

    // ----------------------------------------------------------------------------------------

    // POST
        const std::string& http_client::post_url (const std::string& url, const string_to_stringmap& postvars, const string_to_stringmap& filenames)
        {
            std::string CT;
            std::string postBody = build_post(CT, postvars, filenames);
            set_header("Content-Type", CT);
            set_header("Content-Length", static_cast<long>(postBody.size()));

            grab_url(url, "POST", postBody);

            return returned_body;
        }

    // ----------------------------------------------------------------------------------------

        const std::string& http_client::post_url (const std::string& url, const std::string& postbuffer)
        {
            if ( !is_header_set("Content-Type") ) // Maybe they just forgot it?
                set_header("Content-Type", "application/x-www-form-urlencoded");

            set_header("Content-Length", static_cast<long>(postbuffer.size()));

            grab_url(url, "POST", postbuffer);

            return returned_body;
        }

    // ----------------------------------------------------------------------------------------

        std::string http_client::get_random_string( size_t length ) const
        {
            static bool has_seeded(false);
            static std::string allowed_chars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789");

            if ( !has_seeded )
            {
                has_seeded = true;
                ::srand( static_cast<unsigned int>(::time(NULL)) );
            }

            std::string retVal; retVal.reserve(length);
            while ( retVal.size() < length )
            {
                retVal += allowed_chars[(rand() % allowed_chars.size())];
            }

            return retVal;
        }

    // ----------------------------------------------------------------------------------------

    // static
        std::string http_client::urlencode(const std::string& in, bool post_encode)
        {
            static std::string allowed_chars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_");

            std::stringstream ss;
            ss << std::hex;
            for (std::string::const_iterator ci = in.begin(); ci != in.end(); ++ci)
            {
                if ( allowed_chars.find(*ci) != std::string::npos )
                {
                    ss << *ci;
                }
                else if ( post_encode && *ci == ' ' )
                {
                    ss << '+';
                }
                else
                {
                    ss << '%' << std::setfill('0') << std::setw(2) << std::right << static_cast<int>(*ci);
                }
            }

            return ss.str();
        }

    // ----------------------------------------------------------------------------------------

        std::string http_client::get_basename( const std::string& filename ) const
        {
            std::string::size_type pos = filename.find_last_of("\\/");
            if ( pos == std::string::npos )
                return filename;
            else
                return filename.substr(pos+1);
        }

    // ----------------------------------------------------------------------------------------

        bool http_client::parse_url(
            const std::string& url,
            std::string& scheme,
            std::string& user,
            std::string& pass,
            std::string& host,
            short& port,
            std::string& path
        ) const
        {
            scheme.clear();
            user.clear();
            pass.clear();
            host.clear();
            path.clear();
            port = 0;

            // Find scheme
            std::string::size_type pos_scheme = url.find("://");
            if ( pos_scheme == std::string::npos )
            {
                pos_scheme = 0;
            }
            else
            {
                scheme = strtolower(url.substr(0, pos_scheme));
                pos_scheme += 3;
            }

            std::string::size_type pos_path = url.find('/', pos_scheme);
            if ( pos_path == std::string::npos )
            {
                host = url.substr(pos_scheme);
            }
            else
            {
                host = url.substr(pos_scheme, pos_path - pos_scheme);
                path = url.substr(pos_path);
            }

            std::string::size_type pos_at = host.find('@');
            if ( pos_at != std::string::npos )
            {
                std::string::size_type pos_dp = host.find(':');
                if ( pos_dp != std::string::npos && pos_dp < pos_at )
                {
                    user = host.substr(0, pos_dp);
                    pass = host.substr(pos_dp+1, pos_at-pos_dp-1);
                }
                else
                {
                    user = host.substr(0, pos_at);
                }
                host = host.substr(pos_at+1);
            }

            std::string::size_type pos_dp = host.find(':');
            if ( pos_dp != std::string::npos )
            {
                port = dlib::string_cast<short>(host.substr(pos_dp+1));
                host = host.substr(0, pos_dp);
            }

            host = strtolower(host);

            if ( port == 0 )
            {
                if ( scheme == "http" )
                    port = 80;
                else if ( scheme == "ftp" )
                    port = 21;
                else if ( scheme == "https" )
                    port = 443;
            }

            return !host.empty();
        }

    // ----------------------------------------------------------------------------------------

        std::string http_client::strtolower(const std::string& in) const
        {
            std::string retVal = in;

            for (std::string::iterator ii = retVal.begin(); ii != retVal.end(); ++ii)
            {
                *ii = ::tolower(*ii);
            }

            return retVal;
        }

    // ----------------------------------------------------------------------------------------

        std::string http_client::strtoupper(const std::string& in) const
        {
            std::string retVal = in;

            for (std::string::iterator ii = retVal.begin(); ii != retVal.end(); ++ii)
            {
                *ii = ::toupper(*ii);
            }

            return retVal;
        }

    // ----------------------------------------------------------------------------------------

    // GET
        const std::string& http_client::get_url(const std::string& url)
        {
            std::string CT = get_header("Content-Type");

            // You do a GET with a POST header??
            if ( CT == "application/x-www-form-urlencoded" || CT == "multipart/form-data" )
                remove_header("Content-Type");

            grab_url(url);

            return returned_body;
        }

    // ----------------------------------------------------------------------------------------

        std::string http_client::build_post(std::string& content_type, const string_to_stringmap& postvars, const string_to_stringmap& filenames_in) const
        {
            if ( postvars.empty() && filenames_in.empty() )
                return std::string();

            string_to_stringmap filenames = filenames_in;

            // sanitize the files
            if ( !filenames.empty() )
            {
                string_to_stringmap::iterator var_names = filenames.begin();
                while (var_names != filenames.end())
                {
                    stringmap::iterator fnames = var_names->second.begin();

                    while( fnames != var_names->second.end() )
                    {
                        FILE *fp = ::fopen(fnames->second.c_str(), "rb");
                        if ( fp == NULL )
                        {
                            stringmap::iterator old_one = fnames++;
                            var_names->second.erase(old_one);
                        }
                        else
                        {
                            fclose(fp);
                            ++fnames;
                        }
                    }

                    if ( fnames->second.empty() )
                    {
                        string_to_stringmap::iterator old_one = var_names++;
                        filenames.erase(old_one);
                    }
                    else
                    {
                        ++var_names;
                    }
                }
            }

            content_type = !filenames.empty() ? "multipart/form-data" : "application/x-www-form-urlencoded";
            std::stringstream postBody;
            if ( !filenames.empty() )
            {
                std::string mime_boundary = get_random_string(32);

                // First add the form vars
                for (string_to_stringmap::const_iterator ci = postvars.begin(); ci != postvars.end(); ++ci)
                {
                    for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
                    {
                        postBody << "--" << mime_boundary << "\r\n"
                            "Content-Disposition: form-data; name=\"" << ci->first << "\"\r\n\r\n"
                            << si->second << "\r\n";
                    }
                }

                // Then add the files
                for (string_to_stringmap::const_iterator ci = filenames.begin(); ci != filenames.end(); ++ci)
                {
                    for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
                    {
                        std::ifstream in(si->second.c_str());
                        postBody << "--" << mime_boundary << "\r\n"
                            "Content-Disposition: form-data; name=\"" << ci->first << "\"; filename=\"" << get_basename(si->second) << "\"\r\n\r\n"
                            << in.rdbuf() << "\r\n";
                    }
                }

                postBody << "--" << mime_boundary << "--\r\n";
            }
            else
            {
                // No files...
                for (string_to_stringmap::const_iterator ci = postvars.begin(); ci != postvars.end(); ++ci)
                {
                    for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
                    {
                        postBody << urlencode(ci->first) << '=' << urlencode(si->second) << '&';
                    }
                }

                // read the last '&'
                char c;
                postBody.read(&c, 1);
            }

            return postBody.str();
        }

    // ----------------------------------------------------------------------------------------

        bool http_client::grab_url(const std::string& url, const std::string& method, const std::string& post_body)
        {
            error_field.clear();
            returned_headers.clear();
            http_return = 0;
            returned_body.clear();

            std::string to_use_method = strtoupper(method);

            std::string scheme, user, pass, host, path;
            short port;
            if ( !parse_url(url, scheme, user, pass, host, port, path) )
            {
                error_field = "Couldn't parse the URL!";
                return false;
            }

            // Build request
            std::stringstream ret;
            ret << to_use_method << ' ' << path << " HTTP/1.0\r\n"
                << "Host: " << host;
            if (port != 80 && port != 443) ret << ':' << port;
            ret << "\r\n";

            bool content_length_said = false;

            set_header("Connection", "Close");
            for (stringmap::iterator ci = headers.begin(); ci != headers.end(); ++ci)
            {
                std::string head = strtolower(ci->first);

                if ( head == "content-length" )
                {
                    content_length_said = true;
                }

                ret << ci->first << ':' << ' ' << ci->second << "\r\n";
            }

            if ( !content_length_said && to_use_method != "GET" )
                ret << "Content-Length: " << static_cast<unsigned int>(post_body.size()) << "\r\n";

            std::stringstream cookie_ss;
            for (stringmap::iterator ci = cookies.begin(); ci != cookies.end(); ++ci)
            {
                std::string var = ci->first ; trim(var);
                std::string val = ci->second; trim(val);

                if ( val.empty() || var.empty() )
                    continue;

                if ( !cookie_ss.str().empty() )
                    cookie_ss << ';' << ' ';

                cookie_ss << urlencode(var) << '=' << urlencode(val);
            }

            if ( !cookie_ss.str().empty() )
                ret << "Cookie: " << cookie_ss.str() << "\r\n";

            ret << "\r\n";
            ret << post_body;

            std::string request_build = ret.str();

            std::stringstream ss;
            {
                dlib::connection * conn(0);
                try
                {
                    if (timeout > 0)
                        conn = dlib::connect(host, port, timeout);
                    else
                        conn = dlib::connect(host, port);
                }
                catch (const dlib::socket_error& e)
                {
                    error_field = e.what();
                    return false;
                }

                // Implement a timeout
                timeout_ptr t;
                if ( timeout > 0 )
                    t.reset( new dlib::timeout(*conn, &dlib::connection::shutdown, timeout) );

                // Write our request
                conn->write(request_build.c_str(), static_cast<long>(request_build.size()));

                t.reset();

                // And read the response
                char buf[512];
                long bytes_read(0), bytes_total(0);
                bool read_headers(true);

                if ( timeout > 0 )
                    t.reset( new dlib::timeout(*conn, &dlib::connection::shutdown, timeout) );

                while ( (bytes_read = conn->read(buf, 512)) > 0 )
                {
                    ss.write(buf, bytes_read);

                    // Incremental read headers
                    if ( read_headers )
                    {
                        std::string body_with_headers = ss.str();
                        std::string::size_type ctr(0);

                        while ( true )
                        {
                            std::string::size_type pos = body_with_headers.find("\r\n", ctr);
                            if ( pos == std::string::npos )
                            {
                                // This is our last position of "\r\n"
                                ss.str("");
                                ss.write( body_with_headers.substr(ctr).c_str(), body_with_headers.size() - ctr );
                                break;
                            }

                            std::string header = body_with_headers.substr(ctr, pos-ctr);
                            if ( header.empty() )
                            {
                                // Ok, we're done reading the headers
                                read_headers = false;
                                // What follows now is the body
                                ss.str("");
                                ss.write( body_with_headers.substr(pos + 2).c_str(), body_with_headers.size() - pos - 2 );
                                break;
                            }
                            ctr = pos + 2;

                            if ( returned_headers.empty() )
                            {
                                if (
                                    header[0] == 'H' &&
                                    header[1] == 'T' &&
                                    header[2] == 'T' &&
                                    header[3] == 'P' &&
                                    header[4] == '/' &&
                                    (header[5] >= '0' && header[5] <= '9') &&
                                    header[6] == '.' &&
                                    (header[7] >= '0' && header[7] <= '9') &&
                                    header[8] == ' '
                                )
                                {
                                    http_return = (header[9 ] - '0') * 100 +
                                        (header[10] - '0') * 10 +
                                        (header[11] - '0');
                                    continue;
                                }
                            }

                            std::string::size_type pos_dp = header.find_first_of(':');
                            std::string header_name, header_value;
                            if ( pos_dp == std::string::npos )
                            {
                                // **TODO** what should I do here??
                                header_name = header;
                            }
                            else
                            {
                                header_name  = trim(header.substr(0, pos_dp));
                                header_value = trim(header.substr(pos_dp+1));
                            }

                            returned_headers[ header_name ].push_back(header_value);

                            if ( BR_CASECMP(header_name.c_str(), "Content-Length", 14) == 0 )
                            {
                                bytes_total = atol( header_value.c_str() );
                            }
                            else if ( BR_CASECMP(header_name.c_str(), "Set-Cookie", 10) == 0 )
                            {
                                std::string::size_type cur_pos(0), pos_pk, pos_is;
                                std::string work, var, val;
                                for ( cur_pos = 0; cur_pos < header_value.size(); cur_pos++ )
                                {
                                    pos_pk = header_value.find(';', cur_pos);
                                    work   = trim( header_value.substr(cur_pos, pos_pk - cur_pos) );

                                    pos_is = work.find('=');
                                    if ( pos_is != std::string::npos )
                                    { // Hmmm? what in the else case?
                                        var = trim( http_client::urldecode( work.substr(0, pos_is) ) );
                                        val = trim( http_client::urldecode( work.substr(pos_is + 1) ) );

                                        if ( var != "expires" && var != "domain" && var != "path" )
                                            set_cookie( var, val );
                                    }
                                    cur_pos = pos_pk == std::string::npos ? pos_pk - 1 : pos_pk;
                                }
                            } // Set-Cookie?

                        } // while (true)
                    } // read_headers?

                    // Call the OnDownload function if it's set
                    if ( OnDownload && !read_headers )
                    {
                        if ( (*OnDownload)(static_cast<long>(ss.tellp()), bytes_total, user_info) == false )
                        {
                            t.reset();
                            break;
                        }
                    }

                    if ( bytes_total != 0 && static_cast<long>(ss.tellp()) == bytes_total )
                    {
                        t.reset();
                        break;
                    }

                    if ( timeout > 0 )
                        t.reset( new dlib::timeout(*conn, &dlib::connection::shutdown, timeout) );
                } // while still data to read

                t.reset();

                delete conn;

                switch ( bytes_read )
                {
                    case dlib::TIMEOUT:      error_field = "Timeout";     return false; break;
                    case dlib::WOULDBLOCK:   error_field = "Would block"; return false; break;
                    case dlib::OTHER_ERROR:  error_field = "Other error"; return false; break;
                    case dlib::SHUTDOWN:     error_field = "Timeout";     return false; break;
                    case dlib::PORTINUSE:    error_field = "Port in use"; return false; break;
                }
            }

            returned_body = ss.str();

            return true;
        }

    // ----------------------------------------------------------------------------------------

        void http_client::clear()
        {
            headers.clear();
            cookies.clear();
        }

    // ----------------------------------------------------------------------------------------

        void http_client::prepare_for_next_url( )
        {
            remove_header("Content-Type");
            remove_header("Content-Length");
        }

    // ----------------------------------------------------------------------------------------

    }