/*
 * Copyright (c) 2002, 2003, 2004 Jean-Yves Lefort
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of Jean-Yves Lefort nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include <string.h>
#include <stdlib.h>
#include <curl/curl.h>
#ifdef ST_REGRESSION_TEST
#define _(String)		(String)
#define st_notice		g_warning
#else
#include <glib/gi18n.h>
#include <gtk/gtk.h>
#include "sgtk-auth-dialog.h"
#include "st-dialog-api.h"
#include "st-settings.h"
#include "st-thread.h"
#include "st-util-api.h"
#include "st-shell.h"
#endif /* ST_REGRESSION_TEST */
#include "sg-util.h"
#include "st-transfer.h"

#define AGENT_STRING			"streamtuner"

/* maintain compatibility with old libcurl versions */

#ifndef CURLOPT_WRITEDATA
#define CURLOPT_WRITEDATA		CURLOPT_FILE
#endif

#ifndef CURLOPT_HEADERDATA
#define CURLOPT_HEADERDATA		CURLOPT_WRITEHEADER
#endif

/*
 * RFC 2616 3.7.1 says that if no charset is provided, media subtypes
 * of the "text" type are defined to have a default charset value of
 * "ISO-8859-1".
 */
#define CHARSET(header, body) ((body) ? (body) : ((header) ? (header) : "ISO-8859-1"))

/*** type definitions ********************************************************/

/*
 * libcurl already provides this prototype as curl_write_callback, but
 * we are not gonna rely on it as it's undocumented
 */
typedef size_t (CurlCallback) (const char *buffer,
			       size_t size,
			       size_t nitems,
			       gpointer data);

typedef struct
{
#ifndef ST_REGRESSION_TEST
  STThread			*thread;
#endif
  double			downloaded;

  const char			*url;
  STTransferFlags		flags;

  STTransferSession		*session;
  char				error[CURL_ERROR_SIZE];

  CurlCallback			*header_cb;
  gpointer			header_data;

  CurlCallback			*body_cb;
  gpointer			body_data;

  curl_proxytype		proxy_type;
  char				*proxy_server;
  int				proxy_port;
  char				*proxy_userpwd;

  char				*header_charset;
  char				*body_charset;
} STTransfer;

struct _STTransferSession
{
  CURL				*handle;
};

typedef enum
{
  ST_TRANSFER_SECTION_HEADERS,
  ST_TRANSFER_SECTION_BODY
} STTransferSection;

typedef struct
{
  STTransfer			*transfer;
  STTransferSection		section;

  GString			*line;
  char				terminator[2]; /* \n\0, \r\n or \r\0 */

  STTransferLineCallback	line_cb;
  gpointer			line_cb_data;
} STTransferLineData;

/*** function declarations ***************************************************/

static gboolean st_transfer_perform (STTransfer *transfer, GError **err);

static gboolean st_transfer_set_proxy_settings (STTransfer *transfer);
static void st_transfer_free_proxy_settings (STTransfer *transfer);

#ifndef ST_REGRESSION_TEST
static char *st_transfer_get_hostname (const char *url);
#endif

static gboolean st_transfer_progress_cb (gpointer data,
					 double dltotal,
					 double dlnow,
					 double ultotal,
					 double ulnow);

static gboolean st_transfer_session_get_internal (STTransferSession *session,
						  const char *url,
						  STTransferFlags flags,
						  char **headers,
						  char **body,
						  GError **err);

static char *st_transfer_get_header_charset (const char *header);
static char *st_transfer_get_body_charset (const char *body);

static void st_transfer_line_data_init (STTransferLineData *data,
					STTransfer *transfer,
					STTransferSection section,
					STTransferLineCallback cb,
					gpointer cb_data);
static void st_transfer_line_data_flush (STTransferLineData *data);
static void st_transfer_line_data_free (STTransferLineData *data,
					gboolean flush);

static gboolean st_transfer_session_get_by_line_internal (STTransferSession *session,
							  const char *url,
							  STTransferFlags flags,
							  STTransferLineCallback header_cb,
							  gpointer header_data,
							  STTransferLineCallback body_cb,
							  gpointer body_data,
							  GError **err);

static size_t st_transfer_session_get_cb (const char *buffer,
					  size_t size,
					  size_t nitems,
					  gpointer data);
static size_t st_transfer_session_get_binary_cb (const char *buffer,
						 size_t size,
						 size_t nitems,
						 gpointer data);
static size_t st_transfer_session_get_by_line_cb (const char *buffer,
						  size_t size,
						  size_t nitems,
						  gpointer data);

/*** API implementation ******************************************************/

/**
 * st_transfer_session_new:
 *
 * Creates a new transfer session.
 *
 * Return value: the new session, which should be freed with
 * st_transfer_session_free() when no longer needed.
 **/
STTransferSession *
st_transfer_session_new (void)
{
  STTransferSession *session;

  session = g_new(STTransferSession, 1);
  session->handle = curl_easy_init();

  return session;
}

/**
 * st_transfer_session_free:
 * @session: the transfer session to destroy.
 *
 * Destroys @session.
 **/
void
st_transfer_session_free (STTransferSession *session)
{
  g_return_if_fail(session != NULL);

  curl_easy_cleanup(session->handle);
  g_free(session);
}

/*
 * Deprecated.
 */
char *
st_transfer_get_full (const char *url, GError **err)
{
  STTransferSession *session;
  gboolean status;
  char *body;

  g_return_val_if_fail(url != NULL, FALSE);

  session = st_transfer_session_new();
  status = st_transfer_session_get_internal(session, url, 0, NULL, &body, err);
  st_transfer_session_free(session);

  return status ? body : NULL;
}

/**
 * st_transfer_session_get:
 * @session: a transfer session.
 * @url: the URL of the data to get.
 * @flags: the transfer flags. Valid flags are #ST_TRANSFER_UTF8,
 * #ST_TRANSFER_PARSE_HTTP_CHARSET and
 * #ST_TRANSFER_PARSE_HTML_CHARSET.
 * @headers: a location to return the header data, or %NULL. The
 * returned data is guaranteed to be valid ASCII.
 * @body: a location to return the body data, or %NULL. If
 * #ST_TRANSFER_UTF8 is used, the returned data is guaranteed to be
 * valid UTF-8.
 * @err: a location to store errors, or %NULL.
 *
 * Gets the data located at @url.
 *
 * Return value: %FALSE if there was an error (in such case, @err will
 * be set). If %TRUE is returned, strings will be stored at @headers
 * and @body. They should be freed when no longer needed.
 **/
gboolean
st_transfer_session_get (STTransferSession *session,
			 const char *url,
			 STTransferFlags flags,
			 char **headers,
			 char **body,
			 GError **err)
{
  gboolean status;

  g_return_val_if_fail(session != NULL, FALSE);
  g_return_val_if_fail(url != NULL, FALSE);
  g_return_val_if_fail((flags & ST_TRANSFER_PASS_NEWLINE) == 0, FALSE);

  status = st_transfer_session_get_internal(session, url, flags, headers, body, err);

  return status;
}

/**
 * st_transfer_session_get_binary:
 * @session: a transfer session.
 * @url: the URL of the data to get.
 * @flags: must be 0.
 * @headers: a location to return the header data, or %NULL. The
 * returned data is guaranteed to be valid ASCII.
 * @body: a location to return the body data, or %NULL. If set,
 * @body_len must also be set.
 * @body_len: a location to return the body length, or %NULL.
 * @err: a location to store errors, or %NULL.
 *
 * Gets the binary data located at @url.
 *
 * Return value: %FALSE if there was an error (in such case, @err will
 * be set). If %TRUE is returned, a string will be stored at @headers
 * and a character buffer will be stored at @body. They both should be
 * freed when no longer needed.
 **/
gboolean
st_transfer_session_get_binary (STTransferSession *session,
				const char *url,
				STTransferFlags flags,
				char **headers,
				guint8 **body,
				unsigned int *body_len,
				GError **err)
{
  STTransfer transfer = { 0, };
  GString *header_string;
  GByteArray *body_array;
  gboolean status;

  g_return_val_if_fail(session != NULL, FALSE);
  g_return_val_if_fail(url != NULL, FALSE);
  g_return_val_if_fail((flags & ST_TRANSFER_PASS_NEWLINE) == 0, FALSE);
  g_return_val_if_fail((flags & ST_TRANSFER_UTF8) == 0, FALSE);
  g_return_val_if_fail((flags & ST_TRANSFER_PARSE_HTTP_CHARSET) == 0, FALSE);
  g_return_val_if_fail((flags & ST_TRANSFER_PARSE_HTML_CHARSET) == 0, FALSE);
  
  transfer.url = url;
  transfer.flags = flags;
  transfer.session = session;

  if (headers)
    {
      header_string = g_string_new(NULL);
      transfer.header_cb = st_transfer_session_get_cb;
      transfer.header_data = header_string;
    }
  if (body)
    {
      body_array = g_byte_array_new();
      transfer.body_cb = st_transfer_session_get_binary_cb;
      transfer.body_data = body_array;
    }
  
  status = st_transfer_perform(&transfer, err);
  if (status)
    {
      if (headers)
	{
	  if (sg_ascii_validate(header_string->str))
	    *headers = header_string->str;
	  else
	    {
	      g_set_error(err, 0, 0, _("invalid ASCII in HTTP headers"));
	      status = FALSE;
	    }
	}
      if (body && status)
	{
	  *body = body_array->data;
	  *body_len = body_array->len;
	}
    }

  if (headers)
    g_string_free(header_string, ! status);
  if (body)
    g_byte_array_free(body_array, ! status);

  return status;
}

/*
 * Deprecated
 */
char *
st_transfer_get_full_with_session (STTransferSession *session,
				   const char *url,
				   GError **err)
{
  gboolean status;
  char *body;

  g_return_val_if_fail(session != NULL, FALSE);
  g_return_val_if_fail(url != NULL, FALSE);

  status = st_transfer_session_get_internal(session, url, 0, NULL, &body, err);
  
  return status ? body : NULL;
}

static gboolean
st_transfer_session_get_internal (STTransferSession *session,
				  const char *url,
				  STTransferFlags flags,
				  char **headers,
				  char **body,
				  GError **err)
{
  STTransfer transfer = { 0, };
  GString *header_string = NULL;
  GString *body_string;
  gboolean status;

  g_return_val_if_fail(session != NULL, FALSE);
  g_return_val_if_fail(url != NULL, FALSE);
  
  transfer.url = url;
  transfer.flags = flags;
  transfer.session = session;

  if (headers || flags & (ST_TRANSFER_UTF8 | ST_TRANSFER_PARSE_HTTP_CHARSET))
    {
      header_string = g_string_new(NULL);
      transfer.header_cb = st_transfer_session_get_cb;
      transfer.header_data = header_string;
    }
  if (body)
    {
      body_string = g_string_new(NULL);
      transfer.body_cb = st_transfer_session_get_cb;
      transfer.body_data = body_string;
    }
  
  status = st_transfer_perform(&transfer, err);
  if (status)
    {
      if (header_string && ! sg_ascii_validate(header_string->str))
	{
	  g_set_error(err, 0, 0, _("invalid ASCII in HTTP headers"));
	  status = FALSE;
	}
      
      if (status && flags & ST_TRANSFER_UTF8)
	{
	  char *header_charset = NULL;
	  
	  if (flags & ST_TRANSFER_PARSE_HTTP_CHARSET)
	    {
	      char **lines;
	      int i;
	      
	      lines = g_strsplit(header_string->str, "\r\n", 0);
	      for (i = 0; lines[i] && ! header_charset; i++)
		header_charset = st_transfer_get_header_charset(lines[i]);
	      g_strfreev(lines);
	    }

	  if (body)
	    {
	      char *body_charset;
	      char *converted;

	      body_charset = flags & ST_TRANSFER_PARSE_HTML_CHARSET
		? st_transfer_get_body_charset(body_string->str)
		: NULL;

	      converted = g_convert(body_string->str,
				    body_string->len,
				    "UTF-8",
				    CHARSET(header_charset, body_charset),
				    NULL,
				    NULL,
				    err);
	      if (converted)
		{
		  g_string_assign(body_string, converted);
		  g_free(converted);
		}
	      else
		status = FALSE;

	      g_free(body_charset);
	    }

	  g_free(header_charset);
	}

      if (status)
	{
	  if (headers)
	    *headers = header_string->str;
	  if (body)
	    *body = body_string->str;
	}
    }

  if (header_string)
    g_string_free(header_string, ! (headers && status));
  if (body)
    g_string_free(body_string, ! status);

  return status;
}

/*
 * Deprecated.
 */
gboolean
st_transfer_get_lines (const char *url,
		       STTransferLineCallback cb,
		       gpointer data,
		       GError **err)
{
  STTransferSession *session;
  gboolean status;

  g_return_val_if_fail(url != NULL, FALSE);
  g_return_val_if_fail(cb != NULL, FALSE);

  session = st_transfer_session_new();
  status = st_transfer_session_get_by_line_internal(session, url, 0, NULL, NULL, cb, data, err);
  st_transfer_session_free(session);

  return status;
}

/**
 * st_transfer_session_get_by_line:
 * @session: a transfer session.
 * @url: the URL of the data to get.
 * @flags: the transfer flags. Valid flags are
 * #ST_TRANSFER_PASS_NEWLINE, #ST_TRANSFER_UTF8,
 * #ST_TRANSFER_PARSE_HTTP_CHARSET and
 * #ST_TRANSFER_PARSE_HTML_CHARSET.
 * @header_cb: a function to call when a header line arrives, or
 * %NULL. The line passed to the function is guaranteed to be valid
 * ASCII.
 * @header_data: data to pass to @header_cb.
 * @body_cb: a function to call when a body line arrives, or %NULL. If
 * #ST_TRANSFER_UTF8 is used, the line passed to the function is
 * guaranteed to be valid UTF-8.
 * @body_data: data to pass to @body_cb.
 * @err: a location to store errors, or %NULL.
 *
 * Gets the data located at @url, splitting it into lines and passing
 * them to @header_cb and @body_cb.
 *
 * Return value: %FALSE if there was an error (in such case, @err will
 * be set).
 **/
gboolean
st_transfer_session_get_by_line (STTransferSession *session,
				 const char *url,
				 STTransferFlags flags,
				 STTransferLineCallback header_cb,
				 gpointer header_data,
				 STTransferLineCallback body_cb,
				 gpointer body_data,
				 GError **err)
{
  gboolean status;

  g_return_val_if_fail(session != NULL, FALSE);
  g_return_val_if_fail(url != NULL, FALSE);

  status = st_transfer_session_get_by_line_internal(session, url, flags, header_cb, header_data, body_cb, body_data, err);

  return status;
}

/*
 * Deprecated.
 */
gboolean
st_transfer_get_lines_with_session (STTransferSession *session,
				    const char *url,
				    STTransferLineCallback cb,
				    gpointer data,
				    GError **err)
{
  g_return_val_if_fail(session != NULL, FALSE);
  g_return_val_if_fail(url != NULL, FALSE);
  g_return_val_if_fail(cb != NULL, FALSE);

  return st_transfer_session_get_by_line_internal(session, url, 0, NULL, NULL, cb, data, err);
}

static char *
st_transfer_get_header_charset (const char *header)
{
  char *charset = NULL;

  g_return_val_if_fail(header != NULL, NULL);

  if (g_str_has_prefix(header, "Content-Type: "))
    {
      char *s;

      s = sg_ascii_strcasestr(header + 14, "charset=");
      if (s)
	{
	  char *end;

	  s += 8;
	  end = strstr(s, "\r\n");
	  charset = end ? g_strndup(s, end - s) : g_strdup(s);
	}
    }

  return charset;
}

static char *
st_transfer_get_body_charset (const char *body)
{
  char *charset = NULL;
  char *s;

  g_return_val_if_fail(body != NULL, NULL);
  
  s = sg_ascii_strcasestr(body, "<meta http-equiv=\"content-type");
  if (! s)
    s = sg_ascii_strcasestr(body, "<meta http-equiv=\'content-type");
    
  if (s)
    {
      s = sg_ascii_strcasestr(s + 17, "charset=");
      if (s)
	{
	  char *end;

	  s += 8;
	  end = strpbrk(s, "'\"");

	  if (end)
	    charset = g_strndup(s, end - s);
	}
    }

  return charset;
}

static void
st_transfer_line_data_init (STTransferLineData *data,
			    STTransfer *transfer,
			    STTransferSection section,
			    STTransferLineCallback cb,
			    gpointer cb_data)
{
  g_return_if_fail(data != NULL);
  g_return_if_fail(transfer != NULL);

  data->transfer = transfer;
  data->section = section;
  data->line = g_string_new(NULL);
  *data->terminator = 0;
  data->line_cb = cb;
  data->line_cb_data = cb_data;
}

static void
st_transfer_line_data_flush (STTransferLineData *data)
{
  char *freeme = NULL;
  const char *line = NULL;

  g_return_if_fail(data != NULL);

  if (data->transfer->flags & ST_TRANSFER_PASS_NEWLINE && *data->terminator)
    g_string_append_len(data->line, data->terminator, sizeof(data->terminator));

  switch (data->section)
    {
    case ST_TRANSFER_SECTION_HEADERS:
      if (sg_ascii_validate(data->line->str))
	{
	  line = data->line->str;
	  if (data->transfer->flags & (ST_TRANSFER_UTF8 | ST_TRANSFER_PARSE_HTTP_CHARSET)
	      && ! data->transfer->header_charset)
	    data->transfer->header_charset = st_transfer_get_header_charset(line);
	}
      else
	st_notice(_("invalid ASCII in HTTP header"));
      break;

    case ST_TRANSFER_SECTION_BODY:
      if (data->transfer->flags & ST_TRANSFER_UTF8)
	{
	  GError *err = NULL;

	  if (data->transfer->flags & ST_TRANSFER_PARSE_HTML_CHARSET
	      && ! data->transfer->body_charset)
	    data->transfer->body_charset = st_transfer_get_body_charset(data->line->str);

	  freeme = g_convert(data->line->str,
			     data->line->len,
			     "UTF-8",
			     CHARSET(data->transfer->header_charset, data->transfer->body_charset),
			     NULL,
			     NULL,
			     &err);
	  if (freeme)
	    line = freeme;
	  else
	    {
	      st_notice(_("cannot convert a line of the HTTP body to UTF-8: %s"), err->message);
	      g_error_free(err);
	    }
	}
      else
	line = data->line->str;
      break;

    default:
      g_return_if_reached();
    }
  
  if (data->line_cb && line)
    data->line_cb(line, data->line_cb_data);

  g_string_truncate(data->line, 0);
  *data->terminator = 0;

  g_free(freeme);
}

static void
st_transfer_line_data_free (STTransferLineData *data, gboolean flush)
{
  g_return_if_fail(data != NULL);

  if (flush && *data->line->str)
    st_transfer_line_data_flush(data);

  g_string_free(data->line, TRUE);
}

static gboolean
st_transfer_session_get_by_line_internal (STTransferSession *session,
					  const char *url,
					  STTransferFlags flags,
					  STTransferLineCallback header_cb,
					  gpointer header_data,
					  STTransferLineCallback body_cb,
					  gpointer body_data,
					  GError **err)
{
  STTransfer transfer = { 0, };
  STTransferLineData header_line_data;
  STTransferLineData body_line_data;
  gboolean status;

  g_return_val_if_fail(session != NULL, FALSE);
  g_return_val_if_fail(url != NULL, FALSE);
  
  transfer.url = url;
  transfer.flags = flags;
  transfer.session = session;

  if (header_cb || flags & (ST_TRANSFER_UTF8 | ST_TRANSFER_PARSE_HTTP_CHARSET))
    {
      st_transfer_line_data_init(&header_line_data, &transfer, ST_TRANSFER_SECTION_HEADERS, header_cb, header_data);
      transfer.header_cb = st_transfer_session_get_by_line_cb;
      transfer.header_data = &header_line_data;
    }
  if (body_cb)
    {
      st_transfer_line_data_init(&body_line_data, &transfer, ST_TRANSFER_SECTION_BODY, body_cb, body_data);
      transfer.body_cb = st_transfer_session_get_by_line_cb;
      transfer.body_data = &body_line_data;
    }

  status = st_transfer_perform(&transfer, err);
  
  if (transfer.header_cb)
    st_transfer_line_data_free(&header_line_data, status);
  if (body_cb)
    st_transfer_line_data_free(&body_line_data, status);

  g_free(transfer.header_charset);
  g_free(transfer.body_charset);
  
  return status;
}

static gboolean
st_transfer_perform (STTransfer *transfer, GError **err)
{
  int status;
#ifndef ST_REGRESSION_TEST
  char *hostname;
#endif

  g_return_val_if_fail(transfer != NULL, FALSE);
  g_return_val_if_fail(transfer->url != NULL, FALSE);
  g_return_val_if_fail(transfer->session != NULL, FALSE);

#ifndef ST_REGRESSION_TEST
  transfer->thread = st_thread_get();
  g_return_val_if_fail(transfer->thread != NULL, FALSE);

  if (st_thread_is_aborted(transfer->thread))
    return FALSE;
#endif /* ! ST_REGRESSION_TEST */

#ifndef ST_REGRESSION_TEST
  GDK_THREADS_ENTER();
  st_thread_set_progress(transfer->thread, TRUE, 0);
  gdk_flush();
  GDK_THREADS_LEAVE();
#endif /* ! ST_REGRESSION_TEST */

  if (! st_transfer_set_proxy_settings(transfer))
    {
#ifndef ST_REGRESSION_TEST
      GDK_THREADS_ENTER();
      st_thread_abort(transfer->thread);
      gdk_flush();
      GDK_THREADS_LEAVE();
#endif /* ! ST_REGRESSION_TEST */
      
      return FALSE;		/* aborted */
    }

#ifndef ST_REGRESSION_TEST
  hostname = st_transfer_get_hostname(transfer->url);

  GDK_THREADS_ENTER();
  st_thread_printf(transfer->thread, TRUE, _("Connecting to %s..."), hostname ? hostname : transfer->url);
  gdk_flush();
  GDK_THREADS_LEAVE();

  g_free(hostname);
#endif /* ! ST_REGRESSION_TEST */
  
  curl_easy_setopt(transfer->session->handle, CURLOPT_USERAGENT, AGENT_STRING);
  curl_easy_setopt(transfer->session->handle, CURLOPT_URL, transfer->url);
  curl_easy_setopt(transfer->session->handle, CURLOPT_HEADERFUNCTION, transfer->header_cb);
  curl_easy_setopt(transfer->session->handle, CURLOPT_HEADERDATA, transfer->header_data);
  curl_easy_setopt(transfer->session->handle, CURLOPT_WRITEFUNCTION, transfer->body_cb);
  curl_easy_setopt(transfer->session->handle, CURLOPT_WRITEDATA, transfer->body_data);
  curl_easy_setopt(transfer->session->handle, CURLOPT_NOPROGRESS, FALSE);
  curl_easy_setopt(transfer->session->handle, CURLOPT_PROGRESSFUNCTION, st_transfer_progress_cb);
  curl_easy_setopt(transfer->session->handle, CURLOPT_PROGRESSDATA, transfer);
  curl_easy_setopt(transfer->session->handle, CURLOPT_ERRORBUFFER, transfer->error);
  curl_easy_setopt(transfer->session->handle, CURLOPT_PROXYTYPE, transfer->proxy_type);
  curl_easy_setopt(transfer->session->handle, CURLOPT_PROXY, transfer->proxy_server);
  curl_easy_setopt(transfer->session->handle, CURLOPT_PROXYPORT, transfer->proxy_port);
  curl_easy_setopt(transfer->session->handle, CURLOPT_PROXYUSERPWD, transfer->proxy_userpwd);
  curl_easy_setopt(transfer->session->handle, CURLOPT_COOKIEFILE, "");
  curl_easy_setopt(transfer->session->handle, CURLOPT_FOLLOWLOCATION, TRUE);
  curl_easy_setopt(transfer->session->handle, CURLOPT_NOBODY, transfer->body_cb == NULL);
  curl_easy_setopt(transfer->session->handle, CURLOPT_ENCODING, "");
  curl_easy_setopt(transfer->session->handle, CURLOPT_NOSIGNAL, TRUE);
  
  status = curl_easy_perform(transfer->session->handle);
  
  st_transfer_free_proxy_settings(transfer);

  if (status != CURLE_OK
      && status != CURLE_ABORTED_BY_CALLBACK
#ifndef ST_REGRESSION_TEST
      && ! st_thread_is_aborted(transfer->thread)
#endif
      )
    g_set_error(err, 0, 0, "%s", transfer->error); /* failure which isn't abort */

  return status == CURLE_OK;
}

static gboolean
st_transfer_set_proxy_settings (STTransfer *transfer)
{
  gboolean status = TRUE;
#ifndef ST_REGRESSION_TEST
  gboolean proxy_enabled;
  gboolean auth_enabled;
  char *auth_name;
  char *auth_password;
#endif /* ! ST_REGRESSION_TEST */

  g_return_val_if_fail(transfer != NULL, FALSE);

#ifndef ST_REGRESSION_TEST
  G_LOCK(st_settings);
  proxy_enabled = st_settings.proxy_enabled;
  if (proxy_enabled && st_settings.proxy_server)
    {
      transfer->proxy_type = st_settings.proxy_type;
      transfer->proxy_server = g_strdup(st_settings.proxy_server);
      transfer->proxy_port = st_settings.proxy_port;
    }
  auth_enabled = st_settings.proxy_auth_enabled;
  auth_name = g_strdup(st_settings.proxy_auth_name);
  auth_password = g_strdup(st_settings.proxy_auth_password);
  G_UNLOCK(st_settings);
  
  if (proxy_enabled && st_settings.proxy_server && auth_enabled)
    {
      char *name = NULL;
      char *password = NULL;

      if (auth_name && auth_password)
	{
	  name = g_strdup(auth_name);
	  password = g_strdup(auth_password);
	}
      else
	{
	  GDK_THREADS_ENTER();
	  st_thread_printf(transfer->thread, TRUE, _("Waiting for proxy password..."));
	  gdk_flush();
	  GDK_THREADS_LEAVE();

	  st_auth_dialog(auth_name,
			 auth_password,
			 &name,
			 &password,
			 _("Password required"),
			 _("Enter your username and password for proxy '%s'."),
			 transfer->proxy_server);
	}

      if (name && password)
	transfer->proxy_userpwd = g_strconcat(name, ":", password, NULL);
      else			/* cancelled by user */
	status = FALSE;

      g_free(name);
      g_free(password);
    }
  else
    transfer->proxy_userpwd = g_strdup(""); /* libcurl crashes in handleSock5Proxy() if this is NULL */

  g_free(auth_name);
  g_free(auth_password);
#endif /* ! ST_REGRESSION_TEST */

  return status;
}

static void
st_transfer_free_proxy_settings (STTransfer *transfer)
{
  g_return_if_fail(transfer != NULL);
  
  g_free(transfer->proxy_server);
  g_free(transfer->proxy_userpwd);
}

/**
 * st_transfer_escape:
 * @url: an URL or URL part to escape.
 *
 * Escapes unsafe characters of @url using standard percent notation.
 *
 * Return value: @url with unsafe characters escaped. The returned
 * string must be freed when no longer needed.
 **/
char *
st_transfer_escape (const char *url)
{
  return curl_escape(url, 0);
}

static gboolean
st_transfer_progress_cb (gpointer data,
			 double dltotal,
			 double dlnow,
			 double ultotal,
			 double ulnow)
{
#ifndef ST_REGRESSION_TEST
  STTransfer *transfer = data;
#endif
  gboolean aborted;

#ifdef ST_REGRESSION_TEST
  aborted = FALSE;
#else
  aborted = st_thread_is_aborted(transfer->thread);
  if (! aborted)
    {
      GDK_THREADS_ENTER();

      if (sg_double_equal(dltotal, 0))
	st_thread_printf(transfer->thread,
			 TRUE,
			 ngettext("Receiving (%u byte so far)...",
				  "Receiving (%u bytes so far)...",
				  (unsigned int) dlnow),
			 (unsigned int) dlnow);
      else
	st_thread_printf(transfer->thread,
			 TRUE,
			 ngettext("Receiving (%u byte out of %u)...",
				  "Receiving (%u bytes out of %u)...",
				  (unsigned int) dlnow),
			 (unsigned int) dlnow,
			 (unsigned int) dltotal);
	  
      if (! sg_double_equal(dlnow, transfer->downloaded))
	{
	  st_thread_set_progress(transfer->thread, TRUE, sg_double_equal(dltotal, 0) ? -1 : dlnow / dltotal);
	  transfer->downloaded = dlnow;
	}

      gdk_flush();
      GDK_THREADS_LEAVE();
    }
#endif /* ST_REGRESSION_TEST */

  return aborted;
}

#ifndef ST_REGRESSION_TEST
static char *
st_transfer_get_hostname (const char *url)
{
  char *s1;
  
  g_return_val_if_fail(url != NULL, NULL);

  if ((s1 = st_strstr_span(url, "://")))
    {
      char *s2;
      return (s2 = strpbrk(s1, "/?"))
	? g_strndup(s1, s2 - s1)
	: g_strdup(s1);
    }
  else
    return NULL;
}
#endif /* ! ST_REGRESSION_TEST */

static size_t
st_transfer_session_get_cb (const char *buffer,
			    size_t size,
			    size_t nitems,
			    gpointer data)
{
  GString *string = data;
  size_t len;

  len = size * nitems;
  g_string_append_len(string, buffer, len);
  
  return len;
}

static size_t
st_transfer_session_get_binary_cb (const char *buffer,
				   size_t size,
				   size_t nitems,
				   gpointer data)
{
  GByteArray *array = data;
  size_t len;

  len = size * nitems;
  g_byte_array_append(array, buffer, len);
  
  return len;
}

static size_t
st_transfer_session_get_by_line_cb (const char *buffer,
				    size_t size,
				    size_t nitems,
				    gpointer data)
{
  STTransferLineData *line_data = data;
  size_t len;
  int start = 0;
  int i;

  len = size * nitems;

  /* handles lf (UNIX), crlf (DOS) and cr (Mac) terminators */
  for (i = 0; i < len; i++)
    {
      gboolean processed = FALSE;

      if (*line_data->terminator)
	{
	  if (buffer[i] == '\n' && *line_data->terminator == '\r')
	    {
	      line_data->terminator[1] = '\n';
	      processed = TRUE;
	      start = i + 1;
	    }
	  else
	    start = i;

	  st_transfer_line_data_flush(line_data);
	}

      if (! processed && (buffer[i] == '\n' || buffer[i] == '\r'))
	{
	  g_string_append_len(line_data->line, buffer + start, i - start);
	  line_data->terminator[0] = buffer[i];
	  line_data->terminator[1] = 0;
	}
    }
  if (start < len)
    g_string_append_len(line_data->line, buffer + start, len - start);
  
  return len;
}

/*** private implementation **************************************************/

void
st_transfer_init (void)
{
  if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
    {
#ifdef ST_REGRESSION_TEST
      g_critical("unable to initialize libcurl");
#else
      st_error_dialog(_("A fatal error has occurred"),
		      _("Unable to initialize the curl library."));
      exit(1);
#endif /* ST_REGRESSION_TEST */
    }
}
