src/lib-http/http-client-request.c - main - niceSOFT/dovecot

src/lib-http/http-client-request.c
2 056 строк59 KB
Michael M Slusarz
COPYING: Simplify and standardize copyright handling
03 июл 2026, 02:25
03 июл 2026, 02:252a5809c
/* Copyright (c) Dovecot authors, see top-level COPYING file */

#include "lib.h"
#include "net.h"
#include "str.h"
#include "str-sanitize.h"
#include "hash.h"
#include "array.h"
#include "llist.h"
#include "time-util.h"
#include "istream.h"
#include "ostream.h"
#include "file-lock.h"
#include "dns-lookup.h"
#include "http-url.h"
#include "http-date.h"
#include "http-auth.h"
#include "http-response-parser.h"
#include "http-transfer.h"

#include "http-client-private.h"

const char *http_request_state_names[] = {
	"new",
	"queued",
	"payload_out",
	"waiting",
	"got_response",
	"payload_in",
	"finished",
	"aborted"
};

/*
 * Request
 */

static bool
http_client_request_send_error(struct http_client_request *req,
			       unsigned int status, const char *error);

const char *http_client_request_label(struct http_client_request *req)
{
	if (req->label == NULL) {
		req->label = p_strdup_printf(
			req->pool, "[Req%u: %s %s%s]", req->id, req->method,
			http_url_create_host(&req->origin_url), req->target);
	}
	return req->label;
}

static void http_client_request_update_event(struct http_client_request *req)
{
	event_add_str(req->event, "method", req->method);
	event_add_str(req->event, "dest_host", req->origin_url.host.name);
	event_add_int(req->event, "dest_port",
		      http_url_get_port(&req->origin_url));
	if (req->target != NULL)
		event_add_str(req->event, "target", req->target);
	event_set_append_log_prefix(
		req->event, t_strdup_printf("request %s: ",
			str_sanitize(http_client_request_label(req), 256)));
}

static struct event_passthrough *
http_client_request_result_event(struct http_client_request *req)
{
	struct http_client_connection *conn = req->conn;

	if (conn != NULL) {
		if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) {
			/* Got here prematurely; use bytes written so far */
			i_assert(req->request_offset <
				 conn->conn.output->offset);
			req->bytes_out = conn->conn.output->offset -
				req->request_offset;
		}
		if (conn->incoming_payload != NULL &&
		    (req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		     req->state == HTTP_REQUEST_STATE_PAYLOAD_IN)) {
			/* Got here prematurely; use bytes read so far */
			i_assert(conn->in_req_callback ||
				 conn->pending_request == req);
			i_assert(req->response_offset <
				 conn->conn.input->v_offset);
			req->bytes_in = conn->conn.input->v_offset -
				req->response_offset;
		}
	}

	struct event_passthrough *e = event_create_passthrough(req->event);
	if (req->queue != NULL &&
	    req->queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX)
		e->add_ip("dest_ip", &req->queue->addr.a.tcp.ip);

	return e->add_int("status_code", req->last_status)->
		add_int("attempts", req->attempts)->
		add_int("redirects", req->redirects)->
		add_int("net_in_bytes", req->bytes_in)->
		add_int("net_out_bytes", req->bytes_out);
}

static struct http_client_request *
http_client_request_new(struct http_client *client, const char *method,
			http_client_request_callback_t *callback, void *context)
{
	static unsigned int id_counter = 0;
	pool_t pool;
	struct http_client_request *req;

	pool = pool_alloconly_create("http client request", 2048);
	req = p_new(pool, struct http_client_request, 1);
	req->pool = pool;
	req->refcount = 1;
	req->client = client;
	req->id = ++id_counter;
	req->method = p_strdup(pool, method);
	req->callback = callback;
	req->context = context;
	req->date = (time_t)-1;
	req->event = event_create(client->event);
	event_strlist_copy_recursive(req->event, event_get_global(),
				     EVENT_REASON_CODE);

	/* Default to client-wide settings: */
	req->max_attempts = client->set->request_max_attempts;
	req->attempt_timeout_msecs = client->set->request_timeout_msecs;
	if (strcasecmp(method, "GET") == 0 ||
	    strcasecmp(method, "HEAD") == 0) {
		if (client->set->read_request_max_attempts != 0)
			req->max_attempts = client->set->read_request_max_attempts;
		if (client->set->read_request_timeout_msecs != 0) {
			req->attempt_timeout_msecs =
				client->set->read_request_timeout_msecs;
		}
	} else if (strcasecmp(method, "PUT") == 0 ||
		   strcasecmp(method, "POST") == 0) {
		if (client->set->write_request_max_attempts != 0)
			req->max_attempts = client->set->write_request_max_attempts;
		if (client->set->write_request_timeout_msecs != 0) {
			req->attempt_timeout_msecs =
				client->set->write_request_timeout_msecs;
		}
	} else if (strcasecmp(method, "DELETE") == 0) {
		if (client->set->delete_request_max_attempts != 0)
			req->max_attempts = client->set->delete_request_max_attempts;
		if (client->set->delete_request_timeout_msecs != 0) {
			req->attempt_timeout_msecs =
				client->set->delete_request_timeout_msecs;
		}
	}

	req->state = HTTP_REQUEST_STATE_NEW;
	return req;
}

#undef http_client_request
struct http_client_request *
http_client_request(struct http_client *client,
		    const char *method, const char *host, const char *target,
		    http_client_request_callback_t *callback, void *context)
{
	struct http_client_request *req;

	req = http_client_request_new(client, method, callback, context);
	req->origin_url.host.name = p_strdup(req->pool, host);
	req->target = (target == NULL ? "/" : p_strdup(req->pool, target));
	http_client_request_update_event(req);
	return req;
}

#undef http_client_request_url
struct http_client_request *
http_client_request_url(struct http_client *client,
			const char *method, const struct http_url *target_url,
			http_client_request_callback_t *callback, void *context)
{
	struct http_client_request *req;

	req = http_client_request_new(client, method, callback, context);
	http_url_copy_authority(req->pool, &req->origin_url, target_url);
	req->target = p_strdup(req->pool, http_url_create_target(target_url));
	if (target_url->user != NULL && *target_url->user != '\0' &&
	    target_url->password != NULL) {
		req->username = p_strdup(req->pool, target_url->user);
		req->password = p_strdup(req->pool, target_url->password);
	}
	http_client_request_update_event(req);
	return req;
}

#undef http_client_request_url_str
struct http_client_request *
http_client_request_url_str(struct http_client *client,
			    const char *method, const char *url_str,
			    http_client_request_callback_t *callback,
			    void *context)
{
	struct http_client_request *req, *tmpreq;
	struct http_url *target_url;
	const char *error;

	req = tmpreq = http_client_request_new(client, method,
					       callback, context);

	if (http_url_parse(url_str, NULL, HTTP_URL_ALLOW_USERINFO_PART,
			   req->pool, &target_url, &error) < 0) {
		req->label = p_strdup_printf(req->pool, "[Req%u: %s %s]",
					     req->id, req->method, url_str);
		http_client_request_error(
			&tmpreq, HTTP_CLIENT_REQUEST_ERROR_INVALID_URL,
			t_strdup_printf("Invalid HTTP URL: %s", error));
		http_client_request_update_event(req);
		return req;
	}

	req->origin_url = *target_url;
	req->target = p_strdup(req->pool, http_url_create_target(target_url));
	if (target_url->user != NULL && *target_url->user != '\0' &&
	    target_url->password != NULL) {
		req->username = p_strdup(req->pool, target_url->user);
		req->password = p_strdup(req->pool, target_url->password);
	}
	http_client_request_update_event(req);
	return req;
}

#undef http_client_request_connect
struct http_client_request *
http_client_request_connect(struct http_client *client,
			    const char *host, in_port_t port,
			    http_client_request_callback_t *callback,
			    void *context)
{
	struct http_client_request *req;

	req = http_client_request_new(client, "CONNECT", callback, context);
	req->origin_url.host.name = p_strdup(req->pool, host);
	req->origin_url.port = port;
	req->connect_tunnel = TRUE;
	req->target = req->origin_url.host.name;
	http_client_request_update_event(req);
	return req;
}

#undef http_client_request_connect_ip
struct http_client_request *
http_client_request_connect_ip(struct http_client *client,
			       const struct ip_addr *ip, in_port_t port,
			       http_client_request_callback_t *callback,
			       void *context)
{
	struct http_client_request *req;
	const char *hostname;

	i_assert(ip->family != 0);
	hostname = net_ip2addr(ip);

	req = http_client_request_connect(client, hostname, port,
					  callback, context);
	req->origin_url.host.ip = *ip;
	return req;
}

void http_client_request_set_event(struct http_client_request *req,
				   struct event *event)
{
	event_unref(&req->event);
	req->event = event_create(event);
	event_add_category(req->event, &event_category_http_client);
	event_set_forced_debug(req->event, event_want_debug(req->client->event));
	event_strlist_copy_recursive(req->event, event_get_global(),
				     EVENT_REASON_CODE);
	http_client_request_update_event(req);
}

static void http_client_request_add(struct http_client_request *req)
{
	struct http_client *client = req->client;

	DLLIST_PREPEND(&client->requests_list, req);
	client->requests_count++;
	req->listed = TRUE;
}

static void http_client_request_remove(struct http_client_request *req)
{
	struct http_client *client = req->client;

	if (client == NULL) {
		i_assert(!req->listed);
		return;
	}
	if (req->listed) {
		/* Only decrease pending request counter if this request was
		   submitted */
		DLLIST_REMOVE(&client->requests_list, req);
		client->requests_count--;
	}
	req->listed = FALSE;

	if (client->requests_count == 0 && client->waiting)
		io_loop_stop(client->ioloop);
}

void http_client_request_ref(struct http_client_request *req)
{
	i_assert(req->refcount > 0);
	req->refcount++;
}

bool http_client_request_unref(struct http_client_request **_req)
{
	struct http_client_request *req = *_req;

	if (req == NULL)
		return FALSE;
	*_req = NULL;

	i_assert(req->refcount > 0);
	if (--req->refcount > 0)
		return TRUE;

	struct http_client *client = req->client;

	if (client == NULL) {
		e_debug(req->event, "Free (client already destroyed)");
	} else {
		e_debug(req->event, "Free (requests left=%d)",
			client->requests_count);
	}

	/* Cannot be destroyed while it is still pending */
	i_assert(req->conn == NULL);

	if (req->queue != NULL)
		http_client_queue_drop_request(req->queue, req);

	if (req->destroy_callback != NULL) {
		req->destroy_callback(req->destroy_context);
		req->destroy_callback = NULL;
	}

	http_client_request_remove(req);

	if (client != NULL) {
		if (client->requests_count == 0 && client->waiting)
			io_loop_stop(client->ioloop);
		if (req->delayed_error != NULL)
			http_client_remove_request_error(req->client, req);
	}
	i_stream_unref(&req->payload_input);
	o_stream_unref(&req->payload_output);
	str_free(&req->headers);
	event_unref(&req->event);
	pool_unref(&req->pool);
	return FALSE;
}

void http_client_request_destroy(struct http_client_request **_req)
{
	struct http_client_request *req = *_req, *tmp_req;
	struct http_client *client = req->client;

	*_req = NULL;

	if (client == NULL) {
		e_debug(req->event, "Destroy (client already destroyed)");
	} else {
		e_debug(req->event, "Destroy (requests left=%d)",
			client->requests_count);
	}


	if (req->state < HTTP_REQUEST_STATE_FINISHED)
		req->state = HTTP_REQUEST_STATE_ABORTED;
	req->callback = NULL;

	if (req->queue != NULL)
		http_client_queue_drop_request(req->queue, req);

	if (client != NULL && req->delayed_error != NULL)
		http_client_remove_request_error(req->client, req);
	req->delayed_error = NULL;

	if (req->destroy_callback != NULL) {
		void (*callback)(void *) = req->destroy_callback;

		req->destroy_callback = NULL;
		callback(req->destroy_context);
	}

	if (req->conn != NULL)
		http_client_connection_request_destroyed(req->conn, req);

	tmp_req = req;
	http_client_request_remove(req);
	if (http_client_request_unref(&tmp_req))
		req->client = NULL;
}

void http_client_request_set_port(struct http_client_request *req,
				  in_port_t port)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);
	req->origin_url.port = port;
	event_add_int(req->event, "port", port);
}

void http_client_request_set_ssl(struct http_client_request *req, bool ssl)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);
	req->origin_url.have_ssl = ssl;
}

void http_client_request_set_urgent(struct http_client_request *req)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);
	req->urgent = TRUE;
}

void http_client_request_set_preserve_exact_reason(
	struct http_client_request *req)
{
	req->preserve_exact_reason = TRUE;
}

static bool
http_client_request_lookup_header_pos(struct http_client_request *req,
				      const char *key, size_t *key_pos_r,
				      size_t *value_pos_r, size_t *next_pos_r)
{
	const unsigned char *data, *p;
	size_t size, line_len;
	size_t key_len = strlen(key);

	if (req->headers == NULL)
		return FALSE;

	data = str_data(req->headers);
	size = str_len(req->headers);
	while ((p = memchr(data, '\n', size)) != NULL) {
		line_len = (p+1) - data;
		if (size > key_len && i_memcasecmp(data, key, key_len) == 0 &&
		    data[key_len] == ':' && data[key_len+1] == ' ') {
			/* Key was found from header, replace its value */
			*key_pos_r = str_len(req->headers) - size;
			*value_pos_r = *key_pos_r + key_len + 2;
			*next_pos_r = *key_pos_r + line_len;
			return TRUE;
		}
		size -= line_len;
		data += line_len;
	}
	return FALSE;
}

static void
http_client_request_add_header_full(struct http_client_request *req,
				    const char *key, const char *value,
				    bool replace_existing)
{
	size_t key_pos, value_pos, next_pos;

	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 /* Allow calling for retries */
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);
	/* Make sure key or value can't break HTTP headers entirely */
	i_assert(strpbrk(key, ":\r\n") == NULL);
	i_assert(strpbrk(value, "\r\n") == NULL);

	/* Mark presence of special headers */
	switch (key[0]) {
	case 'a': case 'A':
		if (strcasecmp(key, "Authorization") == 0)
			req->have_hdr_authorization = TRUE;
		break;
	case 'c': case 'C':
		if (strcasecmp(key, "Connection") == 0)
			req->have_hdr_connection = TRUE;
		else if (strcasecmp(key, "Content-Length") == 0)
			req->have_hdr_body_spec = TRUE;
		break;
	case 'd': case 'D':
		if (strcasecmp(key, "Date") == 0)
			req->have_hdr_date = TRUE;
		break;
	case 'e': case 'E':
		if (strcasecmp(key, "Expect") == 0)
			req->have_hdr_expect = TRUE;
		break;
	case 'h': case 'H':
		if (strcasecmp(key, "Host") == 0)
			req->have_hdr_host = TRUE;
		break;
	case 'p': case 'P':
		i_assert(strcasecmp(key, "Proxy-Authorization") != 0);
		break;
	case 't': case 'T':
		if (strcasecmp(key, "Transfer-Encoding") == 0)
			req->have_hdr_body_spec = TRUE;
		break;
	case 'u': case 'U':
		if (strcasecmp(key, "User-Agent") == 0)
			req->have_hdr_user_agent = TRUE;
		break;
	}
	if (req->headers == NULL)
		req->headers = str_new(default_pool, 256);
	if (!http_client_request_lookup_header_pos(req, key, &key_pos,
						   &value_pos, &next_pos))
		str_printfa(req->headers, "%s: %s\r\n", key, value);
	else if (replace_existing) {
		/* Don't delete CRLF */
		size_t old_value_len = next_pos - value_pos - 2;
		str_replace(req->headers, value_pos, old_value_len, value);
	}
}

void http_client_request_add_header(struct http_client_request *req,
				    const char *key, const char *value)
{
	http_client_request_add_header_full(req, key, value, TRUE);
}

void http_client_request_add_missing_header(struct http_client_request *req,
					    const char *key, const char *value)
{
	http_client_request_add_header_full(req, key, value, FALSE);
}

void http_client_request_remove_header(struct http_client_request *req,
				       const char *key)
{
	size_t key_pos, value_pos, next_pos;

	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 /* Allow calling for retries */
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);

	if (http_client_request_lookup_header_pos(req, key, &key_pos,
						  &value_pos, &next_pos))
		str_delete(req->headers, key_pos, next_pos - key_pos);
}

const char *http_client_request_lookup_header(struct http_client_request *req,
					      const char *key)
{
	size_t key_pos, value_pos, next_pos;

	if (!http_client_request_lookup_header_pos(req, key, &key_pos,
						   &value_pos, &next_pos))
		return NULL;

	/* Don't return CRLF */
	return t_strndup(str_data(req->headers) + value_pos,
			 next_pos - value_pos - 2);
}

const char *http_client_request_retrieve_headers(struct http_client_request *req)
{
	return str_c(req->headers);
}

void http_client_request_set_date(struct http_client_request *req, time_t date)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);
	req->date = date;
}

static void
http_client_request_set_payload_real(struct http_client_request *req,
				     struct istream *input, bool sync)
{
	int ret;

	if (req->state == HTTP_REQUEST_STATE_ABORTED)
		return;

	i_assert(req->state == HTTP_REQUEST_STATE_NEW);
	i_assert(req->payload_input == NULL);

	i_stream_ref(input);
	req->payload_input = input;
	if ((ret = i_stream_get_size(input, TRUE, &req->payload_size)) <= 0) {
		if (ret < 0) {
			e_error(req->event, "i_stream_get_size(%s) failed: %s",
				i_stream_get_name(input),
				i_stream_get_error(input));
		}
		req->payload_size = 0;
		req->payload_chunked = TRUE;
	} else {
		i_assert(input->v_offset <= req->payload_size);
		req->payload_size -= input->v_offset;
	}
	req->payload_offset = input->v_offset;

	/* Prepare request payload sync using 100 Continue response from server
	 */
	if ((req->payload_chunked || req->payload_size > 0) && sync)
		req->payload_sync = TRUE;
}

void http_client_request_set_payload(struct http_client_request *req,
				     struct istream *input, bool sync)
{
	input = i_stream_create_limit(input, UOFF_T_MAX);
	http_client_request_set_payload_real(req, input, sync);
	i_stream_unref(&input);
}

void http_client_request_set_payload_data(struct http_client_request *req,
					  const unsigned char *data,
					  size_t size)
{
	struct istream *input;
	unsigned char *payload_data;

	if (req->state == HTTP_REQUEST_STATE_ABORTED)
		return;
	if (size == 0)
		return;

	payload_data = p_malloc(req->pool, size);
	memcpy(payload_data, data, size);
	input = i_stream_create_from_data(payload_data, size);

	http_client_request_set_payload_real(req, input, FALSE);
	i_stream_unref(&input);
}

void http_client_request_set_payload_empty(struct http_client_request *req)
{
	req->payload_empty = TRUE;
}

void http_client_request_set_timeout_msecs(struct http_client_request *req,
					   unsigned int msecs)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);

	req->timeout_msecs = msecs;
}

void http_client_request_set_timeout(struct http_client_request *req,
				     const struct timeval *time)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);

	req->timeout_time = *time;
	req->timeout_msecs = 0;
}

void http_client_request_set_attempt_timeout_msecs(
	struct http_client_request *req, unsigned int msecs)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);

	req->attempt_timeout_msecs = msecs;
}

void http_client_request_set_max_attempts(struct http_client_request *req,
					  unsigned int max_attempts)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);

	req->max_attempts = max_attempts;
}

void http_client_request_set_event_headers(struct http_client_request *req,
					   const char *const *headers)
{
	req->event_headers = p_strarray_dup(req->pool, headers);
}

void http_client_request_set_auth_simple(struct http_client_request *req,
					 const char *username,
					 const char *password)
{
	req->username = p_strdup(req->pool, username);
	req->password = p_strdup(req->pool, password);
}

void http_client_request_set_proxy_url(struct http_client_request *req,
				       const struct http_url *proxy_url)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);

	req->host_url = http_url_clone_authority(req->pool, proxy_url);
	req->host_socket = NULL;
}

void http_client_request_set_proxy_socket(struct http_client_request *req,
					  const char *proxy_socket)
{
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_GOT_RESPONSE ||
		 req->state == HTTP_REQUEST_STATE_ABORTED);

	req->host_socket = p_strdup(req->pool, proxy_socket);
	req->host_url = NULL;
}

void http_client_request_delay_until(struct http_client_request *req,
				     time_t time)
{
	req->release_time.tv_sec = time;
	req->release_time.tv_usec = 0;
}

void http_client_request_delay(struct http_client_request *req, time_t seconds)
{
	req->release_time = ioloop_timeval;
	req->release_time.tv_sec += seconds;
}

void http_client_request_delay_msecs(struct http_client_request *req,
				     unsigned int msecs)
{
	req->release_time = ioloop_timeval;
	timeval_add_msecs(&req->release_time, msecs);
}

int http_client_request_delay_from_response(
	struct http_client_request *req, const struct http_response *response)
{
	time_t retry_after = response->retry_after;
	unsigned int max;

	i_assert(req->client != NULL);

	if (retry_after == (time_t)-1)
		return 0;  /* no delay */
	if (retry_after < ioloop_time)
		return 0;  /* delay already expired */
	max = (req->client->set->max_auto_retry_delay_secs == 0 ?
	       req->attempt_timeout_msecs / 1000 :
	       req->client->set->max_auto_retry_delay_secs);
	if ((unsigned int)(retry_after - ioloop_time) > max)
		return -1; /* delay too long */
	req->release_time.tv_sec = retry_after;
	req->release_time.tv_usec = 0;
	return 1; /* valid delay */
}

const char *
http_client_request_get_method(const struct http_client_request *req)
{
	return req->method;
}

const char *
http_client_request_get_target(const struct http_client_request *req)
{
	return req->target;
}

const struct http_url *
http_client_request_get_origin_url(const struct http_client_request *req)
{
	return &req->origin_url;
}

enum http_request_state
http_client_request_get_state(const struct http_client_request *req)
{
	return req->state;
}

unsigned int
http_client_request_get_attempts(const struct http_client_request *req)
{
	return req->attempts;
}

void http_client_request_get_stats(struct http_client_request *req,
				   struct http_client_request_stats *stats_r)
{
	long long diff_msecs;
	uint64_t wait_usecs;

	i_zero(stats_r);
	if (!req->submitted)
		return;

	/* Total elapsed time since message was submitted */
	diff_msecs = timeval_diff_msecs(&ioloop_timeval, &req->submit_time);
	stats_r->total_msecs = (unsigned int)I_MAX(diff_msecs, 0);

	/* Elapsed time since message was first sent */
	if (req->first_sent_time.tv_sec > 0) {
		diff_msecs = timeval_diff_msecs(&ioloop_timeval,
						&req->first_sent_time);
		stats_r->first_sent_msecs = (unsigned int)I_MAX(diff_msecs, 0);
	}

	/* Elapsed time since message was last sent */
	if (req->sent_time.tv_sec > 0) {
		diff_msecs = timeval_diff_msecs(&ioloop_timeval,
						&req->sent_time);
		stats_r->last_sent_msecs = (unsigned int)I_MAX(diff_msecs, 0);
	}

	/* Time spent in other ioloops */
	i_assert(ioloop_global_wait_usecs >= req->queued_global_ioloop_usecs);
	stats_r->other_ioloop_msecs = (unsigned int)
		(ioloop_global_wait_usecs -
		 req->queued_global_ioloop_usecs + 999) / 1000;

	if (req->conn != NULL) {
		/* Time spent in the http-client's own ioloop */
		wait_usecs = io_wait_timer_get_usecs(req->conn->io_wait_timer);
		i_assert(wait_usecs >= req->sent_http_ioloop_usecs);
		stats_r->http_ioloop_msecs = (unsigned int)
			(wait_usecs - req->sent_http_ioloop_usecs + 999) / 1000;

		i_assert(stats_r->other_ioloop_msecs >=
			 stats_r->http_ioloop_msecs);
		stats_r->other_ioloop_msecs -= stats_r->http_ioloop_msecs;
	}

	/* Total time spent on waiting for file locks */
	wait_usecs = file_lock_wait_get_total_usecs();
	i_assert(wait_usecs >= req->queued_lock_usecs);
	stats_r->lock_msecs = (unsigned int)
		(wait_usecs - req->queued_lock_usecs + 999) / 1000;

	/* Number of attempts for this request */
	stats_r->attempts = req->attempts;
	/* Number of send attempts for this request */
	stats_r->send_attempts = req->send_attempts;
}

void http_client_request_append_stats_text(struct http_client_request *req,
					   string_t *str)
{
	struct http_client_request_stats stats;

	if (!req->submitted) {
		str_append(str, "not yet submitted");
		return;
	}

	http_client_request_get_stats(req, &stats);

	str_printfa(str, "queued %u.%03u secs ago",
		    stats.total_msecs/1000, stats.total_msecs%1000);
	if (stats.attempts > 0)
		str_printfa(str, ", %u times retried", stats.attempts);

	if (stats.send_attempts == 0) {
		str_append(str, ", not yet sent");
	} else {
		str_printfa(str, ", %u send attempts in %u.%03u secs",
			    stats.send_attempts, stats.first_sent_msecs/1000,
			    stats.first_sent_msecs%1000);
		if (stats.send_attempts > 1) {
			str_printfa(str, ", %u.%03u in last attempt",
				    stats.last_sent_msecs/1000,
				    stats.last_sent_msecs%1000);
		}
	}

	if (stats.http_ioloop_msecs > 0) {
		str_printfa(str, ", %u.%03u in http ioloop",
			    stats.http_ioloop_msecs/1000,
			    stats.http_ioloop_msecs%1000);
	}
	str_printfa(str, ", %u.%03u in other ioloops",
		    stats.other_ioloop_msecs/1000,
		    stats.other_ioloop_msecs%1000);

	if (stats.lock_msecs > 0) {
		str_printfa(str, ", %u.%03u in locks",
			    stats.lock_msecs/1000, stats.lock_msecs%1000);
	}
}

enum http_response_payload_type
http_client_request_get_payload_type(struct http_client_request *req)
{
	/* RFC 7230, Section 3.3:

	   The presence of a message body in a response depends on both the
	   request method to which it is responding and the response status code
	   (Section 3.1.2 of [RFC7230]). Responses to the HEAD request method
	   (Section 4.3.2 of [RFC7231]) never include a message body because the
	   associated response header fields (e.g., Transfer-Encoding,
	   Content-Length, etc.), if present, indicate only what their values
	   would have been if the request method had been GET (Section 4.3.1 of
	   [RFC7231]). 2xx (Successful) responses to a CONNECT request method
	   (Section 4.3.6 of [RFC7231]) switch to tunnel mode instead of having
	   a message body.
	 */
	if (strcmp(req->method, "HEAD") == 0)
		return HTTP_RESPONSE_PAYLOAD_TYPE_NOT_PRESENT;
	if (strcmp(req->method, "CONNECT") == 0)
		return HTTP_RESPONSE_PAYLOAD_TYPE_ONLY_UNSUCCESSFUL;
	return HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED;
}

static void http_client_request_do_submit(struct http_client_request *req)
{
	struct http_client *client = req->client;
	struct http_client_host *host;
	const char *proxy_socket_path = client->set->proxy_socket_path;
	const struct http_url *proxy_url = client->set->parsed_proxy_url;
	bool have_proxy =
		((proxy_socket_path != NULL && proxy_socket_path[0] != '\0') ||
		 (proxy_url != NULL) ||
		 (req->host_socket != NULL) || (req->host_url != NULL));
	const char *authority, *target;

	if (req->state == HTTP_REQUEST_STATE_ABORTED)
		return;
	i_assert(client != NULL);
	i_assert(req->state == HTTP_REQUEST_STATE_NEW);

	authority = http_url_create_authority(&req->origin_url);
	if (req->connect_tunnel) {
		/* Connect requests require authority form for request target */
		target = authority;
	} else {
		/* Absolute target URL */
		target = t_strconcat(http_url_create_host(&req->origin_url),
				     req->target, NULL);
	}

	/* Determine what host to contact to submit this request */
	if (have_proxy) {
		if (req->host_socket != NULL) {
			/* Specific socket proxy */
			req->host_url = NULL;
		} else if (req->host_url != NULL) {
			/* Specific normal proxy */
			req->host_socket = NULL;
		} else if (req->origin_url.have_ssl &&
			   client->set->proxy_ssl_tunnel &&
			   !req->connect_tunnel) {
			/* Tunnel to origin server */
			req->host_url = &req->origin_url;
			req->ssl_tunnel = TRUE;
		} else if (proxy_socket_path != NULL &&
			   proxy_socket_path[0] != '\0') {
			/* Proxy on unix socket */
			req->host_socket = proxy_socket_path;
			req->host_url = NULL;
		} else {
			/* Normal proxy server */
			req->host_url = proxy_url;
			req->host_socket = NULL;
		}
	} else {
		/* Origin server */
		req->host_url = &req->origin_url;
	}

	/* Use submission date if no date is set explicitly */
	if (req->date == (time_t)-1)
		req->date = ioloop_time;

	/* Prepare value for Host header */
	req->authority = p_strdup(req->pool, authority);

	/* Debug label */
	req->label = p_strdup_printf(req->pool, "[Req%u: %s %s]",
				     req->id, req->method, target);

	/* Update request target */
	if (req->connect_tunnel || have_proxy)
		req->target = p_strdup(req->pool, target);

	if (!have_proxy) {
		/* If we don't have a proxy, CONNECT requests are handled by
		   creating the requested connection directly */
		req->connect_direct = req->connect_tunnel;
		if (req->connect_direct)
			req->urgent = TRUE;
	}

	if (req->timeout_time.tv_sec == 0) {
		if (req->timeout_msecs > 0) {
			req->timeout_time = ioloop_timeval;
			timeval_add_msecs(&req->timeout_time,
					  req->timeout_msecs);
		} else if (client->set->request_absolute_timeout_msecs > 0) {
			req->timeout_time = ioloop_timeval;
			timeval_add_msecs(&req->timeout_time,
				client->set->request_absolute_timeout_msecs);
		}
	}

	host = http_client_host_get(client, req->host_url);
	req->state = HTTP_REQUEST_STATE_QUEUED;
	req->last_status = 0;

	http_client_host_submit_request(host, req);
}

void http_client_request_submit(struct http_client_request *req)
{
	i_assert(req->client != NULL);

	req->submit_time = ioloop_timeval;
	req->queued_lock_usecs = file_lock_wait_get_total_usecs();
	req->queued_global_ioloop_usecs = ioloop_global_wait_usecs;

	http_client_request_update_event(req);
	http_client_request_do_submit(req);

	req->submitted = TRUE;
	http_client_request_add(req);

	e_debug(req->event, "Submitted (requests left=%d)",
		req->client->requests_count);
}

void http_client_request_get_peer_addr(const struct http_client_request *req,
				       struct http_client_peer_addr *addr)
{
	const char *host_socket = req->host_socket;
	const struct http_url *host_url = req->host_url;

	/* The IP address may be unassigned in the returned peer address, since
	   that is only available at this stage when the target URL has an
	   explicit IP address. */
	i_zero(addr);
	if (host_socket != NULL) {
		addr->type = HTTP_CLIENT_PEER_ADDR_UNIX;
		addr->a.un.path = host_socket;
	} else if (req->connect_direct) {
		addr->type = HTTP_CLIENT_PEER_ADDR_RAW;
		addr->a.tcp.ip = host_url->host.ip;
		addr->a.tcp.port =
			http_url_get_port_default(host_url, HTTPS_DEFAULT_PORT);
	} else if (host_url->have_ssl) {
		if (req->ssl_tunnel)
			addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL;
		else
			addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS;
		addr->a.tcp.ip = host_url->host.ip;
		addr->a.tcp.https_name = host_url->host.name;
		addr->a.tcp.port = http_url_get_port(host_url);
	} else {
		addr->type = HTTP_CLIENT_PEER_ADDR_HTTP;
		addr->a.tcp.ip = host_url->host.ip;
		addr->a.tcp.port = http_url_get_port(host_url);
	}
}

static int http_client_request_flush_payload(struct http_client_request *req)
{
	struct http_client_connection *conn = req->conn;
	int ret;

	if (req->payload_output == conn->conn.output)
		ret = o_stream_flush(req->payload_output);
	else
		ret = o_stream_finish(req->payload_output);

	if (ret < 0)
		http_client_connection_handle_output_error(conn);
	return ret;
}

static int
http_client_request_finish_payload_out(struct http_client_request *req)
{
	struct http_client_connection *conn = req->conn;
	int ret;

	i_assert(conn != NULL);
	req->payload_finished = TRUE;

	/* Drop payload output stream */
	if (req->payload_output != NULL) {
		ret = http_client_request_flush_payload(req);
		if (ret < 0)
			return -1;
		if (ret == 0) {
			e_debug(req->event,
				"Not quite finished sending payload");
			conn->output_locked = TRUE;
			return 0;
		}
		o_stream_unref(&req->payload_output);
		req->payload_output = NULL;
	}
	if (o_stream_get_buffer_used_size(conn->conn.output) > 0) {
		e_debug(req->event,
			"Not quite finished sending request");
		conn->output_locked = TRUE;
		return 0;
	}

	i_assert(req->request_offset < conn->conn.output->offset);
	req->bytes_out = conn->conn.output->offset - req->request_offset;

	/* Advance state only when request didn't get aborted in the mean time
	 */
	if (req->state != HTTP_REQUEST_STATE_ABORTED) {
		i_assert(req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT);

		/* we're now waiting for a response from the server */
		req->state = HTTP_REQUEST_STATE_WAITING;
		http_client_connection_start_request_timeout(conn);
	}

	/* Release connection */
	conn->output_locked = FALSE;

	e_debug(req->event, "Finished sending%s payload",
		(req->state == HTTP_REQUEST_STATE_ABORTED ? " aborted" : ""));
	return 1;
}

static int
http_client_request_continue_payload(struct http_client_request **_req,
				     const unsigned char *data, size_t size)
{
	struct ioloop *prev_ioloop, *client_ioloop, *prev_client_ioloop;
	struct http_client_request *req = *_req;
	struct http_client_connection *conn = req->conn;
	struct http_client *client = req->client;
	int ret;

	i_assert(client != NULL);
	i_assert(req->state == HTTP_REQUEST_STATE_NEW ||
		 req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT);
	i_assert(req->payload_input == NULL);

	if (conn != NULL)
		http_client_connection_ref(conn);
	http_client_request_ref(req);
	req->payload_wait = TRUE;

	if (data == NULL) {
		req->payload_input = NULL;
		if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT)
			(void)http_client_request_finish_payload_out(req);
	} else {
		req->payload_input = i_stream_create_from_data(data, size);
		i_stream_set_name(req->payload_input, "<HTTP request payload>");
	}
	req->payload_size = 0;
	req->payload_chunked = TRUE;

	if (req->state == HTTP_REQUEST_STATE_NEW)
		http_client_request_submit(req);
	if (req->state == HTTP_REQUEST_STATE_ABORTED) {
		/* Request already failed */
		if (req->delayed_error != NULL) {
			struct http_client_request *tmpreq = req;

			/* Handle delayed error outside ioloop; the caller
			   expects callbacks occurring, so there is no need for
			   delay. Also, it is very important that any error
			   triggers a callback before
			   http_client_request_send_payload() finishes, since
			   its return value is not always checked.
			 */
			http_client_remove_request_error(client, req);
			http_client_request_error_delayed(&tmpreq);
		}
	} else {
		/* Wait for payload data to be written */

		prev_ioloop = current_ioloop;
		client_ioloop = io_loop_create();
		prev_client_ioloop = http_client_switch_ioloop(client);
		if (client->dns_client != NULL)
			dns_client_switch_ioloop(client->dns_client);

		client->waiting = TRUE;
		while (req->state < HTTP_REQUEST_STATE_PAYLOAD_IN) {
			e_debug(req->event, "Waiting for request to finish");

			if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) {
				o_stream_set_flush_pending(
					req->payload_output, TRUE);
			}

			io_loop_run(client_ioloop);

			if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT &&
			    req->payload_input != NULL &&
			    req->payload_input->eof) {
				i_stream_unref(&req->payload_input);
				req->payload_input = NULL;
				break;
			}
		}
		client->waiting = FALSE;

		if (prev_client_ioloop != NULL)
			io_loop_set_current(prev_client_ioloop);
		else
			io_loop_set_current(prev_ioloop);
		(void)http_client_switch_ioloop(client);
		if (client->dns_client != NULL)
			dns_client_switch_ioloop(client->dns_client);
		io_loop_set_current(client_ioloop);
		io_loop_destroy(&client_ioloop);
	}

	switch (req->state) {
	case HTTP_REQUEST_STATE_PAYLOAD_IN:
	case HTTP_REQUEST_STATE_FINISHED:
		ret = 1;
		break;
	case HTTP_REQUEST_STATE_ABORTED:
		ret = -1;
		break;
	default:
		ret = 0;
		break;
	}

	req->payload_wait = FALSE;

	/* callback may have messed with our pointer, so unref using local
	   variable */
	if (!http_client_request_unref(&req))
		*_req = NULL;

	if (conn != NULL)
		http_client_connection_unref(&conn);

	return ret;
}

int http_client_request_send_payload(struct http_client_request **_req,
				     const unsigned char *data, size_t size)
{
	struct http_client_request *req = *_req;
	int ret;

	i_assert(data != NULL);

	ret = http_client_request_continue_payload(&req, data, size);
	if (ret < 0) {
		/* Failed to send payload */
		*_req = NULL;
	} else if (ret > 0) {
		/* Premature end of request;
		   server sent error before all payload could be sent */
		ret = -1;
		*_req = NULL;
	} else {
		/* Not finished sending payload */
		i_assert(req != NULL);
	}
	return ret;
}

int http_client_request_finish_payload(struct http_client_request **_req)
{
	struct http_client_request *req = *_req;
	int ret;

	*_req = NULL;
	ret = http_client_request_continue_payload(&req, NULL, 0);
	i_assert(ret != 0);
	return ret < 0 ? -1 : 0;
}

static void http_client_request_payload_input(struct http_client_request *req)
{
	struct http_client_connection *conn = req->conn;

	io_remove(&conn->io_req_payload);

	(void)http_client_connection_output(conn);
}

int http_client_request_send_more(struct http_client_request *req,
				  bool pipelined)
{
	struct http_client_connection *conn = req->conn;
	struct http_client_context *cctx = conn->ppool->peer->cctx;
	struct ostream *output = req->payload_output;
	enum ostream_send_istream_result res;
	const char *error;
	uoff_t offset;

	if (req->payload_finished)
		return http_client_request_finish_payload_out(req);

	i_assert(req->payload_input != NULL);
	i_assert(req->payload_output != NULL);

	io_remove(&conn->io_req_payload);

	/* Chunked ostream needs to write to the parent stream's buffer */
	offset = req->payload_input->v_offset;
	o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE);
	res = o_stream_send_istream(output, req->payload_input);
	o_stream_set_max_buffer_size(output, SIZE_MAX);

	i_assert(req->payload_input->v_offset >= offset);
	e_debug(req->event, "Send more (sent %"PRIuUOFF_T", buffered=%zu)",
		(uoff_t)(req->payload_input->v_offset - offset),
		o_stream_get_buffer_used_size(output));

	switch (res) {
	case OSTREAM_SEND_ISTREAM_RESULT_FINISHED:
		/* Finished sending */
		if (!req->payload_chunked &&
		    (req->payload_input->v_offset - req->payload_offset) !=
			req->payload_size) {
			error = t_strdup_printf(
				"BUG: stream '%s' input size changed: "
				"%"PRIuUOFF_T"-%"PRIuUOFF_T" != %"PRIuUOFF_T,
				i_stream_get_name(req->payload_input),
				req->payload_input->v_offset,
				req->payload_offset, req->payload_size);
			i_error("%s", error); //FIXME: remove?
			http_client_connection_lost(&conn, error);
			return -1;
		}

		if (req->payload_wait) {
			/* This chunk of input is finished
			   (client needs to act; disable timeout) */
			i_assert(!pipelined);
			conn->output_locked = TRUE;
			http_client_connection_stop_request_timeout(conn);
			if (req->client != NULL && req->client->waiting)
				io_loop_stop(req->client->ioloop);
			return 0;
		}
		/* Finished sending payload */
		return http_client_request_finish_payload_out(req);
	case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT:
		/* Input is blocking (client needs to act; disable timeout) */
		conn->output_locked = TRUE;
		if (!pipelined)
			http_client_connection_stop_request_timeout(conn);
		conn->io_req_payload = io_add_istream_to(
			cctx->ioloop, req->payload_input,
			http_client_request_payload_input, req);
		return 1;
	case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT:
		/* Output is blocking (server needs to act; enable timeout) */
		conn->output_locked = TRUE;
		if (!pipelined)
			http_client_connection_start_request_timeout(conn);
		e_debug(req->event, "Partially sent payload");
		return 0;
	case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT:
		/* We're in the middle of sending a request, so the connection
		   will also have to be aborted */
		error = t_strdup_printf("read(%s) failed: %s",
					i_stream_get_name(req->payload_input),
					i_stream_get_error(req->payload_input));

		/* The payload stream assigned to this request is broken, fail
		   this the request immediately */
		http_client_request_error(&req,
			HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD,
			"Broken payload stream");

		http_client_connection_lost(&conn, error);
		return -1;
	case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT:
		/* Failed to send request */
		http_client_connection_handle_output_error(conn);
		return -1;
	}
	i_unreached();
}

static int
http_client_request_send_real(struct http_client_request *req, bool pipelined)
{
	const struct http_client_settings *set = req->client->set;
	struct http_client_connection *conn = req->conn;
	string_t *rtext = t_str_new(256);
	struct const_iovec iov[3];
	int ret;

	i_assert(!req->conn->output_locked);
	i_assert(req->payload_output == NULL);

	/* Create request line */
	str_append(rtext, req->method);
	str_append(rtext, " ");
	str_append(rtext, req->target);
	str_append(rtext, " HTTP/1.1\r\n");

	/* Create special headers implicitly if not set explicitly using
	   http_client_request_add_header() */
	if (!req->have_hdr_host) {
		str_append(rtext, "Host: ");
		str_append(rtext, req->authority);
		str_append(rtext, "\r\n");
	}
	if (!req->have_hdr_date) {
		str_append(rtext, "Date: ");
		str_append(rtext, http_date_create(req->date));
		str_append(rtext, "\r\n");
	}
	if (!req->have_hdr_authorization &&
		req->username != NULL && req->password != NULL) {
		struct http_auth_credentials auth_creds;

		http_auth_basic_credentials_init(&auth_creds,
			req->username, req->password);

		str_append(rtext, "Authorization: ");
		http_auth_create_credentials(rtext, &auth_creds);
		str_append(rtext, "\r\n");
	}
	if (http_client_request_to_proxy(req) &&
	    set->proxy_username != NULL && set->proxy_username[0] != '\0' &&
	    set->proxy_password != NULL) {
		struct http_auth_credentials auth_creds;

		http_auth_basic_credentials_init(&auth_creds,
			set->proxy_username, set->proxy_password);

		str_append(rtext, "Proxy-Authorization: ");
		http_auth_create_credentials(rtext, &auth_creds);
		str_append(rtext, "\r\n");
	}
	if (!req->have_hdr_user_agent && req->client->set->user_agent != NULL &&
	    req->client->set->user_agent[0] != '\0') {
		str_printfa(rtext, "User-Agent: %s\r\n",
			    req->client->set->user_agent);
	}
	if (!req->have_hdr_expect && req->payload_sync) {
		str_append(rtext, "Expect: 100-continue\r\n");
	}
	if (req->payload_input != NULL && req->payload_chunked) {
		// FIXME: can't do this for a HTTP/1.0 server
		if (!req->have_hdr_body_spec)
			str_append(rtext, "Transfer-Encoding: chunked\r\n");
		req->payload_output =
			http_transfer_chunked_ostream_create(conn->conn.output);
		o_stream_set_finish_also_parent(req->payload_output, FALSE);
	} else if (req->payload_input != NULL ||
		req->payload_empty ||
		strcasecmp(req->method, "POST") == 0 ||
		strcasecmp(req->method, "PUT") == 0) {

		/* Send Content-Length if we have specified a payload or when
		   one is normally expected, even if it's 0 bytes. */
		i_assert(req->payload_input != NULL || req->payload_size == 0);
		if (!req->have_hdr_body_spec) {
			str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n",
				req->payload_size);
		}
		if (req->payload_input != NULL) {
			req->payload_output = conn->conn.output;
			o_stream_ref(conn->conn.output);
		}
	}
	if (!req->have_hdr_connection &&
		!http_client_request_to_proxy(req)) {
		/* RFC 2068, Section 19.7.1:

		   A client MUST NOT send the Keep-Alive connection token to a
		   proxy server as HTTP/1.0 proxy servers do not obey the rules
		   of HTTP/1.1 for parsing the Connection header field.
		 */
		str_append(rtext, "Connection: Keep-Alive\r\n");
	}

	/* Request line + implicit headers */
	iov[0].iov_base = str_data(rtext);
	iov[0].iov_len = str_len(rtext);
	/* Explicit headers */
	if (req->headers != NULL) {
		iov[1].iov_base = str_data(req->headers);
		iov[1].iov_len = str_len(req->headers);
	} else {
		iov[1].iov_base = "";
		iov[1].iov_len = 0;
	}
	/* End of header */
	iov[2].iov_base = "\r\n";
	iov[2].iov_len = 2;

	req->state = HTTP_REQUEST_STATE_PAYLOAD_OUT;
	req->payload_finished = FALSE;

	req->send_attempts++;
	if (req->first_sent_time.tv_sec == 0)
		req->first_sent_time = ioloop_timeval;
	req->sent_time = ioloop_timeval;
	req->sent_http_ioloop_usecs =
		io_wait_timer_get_usecs(req->conn->io_wait_timer);

	ret = 1;
	o_stream_cork(conn->conn.output);
	req->request_offset = conn->conn.output->offset;

	if (o_stream_sendv(conn->conn.output, iov, N_ELEMENTS(iov)) < 0) {
		http_client_connection_handle_output_error(conn);
		return -1;
	}

	e_debug(req->event, "Sent header");

	if (req->payload_output != NULL) {
		if (!req->payload_sync) {
			ret = http_client_request_send_more(req, pipelined);
			if (ret < 0)
				return -1;
		} else {
			e_debug(req->event, "Waiting for 100-continue");
			conn->output_locked = TRUE;
		}
	} else {
		req->state = HTTP_REQUEST_STATE_WAITING;
		if (!pipelined)
			http_client_connection_start_request_timeout(req->conn);
		conn->output_locked = FALSE;
	}
	if (conn->conn.output != NULL) {
		i_assert(req->request_offset < conn->conn.output->offset);
		req->bytes_out = conn->conn.output->offset - req->request_offset;
		if (o_stream_uncork_flush(conn->conn.output) < 0) {
			http_client_connection_handle_output_error(conn);
			return -1;
		}
	}
	return ret;
}

int http_client_request_send(struct http_client_request *req, bool pipelined)
{
	int ret;

	T_BEGIN {
		ret = http_client_request_send_real(req, pipelined);
	} T_END;

	return ret;
}

static const char *
http_client_request_add_event_headers(struct http_client_request *req,
				      const struct http_response *response)
{
	if (req->event_headers == NULL)
		return "";

	string_t *str = t_str_new(128);
	for (unsigned int i = 0; req->event_headers[i] != NULL; i++) {
		const char *hdr_name = req->event_headers[i];
		const char *value =
			http_response_header_get(response, hdr_name);

		if (value == NULL)
			continue;

		str_append(str, str_len(str) == 0 ? " (" : ", ");
		event_add_str(req->event,
			      t_strconcat("http_hdr_", hdr_name, NULL), value);
		str_printfa(str, "%s:%s", hdr_name, value);
	}
	if (str_len(str) > 0)
		str_append_c(str, ')');
	return str_c(str);
}

static int
http_client_request_1xx_response(struct http_client_request *req,
				 struct http_response *resp)
{
	struct http_client_connection *conn = req->conn;

	/* RFC 7231, Section 6.2:

	   A client MUST be able to parse one or more 1xx responses received
	   prior to a final response, even if the client does not expect one.
	   A user agent MAY ignore unexpected 1xx responses.
	 */

	if (req->payload_sync && resp->status == 100) {
		struct http_client_peer_shared *pshared = conn->ppool->peer;

		if (req->payload_sync_continue) {
			e_debug(req->event,
				"Got 100-continue response after timeout");
			return 0;
		}

		pshared->no_payload_sync = FALSE;
		pshared->seen_100_response = TRUE;
		req->payload_sync_continue = TRUE;

		e_debug(req->event,
			"Got expected 100-continue response");

		if (req->state == HTTP_REQUEST_STATE_ABORTED) {
			e_debug(req->event,
				"Request aborted before sending payload was complete.");
			http_client_connection_close(&conn);
			return -1;
		}

		i_assert(conn->output_locked);
		if (conn->conn.output != NULL)
			o_stream_set_flush_pending(conn->conn.output, TRUE);
		return -1;
	}

	/* Ignore other 1xx for now */
	e_debug(req->event,
		"Got unexpected %u response; ignoring", resp->status);
	return 0;
}

int http_client_request_check_response(struct http_client_request *req,
				       struct http_response *resp,
				       bool *early_r)
{
	struct http_client_connection *conn = req->conn;

	if (resp->status / 100 == 1)
		return http_client_request_1xx_response(req, resp);
	if (!req->payload_sync && !req->payload_finished &&
	    req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT) {
		/* Got early response from server while we're still sending
		   request payload. we cannot recover from this reliably, so we
		   stop sending payload and close the connection once the
		   response is processed */
		e_debug(req->event,
			"Got early input from server; "
			"request payload not completely sent "
			"(will close connection)");
		o_stream_unset_flush_callback(conn->conn.output);
		conn->output_broken = *early_r = TRUE;
	}

	const char *suffix =
		http_client_request_add_event_headers(req, resp);
	e_debug(req->event,
		"Got %u response: %s%s (took %lld ms + %lld ms in queue)",
		resp->status, resp->reason, suffix,
		timeval_diff_msecs(&req->response_time, &req->sent_time),
		timeval_diff_msecs(&req->sent_time, &req->submit_time));

	/* Make sure connection output is unlocked if 100-continue failed */
	if (req->payload_sync && !req->payload_sync_continue &&
	    array_count(&conn->request_wait_list) == 1) {
		e_debug(req->event, "Unlocked output");
		conn->output_locked = FALSE;
	}
	return 1;
}

bool http_client_request_callback(struct http_client_request *req,
				  struct http_response *response)
{
	http_client_request_callback_t *callback = req->callback;
	unsigned int orig_attempts = req->attempts;

	i_assert(req->state >= HTTP_REQUEST_STATE_PAYLOAD_OUT);
	i_assert(req->conn != NULL);
	if (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT)
		req->conn->output_locked = FALSE;
	req->state = HTTP_REQUEST_STATE_GOT_RESPONSE;
	req->last_status = response->status;

	req->callback = NULL;
	if (callback != NULL) {
		struct http_response response_copy = *response;

		if (req->attempts > 0 && !req->preserve_exact_reason) {
			long long total_msecs =
				timeval_diff_msecs(&ioloop_timeval,
						   &req->submit_time);
			response_copy.reason = t_strdup_printf(
				"%s (%u retries in %lld.%03lld secs)",
				response_copy.reason, req->attempts,
				total_msecs/1000, total_msecs%1000);
		}

		callback(&response_copy, req->context);
		if (req->attempts != orig_attempts) {
			/* Retrying */
			req->callback = callback;
			http_client_request_resubmit(req);
			return FALSE;
		} else {
			/* Release payload early
			   (prevents server/client deadlock in proxy) */
			i_stream_unref(&req->payload_input);
		}
	}
	return TRUE;
}

static bool
http_client_request_send_error(struct http_client_request *req,
			       unsigned int status, const char *error)
{
	http_client_request_callback_t *callback;
	bool sending = (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT);
	unsigned int orig_attempts = req->attempts;

	req->state = HTTP_REQUEST_STATE_ABORTED;

	callback = req->callback;
	req->callback = NULL;
	if (callback != NULL) {
		struct http_response response;

		http_response_init(&response, status, error);
		(void)callback(&response, req->context);

		if (req->attempts != orig_attempts) {
			/* Retrying */
			req->callback = callback;
			http_client_request_resubmit(req);
			return FALSE;
		} else {
			/* Release payload early
			   (prevents server/client deadlock in proxy) */
			if (!sending && req->payload_input != NULL)
				i_stream_unref(&req->payload_input);
		}
	}
	if (req->payload_wait) {
		i_assert(req->client != NULL);
		io_loop_stop(req->client->ioloop);
	}
	return TRUE;
}

void http_client_request_error_delayed(struct http_client_request **_req)
{
	struct http_client_request *req = *_req;
	const char *error = req->delayed_error;
	unsigned int status = req->delayed_error_status;
	bool destroy;

	i_assert(req->state == HTTP_REQUEST_STATE_ABORTED);

	*_req = NULL;
	req->delayed_error = NULL;
	req->delayed_error_status = 0;

	i_assert(error != NULL && status != 0);
	destroy = http_client_request_send_error(req, status, error);
	if (req->queue != NULL)
		http_client_queue_drop_request(req->queue, req);
	if (destroy)
		http_client_request_destroy(&req);
}

void http_client_request_error(struct http_client_request **_req,
			       unsigned int status, const char *error)
{
	struct http_client_request *req = *_req;

	*_req = NULL;

	if (req->state >= HTTP_REQUEST_STATE_ABORTED)
		return;
	i_assert(req->delayed_error_status == 0);
	i_assert(req->state < HTTP_REQUEST_STATE_FINISHED);

	req->state = HTTP_REQUEST_STATE_ABORTED;
	req->last_status = status;

	e_debug(http_client_request_result_event(req)->
			set_name("http_request_finished")->event(),
		"Error: %u %s", status, error);

	if (req->queue != NULL)
		http_client_queue_drop_request(req->queue, req);

	if (req->client != NULL &&
	    (!req->submitted ||
	     req->state == HTTP_REQUEST_STATE_GOT_RESPONSE)) {
		/* We're still in http_client_request_submit() or in the
		   callback during a retry attempt. delay reporting the error,
		   so the caller doesn't have to handle immediate or nested
		   callbacks. */
		req->delayed_error = p_strdup(req->pool, error);
		req->delayed_error_status = status;
		http_client_delay_request_error(req->client, req);
	} else {
		if (http_client_request_send_error(req, status, error))
			http_client_request_destroy(&req);
	}
}

void http_client_request_abort(struct http_client_request **_req)
{
	struct http_client_request *req = *_req;
	bool sending;

	if (req == NULL)
		return;

	sending = (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT);

	*_req = NULL;

	if (req->state >= HTTP_REQUEST_STATE_FINISHED &&
	    req->delayed_error_status == 0)
		return;

	req->callback = NULL;
	req->state = HTTP_REQUEST_STATE_ABORTED;
	if (req->last_status == 0)
		req->last_status = HTTP_CLIENT_REQUEST_ERROR_ABORTED;

	if (req->state > HTTP_REQUEST_STATE_NEW &&
	    req->delayed_error_status == 0) {
		e_debug(http_client_request_result_event(req)->
			set_name("http_request_finished")->event(),
			"Aborted");
	}

	/* Release payload early (prevents server/client deadlock in proxy) */
	if (!sending && req->payload_input != NULL)
		i_stream_unref(&req->payload_input);

	if (req->queue != NULL)
		http_client_queue_drop_request(req->queue, req);
	if (req->payload_wait) {
		i_assert(req->client != NULL);
		i_assert(req->client->ioloop != NULL);
		io_loop_stop(req->client->ioloop);
	}
	http_client_request_destroy(&req);
}

void http_client_request_finish(struct http_client_request *req)
{
	if (req->state >= HTTP_REQUEST_STATE_FINISHED)
		return;

	i_assert(req->refcount > 0);

	e_debug(http_client_request_result_event(req)->
		set_name("http_request_finished")->event(),
		"Finished");

	req->callback = NULL;
	req->state = HTTP_REQUEST_STATE_FINISHED;

	if (req->queue != NULL)
		http_client_queue_drop_request(req->queue, req);
	if (req->payload_wait) {
		i_assert(req->client != NULL);
		i_assert(req->client->ioloop != NULL);
		io_loop_stop(req->client->ioloop);
	}
	http_client_request_unref(&req);
}

static int
http_client_request_reset(struct http_client_request *req, bool rewind,
			  const char **error_r)
{
	/* Rewind payload stream */
	if (rewind && req->payload_input != NULL && req->payload_size > 0) {
		if (req->payload_input->v_offset != req->payload_offset &&
		    !req->payload_input->seekable) {
			*error_r = "Cannot resend payload; "
				   "stream is not seekable";
			return -1;
		}
		i_stream_seek(req->payload_input, req->payload_offset);
	}

	/* Drop payload output stream from previous attempt */
	o_stream_unref(&req->payload_output);

	/* Reset payload state */
	req->payload_finished = FALSE;

	return 0;
}

void http_client_request_redirect(struct http_client_request *req,
				  unsigned int status, const char *location)
{
	struct http_url *url;
	const char *error, *target, *origin_url;

	i_assert(req->client != NULL);
	i_assert(!req->payload_wait);

	req->last_status = status;

	/* parse URL */
	if (http_url_parse(location, NULL, 0,
			   pool_datastack_create(), &url, &error) < 0) {
		http_client_request_error(
			&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
			t_strdup_printf("Invalid redirect location: %s",
					error));
		return;
	}

	i_assert(req->redirects <= req->client->set->request_max_redirects);
	if (++req->redirects > req->client->set->request_max_redirects) {
		if (req->client->set->request_max_redirects > 0) {
			http_client_request_error(
				&req,
				HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
				t_strdup_printf(
					"Redirected more than %d times",
					req->client->set->request_max_redirects));
		} else {
			http_client_request_error(
				&req,
				HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
				"Redirect refused");
		}
		return;
	}

	if (http_client_request_reset(req, (status != 303), &error) < 0) {
		http_client_request_error(
			&req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
			t_strdup_printf("Redirect failed: %s", error));
		return;
	}

	target = http_url_create_target(url);

	http_url_copy(req->pool, &req->origin_url, url);
	req->target = p_strdup(req->pool, target);

	req->host = NULL;

	origin_url = http_url_create(&req->origin_url);

	e_debug(http_client_request_result_event(req)->
			set_name("http_request_redirected")->event(),
		"Redirecting to %s%s (redirects=%u)",
		origin_url, target, req->redirects);

	req->label = p_strdup_printf(req->pool, "[%s %s%s]",
				     req->method, origin_url, req->target);

	/* RFC 7231, Section 6.4.4:

	   -> A 303 `See Other' redirect status response is handled a bit
	   differently. Basically, the response content is located elsewhere,
	   but the original (POST) request is handled already.
	 */
	if (status == 303 && strcasecmp(req->method, "HEAD") != 0 &&
	    strcasecmp(req->method, "GET") != 0) {
		// FIXME: should we provide the means to skip this step? The
		// original request was already handled at this point.
		req->method = p_strdup(req->pool, "GET");

		/* drop payload */
		i_stream_unref(&req->payload_input);
		req->payload_size = 0;
		req->payload_offset = 0;
	}

	/* Resubmit */
	req->state = HTTP_REQUEST_STATE_NEW;
	http_client_request_do_submit(req);
}

void http_client_request_resubmit(struct http_client_request *req)
{
	const char *error;

	i_assert(!req->payload_wait);

	e_debug(req->event, "Resubmitting request");

	if (http_client_request_reset(req, TRUE, &error) < 0) {
		http_client_request_error(
			&req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
			t_strdup_printf("Resubmission failed: %s", error));
		return;
	}

	req->peer = NULL;
	req->state = HTTP_REQUEST_STATE_QUEUED;
	req->redirects = 0;
	req->last_status = 0;
	http_client_host_submit_request(req->host, req);
}

void http_client_request_retry(struct http_client_request *req,
			       unsigned int status, const char *error)
{
	if (req->client == NULL || !req->client->set->auto_retry ||
	    !http_client_request_try_retry(req))
		http_client_request_error(&req, status, error);
}

bool http_client_request_try_retry(struct http_client_request *req)
{
	/* Don't ever retry if we're sending data in small blocks via
	   http_client_request_send_payload() and we're not waiting for a
	   100 continue (there's no way to rewind the payload for a retry)
	 */
	if (req->payload_wait &&
	    (!req->payload_sync || req->payload_sync_continue))
		return FALSE;
	/* Limit the number of attempts for each request */
	if (req->attempts+1 >= req->max_attempts)
		return FALSE;
	req->attempts++;

	e_debug(http_client_request_result_event(req)->
		set_name("http_request_retried")->event(),
		"Retrying (attempts=%d)", req->attempts);

	if (req->callback != NULL)
		http_client_request_resubmit(req);
	return TRUE;
}

#undef http_client_request_set_destroy_callback
void http_client_request_set_destroy_callback(struct http_client_request *req,
					      void (*callback)(void *),
					      void *context)
{
	req->destroy_callback = callback;
	req->destroy_context = context;
}

void http_client_request_start_tunnel(struct http_client_request *req,
				      struct http_client_tunnel *tunnel)
{
	struct http_client_connection *conn = req->conn;

	i_assert(req->state == HTTP_REQUEST_STATE_GOT_RESPONSE);

	http_client_connection_start_tunnel(&conn, tunnel);
}