1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/src/gt_http_client.c	Sat Feb 20 21:18:28 2010 -0800
     1.3 @@ -0,0 +1,909 @@
     1.4 +/*
     1.5 + * $Id: gt_http_client.c,v 1.57 2006/02/03 20:11:40 mkern Exp $
     1.6 + *
     1.7 + * Copyright (C) 2001-2003 giFT project (gift.sourceforge.net)
     1.8 + *
     1.9 + * This program is free software; you can redistribute it and/or modify it
    1.10 + * under the terms of the GNU General Public License as published by the
    1.11 + * Free Software Foundation; either version 2, or (at your option) any
    1.12 + * later version.
    1.13 + *
    1.14 + * This program is distributed in the hope that it will be useful, but
    1.15 + * WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    1.17 + * General Public License for more details.
    1.18 + */
    1.19 +
    1.20 +#include "gt_gnutella.h"
    1.21 +#include "gt_version.h"
    1.22 +
    1.23 +#include "gt_xfer_obj.h"
    1.24 +#include "gt_xfer.h"
    1.25 +
    1.26 +#include "gt_http_client.h"
    1.27 +#include "gt_http_server.h"
    1.28 +
    1.29 +#include "gt_accept.h"
    1.30 +
    1.31 +#include "transfer/source.h"
    1.32 +
    1.33 +/*****************************************************************************/
    1.34 +
    1.35 +/* prototyping this function effectively provides the non-blocking flow of the
    1.36 + * program and helps to self-document this file */
    1.37 +static void get_server_reply     (int fd, input_id id, GtTransfer *xfer);
    1.38 +
    1.39 +/*****************************************************************************/
    1.40 +/* CLIENT HELPERS */
    1.41 +
    1.42 +static int gt_http_client_send (TCPC *c, char *command, char *request, ...)
    1.43 +{
    1.44 +	char        *key;
    1.45 +	char        *value;
    1.46 +	String      *s;
    1.47 +	int          ret;
    1.48 +	va_list      args;
    1.49 +
    1.50 +	if (!command || !request)
    1.51 +		return -1;
    1.52 +
    1.53 +	if (!(s = string_new (NULL, 0, 0, TRUE)))
    1.54 +		return -1;
    1.55 +
    1.56 +	string_appendf (s, "%s %s HTTP/1.1\r\n", command, request);
    1.57 +
    1.58 +	va_start (args, request);
    1.59 +
    1.60 +	for (;;)
    1.61 +	{
    1.62 +		/* if we receive the sentinel, bail out */
    1.63 +		if (!(key = va_arg (args, char *)))
    1.64 +			break;
    1.65 +
    1.66 +		if (!(value = va_arg (args, char *)))
    1.67 +			continue;
    1.68 +
    1.69 +		string_appendf (s, "%s: %s\r\n", key, value);
    1.70 +	}
    1.71 +
    1.72 +	va_end (args);
    1.73 +
    1.74 +	/* append final message terminator */
    1.75 +	string_append (s, "\r\n");
    1.76 +
    1.77 +	if (HTTP_DEBUG)
    1.78 +		GT->DBGSOCK (GT, c, "sending client request:\n%s", s->str);
    1.79 +
    1.80 +	ret = tcp_send (c, s->str, s->len);
    1.81 +	string_free (s);
    1.82 +
    1.83 +	return ret;
    1.84 +}
    1.85 +
    1.86 +/* parse an HTTP server reply */
    1.87 +static BOOL parse_server_reply (GtTransfer *xfer, TCPC *c, char *reply)
    1.88 +{
    1.89 +	char  *response; /* HTTP/1.1 200 OK */
    1.90 +	char  *version;
    1.91 +	int    code;     /* 200, 404, ... */
    1.92 +
    1.93 +	if (!xfer || !reply)
    1.94 +		return FALSE;
    1.95 +
    1.96 +	if (HTTP_DEBUG)
    1.97 +		GT->DBGSOCK (GT, c, "reply:\n%s", reply);
    1.98 +
    1.99 +	response = string_sep_set (&reply, "\r\n");
   1.100 +
   1.101 +	if (!response)
   1.102 +		return FALSE;
   1.103 +
   1.104 +	version =       string_sep (&response, " ");  /* shift past HTTP/1.1 */
   1.105 +	code    = ATOI (string_sep (&response, " ")); /* shift past 200 */
   1.106 +
   1.107 +	/* parse the rest of the key/value fields */
   1.108 +	gt_http_header_parse (reply, &xfer->header);
   1.109 +
   1.110 +	xfer->code    = code;
   1.111 +	xfer->version = STRDUP (version);
   1.112 +
   1.113 +	return TRUE;
   1.114 +}
   1.115 +
   1.116 +/*****************************************************************************/
   1.117 +
   1.118 +/*
   1.119 + * Complete interface to the standard HTTP GET / with a server.  These routines
   1.120 + * require non of OpenFT's extensions and communicate perfectly valid
   1.121 + * HTTP/1.1 (to my knowledge).  You could use this to transfer non-OpenFT,
   1.122 + * hopefully. ;)
   1.123 + *
   1.124 + * NOTE:
   1.125 + * I need to add more text here so this stands out better as one of the two
   1.126 + * core subsystems within this file.  So here it is. :P
   1.127 + */
   1.128 +void gt_http_client_get (Chunk *chunk, GtTransfer *xfer)
   1.129 +{
   1.130 +	TCPC *c;
   1.131 +
   1.132 +	if (!chunk || !xfer)
   1.133 +	{
   1.134 +		GT->DBGFN (GT, "uhm.");
   1.135 +		return;
   1.136 +	}
   1.137 +
   1.138 +	xfer->command = STRDUP ("GET");
   1.139 +
   1.140 +	if (!(c = gt_http_connection_open (GT_TRANSFER_DOWNLOAD, xfer->ip,
   1.141 +	                                   xfer->port)))
   1.142 +	{
   1.143 +		gt_transfer_close (xfer, TRUE);
   1.144 +		return;
   1.145 +	}
   1.146 +
   1.147 +	/* pass along the connection with the xfer */
   1.148 +	gt_transfer_set_tcpc (xfer, c);
   1.149 +	assert (xfer->chunk == chunk);
   1.150 +	assert (chunk->udata == xfer);
   1.151 +
   1.152 +	gt_transfer_status (xfer, SOURCE_WAITING, "Connecting");
   1.153 +
   1.154 +	/* be a little more aggressive timing out HTTP connections (TIMEOUT_DEF /
   1.155 +	 * 2 + 5), so that useless sources don't occupy Chunks so often */
   1.156 +	input_add (c->fd, xfer, INPUT_WRITE,
   1.157 +	           (InputCallback)gt_http_client_start, TIMEOUT_DEF / 2 + 5);
   1.158 +}
   1.159 +
   1.160 +static int client_get_request (GtTransfer *xfer)
   1.161 +{
   1.162 +	TCPC  *c;
   1.163 +	char   host[128];
   1.164 +	char   range_hdr[64];
   1.165 +	int    ret;
   1.166 +
   1.167 +	if (!xfer)
   1.168 +		return FALSE;
   1.169 +
   1.170 +	c = gt_transfer_get_tcpc (xfer);
   1.171 +
   1.172 +	snprintf (range_hdr, sizeof (range_hdr) - 1, "bytes=%i-%i",
   1.173 +	          (int) xfer->start, (int) xfer->stop - 1);
   1.174 +
   1.175 +	snprintf (host, sizeof (host) - 1, "%s:%hu", net_ip_str (xfer->ip),
   1.176 +	          xfer->port);
   1.177 +
   1.178 +	/* always send the Range request just because we always know the full size
   1.179 +	 * we want */
   1.180 +	ret = gt_http_client_send (c, "GET",        xfer->request,
   1.181 +	                           "Range",         range_hdr,
   1.182 +	                           "Host",          host,
   1.183 +	                           "User-Agent",    gt_version(),
   1.184 +	                           "X-Queue",       "0.1",
   1.185 +	                           NULL);
   1.186 +
   1.187 +	return ret;
   1.188 +}
   1.189 +
   1.190 +/*
   1.191 + * Verify connection status and Send the GET request to the server.
   1.192 + */
   1.193 +void gt_http_client_start (int fd, input_id id, GtTransfer *xfer)
   1.194 +{
   1.195 +	Chunk    *chunk;
   1.196 +	TCPC     *c;
   1.197 +
   1.198 +	c     = gt_transfer_get_tcpc  (xfer);
   1.199 +	chunk = gt_transfer_get_chunk (xfer);
   1.200 +
   1.201 +	if (net_sock_error (c->fd))
   1.202 +	{
   1.203 +		GtSource *gt;
   1.204 +
   1.205 +		gt = gt_transfer_get_source (xfer);
   1.206 +
   1.207 +		/* set the connection as having failed, and retry w/ a push */
   1.208 +		gt->connect_failed = TRUE;
   1.209 +
   1.210 +		gt_transfer_status (xfer, SOURCE_CANCELLED, (fd == -1 ?
   1.211 +		                                             "Connect timeout" :
   1.212 +		                                             "Connect failed"));
   1.213 +		gt_transfer_close (xfer, TRUE);
   1.214 +		return;
   1.215 +	}
   1.216 +
   1.217 +	/*
   1.218 +	 * Update the length of the chunk in the GtTransfer. We do this
   1.219 +	 * because giftd may change the range of the chunk while we are
   1.220 +	 * playing with it, and this is the last point where we can update
   1.221 +	 * the range.
   1.222 +	 *
   1.223 +	 * If giftd changes the range after this point, we'll be forced to break
   1.224 +	 * this connection :(
   1.225 +	 */
   1.226 +	gt_transfer_set_length (xfer, chunk);
   1.227 +
   1.228 +	/* send the GET / request to the server */
   1.229 +	if (client_get_request (xfer) <= 0)
   1.230 +	{
   1.231 +		gt_transfer_status (xfer, SOURCE_CANCELLED, "GET send failed");
   1.232 +		gt_transfer_close (xfer, TRUE);
   1.233 +		return;
   1.234 +	}
   1.235 +
   1.236 +	gt_transfer_status (xfer, SOURCE_WAITING, "Sent HTTP request");
   1.237 +
   1.238 +	/* do not remove all fds associated with this socket until we destroy it */
   1.239 +	input_remove (id);
   1.240 +
   1.241 +	/* wait for the server response */
   1.242 +	input_add (fd, xfer, INPUT_READ,
   1.243 +			   (InputCallback)get_server_reply, TIMEOUT_DEF);
   1.244 +}
   1.245 +
   1.246 +/*
   1.247 + * Read the response body, if any, so persistent HTTP will work.
   1.248 + * Note that the GtTransfer can still timeout, in which case
   1.249 + * the connection will get closed and so will the xfer.
   1.250 + */
   1.251 +static void read_response_body (int fd, input_id id, GtTransfer *xfer)
   1.252 +{
   1.253 +	Chunk       *chunk;
   1.254 +	TCPC        *c;
   1.255 +	FDBuf       *buf;
   1.256 +	char        *response;
   1.257 +	int          n;
   1.258 +	int          len;
   1.259 +
   1.260 +	c     = gt_transfer_get_tcpc  (xfer);
   1.261 +	chunk = gt_transfer_get_chunk (xfer);
   1.262 +
   1.263 +	len = xfer->stop - xfer->start;
   1.264 +
   1.265 +	/* since the body isnt important, close if its too large */
   1.266 +	if (len >= 16384)
   1.267 +	{
   1.268 +		GT->DBGFN (GT, "[%s:%hu] response body too large (%d)",
   1.269 +		           net_ip_str (xfer->ip), xfer->port);
   1.270 +		gt_transfer_close (xfer, TRUE);
   1.271 +		return;
   1.272 +	}
   1.273 +
   1.274 +	buf = tcp_readbuf (c);
   1.275 +
   1.276 +	if ((n = fdbuf_fill (buf, len)) < 0)
   1.277 +	{
   1.278 +		GT->DBGFN (GT, "error [%s:%hu]: %s",
   1.279 +		           net_ip_str (xfer->ip), xfer->port, GIFT_NETERROR ());
   1.280 +		gt_transfer_close (xfer, TRUE);
   1.281 +		return;
   1.282 +	}
   1.283 +
   1.284 +	if (n > 0)
   1.285 +		return;
   1.286 +
   1.287 +	/*
   1.288 +	 * Set the body as having been completely read.
   1.289 +	 * This allows the connection to be cached.
   1.290 +	 */
   1.291 +	xfer->remaining_len -= len;
   1.292 +	assert (xfer->remaining_len == 0);
   1.293 +
   1.294 +	response = fdbuf_data (buf, NULL);
   1.295 +	fdbuf_release (buf);
   1.296 +
   1.297 +	if (HTTP_DEBUG)
   1.298 +		GT->DBGSOCK (GT, c, "body:\n%s", response);
   1.299 +
   1.300 +	input_remove (id);
   1.301 +
   1.302 +	/* perform an orderly close */
   1.303 +	gt_transfer_close (xfer, FALSE);
   1.304 +}
   1.305 +
   1.306 +static time_t queue_interval (time_t interval)
   1.307 +{
   1.308 +	/*
   1.309 +	 * HACK: giftd will retry the queued download every 49 seconds,
   1.310 +	 * so round the next retry time to coincide with that interval.
   1.311 +	 */
   1.312 +	if (interval > 49)
   1.313 +		interval = (interval / 49 + 1) * 49;
   1.314 +
   1.315 +	return interval;
   1.316 +}
   1.317 +
   1.318 +/*****************************************************************************/
   1.319 +
   1.320 +/* set the next time to retry from the Retry-After: header */
   1.321 +static void set_retry_after (GtTransfer *xfer)
   1.322 +{
   1.323 +	int          seconds;
   1.324 +	char        *retry_after;
   1.325 +	GtSource    *gt;
   1.326 +#if 0
   1.327 +	char        *msg;
   1.328 +	struct tm   *tm;
   1.329 +#endif
   1.330 +
   1.331 +	if (!(retry_after = dataset_lookupstr (xfer->header, "retry-after")))
   1.332 +		return;
   1.333 +
   1.334 +	/*
   1.335 +	 * This can be either a HTTP date or a number of seconds. We only
   1.336 +	 * support the number of seconds right now.
   1.337 +	 */
   1.338 +	seconds = ATOI (retry_after);
   1.339 +
   1.340 +	if (seconds <= 0)
   1.341 +		return;
   1.342 +
   1.343 +	if (!(gt = gt_transfer_get_source (xfer)))
   1.344 +		return;
   1.345 +
   1.346 +	/* set the retry time for the next download */
   1.347 +	gt->retry_time = time (NULL) + queue_interval (seconds);
   1.348 +
   1.349 +#if 0
   1.350 +	/* get the absolute time */
   1.351 +	tm = localtime (&gt->retry_time);
   1.352 +
   1.353 +	/* let the user know when we are going to retry */
   1.354 +	msg = stringf_dup ("Queued (retry at %d:%02d:%02d)", tm->tm_hour,
   1.355 +	                   tm->tm_min, tm->tm_sec);
   1.356 +
   1.357 +	gt_transfer_status (xfer, SOURCE_QUEUED_REMOTE, msg);
   1.358 +	free (msg);
   1.359 +#endif
   1.360 +}
   1.361 +
   1.362 +/*
   1.363 + * Check for both the active-queueing style "X-Queue:" and PARQ-style
   1.364 + * "X-Queued:".
   1.365 + *
   1.366 + * We avoid having to parse the X-Queue: line in the PARQ case (which would
   1.367 + * be "1.0") by allowing X-Queued to override X-Queue.
   1.368 + */
   1.369 +static size_t find_queue_key (Dataset *header, const char *key)
   1.370 +{
   1.371 +	size_t  pos = 0;
   1.372 +	char   *val;
   1.373 +	char   *line0, *line;
   1.374 +	char   *active_queue_line;
   1.375 +	char   *parq_queue_line;
   1.376 +	char   *sep;
   1.377 +
   1.378 +	active_queue_line = dataset_lookupstr (header, "x-queue");
   1.379 +	parq_queue_line   = dataset_lookupstr (header, "x-queued");
   1.380 +
   1.381 +	if (!active_queue_line && !parq_queue_line)
   1.382 +		return 0;
   1.383 +
   1.384 +	line = active_queue_line;
   1.385 +	sep  = ", ";
   1.386 +
   1.387 +	if (parq_queue_line)
   1.388 +	{
   1.389 +		line = parq_queue_line;
   1.390 +		sep  = "; ";
   1.391 +	}
   1.392 +
   1.393 +	line0 = line = STRDUP (line);
   1.394 +
   1.395 +	while ((val = string_sep_set (&line, sep)))
   1.396 +	{
   1.397 +		char *str;
   1.398 +
   1.399 +		str = string_sep_set (&val, "= ");
   1.400 +
   1.401 +		if (!str || !val)
   1.402 +			continue;
   1.403 +
   1.404 +		if (!strcasecmp (str, key))
   1.405 +			pos = ATOI (val);
   1.406 +	}
   1.407 +
   1.408 +	free (line0);
   1.409 +	return pos;
   1.410 +}
   1.411 +
   1.412 +/* Create a message describing our position in the remote node's
   1.413 + * upload queue. */
   1.414 +static char *get_queue_status (GtTransfer *xfer, char *msg)
   1.415 +{
   1.416 +	size_t  len   = 0;
   1.417 +	size_t  pos   = 0;
   1.418 +
   1.419 +	pos = find_queue_key (xfer->header, "position");
   1.420 +	len = find_queue_key (xfer->header, "length");
   1.421 +
   1.422 +	msg = STRDUP (msg);
   1.423 +
   1.424 +	if (pos != 0)
   1.425 +	{
   1.426 +		free (msg);
   1.427 +
   1.428 +		if (len != 0)
   1.429 +			msg = stringf_dup ("Queued (%u/%u)", pos, len);
   1.430 +		else
   1.431 +			msg = stringf_dup ("Queued (position %u)", pos);
   1.432 +	}
   1.433 +
   1.434 +	return msg;
   1.435 +}
   1.436 +
   1.437 +/* set the next time to retry the download based on the X-Queue: field */
   1.438 +static void set_queue_status (GtTransfer *xfer)
   1.439 +{
   1.440 +	GtSource    *gt;
   1.441 +	char        *queue_line;
   1.442 +	int          poll_min;
   1.443 +
   1.444 +	if (!(gt = gt_transfer_get_source (xfer)))
   1.445 +		return;
   1.446 +
   1.447 +	if (!(queue_line = dataset_lookupstr (xfer->header, "x-queue")))
   1.448 +		return;
   1.449 +
   1.450 +	if ((poll_min = find_queue_key (xfer->header, "pollmin")) <= 0)
   1.451 +		return;
   1.452 +
   1.453 +	gt->retry_time = time (NULL) + queue_interval (poll_min);
   1.454 +}
   1.455 +
   1.456 +/*
   1.457 + * Try to read any content-body in the response, so that persistent
   1.458 + * HTTP can work correctly, which is really important for push downloads.
   1.459 + */
   1.460 +static void handle_http_error (GtTransfer *xfer, SourceStatus status,
   1.461 +                               char *status_txt)
   1.462 +{
   1.463 +	TCPC    *c;
   1.464 +	Chunk   *chunk;
   1.465 +	int      len         = 0;
   1.466 +	char    *content_len;
   1.467 +	char    *conn_hdr;
   1.468 +
   1.469 +	/* update the interface protocol with the error status of this xfer */
   1.470 +	status_txt = get_queue_status (xfer, status_txt);
   1.471 +	gt_transfer_status (xfer, status, status_txt);
   1.472 +
   1.473 +	free (status_txt);
   1.474 +
   1.475 +	c     = gt_transfer_get_tcpc  (xfer);
   1.476 +	chunk = gt_transfer_get_chunk (xfer);
   1.477 +
   1.478 +	/*
   1.479 +	 * Check for a Content-Length: field, and use that for the
   1.480 +	 * length of the response body, if any.
   1.481 +	 */
   1.482 +	content_len = dataset_lookupstr (xfer->header, "content-length");
   1.483 +	conn_hdr    = dataset_lookupstr (xfer->header, "connection");
   1.484 +
   1.485 +	/* look at the Retry-After: header and set download retry time */
   1.486 +	set_retry_after (xfer);
   1.487 +
   1.488 +	/* parse the X-Queue: and X-Queued: headers for the queue position/
   1.489 +	 * retry time */
   1.490 +	set_queue_status (xfer);
   1.491 +
   1.492 +	/*
   1.493 +	 * Don't read the body if they supplied "Connection: Close".
   1.494 +	 * Close if the HTTP version is 1.0 (TODO: this should
   1.495 +	 * not close 1.0 connections if they supplied Connection:
   1.496 +	 * Keep-Alive, I guess)
   1.497 +	 */
   1.498 +	if (!STRCASECMP (xfer->version, "HTTP/1.0") ||
   1.499 +	    !STRCASECMP (xfer->version, "HTTP") ||
   1.500 +	    !STRCASECMP (conn_hdr, "close"))
   1.501 +	{
   1.502 +		gt_transfer_close (xfer, TRUE);
   1.503 +		return;
   1.504 +	}
   1.505 +
   1.506 +	if (content_len)
   1.507 +		len = ATOUL (content_len);
   1.508 +
   1.509 +	/* abuse xfer->{start,stop} fields for the length of the response body */
   1.510 +	xfer->start = 0;
   1.511 +	xfer->stop  = len;
   1.512 +
   1.513 +	/* set this flag to let gt_transfer_close() know the headers
   1.514 +	 * have been parsed */
   1.515 +	xfer->transmitted_hdrs = TRUE;
   1.516 +
   1.517 +	/* set the length so that if the connection times out, it will be
   1.518 +	 * force closed when we havent read the entire body */
   1.519 +	xfer->remaining_len = len;
   1.520 +
   1.521 +	/* if there is no length to read, we are done */
   1.522 +	if (len == 0)
   1.523 +	{
   1.524 +		gt_transfer_close (xfer, FALSE);
   1.525 +		return;
   1.526 +	}
   1.527 +
   1.528 +	input_remove_all (c->fd);
   1.529 +	input_add (c->fd, xfer, INPUT_READ,
   1.530 +	           (InputCallback)read_response_body, TIMEOUT_DEF);
   1.531 +}
   1.532 +
   1.533 +/*****************************************************************************/
   1.534 +
   1.535 +/*
   1.536 + * Process an HTTP return code (either client or server [push]) and attempt
   1.537 + * to appropriately handle/expunge the transfer structure accordingly.  The
   1.538 + * return value indicates whether or not we may continue after the code
   1.539 + * has been processed.  Some error codes (404) are considered fatal here
   1.540 + * and you should abort after this call.
   1.541 + *
   1.542 + * NOTE:
   1.543 + * If this function returns FALSE, the calling argument is free'd and you
   1.544 + * should not access it further.
   1.545 + */
   1.546 +int gt_http_handle_code (GtTransfer *xfer, int code)
   1.547 +{
   1.548 +	TCPC     *c;
   1.549 +	Chunk    *chunk;
   1.550 +	GtSource *gt;
   1.551 +
   1.552 +	/* successful code, do nothing */
   1.553 +	if (code >= 200 && code <= 299)
   1.554 +		return TRUE;
   1.555 +
   1.556 +	c     = gt_transfer_get_tcpc  (xfer);
   1.557 +	chunk = gt_transfer_get_chunk (xfer);
   1.558 +
   1.559 +	gt = gt_transfer_get_source (xfer);
   1.560 +	assert (gt != NULL);
   1.561 +
   1.562 +	/*
   1.563 +	 * We have to be careful not to access the transfer or any
   1.564 +	 * data related to it after calling p->source_abort, which
   1.565 +	 * will cancel the transfer.
   1.566 +	 */
   1.567 +	switch (code)
   1.568 +	{
   1.569 +	 default:
   1.570 +	 case 404:                     /* not found */
   1.571 +		{
   1.572 +			/*
   1.573 +			 * If this was a uri-res request, retry with an index request.
   1.574 +			 * Otherwise, remove the source.
   1.575 +			 *
   1.576 +			 * TODO: perhaps this should retry with url-encoded index
   1.577 +			 * request, before removing?
   1.578 +			 */
   1.579 +			if (gt->uri_res_failed)
   1.580 +			{
   1.581 +				GT->source_abort (GT, chunk->transfer, xfer->source);
   1.582 +			}
   1.583 +			else
   1.584 +			{
   1.585 +				handle_http_error (xfer, SOURCE_QUEUED_REMOTE,
   1.586 +				                   "File not found");
   1.587 +				gt->uri_res_failed = TRUE;
   1.588 +			}
   1.589 +		}
   1.590 +		break;
   1.591 +	 case 401:                     /* unauthorized */
   1.592 +		handle_http_error (xfer, SOURCE_CANCELLED, "Access denied");
   1.593 +		break;
   1.594 +	 case 503:                     /* remotely queued */
   1.595 +		handle_http_error (xfer, SOURCE_QUEUED_REMOTE, "Queued (Remotely)");
   1.596 +		break;
   1.597 +	 case 500:                     /* source may not match, check later */
   1.598 +		/*
   1.599 +		 * The remote node has reported that this file has changed
   1.600 +		 * since the last time we received results for it.  This more
   1.601 +		 * than likely indicates a hash change, in which case we should
   1.602 +		 * not keep this associated with this transfer.  Unfortunately,
   1.603 +		 * giFT fails to provide any sort of abstraction for us
   1.604 +		 * to remove sources ourselves, so we need this hopefully
   1.605 +		 * temporary hack to remove the source.
   1.606 +		 */
   1.607 +		GT->source_abort (GT, chunk->transfer, xfer->source);
   1.608 +		break;
   1.609 +	}
   1.610 +
   1.611 +	return FALSE;
   1.612 +}
   1.613 +
   1.614 +static int parse_content_range (char *range, off_t *r_start, off_t *r_end,
   1.615 +                                off_t *r_size)
   1.616 +{
   1.617 +	char *start, *end, *size;
   1.618 +
   1.619 +	string_sep (&range, "bytes");
   1.620 +	string_sep_set (&range, " ="); /* accept "bytes=" and "bytes " */
   1.621 +
   1.622 +	if (r_end)
   1.623 +		*r_end   = -1;
   1.624 +	if (r_start)
   1.625 +		*r_start = -1;
   1.626 +	if (r_size)
   1.627 +		*r_size  = -1;
   1.628 +
   1.629 +	if (!range)
   1.630 +		return FALSE;
   1.631 +
   1.632 +	start = string_sep (&range, "-");
   1.633 +	end   = string_sep (&range, "/");
   1.634 +	size  = range;
   1.635 +
   1.636 +	if (r_start && start)
   1.637 +		*r_start = ATOUL (start);
   1.638 +	if (r_end && end)
   1.639 +		*r_end   = ATOUL (end);
   1.640 +	if (r_size && size)
   1.641 +		*r_size  = ATOUL (size);
   1.642 +
   1.643 +	if (start && end && size)
   1.644 +		return TRUE;
   1.645 +
   1.646 +	return FALSE;
   1.647 +}
   1.648 +
   1.649 +static int verify_range_response (GtTransfer *xfer, Chunk *chunk)
   1.650 +{
   1.651 +	char    *user_agent;
   1.652 +	char    *content_range;
   1.653 +	char    *content_len;
   1.654 +	off_t    start, stop, size;
   1.655 +#if 0
   1.656 +	off_t    file_size;
   1.657 +#endif
   1.658 +	off_t    xfer_size;
   1.659 +	int      error = FALSE;
   1.660 +
   1.661 +#if 0
   1.662 +	file_size = chunk->transfer->total;
   1.663 +#endif
   1.664 +	xfer_size = xfer->stop - xfer->start;
   1.665 +
   1.666 +	if ((content_len = dataset_lookupstr (xfer->header, "content-length")))
   1.667 +	{
   1.668 +		size = ATOUL (content_len);
   1.669 +
   1.670 +		if (size != xfer_size)
   1.671 +		{
   1.672 +			GIFT_ERROR (("bad content len=%lu, expected %lu", size, xfer_size));
   1.673 +			error = TRUE;
   1.674 +			gt_transfer_status (xfer, SOURCE_CANCELLED, "Bad Content-Length");
   1.675 +		}
   1.676 +	}
   1.677 +
   1.678 +	if ((content_range = dataset_lookupstr (xfer->header, "content-range")))
   1.679 +	{
   1.680 +		if (HTTP_DEBUG)
   1.681 +		{
   1.682 +			GT->dbg (GT, "Content-Range: %s, start=%lu, stop=%lu",
   1.683 +			         content_range, chunk->start, chunk->stop);
   1.684 +		}
   1.685 +
   1.686 +		if (parse_content_range (content_range, &start, &stop, &size))
   1.687 +		{
   1.688 +			if (start != xfer->start)
   1.689 +			{
   1.690 +				GIFT_ERROR (("bad xfer start: %lu %lu", xfer->start, start));
   1.691 +				error = TRUE;
   1.692 +			}
   1.693 +			if (stop != xfer->stop - 1)
   1.694 +			{
   1.695 +				GIFT_ERROR (("bad xfer end: %lu %lu", xfer->stop, stop));
   1.696 +				error = TRUE;
   1.697 +			}
   1.698 +#if 0
   1.699 +			if (size != file_size)
   1.700 +			{
   1.701 +				GIFT_ERROR (("bad xfer size: %lu, expected %lu", file_size,
   1.702 +				             size));
   1.703 +				error = TRUE;
   1.704 +			}
   1.705 +#endif
   1.706 +		}
   1.707 +		else
   1.708 +		{
   1.709 +			GIFT_ERROR (("error parsing content-range hdr"));
   1.710 +			error = TRUE;
   1.711 +		}
   1.712 +	}
   1.713 +
   1.714 +	if (!content_len && !content_range)
   1.715 +	{
   1.716 +		if (!(user_agent = dataset_lookupstr (xfer->header, "Server")))
   1.717 +			user_agent = dataset_lookupstr (xfer->header, "User-Agent");
   1.718 +
   1.719 +		GIFT_ERROR (("missing Content-Range/Length, start=%lu, stop=%lu, "
   1.720 +		             "culprit=%s", xfer->start, xfer->stop, user_agent));
   1.721 +
   1.722 +		error = TRUE;
   1.723 +	}
   1.724 +
   1.725 +	if (error)
   1.726 +	{
   1.727 +		GT->DBGFN (GT, "removing source %s", chunk->source->url);
   1.728 +		GT->source_abort (GT, chunk->transfer, chunk->source);
   1.729 +		return FALSE;
   1.730 +	}
   1.731 +
   1.732 +	return TRUE;
   1.733 +}
   1.734 +
   1.735 +/*
   1.736 + * Receive and process the HTTP response
   1.737 + */
   1.738 +static void get_server_reply (int fd, input_id id, GtTransfer *xfer)
   1.739 +{
   1.740 +	Chunk         *chunk;
   1.741 +	TCPC          *c;
   1.742 +	GtSource      *gt;
   1.743 +	FDBuf         *buf;
   1.744 +	unsigned char *data;
   1.745 +	char          *msg;
   1.746 +	size_t         data_len = 0;
   1.747 +	int            n;
   1.748 +
   1.749 +	chunk = gt_transfer_get_chunk (xfer);
   1.750 +	c     = gt_transfer_get_tcpc  (xfer);
   1.751 +
   1.752 +	gt = gt_transfer_get_source (xfer);
   1.753 +
   1.754 +	buf = tcp_readbuf (c);
   1.755 +
   1.756 +	/* attempt to read the complete server response */
   1.757 +	if ((n = fdbuf_delim (buf, "\n")) < 0)
   1.758 +	{
   1.759 +		msg = "Timed out";
   1.760 +
   1.761 +		/*
   1.762 +		 * If the peer abruptly closed the connection, its possible it
   1.763 +		 * didn't understand our request if it was a uri-res request.
   1.764 +		 * So, disable uri-res in that case.
   1.765 +		 */
   1.766 +		if (fd != -1)
   1.767 +		{
   1.768 +			gt->uri_res_failed = TRUE;
   1.769 +			msg = "Connection closed";
   1.770 +		}
   1.771 +
   1.772 +		gt_transfer_status (xfer, SOURCE_CANCELLED, msg);
   1.773 +		gt_transfer_close (xfer, TRUE);
   1.774 +		return;
   1.775 +	}
   1.776 +
   1.777 +	if (gt_fdbuf_full (buf))
   1.778 +	{
   1.779 +		gt_transfer_close (xfer, TRUE);
   1.780 +		return;
   1.781 +	}
   1.782 +
   1.783 +	if (n > 0)
   1.784 +		return;
   1.785 +
   1.786 +	data = fdbuf_data (buf, &data_len);
   1.787 +
   1.788 +	if (!gt_http_header_terminated (data, data_len))
   1.789 +		return;
   1.790 +
   1.791 +	fdbuf_release (buf);
   1.792 +
   1.793 +	/* parse the server response */
   1.794 +	if (!parse_server_reply (xfer, c, data))
   1.795 +	{
   1.796 +		gt_transfer_status (xfer, SOURCE_CANCELLED, "Malformed HTTP header");
   1.797 +		gt_transfer_close (xfer, TRUE);
   1.798 +		return;
   1.799 +	}
   1.800 +
   1.801 +	/*
   1.802 +	 * NOTE: if we wanted to do any further processing of the server reply
   1.803 +	 * after GET /, this is where it would be
   1.804 +	 */
   1.805 +
   1.806 +	/* determine what to do with the HTTP code reply */
   1.807 +	if (!gt_http_handle_code (xfer, xfer->code))
   1.808 +		return;
   1.809 +
   1.810 +	/* make sure the server understood our "Range:" request */
   1.811 +	if (!verify_range_response (xfer, chunk))
   1.812 +		return;
   1.813 +
   1.814 +	/*
   1.815 +	 * Received HTTP headers, ...and now we are waiting for the file.  This
   1.816 +	 * should be a very short wait :)
   1.817 +	 */
   1.818 +	gt_transfer_status (xfer, SOURCE_WAITING, "Received HTTP headers");
   1.819 +	xfer->transmitted_hdrs = TRUE;
   1.820 +
   1.821 +	/* special case if the request size is 0 */
   1.822 +	if (xfer->remaining_len == 0)
   1.823 +	{
   1.824 +		gt_transfer_close (xfer, FALSE);
   1.825 +		return;
   1.826 +	}
   1.827 +
   1.828 +	/* disable the header read timeout */
   1.829 +	timer_remove_zero (&xfer->header_timer);
   1.830 +
   1.831 +	/* wait for the file to be sent by the server */
   1.832 +	input_remove (id);
   1.833 +	input_add (fd, xfer, INPUT_READ,
   1.834 +	           (InputCallback)gt_get_read_file, 0);
   1.835 +}
   1.836 +
   1.837 +/*
   1.838 + * Receive the requested file from the server.
   1.839 + */
   1.840 +void gt_get_read_file (int fd, input_id id, GtTransfer *xfer)
   1.841 +{
   1.842 +	TCPC         *c;
   1.843 +	Chunk        *chunk;
   1.844 +	char          buf[RW_BUFFER];
   1.845 +	size_t        size;
   1.846 +	int           recv_len;
   1.847 +
   1.848 +	c     = gt_transfer_get_tcpc  (xfer);
   1.849 +	chunk = gt_transfer_get_chunk (xfer);
   1.850 +
   1.851 +	/* set the max initial size of the request */
   1.852 +	size = sizeof (buf);
   1.853 +
   1.854 +	/*
   1.855 +	 * Cap the size of the received length at the length
   1.856 +	 * of the request.
   1.857 +	 *
   1.858 +	 * This way we can't receive part of the next request
   1.859 +	 * header with persistent HTTP.
   1.860 +	 */
   1.861 +	if (size > xfer->remaining_len)
   1.862 +		size = xfer->remaining_len;
   1.863 +
   1.864 +	/*
   1.865 +	 * Ask giFT for the max size we should read.  If this returns 0, the
   1.866 +	 * download was suspended.
   1.867 +	 */
   1.868 +	if ((size = download_throttle (chunk, size)) == 0)
   1.869 +		return;
   1.870 +
   1.871 +	if ((recv_len = tcp_recv (c, buf, size)) <= 0)
   1.872 +	{
   1.873 +		GT->DBGFN (GT, "tcp_recv error (%d) from %s:%hu: %s", recv_len,
   1.874 +		           net_ip_str (c->host), c->port,  GIFT_NETERROR());
   1.875 +
   1.876 +		gt_transfer_status (xfer, SOURCE_CANCELLED, "Cancelled remotely");
   1.877 +		gt_transfer_close (xfer, TRUE);
   1.878 +		return;
   1.879 +	}
   1.880 +
   1.881 +	/*
   1.882 +	 * We are receiving a file here, so this will always be calling
   1.883 +	 * gt_download.
   1.884 +	 */
   1.885 +	gt_transfer_write (xfer, chunk, buf, (size_t)recv_len);
   1.886 +}
   1.887 +
   1.888 +/*****************************************************************************/
   1.889 +
   1.890 +static void client_reset_timeout (int fd, input_id id, TCPC *c)
   1.891 +{
   1.892 +	/* normally we would call recv () here but there's no reason why the server
   1.893 +	 * should be sending stuff down to use...so, disconnect */
   1.894 +	if (HTTP_DEBUG)
   1.895 +		GT->DBGSOCK (GT, c, "closing client HTTP connection");
   1.896 +
   1.897 +	gt_http_connection_close (GT_TRANSFER_DOWNLOAD, c, TRUE);
   1.898 +}
   1.899 +
   1.900 +void gt_http_client_reset (TCPC *c)
   1.901 +{
   1.902 +	/* This can happen when the GtTransfer and TCPC have been decoupled,
   1.903 +	 * such as when we are waiting for a GIV response from a node. */
   1.904 +	if (!c)
   1.905 +		return;
   1.906 +
   1.907 +	tcp_flush (c, TRUE);
   1.908 +
   1.909 +	input_remove_all (c->fd);
   1.910 +	input_add (c->fd, c, INPUT_READ,
   1.911 +	           (InputCallback)client_reset_timeout, 2 * MINUTES);
   1.912 +}