wok rev 8913

busybox: wget speedup
author Pascal Bellard <pascal.bellard@slitaz.org>
date Mon Feb 28 15:05:05 2011 +0100 (2011-02-28)
parents 1e7c52662662
children bd8206e828b5
files busybox/receipt busybox/stuff/busybox-1.18-wget.u
line diff
     1.1 --- a/busybox/receipt	Mon Feb 28 14:45:49 2011 +0100
     1.2 +++ b/busybox/receipt	Mon Feb 28 15:05:05 2011 +0100
     1.3 @@ -21,6 +21,7 @@
     1.4      	patch -p1 < ../stuff/$PACKAGE-${VERSION%.*}-$file || return 1
     1.5  	touch done.$file
     1.6      done <<EOT
     1.7 +wget.u
     1.8  tar.u
     1.9  stat.u
    1.10  ris.u
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/busybox/stuff/busybox-1.18-wget.u	Mon Feb 28 15:05:05 2011 +0100
     2.3 @@ -0,0 +1,128 @@
     2.4 +--- busybox-1.18.3/networking/wget.c
     2.5 ++++ busybox-1.18.3-wget/networking/wget.c
     2.6 +@@ -446,7 +446,7 @@ static FILE* prepare_ftp_session(FILE **
     2.7 + 
     2.8 + static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
     2.9 + {
    2.10 +-	char buf[512];
    2.11 ++	char buf[4*1024]; /* made bigger to speed up local xfers */
    2.12 + #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
    2.13 + # if ENABLE_FEATURE_WGET_TIMEOUT
    2.14 + 	unsigned second_cnt;
    2.15 +@@ -455,7 +455,6 @@ static void NOINLINE retrieve_file_data(
    2.16 + 
    2.17 + 	polldata.fd = fileno(dfp);
    2.18 + 	polldata.events = POLLIN | POLLPRI;
    2.19 +-	ndelay_on(polldata.fd);
    2.20 + #endif
    2.21 + 	progress_meter(PROGRESS_START);
    2.22 + 
    2.23 +@@ -464,6 +463,10 @@ static void NOINLINE retrieve_file_data(
    2.24 + 
    2.25 + 	/* Loops only if chunked */
    2.26 + 	while (1) {
    2.27 ++
    2.28 ++#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
    2.29 ++		ndelay_on(polldata.fd);
    2.30 ++#endif
    2.31 + 		while (1) {
    2.32 + 			int n;
    2.33 + 			unsigned rdsz;
    2.34 +@@ -493,22 +496,46 @@ static void NOINLINE retrieve_file_data(
    2.35 + 				progress_meter(PROGRESS_BUMP);
    2.36 + 			}
    2.37 + #endif
    2.38 ++			/* fread internally uses read loop, which in our case
    2.39 ++			 * is usually exited when we get EAGAIN.
    2.40 ++			 * In this case, libc sets error marker on the stream.
    2.41 ++			 * Need to clear it before next fread to avoid possible
    2.42 ++			 * rare false positive ferror below. Rare because usually
    2.43 ++			 * fread gets more than zero bytes, and we don't fall
    2.44 ++			 * into if (n <= 0) ...
    2.45 ++			 */
    2.46 ++			clearerr(dfp);
    2.47 ++			errno = 0;
    2.48 + 			n = safe_fread(buf, rdsz, dfp);
    2.49 ++			/* man fread:
    2.50 ++			 * If error occurs, or EOF is reached, the return value
    2.51 ++			 * is a short item count (or zero).
    2.52 ++			 * fread does not distinguish between EOF and error.
    2.53 ++			 */
    2.54 + 			if (n <= 0) {
    2.55 +-				if (ferror(dfp)) {
    2.56 +-					/* perror will not work: ferror doesn't set errno */
    2.57 +-					bb_error_msg_and_die(bb_msg_read_error);
    2.58 +-				}
    2.59 +-				break;
    2.60 ++#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
    2.61 ++				if (errno == EAGAIN) /* poll lied, there is no data? */
    2.62 ++					continue; /* yes */
    2.63 ++#endif
    2.64 ++				if (ferror(dfp))
    2.65 ++					bb_perror_msg_and_die(bb_msg_read_error);
    2.66 ++				break; /* EOF, not error */
    2.67 + 			}
    2.68 ++
    2.69 + 			xwrite(output_fd, buf, n);
    2.70 + #if ENABLE_FEATURE_WGET_STATUSBAR
    2.71 + 			G.transferred += n;
    2.72 + 			progress_meter(PROGRESS_BUMP);
    2.73 + #endif
    2.74 +-			if (G.got_clen)
    2.75 ++			if (G.got_clen) {
    2.76 + 				G.content_len -= n;
    2.77 ++				if (G.content_len == 0)
    2.78 ++					break;
    2.79 ++			}
    2.80 + 		}
    2.81 ++#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
    2.82 ++		ndelay_off(polldata.fd);
    2.83 ++#endif
    2.84 + 
    2.85 + 		if (!G.chunked)
    2.86 + 			break;
    2.87 +@@ -706,6 +733,11 @@ int wget_main(int argc UNUSED_PARAM, cha
    2.88 + 		fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n",
    2.89 + 			target.host, user_agent);
    2.90 + 
    2.91 ++		/* Ask server to close the connection as soon as we are done
    2.92 ++		 * (IOW: we do not intend to send more requests)
    2.93 ++		 */
    2.94 ++		fprintf(sfp, "Connection: close\r\n");
    2.95 ++
    2.96 + #if ENABLE_FEATURE_WGET_AUTHENTICATION
    2.97 + 		if (target.user) {
    2.98 + 			fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
    2.99 +@@ -719,22 +751,25 @@ int wget_main(int argc UNUSED_PARAM, cha
   2.100 + 
   2.101 + 		if (G.beg_range)
   2.102 + 			fprintf(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range);
   2.103 ++
   2.104 + #if ENABLE_FEATURE_WGET_LONG_OPTIONS
   2.105 + 		if (extra_headers)
   2.106 + 			fputs(extra_headers, sfp);
   2.107 + 
   2.108 + 		if (opt & WGET_OPT_POST_DATA) {
   2.109 + 			char *estr = URL_escape(post_data);
   2.110 +-			fprintf(sfp, "Content-Type: application/x-www-form-urlencoded\r\n");
   2.111 +-			fprintf(sfp, "Content-Length: %u\r\n" "\r\n" "%s",
   2.112 +-					(int) strlen(estr), estr);
   2.113 +-			/*fprintf(sfp, "Connection: Keep-Alive\r\n\r\n");*/
   2.114 +-			/*fprintf(sfp, "%s\r\n", estr);*/
   2.115 ++			fprintf(sfp,
   2.116 ++				"Content-Type: application/x-www-form-urlencoded\r\n"
   2.117 ++				"Content-Length: %u\r\n"
   2.118 ++				"\r\n"
   2.119 ++				"%s",
   2.120 ++				(int) strlen(estr), estr
   2.121 ++			);
   2.122 + 			free(estr);
   2.123 + 		} else
   2.124 + #endif
   2.125 +-		{ /* If "Connection:" is needed, document why */
   2.126 +-			fprintf(sfp, /* "Connection: close\r\n" */ "\r\n");
   2.127 ++		{
   2.128 ++			fprintf(sfp, "\r\n");
   2.129 + 		}
   2.130 + 
   2.131 + 		fflush(sfp);