[svn] New option `--limit-rate'.

Published in <sxsd725cn56.fsf@florida.arsdigita.de>.

ChangeLog

@@ -1,3 +1,7 @@
+2001-11-26  Hrvoje Niksic  <hniksic@arsdigita.com>
+
+	* configure.in: Check for usleep.
+
 2001-11-25  Hrvoje Niksic  <hniksic@arsdigita.com>
 
 	* util/dist-wget: New file: the script used for building Wget.

configure.in

@@ -175,7 +175,7 @@ AC_FUNC_MMAP
 AC_CHECK_FUNCS(strdup strstr strcasecmp strncasecmp strpbrk)
 AC_CHECK_FUNCS(gettimeofday mktime strptime)
 AC_CHECK_FUNCS(strerror snprintf vsnprintf select signal symlink access isatty)
-AC_CHECK_FUNCS(uname gethostname)
+AC_CHECK_FUNCS(uname gethostname usleep)
 
 dnl
 dnl Check if we need to compile in getopt.c.

src/ChangeLog

@@ -1,3 +1,22 @@
+2001-11-26  Hrvoje Niksic  <hniksic@arsdigita.com>
+
+	* config.h.in: Put a HAVE_USLEEP stub.
+
+	* cmpt.c (usleep): Replacement implementation of usleep using
+	select.
+
+	* init.c: New option init_rate.
+
+	* main.c (main): New option --limit-rate.
+
+	* retr.c (limit_bandwidth): New function.
+	(get_contents): Call it to limit the bandwidth used when
+	downloading.
+
+	* progress.c (dot_update): Would print the wrong download speed on
+	rows other than the first one when the download was continued.
+	(dot_finish): Ditto.
+
 2001-11-26  Ian Abbott <abbotti@mev.co.uk>
 
 	* http.c (gethttp): fix undeclared variable 'err' when compiled

src/cmpt.c

@@ -1206,3 +1206,19 @@ strptime (buf, format, tm)
   return strptime_internal (buf, format, tm, &decided);
 }
 #endif /* not HAVE_STRPTIME */
+
+
+#ifndef HAVE_USLEEP
+/* A simple usleep implementation based on select().  This will
+   probably not work on Windows.  */
+
+int
+usleep (unsigned long usec)
+{
+  struct timeval tm;
+  tm.tv_sec = 0;
+  tm.tv_usec = usec;
+  select (0, NULL, NULL, NULL, &tm);
+  return 0;
+}
+#endif /* not HAVE_USLEEP */

src/config.h.in

@@ -165,6 +165,9 @@ char *alloca ();
 /* Define if you have the gettext function.  */
 #undef HAVE_GETTEXT
 
+/* Define if you have the usleep function.  */
+#undef HAVE_USLEEP
+
 /* Define if you have the <string.h> header file.  */
 #undef HAVE_STRING_H
 

src/init.c

@@ -142,6 +142,7 @@ static struct {
   { "includedirectories", &opt.includes,	cmd_directory_vector },
   { "input",		&opt.input_filename,	cmd_file },
   { "killlonger",	&opt.kill_longer,	cmd_boolean },
+  { "limitrate",	&opt.limit_rate,	cmd_bytes },
   { "loadcookies",	&opt.cookies_input,	cmd_file },
   { "logfile",		&opt.lfilename,		cmd_file },
   { "login",		&opt.ftp_acc,		cmd_string },

src/main.c

@@ -164,6 +164,7 @@ Download:\n\
        --random-wait            wait from 0...2*WAIT secs between retrievals.\n\
   -Y,  --proxy=on/off           turn proxy on or off.\n\
   -Q,  --quota=NUMBER           set retrieval quota to NUMBER.\n\
+       --limit-rate=RATE        limit download rate to RATE.\n\
 \n"), stdout);
 #ifdef HAVE_RANDOM
   fputs (_("\
@@ -303,6 +304,7 @@ main (int argc, char *const *argv)
     { "include-directories", required_argument, NULL, 'I' },
     { "input-file", required_argument, NULL, 'i' },
     { "level", required_argument, NULL, 'l' },
+    { "limit-rate", required_argument, NULL, 164 },
     { "load-cookies", required_argument, NULL, 161 },
     { "no", required_argument, NULL, 'n' },
     { "output-document", required_argument, NULL, 'O' },
@@ -535,6 +537,9 @@ GNU General Public License for more details.\n"));
 	case 163:
 	  setval ("progress", optarg);
 	  break;
+	case 164:
+	  setval ("limitrate", optarg);
+	  break;
 	case 157:
 	  setval ("referer", optarg);
 	  break;

src/options.h

@@ -104,6 +104,8 @@ struct options
   long waitretry;		/* The wait period between retries. - HEH */
   int use_robots;		/* Do we heed robots.txt? */
 
+  long limit_rate;		/* Limit the download rate to this
+				   many bps. */
   long quota;			/* Maximum number of bytes to
 				   retrieve. */
   VERY_LONG_TYPE downloaded;	/* How much we downloaded already. */

src/progress.c

@@ -195,8 +195,8 @@ dot_create (long initial, long total)
 	  /* Align the [ skipping ... ] line with the dots.  To do
 	     that, insert the number of spaces equal to the number of
 	     digits in the skipped amount in K.  */
-	  logprintf (LOG_VERBOSE, "\n%*s%s",
-		     2 + skipped_k_len, "", _("[ skipping %dK ]"));
+	  logprintf (LOG_VERBOSE, _("\n%*s[ skipping %dK ]"),
+		     2 + skipped_k_len, "", skipped_k);
 	}
 
       logprintf (LOG_VERBOSE, "\n%5ldK", skipped / 1024);
@@ -255,15 +255,16 @@ dot_update (void *progress, long howmuch, long dltime)
       ++dp->dots;
       if (dp->dots >= opt.dots_in_line)
 	{
+	  long row_qty = row_bytes;
+	  if (dp->rows == dp->initial_length / row_bytes)
+	    row_qty -= dp->initial_length % row_bytes;
+
 	  ++dp->rows;
 	  dp->dots = 0;
 
 	  if (dp->total_length)
 	    print_percentage (dp->rows * row_bytes, dp->total_length);
-
-	  print_download_speed (dp,
-				row_bytes - (dp->initial_length % row_bytes),
-				dltime);
+	  print_download_speed (dp, row_qty, dltime);
 	}
     }
 
@@ -296,12 +297,14 @@ dot_finish (void *progress, long dltime)
 			dp->total_length);
     }
 
-  print_download_speed (dp, dp->dots * dot_bytes
-			+ dp->accumulated
-			- dp->initial_length % row_bytes,
-			dltime);
-  logputs (LOG_VERBOSE, "\n\n");
+  {
+    long row_qty = dp->dots * dot_bytes + dp->accumulated;
+    if (dp->rows == dp->initial_length / row_bytes)
+      row_qty -= dp->initial_length % row_bytes;
+    print_download_speed (dp, row_qty, dltime);
+  }
 
+  logputs (LOG_VERBOSE, "\n\n");
   log_set_flush (0);
 
   xfree (dp);
@@ -401,7 +404,7 @@ bar_create (long initial, long total)
   bp->width = screen_width;
   bp->buffer = xmalloc (bp->width + 1);
 
-  logputs (LOG_VERBOSE, "\n\n");
+  logputs (LOG_VERBOSE, "\n");
 
   create_image (bp, 0);
   display_image (bp->buffer);

src/retr.c

@@ -52,6 +52,50 @@ extern int errno;
 int global_download_count;
 
 
+static struct {
+  long bytes;
+  long dltime;
+} limit_data;
+
+static void
+limit_bandwidth_reset (void)
+{
+  limit_data.bytes  = 0;
+  limit_data.dltime = 0;
+}
+
+/* Limit the bandwidth by pausing the download for an amount of time.
+   BYTES is the number of bytes received from the network, DELTA is
+   how long it took to receive them, DLTIME the current download time,
+   TIMER the timer, and ADJUSTMENT the previous.  */
+
+static void
+limit_bandwidth (long bytes, long delta)
+{
+  long expected;
+
+  limit_data.bytes += bytes;
+  limit_data.dltime += delta;
+
+  expected = (long)(1000.0 * limit_data.bytes / opt.limit_rate);
+
+  if (expected > limit_data.dltime)
+    {
+      long slp = expected - limit_data.dltime;
+      if (slp < 200)
+	{
+	  DEBUGP (("deferring a %ld ms sleep (%ld/%ld) until later.\n",
+		   slp, limit_data.bytes, limit_data.dltime));
+	  return;
+	}
+      DEBUGP (("sleeping %ld ms\n", slp));
+      usleep (1000 * slp);
+    }
+
+  limit_data.bytes = 0;
+  limit_data.dltime = 0;
+}
+
 #define MIN(i, j) ((i) <= (j) ? (i) : (j))
 
 /* Reads the contents of file descriptor FD, until it is closed, or a
@@ -82,14 +126,13 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
   int res = 0;
   static char c[8192];
   void *progress = NULL;
-  struct wget_timer *timer = NULL;
+  struct wget_timer *timer = wtimer_allocate ();
+  long dltime = 0, last_dltime = 0;
 
   *len = restval;
 
   if (opt.verbose)
     progress = progress_create (restval, expected);
-  if (opt.verbose || elapsed != NULL)
-    timer = wtimer_new ();
 
   if (rbuf && RBUF_FD (rbuf) == fd)
     {
@@ -108,8 +151,13 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
 	  goto out;
 	}
       if (opt.verbose)
-	progress_update (progress, sz, wtimer_elapsed (timer));
+	progress_update (progress, sz, 0);
     }
+
+  if (opt.limit_rate)
+    limit_bandwidth_reset ();
+  wtimer_reset (timer);
+
   /* Read from fd while there is available data.
 
      Normally, if expected is 0, it means that it is not known how
@@ -121,14 +169,12 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
 			    ? MIN (expected - *len, sizeof (c))
 			    : sizeof (c));
 #ifdef HAVE_SSL
-		if (rbuf->ssl!=NULL) {
-		  res = ssl_iread (rbuf->ssl, c, amount_to_read);
-		} else {
-#endif /* HAVE_SSL */
-		  res = iread (fd, c, amount_to_read);
-#ifdef HAVE_SSL
-		}
+      if (rbuf->ssl!=NULL)
+	res = ssl_iread (rbuf->ssl, c, amount_to_read);
+      else
 #endif /* HAVE_SSL */
+	res = iread (fd, c, amount_to_read);
+
       if (res > 0)
 	{
 	  fwrite (c, sizeof (char), res, fp);
@@ -141,8 +187,19 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
 	      res = -2;
 	      goto out;
 	    }
+
+	  /* If bandwidth is not limited, one call to wtimer_elapsed
+	     is sufficient.  */
+	  dltime = wtimer_elapsed (timer);
+	  if (opt.limit_rate)
+	    {
+	      limit_bandwidth (res, dltime - last_dltime);
+	      dltime = wtimer_elapsed (timer);
+	      last_dltime = dltime;
+	    }
+
 	  if (opt.verbose)
-	    progress_update (progress, res, wtimer_elapsed (timer));
+	    progress_update (progress, res, dltime);
 	  *len += res;
 	}
       else
@@ -152,15 +209,12 @@ get_contents (int fd, FILE *fp, long *len, long restval, long expected,
     res = -1;
 
  out:
-  if (timer)
-    {
-      long dltime = wtimer_elapsed (timer);
-      if (opt.verbose)
-	progress_finish (progress, dltime);
-      if (elapsed)
-	*elapsed = dltime;
-      wtimer_delete (timer);
-    }
+  if (opt.verbose)
+    progress_finish (progress, dltime);
+  if (elapsed)
+    *elapsed = dltime;
+  wtimer_delete (timer);
+
   return res;
 }
 

src/sysdep.h

@@ -163,6 +163,9 @@ char *strptime ();
 #ifndef HAVE_VSNPRINTF
 int vsnprintf ();
 #endif
+#ifndef HAVE_USLEEP
+int usleep ();
+#endif
 
 /* SunOS brain damage -- for some reason, SunOS header files fail to
    declare the functions below, which causes all kinds of problems