From 711bf72609e5d49ff7f5366a4664377cd3208d70 Mon Sep 17 00:00:00 2001
From: hniksic <devnull@localhost>
Date: Sat, 11 Oct 2003 06:57:11 -0700
Subject: [PATCH] [svn] Remove VERY_LONG_TYPE; use LARGE_INT instead. Remove
special code for handling VERY_LONG_TYPE overflows. Make opt.quota a
LARGE_INT.
---
src/ChangeLog | 26 +++++++++++
src/ftp.c | 14 +++---
src/http.c | 13 +++---
src/init.c | 123 ++++++++++++++++++++++++++++++++------------------
src/main.c | 20 ++++----
src/options.h | 6 +--
src/recur.c | 5 +-
src/retr.c | 39 ++--------------
src/retr.h | 3 --
src/sysdep.h | 44 ++++++++++--------
src/utils.c | 65 +++++++++-----------------
src/utils.h | 2 +-
12 files changed, 185 insertions(+), 175 deletions(-)
diff --git a/src/ChangeLog b/src/ChangeLog
index eaa9ec49..21a91547 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,29 @@
+2003-10-11 Hrvoje Niksic <hniksic@xemacs.org>
+
+ * utils.c (large_int_to_string): Use snprintf() to print the
+ number. This will work even on systems where libc doesn't
+ understand %lld, but the compiler does, because it will use our
+ snprintf replacement.
+
+ * init.c (parse_bytes_helper): New function.
+ (cmd_bytes): Use it to parse bytes, but cast the result to long.
+ (cmd_bytes_large): Ditto, but store the result to LARGE_INT. Used
+ for --quota so that --quota=10G works even on machines without
+ long long.
+
+ * options.h (struct options): Declare quota as LARGE_INT.
+
+ * retr.c (downloaded_exceeds_quota): Removed.
+ (downloaded_increase): Ditto.
+ (total_downloaded_bytes): New variable, replaces opt.downloaded,
+ which was the wrong place for it anyway. Updated callers of
+ downloaded_exceeds_quota and downloaded_increase to check this
+ variable directly.
+
+ * sysdep.h: Get rid of VERY_LONG_TYPE. Use LARGE_INT for the same
+ purpose, defined as `long', `long long' or `double', depending on
+ size of long and whether long long is available.
+
2003-10-11 Hrvoje Niksic <hniksic@xemacs.org>
* sysdep.h: Also check size of short for int32_t.
diff --git a/src/ftp.c b/src/ftp.c
index 5061d9cf..5a84a9d8 100644
--- a/src/ftp.c
+++ b/src/ftp.c
@@ -59,6 +59,8 @@ so, delete this exception statement from your version. */
extern int errno;
#endif
+extern LARGE_INT total_downloaded_bytes;
+
/* File where the "ls -al" listing will be saved. */
#define LIST_FILENAME ".listing"
@@ -1195,7 +1197,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
/* --dont-remove-listing was specified, so do count this towards the
number of bytes and files downloaded. */
{
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
}
@@ -1210,7 +1212,7 @@ ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
downloaded if they're going to be deleted. People seeding proxies,
for instance, may want to know how many bytes and files they've
downloaded through it. */
- downloaded_increase (len);
+ total_downloaded_bytes += len;
opt.numurls++;
if (opt.delete_after)
@@ -1336,7 +1338,7 @@ ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
{
char *old_target, *ofile;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
{
--depth;
return QUOTEXC;
@@ -1540,7 +1542,7 @@ ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
int size;
char *odir, *newdir;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (f->type != FT_DIRECTORY)
continue;
@@ -1586,7 +1588,7 @@ Not descending to `%s' as it is excluded/not-included.\n"), newdir);
/* Set the time-stamp? */
}
- if (opt.quota && opt.downloaded > opt.quota)
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
return RETROK;
@@ -1704,7 +1706,7 @@ ftp_retrieve_glob (struct url *u, ccon *con, int action)
}
}
freefileinfo (start);
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else
/* #### Should we return `res' here? */
diff --git a/src/http.c b/src/http.c
index 20010de0..4be6299d 100644
--- a/src/http.c
+++ b/src/http.c
@@ -53,6 +53,9 @@ so, delete this exception statement from your version. */
# include <time.h>
# endif
#endif
+#ifndef errno
+extern int errno;
+#endif
#include "wget.h"
#include "utils.h"
@@ -73,10 +76,8 @@ so, delete this exception statement from your version. */
#include "convert.h"
extern char *version_string;
+extern LARGE_INT total_downloaded_bytes;
-#ifndef errno
-extern int errno;
-#endif
static int cookies_loaded_p;
struct cookie_jar *wget_cookie_jar;
@@ -1951,7 +1952,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
tms, u->url, hstat.len, hstat.contlen, locf, count);
}
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
@@ -1978,7 +1979,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
tms, u->url, hstat.len, locf, count);
}
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
@@ -2009,7 +2010,7 @@ The sizes do not match (local %ld) -- retrieving.\n"), local_size);
"%s URL:%s [%ld/%ld] -> \"%s\" [%d]\n",
tms, u->url, hstat.len, hstat.contlen, locf, count);
++opt.numurls;
- downloaded_increase (hstat.len);
+ total_downloaded_bytes += hstat.len;
/* Remember that we downloaded the file for later ".orig" code. */
if (*dt & ADDED_HTML_EXTENSION)
diff --git a/src/init.c b/src/init.c
index 0cd65ae5..50c66c44 100644
--- a/src/init.c
+++ b/src/init.c
@@ -84,6 +84,7 @@ static int enable_tilde_expansion;
CMD_DECLARE (cmd_boolean);
CMD_DECLARE (cmd_bytes);
+CMD_DECLARE (cmd_bytes_large);
CMD_DECLARE (cmd_directory_vector);
CMD_DECLARE (cmd_lockable_boolean);
CMD_DECLARE (cmd_number);
@@ -184,7 +185,7 @@ static struct {
{ "proxypasswd", &opt.proxy_passwd, cmd_string },
{ "proxyuser", &opt.proxy_user, cmd_string },
{ "quiet", &opt.quiet, cmd_boolean },
- { "quota", &opt.quota, cmd_bytes },
+ { "quota", &opt.quota, cmd_bytes_large },
{ "randomwait", &opt.random_wait, cmd_boolean },
{ "readtimeout", &opt.read_timeout, cmd_time },
{ "reclevel", &opt.reclevel, cmd_number_inf },
@@ -849,6 +850,63 @@ cmd_directory_vector (const char *com, const char *val, void *closure)
static int simple_atof PARAMS ((const char *, const char *, double *));
+/* Enginge for cmd_bytes and cmd_bytes_large: converts a string such
+ as "100k" or "2.5G" to a floating point number. */
+
+static int
+parse_bytes_helper (const char *val, double *result)
+{
+ double number, mult;
+ const char *end = val + strlen (val);
+
+ /* Check for "inf". */
+ if (0 == strcmp (val, "inf"))
+ {
+ *result = 0;
+ return 1;
+ }
+
+ /* Strip trailing whitespace. */
+ while (val < end && ISSPACE (end[-1]))
+ --end;
+ if (val == end)
+ return 0;
+
+ switch (TOLOWER (end[-1]))
+ {
+ case 'k':
+ --end, mult = 1024.0;
+ break;
+ case 'm':
+ --end, mult = 1048576.0;
+ break;
+ case 'g':
+ --end, mult = 1073741824.0;
+ break;
+ case 't':
+ --end, mult = 1099511627776.0;
+ break;
+ default:
+ /* Not a recognized suffix: assume it's a digit. (If not,
+ simple_atof will raise an error.) */
+ mult = 1;
+ }
+
+ /* Skip leading and trailing whitespace. */
+ while (val < end && ISSPACE (*val))
+ ++val;
+ while (val < end && ISSPACE (end[-1]))
+ --end;
+ if (val == end)
+ return 0;
+
+ if (!simple_atof (val, end, &number))
+ return 0;
+
+ *result = number * mult;
+ return 1;
+}
+
/* Parse VAL as a number and set its value to CLOSURE (which should
point to a long int).
@@ -866,58 +924,33 @@ static int simple_atof PARAMS ((const char *, const char *, double *));
static int
cmd_bytes (const char *com, const char *val, void *closure)
{
- long mult;
- double number;
- const char *end = val + strlen (val);
-
- /* Check for "inf". */
- if (0 == strcmp (val, "inf"))
+ double byte_value;
+ if (!parse_bytes_helper (val, &byte_value))
{
- *(long *)closure = 0;
- return 1;
- }
-
- /* Strip trailing whitespace. */
- while (val < end && ISSPACE (end[-1]))
- --end;
-
- if (val == end)
- {
- err:
fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
exec_name, com, val);
return 0;
}
+ *(long *)closure = (long)byte_value;
+ return 1;
+}
- switch (TOLOWER (end[-1]))
+/* Like cmd_bytes, but CLOSURE is interpreted as a pointer to
+ LARGE_INT. It works by converting the string to double, therefore
+ working with values up to 2^53-1 without loss of precision. This
+ value (8192 TB) is large enough to serve for a while. */
+
+static int
+cmd_bytes_large (const char *com, const char *val, void *closure)
+{
+ double byte_value;
+ if (!parse_bytes_helper (val, &byte_value))
{
- case 'k':
- --end, mult = 1L<<10;
- break;
- case 'm':
- --end, mult = 1L<<20;
- break;
- case 'g':
- --end, mult = 1L<<30;
- break;
- default:
- /* Not a recognized suffix: assume it belongs to the number.
- (If not, atof simple_atof will raise an error.) */
- mult = 1;
+ fprintf (stderr, _("%s: %s: Invalid byte value `%s'\n"),
+ exec_name, com, val);
+ return 0;
}
-
- /* Skip leading and trailing whitespace. */
- while (val < end && ISSPACE (*val))
- ++val;
- while (val < end && ISSPACE (end[-1]))
- --end;
- if (val == end)
- goto err;
-
- if (!simple_atof (val, end, &number))
- goto err;
-
- *(long *)closure = (long)(number * mult);
+ *(LARGE_INT *)closure = (LARGE_INT)byte_value;
return 1;
}
diff --git a/src/main.c b/src/main.c
index 59367ef7..bec8561d 100644
--- a/src/main.c
+++ b/src/main.c
@@ -50,6 +50,9 @@ so, delete this exception statement from your version. */
#endif /* HAVE_LOCALE_H */
#endif /* HAVE_NLS */
#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
#include "wget.h"
#include "utils.h"
@@ -73,14 +76,11 @@ so, delete this exception statement from your version. */
# define PATH_SEPARATOR '/'
#endif
-extern char *version_string;
-
-#ifndef errno
-extern int errno;
-#endif
-
struct options opt;
+extern LARGE_INT total_downloaded_bytes;
+extern char *version_string;
+
extern struct cookie_jar *wget_cookie_jar;
/* From log.c. */
@@ -908,16 +908,14 @@ Can't timestamp and not clobber old files at the same time.\n"));
/* Print the downloaded sum. */
if (opt.recursive
|| nurl > 1
- || (opt.input_filename && opt.downloaded != 0))
+ || (opt.input_filename && total_downloaded_bytes != 0))
{
logprintf (LOG_NOTQUIET,
_("\nFINISHED --%s--\nDownloaded: %s bytes in %d files\n"),
- time_str (NULL),
- (opt.downloaded_overflow ?
- "<overflow>" : legible_very_long (opt.downloaded)),
+ time_str (NULL), legible_large_int (total_downloaded_bytes),
opt.numurls);
/* Print quota warning, if exceeded. */
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
logprintf (LOG_NOTQUIET,
_("Download quota (%s bytes) EXCEEDED!\n"),
legible (opt.quota));
diff --git a/src/options.h b/src/options.h
index 9895e825..55ee968e 100644
--- a/src/options.h
+++ b/src/options.h
@@ -119,10 +119,8 @@ struct options
long limit_rate; /* Limit the download rate to this
many bps. */
- long quota; /* Maximum number of bytes to
- retrieve. */
- VERY_LONG_TYPE downloaded; /* How much we downloaded already. */
- int downloaded_overflow; /* Whether the above overflowed. */
+ LARGE_INT quota; /* Maximum file size to download and
+ store. */
int numurls; /* Number of successfully downloaded
URLs */
diff --git a/src/recur.c b/src/recur.c
index bf367074..d90aeadc 100644
--- a/src/recur.c
+++ b/src/recur.c
@@ -59,6 +59,7 @@ extern int errno;
#endif
extern char *version_string;
+extern LARGE_INT total_downloaded_bytes;
extern struct hash_table *dl_url_file_map;
extern struct hash_table *downloaded_html_set;
@@ -224,7 +225,7 @@ retrieve_tree (const char *start_url)
int depth, html_allowed;
boolean dash_p_leaf_HTML = FALSE;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (status == FWRITEERR)
break;
@@ -404,7 +405,7 @@ retrieve_tree (const char *start_url)
url_free (start_url_parsed);
string_set_free (blacklist);
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
return QUOTEXC;
else if (status == FWRITEERR)
return FWRITEERR;
diff --git a/src/retr.c b/src/retr.c
index c0b439e8..35286111 100644
--- a/src/retr.c
+++ b/src/retr.c
@@ -66,6 +66,9 @@ extern int errno;
/* See the comment in gethttp() why this is needed. */
int global_download_count;
+/* Total size of downloaded files. Used to enforce quota. */
+LARGE_INT total_downloaded_bytes;
+
static struct {
long chunk_bytes;
@@ -576,7 +579,7 @@ retrieve_from_file (const char *file, int html, int *count)
if (cur_url->ignore_when_downloading)
continue;
- if (downloaded_exceeds_quota ())
+ if (opt.quota && total_downloaded_bytes > opt.quota)
{
status = QUOTEXC;
break;
@@ -614,40 +617,6 @@ printwhat (int n1, int n2)
logputs (LOG_VERBOSE, (n1 == n2) ? _("Giving up.\n\n") : _("Retrying.\n\n"));
}
-/* Increment opt.downloaded by BY_HOW_MUCH. If an overflow occurs,
- set opt.downloaded_overflow to 1. */
-void
-downloaded_increase (unsigned long by_how_much)
-{
- VERY_LONG_TYPE old;
- if (opt.downloaded_overflow)
- return;
- old = opt.downloaded;
- opt.downloaded += by_how_much;
- if (opt.downloaded < old) /* carry flag, where are you when I
- need you? */
- {
- /* Overflow. */
- opt.downloaded_overflow = 1;
- opt.downloaded = ~((VERY_LONG_TYPE)0);
- }
-}
-
-/* Return non-zero if the downloaded amount of bytes exceeds the
- desired quota. If quota is not set or if the amount overflowed, 0
- is returned. */
-int
-downloaded_exceeds_quota (void)
-{
- if (!opt.quota)
- return 0;
- if (opt.downloaded_overflow)
- /* We don't really know. (Wildly) assume not. */
- return 0;
-
- return opt.downloaded > opt.quota;
-}
-
/* If opt.wait or opt.waitretry are specified, and if certain
conditions are met, sleep the appropriate number of seconds. See
the documentation of --wait and --waitretry for more information.
diff --git a/src/retr.h b/src/retr.h
index 2b804754..95090721 100644
--- a/src/retr.h
+++ b/src/retr.h
@@ -43,9 +43,6 @@ char *retr_rate PARAMS ((long, double, int));
double calc_rate PARAMS ((long, double, int *));
void printwhat PARAMS ((int, int));
-void downloaded_increase PARAMS ((unsigned long));
-int downloaded_exceeds_quota PARAMS ((void));
-
void sleep_between_retrievals PARAMS ((int));
void rotate_backups PARAMS ((const char *));
diff --git a/src/sysdep.h b/src/sysdep.h
index 05a0e773..5d74f48a 100644
--- a/src/sysdep.h
+++ b/src/sysdep.h
@@ -128,27 +128,30 @@ do { \
DEBUGP (("Closing fd %d\n", x)); \
} while (0)
-/* Define a large ("very long") type useful for storing large
- non-negative quantities that exceed sizes of normal download. Note
- that this has nothing to do with large file support. For example,
- one should be able to say `--quota=10G', large files
- notwithstanding.
+/* Define a large integral type useful for storing large sizes that
+ exceed sizes of one download, such as when printing the sum of all
+ downloads. Note that this has nothing to do with large file
+ support, yet.
- On the machines where `long' is 64-bit, we use long. Otherwise, we
- check whether `long long' is available and if yes, use that. If
- long long is unavailable, we give up and just use `long'.
+ We use a 64-bit integral type where available, `double' otherwise.
+ It's hard to print LARGE_INT's portably, but fortunately it's
+ rarely needed. */
- Note: you cannot use VERY_LONG_TYPE along with printf(). When you
- need to print it, use very_long_to_string(). */
-
-#if SIZEOF_LONG >= 8 || SIZEOF_LONG_LONG == 0
-/* either long is "big enough", or long long is unavailable which
- leaves long as the only choice. */
-# define VERY_LONG_TYPE unsigned long
-#else /* use long long */
-/* long is smaller than 8 bytes, but long long is available. */
-# define VERY_LONG_TYPE unsigned long long
-#endif /* use long long */
+#if SIZEOF_LONG >= 8
+/* Long is large enough: use it. */
+typedef long LARGE_INT;
+# define LARGE_INT_FMT "%ld"
+#else
+# if SIZEOF_LONG_LONG == 8
+/* Long long is large enough: use it. */
+typedef long long LARGE_INT;
+# define LARGE_INT_FMT "%lld"
+# else
+/* Use `double'. */
+typedef double LARGE_INT;
+# define LARGE_INT_FMT "%.0f"
+# endif
+#endif
/* These are defined in cmpt.c if missing, therefore it's generally
safe to declare their parameters. */
@@ -167,6 +170,9 @@ char *strstr ();
#ifndef HAVE_STRPTIME
char *strptime ();
#endif
+#ifndef HAVE_SNPRINTF
+int snprintf ();
+#endif
#ifndef HAVE_VSNPRINTF
int vsnprintf ();
#endif
diff --git a/src/utils.c b/src/utils.c
index c78b1708..85dd5fcd 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -1290,13 +1290,13 @@ free_keys_and_values (struct hash_table *ht)
}
-/* Engine for legible and legible_very_long; this function works on
- strings. */
+/* Engine for legible and legible_large_int; add thousand separators
+ to numbers printed in strings. */
static char *
legible_1 (const char *repr)
{
- static char outbuf[128];
+ static char outbuf[48];
int i, i1, mod;
char *outptr;
const char *inptr;
@@ -1304,7 +1304,9 @@ legible_1 (const char *repr)
/* Reset the pointers. */
outptr = outbuf;
inptr = repr;
- /* If the number is negative, shift the pointers. */
+
+ /* Ignore the sign for the purpose of adding thousand
+ separators. */
if (*inptr == '-')
{
*outptr++ = '-';
@@ -1329,6 +1331,7 @@ legible_1 (const char *repr)
}
/* Legible -- return a static pointer to the legibly printed long. */
+
char *
legible (long l)
{
@@ -1338,53 +1341,29 @@ legible (long l)
return legible_1 (inbuf);
}
-/* Write a string representation of NUMBER into the provided buffer.
- We cannot use sprintf() because we cannot be sure whether the
- platform supports printing of what we chose for VERY_LONG_TYPE.
+/* Write a string representation of LARGE_INT NUMBER into the provided
+ buffer. The buffer should be able to accept 24 characters,
+ including the terminating zero.
- Example: Gcc supports `long long' under many platforms, but on many
- of those the native libc knows nothing of it and therefore cannot
- print it.
-
- How long BUFFER needs to be depends on the platform and the content
- of NUMBER. For 64-bit VERY_LONG_TYPE (the most common case), 24
- bytes are sufficient. Using more might be a good idea.
-
- This function does not go through the hoops that long_to_string
- goes to because it doesn't aspire to be fast. (It's called perhaps
- once in a Wget run.) */
+ It would be dangerous to use sprintf, because the code wouldn't
+ work on a machine with gcc-provided long long support, but without
+ libc support for "%lld". However, such platforms will typically
+ not have snprintf and will use our version, which does support
+ "%lld" where long longs are available. */
static void
-very_long_to_string (char *buffer, VERY_LONG_TYPE number)
+large_int_to_string (char *buffer, LARGE_INT number)
{
- int i = 0;
- int j;
-
- /* Print the number backwards... */
- do
- {
- buffer[i++] = '0' + number % 10;
- number /= 10;
- }
- while (number);
-
- /* ...and reverse the order of the digits. */
- for (j = 0; j < i / 2; j++)
- {
- char c = buffer[j];
- buffer[j] = buffer[i - 1 - j];
- buffer[i - 1 - j] = c;
- }
- buffer[i] = '\0';
+ snprintf (buffer, 24, LARGE_INT_FMT, number);
}
-/* The same as legible(), but works on VERY_LONG_TYPE. See sysdep.h. */
+/* The same as legible(), but works on LARGE_INT. */
+
char *
-legible_very_long (VERY_LONG_TYPE l)
+legible_large_int (LARGE_INT l)
{
- char inbuf[128];
- /* Print the number into the buffer. */
- very_long_to_string (inbuf, l);
+ char inbuf[48];
+ large_int_to_string (inbuf, l);
return legible_1 (inbuf);
}
diff --git a/src/utils.h b/src/utils.h
index 01a1d3de..d49847b7 100644
--- a/src/utils.h
+++ b/src/utils.h
@@ -102,7 +102,7 @@ void string_set_free PARAMS ((struct hash_table *));
void free_keys_and_values PARAMS ((struct hash_table *));
char *legible PARAMS ((long));
-char *legible_very_long PARAMS ((VERY_LONG_TYPE));
+char *legible_large_int PARAMS ((LARGE_INT));
int numdigit PARAMS ((long));
char *number_to_string PARAMS ((char *, long));