Add support for --retry-on-http-error

* doc/wget.text: Add documentation
* src/http.c: Add function check_retry_on_http_error ()
* src/init.c: Add opt.retry_on_http_error
* src/main.c: Add struct for retry-on-http-error to option_data[]
* src/options.h: Add retry_on_http_error to struct options

doc/wget.texi

@@ -1718,6 +1718,21 @@ some few obscure servers, which never send HTTP authentication
 challenges, but accept unsolicited auth info, say, in addition to
 form-based authentication.
 
+@item --retry-on-http-error=@var{code[,code,...]}
+Consider given HTTP response codes as non-fatal, transient errors.
+Supply a comma-separated list of 3-digit HTTP response codes as
+argument. Useful to work around special circumstances where retries
+are required, but the server responds with an error code normally not
+retried by Wget. Such errors might be 503 (Service Unavailable) and
+429 (Too Many Requests). Retries enabled by this option are performed
+subject to the normal retry timing and retry count limitations of
+Wget.
+
+Using this option is intended to support special use cases only and is
+generally not recommended, as it can force retries even in cases where
+the server is actually trying to decrease its load. Please use wisely
+and only if you know what you are doing.
+
 @end table
 
 @node HTTPS (SSL/TLS) Options, FTP Options, HTTP Options, Invoking

src/http.c

@@ -3982,6 +3982,22 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
   return retval;
 }
 
+/* Check whether the supplied HTTP status code is among those
+   listed for the --retry-on-http-error option. */
+static bool
+check_retry_on_http_error (const int statcode)
+{
+  const char *tok = opt.retry_on_http_error;
+  while (tok && *tok)
+    {
+      if (atoi (tok) == statcode)
+        return true;
+      if (tok = strchr (tok, ','))
+        ++tok;
+    }
+  return false;
+}
+
 /* The genuine HTTP loop!  This is the part where the retrieval is
    retried, and retried, and retried, and...  */
 uerr_t
@@ -4319,6 +4335,11 @@ http_loop (const struct url *u, struct url *original_url, char **newloc,
               logprintf (LOG_NOTQUIET, _("\
 Remote file does not exist -- broken link!!!\n"));
             }
+          else if (check_retry_on_http_error (hstat.statcode))
+            {
+              printwhat (count, opt.ntry);
+              continue;
+            }
           else
             {
               logprintf (LOG_NOTQUIET, _("%s ERROR %d: %s.\n"),

src/init.c

@@ -304,6 +304,7 @@ static const struct {
   { "restrictfilenames", NULL,                  cmd_spec_restrict_file_names },
   { "retrsymlinks",     &opt.retr_symlinks,     cmd_boolean },
   { "retryconnrefused", &opt.retry_connrefused, cmd_boolean },
+  { "retryonhttperror", &opt.retry_on_http_error, cmd_string },
   { "robots",           &opt.use_robots,        cmd_boolean },
   { "savecookies",      &opt.cookies_output,    cmd_file },
   { "saveheaders",      &opt.save_headers,      cmd_boolean },
@@ -1979,6 +1980,7 @@ cleanup (void)
   xfree (opt.body_file);
   xfree (opt.rejected_log);
   xfree (opt.use_askpass);
+  xfree (opt.retry_on_http_error);
 
 #ifdef HAVE_LIBCARES
 #include <ares.h>

src/main.c

@@ -404,6 +404,7 @@ static struct cmdline_option option_data[] =
     { "restrict-file-names", 0, OPT_BOOLEAN, "restrictfilenames", -1 },
     { "retr-symlinks", 0, OPT_BOOLEAN, "retrsymlinks", -1 },
     { "retry-connrefused", 0, OPT_BOOLEAN, "retryconnrefused", -1 },
+    { "retry-on-http-error", 0, OPT_VALUE, "retryonhttperror", -1 },
     { "save-cookies", 0, OPT_VALUE, "savecookies", -1 },
     { "save-headers", 0, OPT_BOOLEAN, "saveheaders", -1 },
     { IF_SSL ("secure-protocol"), 0, OPT_VALUE, "secureprotocol", -1 },

src/options.h

@@ -43,6 +43,7 @@ struct options
   bool quiet;                   /* Are we quiet? */
   int ntry;                     /* Number of tries per URL */
   bool retry_connrefused;       /* Treat CONNREFUSED as non-fatal. */
+  char *retry_on_http_error;    /* Treat given HTTP errors as non-fatal. */
   bool background;              /* Whether we should work in background. */
   bool ignore_length;           /* Do we heed content-length at all?  */
   bool recursive;               /* Are we recursive? */