Fix 504 status handling

* src/http.c (gethttp): Move 504 handling to correct place.
  (http_loop): Fix memeory leak.
* testenv/server/http/http_server.py: Add Content-Length header on non-2xx
  status codes with a body

Reported-by: Adam Sampson

src/http.c

@@ -3476,7 +3476,7 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
 
 #ifdef HAVE_METALINK
   /* We need to check for the Metalink data in the very first response
-     we get from the server (before redirectionrs, authorization, etc.).  */
+     we get from the server (before redirections, authorization, etc.).  */
   if (metalink)
     {
       hs->metalink = metalink_from_http (resp, hs, u);
@@ -3496,7 +3496,7 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
       uerr_t auth_err = RETROK;
       bool retry;
       /* Normally we are not interested in the response body.
-         But if we are writing a WARC file we are: we like to keep everyting.  */
+         But if we are writing a WARC file we are: we like to keep everything.  */
       if (warc_enabled)
         {
           int _err;
@@ -3556,20 +3556,6 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
         pconn.authorized = true;
     }
 
-  if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT)
-    {
-      hs->len = 0;
-      hs->res = 0;
-      hs->restval = 0;
-
-      CLOSE_FINISH (sock);
-      xfree (hs->message);
-
-      retval = GATEWAYTIMEOUT;
-      goto cleanup;
-    }
-
-
   {
     uerr_t ret = check_file_output (u, hs, resp, hdrval, sizeof hdrval);
     if (ret != RETROK)
@@ -3910,7 +3896,7 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
               retval = _err;
               goto cleanup;
             }
-          else
+
           CLOSE_FINISH (sock);
         }
       else
@@ -3934,7 +3920,11 @@ gethttp (const struct url *u, struct url *original_url, struct http_stat *hs,
             CLOSE_INVALIDATE (sock);
         }
 
+      if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT)
+        retval = GATEWAYTIMEOUT;
+      else
         retval = RETRFINISHED;
+
       goto cleanup;
     }
 
@@ -4208,6 +4198,8 @@ http_loop (const struct url *u, struct url *original_url, char **newloc,
              bring them to "while" statement at the end, to judge
              whether the number of tries was exceeded.  */
           printwhat (count, opt.ntry);
+          xfree (hstat.message);
+          xfree (hstat.error);
           continue;
         case FWRITEERR: case FOPENERR:
           /* Another fatal error.  */

testenv/server/http/http_server.py

@@ -204,7 +204,6 @@ class _Handler(BaseHTTPRequestHandler):
 
     def Response(self, resp_obj):
         self.send_response(resp_obj.response_code)
-        self.finish_headers()
         if resp_obj.response_code == 304:
             raise NoBodyServerError("Conditional get falling to head")
         raise ServerError("Custom Response code sent.")
@@ -329,7 +328,6 @@ class _Handler(BaseHTTPRequestHandler):
         except AuthError as se:
             self.send_response(401, "Authorization Required")
             self.send_challenge(auth_rule.auth_type, auth_rule.auth_parm)
-            self.finish_headers()
             raise se
 
     def handle_auth(self, auth_rule):
@@ -362,7 +360,6 @@ class _Handler(BaseHTTPRequestHandler):
             if header_recd is None or header_recd != exp_headers[header_line]:
                 self.send_error(400, "Expected Header %s not found" %
                                 header_line)
-                self.finish_headers()
                 raise ServerError("Header " + header_line + " not found")
 
     def RejectHeader(self, header_obj):
@@ -372,7 +369,6 @@ class _Handler(BaseHTTPRequestHandler):
             if header_recd and header_recd == rej_headers[header_line]:
                 self.send_error(400, 'Blacklisted Header %s received' %
                                 header_line)
-                self.finish_headers()
                 raise ServerError("Header " + header_line + ' received')
 
     def __log_request(self, method):
@@ -400,6 +396,7 @@ class _Handler(BaseHTTPRequestHandler):
 
             content = self.server.fileSys.get(path)
             content_length = len(content)
+
             for rule_name in self.rules:
                 try:
                     assert hasattr(self, rule_name)
@@ -410,12 +407,16 @@ class _Handler(BaseHTTPRequestHandler):
                     return(None, None)
                 except AuthError as ae:
                     print(ae.__str__())
+                    self.finish_headers()
                     return(None, None)
                 except NoBodyServerError as nbse:
                     print(nbse.__str__())
+                    self.finish_headers()
                     return(None, None)
                 except ServerError as se:
                     print(se.__str__())
+                    self.add_header("Content-Length", content_length)
+                    self.finish_headers()
                     return(content, None)
 
             try: