IRI requirement: do not percent-encode already percent-encoded values (try1)

src/html-url.c

@@ -288,7 +288,7 @@ append_url (const char *link_uri, int position, int size,
           return NULL;
         }
 
-      url = url_parse (link_uri, NULL, NULL);
+      url = url_parse (link_uri, NULL, NULL, false);
       if (!url)
         {
           DEBUGP (("%s: link \"%s\" doesn't parse.\n",
@@ -307,7 +307,7 @@ append_url (const char *link_uri, int position, int size,
       DEBUGP (("%s: merge(\"%s\", \"%s\") -> %s\n",
                ctx->document_file, base, link_uri, complete_uri));
 
-      url = url_parse (complete_uri, NULL, NULL);
+      url = url_parse (complete_uri, NULL, NULL, false);
       if (!url)
         {
           DEBUGP (("%s: merged link \"%s\" doesn't parse.\n",
@@ -752,7 +752,7 @@ get_urls_file (const char *file)
           url_text = merged;
         }
 
-      url = url_parse (url_text, &up_error_code, NULL);
+      url = url_parse (url_text, &up_error_code, NULL, false);
       if (!url)
         {
           char *error = url_error (url_text, up_error_code);

src/iri.c

@@ -298,6 +298,7 @@ iri_new (void)
   struct iri *i = xmalloc (sizeof (struct iri));
   i->uri_encoding = opt.encoding_remote ? xstrdup (opt.encoding_remote) : NULL;
   i->content_encoding = NULL;
+  i->orig_url = NULL;
   i->utf8_encode = opt.enable_iri;
   return i;
 }
@@ -308,6 +309,7 @@ iri_free (struct iri *i)
 {
   xfree_null (i->uri_encoding);
   xfree_null (i->content_encoding);
+  xfree_null (i->orig_url);
   xfree (i);
 }
 

src/iri.h

@@ -33,6 +33,7 @@ as that of the covered work.  */
 struct iri {
   char *uri_encoding;      /* Encoding of the uri to fetch */
   char *content_encoding;  /* Encoding of links inside the fetched file */
+  char *orig_url;          /* */
   bool utf8_encode;        /* Will/Is the current url encoded in utf8 */
 };
 

src/recur.c

@@ -214,7 +214,7 @@ retrieve_tree (const char *start_url, struct iri *pi)
     set_uri_encoding (i, opt.locale, true);
 #undef COPYSTR
 
-  start_url_parsed = url_parse (start_url, &up_error_code, i);
+  start_url_parsed = url_parse (start_url, &up_error_code, i, true);
   if (!start_url_parsed)
     {
       char *error = url_error (start_url, up_error_code);
@@ -381,7 +381,7 @@ retrieve_tree (const char *start_url, struct iri *pi)
           if (children)
             {
               struct urlpos *child = children;
-              struct url *url_parsed = url_parse (url, NULL, i);
+              struct url *url_parsed = url_parse (url, NULL, i, false);
               struct iri *ci;
               char *referer_url = url;
               bool strip_auth = (url_parsed != NULL
@@ -694,10 +694,10 @@ descend_redirect_p (const char *redirected, const char *original, int depth,
   struct urlpos *upos;
   bool success;
 
-  orig_parsed = url_parse (original, NULL, NULL);
+  orig_parsed = url_parse (original, NULL, NULL, false);
   assert (orig_parsed != NULL);
 
-  new_parsed = url_parse (redirected, NULL, NULL);
+  new_parsed = url_parse (redirected, NULL, NULL, false);
   assert (new_parsed != NULL);
 
   upos = xnew0 (struct urlpos);

src/retr.c

@@ -626,7 +626,7 @@ retrieve_url (const char *origurl, char **file, char **newloc,
     *file = NULL;
 
  second_try:
-  u = url_parse (url, &up_error_code, iri);
+  u = url_parse (url, &up_error_code, iri, true);
   if (!u)
     {
       char *error = url_error (url, up_error_code);
@@ -658,7 +658,7 @@ retrieve_url (const char *origurl, char **file, char **newloc,
       pi->utf8_encode = false;
 
       /* Parse the proxy URL.  */
-      proxy_url = url_parse (proxy, &up_error_code, NULL);
+      proxy_url = url_parse (proxy, &up_error_code, NULL, true);
       if (!proxy_url)
         {
           char *error = url_error (proxy, up_error_code);
@@ -739,9 +739,10 @@ retrieve_url (const char *origurl, char **file, char **newloc,
          the content encoding. */
       iri->utf8_encode = opt.enable_iri;
       set_content_encoding (iri, NULL);
+      xfree_null (iri->orig_url);
 
       /* Now, see if this new location makes sense. */
-      newloc_parsed = url_parse (mynewloc, &up_error_code, iri);
+      newloc_parsed = url_parse (mynewloc, &up_error_code, iri, true);
       if (!newloc_parsed)
         {
           char *error = url_error (mynewloc, up_error_code);
@@ -794,7 +795,7 @@ retrieve_url (const char *origurl, char **file, char **newloc,
   if (!(*dt & RETROKF) && iri->utf8_encode)
     {
       iri->utf8_encode = false;
-      DEBUGP (("[IRI Fallbacking to non-utf8 for %s\n", quote (url)));
+      DEBUGP (("[IRI fallbacking to non-utf8 for %s\n", quote (url)));
       goto second_try;
     }
 
@@ -907,6 +908,8 @@ retrieve_from_file (const char *file, bool html, int *count)
 
       /* Reset UTF-8 encode status */
       iri->utf8_encode = opt.enable_iri;
+      xfree_null (iri->orig_url);
+      iri->orig_url = NULL;
 
       if ((opt.recursive || opt.page_requisites)
           && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
@@ -1100,7 +1103,7 @@ url_uses_proxy (const char *url)
   struct iri *i = iri_new();
   /* url was given in the command line, so use locale as encoding */
   set_uri_encoding (i, opt.locale, true);
-  u= url_parse (url, NULL, i);
+  u= url_parse (url, NULL, i, false);
   if (!u)
     return false;
   ret = getproxy (u) != NULL;

src/url.c

@@ -640,7 +640,7 @@ static const char *parse_errors[] = {
    error, and if ERROR is not NULL, also set *ERROR to the appropriate
    error code. */
 struct url *
-url_parse (const char *url, int *error, struct iri *iri)
+url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
 {
   struct url *u;
   const char *p;
@@ -672,13 +672,19 @@ url_parse (const char *url, int *error, struct iri *iri)
 
   if (iri && iri->utf8_encode)
     {
-      url_unescape ((char *) url);
+      iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
-      iri->utf8_encode = remote_to_utf8 (iri, url, (const char **) &new_url);
       if (!iri->utf8_encode)
         new_url = NULL;
+      else
+        iri->orig_url = xstrdup (url);
     }
 
-  url_encoded = reencode_escapes (new_url ? new_url : url);
+  /* XXX XXX Could that change introduce (security) bugs ???  XXX XXX*/
+  if (percent_encode)
+    url_encoded = reencode_escapes (new_url ? new_url : url);
+  else
+     url_encoded = new_url ? new_url : url;
+
   p = url_encoded;
 
   if (new_url && url_encoded != new_url)
@@ -1992,12 +1998,12 @@ schemes_are_similar_p (enum url_scheme a, enum url_scheme b)
 
 static int
 getchar_from_escaped_string (const char *str, char *c)
-{  
+{
   const char *p = str;
 
   assert (str && *str);
   assert (c);
-  
+
   if (p[0] == '%')
     {
       if (!c_isxdigit(p[1]) || !c_isxdigit(p[2]))
@@ -2047,7 +2053,7 @@ are_urls_equal (const char *u1, const char *u2)
       p += pp;
       q += qq;
     }
-  
+
   return (*p == 0 && *q == 0 ? true : false);
 }
 
@@ -2156,7 +2162,7 @@ test_append_uri_pathel()
   } test_array[] = {
     { "http://www.yoyodyne.com/path/", "somepage.html", false, "http://www.yoyodyne.com/path/somepage.html" },
   };
-  
+
   for (i = 0; i < sizeof(test_array)/sizeof(test_array[0]); ++i) 
     {
       struct growable dest;

src/url.h

@@ -84,7 +84,7 @@ struct url
 
 char *url_escape (const char *);
 
-struct url *url_parse (const char *, int *, struct iri *iri);
+struct url *url_parse (const char *, int *, struct iri *iri, bool percent_encode);
 char *url_error (const char *, int);
 char *url_full_path (const struct url *);
 void url_set_dir (struct url *, const char *);

tests/Test-iri.px

@@ -214,9 +214,9 @@ my %expected_downloaded_files = (
 ###############################################################################
 
 my $the_test = HTTPTest->new (name => "Test-iri",
-                              input => \%urls, 
+                              input => \%urls,
-                              cmdline => $cmdline, 
+                              cmdline => $cmdline,
-                              errcode => $expected_error_code, 
+                              errcode => $expected_error_code,
                               output => \%expected_downloaded_files);
 exit $the_test->run();