github-repos/wget
1
0
Fork 0
mirror of https://github.com/mirror/wget.git synced 2025-03-14 20:00:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make thread support optional. - No check for semaphore in configure.ac.

Browse Source
This commit is contained in:
Ilim Ugur 2012-07-29 23:43:03 +03:00
parent f07e0a67b4
commit 38b1df95f9
12 changed files with 318 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
configure.ac
View File

@ -350,6 +350,29 @@ else
fi
fi
AS_IF([test x"$ENABLE_THREADS" != xno],[
AC_CHECK_LIB(pthread, pthread_create, [
dnl AC_CHECK_LIB(semaphore, sem_init, [
threads=yes
AC_DEFINE([ENABLE_THREADS], 1,
[Define if you want the thread support compiled in.])
AC_MSG_NOTICE([compiling with support for threads])
dnl ],[
dnl threads=no
dnl if test x"$ENABLE_THREADS" == xyes
dnl then
dnl AC_MSG_ERROR([Could not find the sem_init function in semaphore.])
dnl fi
dnl ])
],[
threads=no
if test x"$ENABLE_THREADS" == xyes
then
AC_MSG_ERROR([Could not find the pthread_create function in pthread.])
fi
])
])
AS_IF([test x"$ENABLE_METALINK" != xno],[
dnl If metalink support is requested
PKG_CHECK_MODULES(LIBMETALINK, libmetalink, [
@ -363,6 +386,9 @@ AS_IF([test x"$ENABLE_METALINK" != xno],[
if test x"$DISABLE_THREADS" == xyes
then
AC_MSG_ERROR([Metalink support depends on threads, which are requested to be disabled.])
elif test "X$threads" != "Xyes"
then
AC_MSG_ERROR([Metalink support depends on threads, support for which could not be enabled.])
else
metalink=yes
AC_MSG_NOTICE([compiling in support for metalink])
@ -564,6 +590,7 @@ fi
dnl Needed by src/Makefile.am
AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"])
AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$metalink" != "Xno"])
AM_CONDITIONAL([THREADS_ARE_ENABLED], [test "X$threads" != "Xno"])
dnl
dnl Create output

8
src/Makefile.am
View File

@ -37,6 +37,9 @@ endif
if METALINK_IS_ENABLED
METALINK_OBJ = metalink.c
endif
if THREADS_ARE_ENABLED
THREAD_OBJ = multi.c
endif
# The following line is losing on some versions of make!
DEFS = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\"
@ -49,8 +52,9 @@ wget_SOURCES = cmpt.c connect.c convert.c cookies.c ftp.c \
css_.c css-url.c \
ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c \
http.c init.c log.c main.c netrc.c progress.c ptimer.c \
recur.c res.c retr.c spider.c url.c multi.c \
utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ) \
recur.c res.c retr.c spider.c url.c \
utils.c exits.c build_info.c $(IRI_OBJ) \
$(THREAD_OBJ) $(METALINK_OBJ) \
css-url.h css-tokens.h connect.h convert.h cookies.h \
ftp.h hash.h host.h html-parse.h html-url.h \
http.h http-ntlm.h init.h log.h mswindows.h netrc.h \

1
src/build_info.c.in
View File

@ -7,6 +7,7 @@ large-file SIZEOF_OFF_T >= 8
nls defined ENABLE_NLS
ntlm defined ENABLE_NTLM
opie defined ENABLE_OPIE
threads defined ENABLE_THREADS
metalink defined ENABLE_METALINK
ssl choice:

4
src/connect.h
View File

@ -59,7 +59,11 @@ enum {
WAIT_FOR_READ = 1,
WAIT_FOR_WRITE = 2
};
#ifdef ENABLE_THREADS
int select_fds (int *, int *, int, double, int);
#else
int select_fd (int, double, int);
#endif
bool test_socket_open (int);
struct transport_implementation {

29
src/convert.c
View File

@ -36,8 +36,10 @@ as that of the covered work. */
#include <unistd.h>
#include <errno.h>
#include <assert.h>
#include "convert.h"
#ifdef ENABLE_THREADS
#include <pthread.h>
#endif
#include "convert.h"
#include "url.h"
#include "recur.h"
#include "utils.h"
@ -56,6 +58,7 @@ struct hash_table *dl_url_file_map;
struct hash_table *downloaded_html_set;
struct hash_table *downloaded_css_set;
#ifdef ENABLE_THREADS
static pthread_mutex_t convert_mutex = PTHREAD_MUTEX_INITIALIZER;
#define THREAD_SAFE(ret, fn, args, argv) \
@ -76,6 +79,11 @@ static pthread_mutex_t convert_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_unlock (&convert_mutex); \
}
#define FNNAME_WTHREADS(fn) fn##_1
#else
#define FNNAME_WTHREADS(fn) fn
#endif
static void convert_links (const char *, struct urlpos *);
@ -200,7 +208,7 @@ convert_links_in_hashtable (struct hash_table *downloaded_set,
extracted from these two lists. */
static void
convert_all_links_1 (void)
FNNAME_WTHREADS(convert_all_links) (void)
{
double secs;
int file_count = 0;
@ -778,7 +786,7 @@ dissociate_urls_from_file (const char *file)
URL has already been downloaded. */
static void
register_download_1 (const char *url, const char *file)
FNNAME_WTHREADS(register_download) (const char *url, const char *file)
{
char *old_file, *old_url;
@ -858,7 +866,7 @@ register_download_1 (const char *url, const char *file)
register_download() above. */
static void
register_redirection_1 (const char *from, const char *to)
FNNAME_WTHREADS(register_redirection) (const char *from, const char *to)
{
char *file;
@ -873,7 +881,7 @@ register_redirection_1 (const char *from, const char *to)
/* Register that the file has been deleted. */
static void
register_delete_file_1 (const char *file)
FNNAME_WTHREADS(register_delete_file) (const char *file)
{
char *old_url, *old_file;
@ -891,7 +899,7 @@ register_delete_file_1 (const char *file)
/* Register that FILE is an HTML file that has been downloaded. */
static void
register_html_1 (const char *url, const char *file)
FNNAME_WTHREADS(register_html) (const char *url, const char *file)
{
if (!downloaded_html_set)
downloaded_html_set = make_string_hash_table (0);
@ -901,7 +909,7 @@ register_html_1 (const char *url, const char *file)
/* Register that FILE is a CSS file that has been downloaded. */
static void
register_css_1 (const char *url, const char *file)
FNNAME_WTHREADS(register_css) (const char *url, const char *file)
{
if (!downloaded_css_set)
downloaded_css_set = make_string_hash_table (0);
@ -913,7 +921,7 @@ static void downloaded_files_free (void);
/* Cleanup the data structures associated with this file. */
static void
convert_cleanup_1 (void)
FNNAME_WTHREADS(convert_cleanup) (void)
{
if (dl_file_url_map)
{
@ -988,7 +996,7 @@ downloaded_mode_to_ptr (downloaded_file_t mode)
URLs. */
static downloaded_file_t
downloaded_file_1 (downloaded_file_t mode, const char *file)
FNNAME_WTHREADS(downloaded_file) (downloaded_file_t mode, const char *file)
{
downloaded_file_t *ptr;
@ -1102,7 +1110,7 @@ html_quote_string (const char *s)
return res;
}
#ifdef ENABLE_THREADS
THREAD_SAFE (downloaded_file_t, downloaded_file, (downloaded_file_t a, const char *b), (a, b));
THREAD_SAFE_VOID (register_download, (const char *a, const char *b), (a, b));
THREAD_SAFE_VOID (register_redirection, (const char *a, const char *b), (a, b));
@ -1111,6 +1119,7 @@ THREAD_SAFE_VOID (register_css, (const char *a, const char *b), (a, b));
THREAD_SAFE_VOID (register_delete_file, (const char *a), (a));
THREAD_SAFE_VOID (convert_cleanup, (void), ());
THREAD_SAFE_VOID (convert_all_links, (void), ());
#endif
/*
* vim: et ts=2 sw=2

180
src/http.c
View File

@ -39,7 +39,9 @@ as that of the covered work. */
#include <errno.h>
#include <time.h>
#include <locale.h>
#ifdef ENABLE_THREADS
#include <pthread.h>
#endif
#include "hash.h"
#include "http.h"
@ -1193,7 +1195,12 @@ parse_content_disposition (const char *hdr, char **filename)
below. Ideally, it should be possible to cache an arbitrary fixed
number of these connections. */
#ifndef ENABLE_THREADS
/* Whether a persistent connection is active. */
static bool pconn_active;
static struct {
#else
static pthread_mutex_t pconn_mutex = PTHREAD_MUTEX_INITIALIZER;
#define PCONN_LOCK() pthread_mutex_lock (&pconn_mutex)
@ -1203,6 +1210,7 @@ static pthread_mutex_t pconn_mutex = PTHREAD_MUTEX_INITIALIZER;
static struct s_pconn {
struct s_pconn *next;
#endif
/* The socket of the connection. */
int socket;
@ -1224,14 +1232,29 @@ static struct s_pconn {
/* NTLM data of the current connection. */
struct ntlmdata ntlm;
#endif
#ifndef ENABLE_THREADS
} pconn;
#else
} *pconn = NULL;
size_t pconn_length = 0;
#endif
/* Mark the persistent connection as invalid and free the resources it
uses. This is used by the CLOSE_* macros after they forcefully
close a registered persistent connection. */
#ifndef ENABLE_THREADS
static void
invalidate_persistent (void)
{
DEBUGP (("Disabling further reuse of socket %d.\n", pconn.socket));
pconn_active = false;
fd_close (pconn.socket);
xfree (pconn.host);
xzero (pconn);
}
#else
static void
invalidate_persistent (int fd)
{
@ -1264,7 +1287,7 @@ invalidate_persistent (int fd)
PCONN_UNLOCK ();
}
#endif
/* Register FD, which should be a TCP/IP connection to HOST:PORT, as
persistent. This will enable someone to use the same connection
later. In the context of HTTP, this must be called only AFTER the
@ -1276,6 +1299,34 @@ invalidate_persistent (int fd)
static void
register_persistent (const char *host, int port, int fd, bool ssl)
{
#ifndef ENABLE_THREADS
if (pconn_active)
{
if (pconn.socket == fd)
{
/* The connection FD is already registered. */
return;
}
else
{
/* The old persistent connection is still active; close it
first. This situation arises whenever a persistent
connection exists, but we then connect to a different
host, and try to register a persistent connection to that
one. */
invalidate_persistent ();
}
}
pconn_active = true;
pconn.socket = fd;
pconn.host = xstrdup (host);
pconn.port = port;
pconn.ssl = ssl;
pconn.authorized = false;
DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
#else
PCONN_LOCK ();
struct s_pconn *it;
@ -1321,29 +1372,48 @@ register_persistent (const char *host, int port, int fd, bool ssl)
DEBUGP (("Registered socket %d for persistent reuse.\n", fd));
exit:
PCONN_UNLOCK ();
#endif
}
/* Return true if a persistent connection is available for connecting
to HOST:PORT. */
static bool
#ifndef ENABLE_THREADS
persistent_available_p (const char *host, int port, bool ssl,
bool *host_lookup_failed)
#else
persistent_available_p (struct s_pconn *pconn, const char *host, int port,
bool ssl, bool *host_lookup_failed)
#endif
{
#ifndef ENABLE_THREADS
/* First, check whether a persistent connection is active at all. */
if (!pconn_active)
return false;
/* If we want SSL and the last connection wasn't or vice versa,
don't use it. Checking for host and port is not enough because
HTTP and HTTPS can apparently coexist on the same port. */
if (ssl != pconn->ssl)
if (ssl != pconn.ssl)
return false;
/* If we're not connecting to the same port, we're not interested. */
if (port != pconn->port)
if (port != pconn.port)
return false;
/* If the host is the same, we're in business. If not, there is
still hope -- read below. */
if (0 != strcasecmp (host, pconn.host))
#else
if (ssl != pconn->ssl)
return false;
if (port != pconn->port)
return false;
if (0 != strcasecmp (host, pconn->host))
#endif
{
/* Check if pconn.socket is talking to HOST under another name.
This happens often when both sites are virtual hosts
@ -1367,13 +1437,21 @@ persistent_available_p (struct s_pconn *pconn, const char *host, int port,
resolves to, pconn.socket is for all intents and purposes
already talking to HOST. */
if (!socket_ip_address (pconn->socket, &ip, ENDPOINT_PEER))
#ifndef ENABLE_THREADS
if (!socket_ip_address (pconn.socket, &ip, ENDPOINT_PEER))
{
/* Can't get the peer's address -- something must be very
wrong with the connection. */
wrong with the connection. */
invalidate_persistent ();
return false;
}
#else
if (!socket_ip_address (pconn->socket, &ip, ENDPOINT_PEER))
{
invalidate_persistent (pconn->socket);
return false;
}
#endif
al = lookup_host (host, 0);
if (!al)
{
@ -1405,19 +1483,27 @@ persistent_available_p (struct s_pconn *pconn, const char *host, int port,
body in response to HEAD, or if it sends more than conent-length
data, we won't reuse the corrupted connection.) */
if (!test_socket_open (pconn->socket))
#ifndef ENABLE_THREADS
if (!test_socket_open (pconn.socket))
{
/* Oops, the socket is no longer open. Now that we know that,
let's invalidate the persistent connection before returning
0. */
invalidate_persistent ();
return false;
}
#else
if (!test_socket_open (pconn->socket))
{
invalidate_persistent (pconn->socket);
return false;
}
#endif
return true;
}
#ifdef ENABLE_THREADS
static struct s_pconn *
get_persistent (const char *host, int port, bool ssl, bool *host_lookup_failed)
{
@ -1448,6 +1534,7 @@ get_persistent (const char *host, int port, bool ssl, bool *host_lookup_failed)
return it;
}
#endif
/* The idea behind these two CLOSE macros is to distinguish between
@ -1465,6 +1552,31 @@ get_persistent (const char *host, int port, bool ssl, bool *host_lookup_failed)
`pc_active_p && (fd) == pc_last_fd' is "we're *now* using an
active, registered connection". */
#ifndef ENABLE_THREADS
#define CLOSE_FINISH(fd) do { \
if (!keep_alive) \
{ \
if (pconn_active && (fd) == pconn.socket) \
invalidate_persistent (); \
else \
{ \
fd_close (fd); \
fd = -1; \
} \
} \
} while (0)
#define CLOSE_INVALIDATE(fd) do { \
if (pconn_active && (fd) == pconn.socket) \
invalidate_persistent (); \
else \
fd_close (fd); \
fd = -1; \
} while (0)
#else
#define CLOSE_FINISH(fd) do { \
if (!keep_alive) \
{ \
@ -1475,6 +1587,8 @@ get_persistent (const char *host, int port, bool ssl, bool *host_lookup_failed)
#define CLOSE_INVALIDATE(fd) do { \
invalidate_persistent (fd); \
} while (0)
#endif
struct http_stat
{
@ -1598,7 +1712,10 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
int sock = -1;
int flags;
#ifdef ENABLE_THREADS
struct s_pconn *pconn = NULL;
#endif
/* Set to 1 when the authorization has already been sent and should
not be tried again. */
@ -1870,6 +1987,25 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
relevant = u;
#endif
#ifndef ENABLE_THREADS
if (persistent_available_p (relevant->host, relevant->port,
#ifdef HAVE_SSL
relevant->scheme == SCHEME_HTTPS,
#else
0,
#endif
&host_lookup_failed))
{
sock = pconn.socket;
using_ssl = pconn.ssl;
logprintf (LOG_VERBOSE, _("Reusing existing connection to %s:%d.\n"),
quotearg_style (escape_quoting_style, pconn.host),
pconn.port);
DEBUGP (("Reusing fd %d.\n", sock));
if (pconn.authorized)
#else
pconn = get_persistent (relevant->host, relevant->port,
#ifdef HAVE_SSL
relevant->scheme == SCHEME_HTTPS,
@ -1887,6 +2023,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
pconn->port);
DEBUGP (("Reusing fd %d.\n", sock));
if (pconn->authorized)
#endif
/* If the connection is already authorized, the "Basic"
authorization added by code above is unnecessary and
only hurts us. */
@ -2166,7 +2303,11 @@ read_header:
CLOSE_FINISH (sock);
else
CLOSE_INVALIDATE (sock);
#ifndef ENABLE_THREADS
pconn.authorized = false;
#else
pconn->authorized = false;
#endif
if (!auth_finished && (user && passwd))
{
/* IIS sends multiple copies of WWW-Authenticate, one with
@ -2234,7 +2375,13 @@ read_header:
{
/* Kludge: if NTLM is used, mark the TCP connection as authorized. */
if (ntlm_seen)
{
#ifndef ENABLE_THREADS
pconn.authorized = true;
#else
pconn->authorized = true;
#endif
}
}
/* Determine the local filename if needed. Notice that if -O is used
@ -3571,6 +3718,17 @@ create_authorization_line (const char *au, const char *user,
*finished = true;
return digest_authentication_encode (au, user, passwd, method, path);
#endif
#ifdef ENABLE_NTLM && ENABLE_THREADS
case 'N': /* NTLM */
if (!ntlm_input (&pconn.ntlm, au))
{
*finished = true;
return NULL;
}
return ntlm_output (&pconn.ntlm, user, passwd, finished);
#endif
#ifdef ENABLE_NTLM
case 'N': /* NTLM */
if (!ntlm_input (&pconn->ntlm, au))
@ -3609,6 +3767,9 @@ save_cookies (void)
void
http_cleanup (void)
{
#ifndef ENABLE_THREADS
xfree_null (pconn.host);
#else
struct s_pconn *it = pconn;
for (; it; it = it->next)
@ -3616,6 +3777,7 @@ http_cleanup (void)
xfree_null (it->host);
xzero (it);
}
#endif
if (wget_cookie_jar)
cookie_jar_delete (wget_cookie_jar);

2
src/init.c
View File

@ -193,7 +193,9 @@ static const struct {
#endif
{ "input", &opt.input_filename, cmd_file },
{ "iri", &opt.enable_iri, cmd_boolean },
#ifdef HAVE_SSL
{ "jobs", &opt.jobs, cmd_number },
#endif
{ "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean },
{ "limitrate", &opt.limit_rate, cmd_bytes },
{ "loadcookies", &opt.cookies_input, cmd_file },

4
src/main.c
View File

@ -224,7 +224,9 @@ static struct cmdline_option option_data[] =
#endif
{ "input-file", 'i', OPT_VALUE, "input", -1 },
{ "iri", 0, OPT_BOOLEAN, "iri", -1 },
#ifdef ENABLE_THREADS
{ "jobs", 0, OPT_VALUE, "jobs", 1 },
#endif
{ "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 },
{ "level", 'l', OPT_VALUE, "reclevel", -1 },
{ "limit-rate", 0, OPT_VALUE, "limitrate", -1 },
@ -655,8 +657,10 @@ FTP options:\n"),
Recursive download:\n"),
N_("\
-r, --recursive specify recursive download.\n"),
#ifdef ENABLE_THREADS
N_("\
--jobs specify how many threads use.\n"),
#endif
N_("\
-l, --level=NUMBER maximum recursion depth (inf or 0 for infinite).\n"),
N_("\

4
src/multi.c
View File

@ -1,8 +1,10 @@
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#ifdef ENABLE_THREADS
#include <pthread.h>
#include <semaphore.h>
#include <errno.h>
#endif
#include "wget.h"

3
src/options.h
View File

@ -257,8 +257,9 @@ struct options
bool show_all_dns_entries; /* Show all the DNS entries when resolving a
name. */
#ifdef ENABLE_THREADS
int jobs; /* How many threads use at the same time. */
#endif
};
extern struct options opt;

84
src/recur.c
View File

@ -37,8 +37,10 @@ as that of the covered work. */
#include <unistd.h>
#include <errno.h>
#include <assert.h>
#ifdef ENABLE_THREADS
#include <pthread.h>
#include <semaphore.h>
#endif
#include "url.h"
#include "recur.h"
@ -168,6 +170,7 @@ static bool download_child_p (const struct urlpos *, struct url *, int,
static bool descend_redirect_p (const char *, struct url *, int,
struct url *, struct hash_table *, struct iri *);
#ifdef ENABLE_THREADS
static void *
start_retrieve_url (void *arg)
{
@ -180,6 +183,7 @@ start_retrieve_url (void *arg)
ctx->terminated = 1;
sem_post (ctx->retr_sem);
}
#endif
/* Retrieve a part of the web beginning with START_URL. This used to
be called "recursive retrieval", because the old function was
@ -205,14 +209,18 @@ start_retrieve_url (void *arg)
uerr_t
retrieve_tree (struct url *start_url_parsed, struct iri *pi)
{
uerr_t status = RETROK;
#ifdef ENABLE_THREADS
const int N_THREADS = opt.jobs > 0 ? opt.jobs : 1;
sem_t retr_sem;
int free_threads = N_THREADS;
uerr_t status = RETROK;
char *next_url = NULL, *next_referer;
int next_depth;
bool next_html_allowed, next_css_allowed;
struct s_thread_ctx *thread_ctx;
#endif
/* The queue of URLs we need to load. */
struct url_queue *queue;
@ -222,8 +230,6 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
struct iri *i = iri_new ();
struct s_thread_ctx *thread_ctx;
#define COPYSTR(x) (x) ? xstrdup(x) : NULL;
/* Duplicate pi struct if not NULL */
if (pi)
@ -236,9 +242,11 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
set_uri_encoding (i, opt.locale, true);
#undef COPYSTR
#ifdef ENABLE_THREADS
thread_ctx = calloc (N_THREADS, sizeof *thread_ctx);
sem_init (&retr_sem, 0, 0);
/* FIXME: CHECK FOR ERRORS. */
#endif
queue = url_queue_new ();
blacklist = make_string_hash_table (0);
@ -255,11 +263,16 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
char *file = NULL;
bool is_css = false;
bool dash_p_leaf_HTML = false;
#ifndef ENABLE_THREADS
char *url, *referer;
int depth;
bool html_allowed, css_allowed;
#else
int index = 0;
char *url = NULL, *referer;
int depth;
bool html_allowed, css_allowed;
#endif
if (opt.quota && total_downloaded_bytes > opt.quota)
break;
if (status == FWRITEERR)
@ -267,6 +280,12 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
/* Get the next URL from the queue... */
#ifndef ENABLE_THREADS
if (!url_dequeue (queue, (struct iri **) &i,
(const char **)&url, (const char **)&referer,
&depth, &html_allowed, &css_allowed))
break;
#else
if (next_url == NULL)
{
if (! url_dequeue (queue, (struct iri **) &i,
@ -280,6 +299,7 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
depth = next_depth;
html_allowed = next_html_allowed;
css_allowed = next_css_allowed;
#endif
/* ...and download it. Note that this download is in most cases
unconditional, as download_child_p already makes sure a file
@ -289,7 +309,11 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
and again under URL2, but at a different (possibly smaller)
depth, we want the URL's children to be taken into account
the second time. */
if (0 && url && dl_url_file_map && hash_table_contains (dl_url_file_map, url))
if (
#ifdef ENABLE_THREADS
0 && url &&
#endif
dl_url_file_map && hash_table_contains (dl_url_file_map, url))
{
file = xstrdup (hash_table_get (dl_url_file_map, url));
@ -312,10 +336,20 @@ retrieve_tree (struct url *start_url_parsed, struct iri *pi)
is_css = true;
}
#ifdef ENABLE_THREADS
next_url = NULL;
#endif
}
else
{
#ifndef ENABLE_THREADS
int dt = 0, url_err;
char *redirected = NULL;
struct url *url_parsed = url_parse (url, &url_err, i, true);
status = retrieve_url (url_parsed, url, &file, &redirected, referer,
&dt, false, i, true);
#else
pthread_t thread;
int j;
retry:
@ -389,9 +423,15 @@ retry:
referer = thread_ctx[index].referer;
i = thread_ctx[index].i;
url = thread_ctx[index].url;
#endif
if (html_allowed && file && status == RETROK
&& (thread_ctx[index].dt & RETROKF) && (thread_ctx[index].dt & TEXTHTML))
#ifndef ENABLE_THREADS
&& (dt & RETROKF) && (dt & TEXTHTML)
#else
&& (thread_ctx[index].dt & RETROKF) && (thread_ctx[index].dt & TEXTHTML)
#endif
)
{
descend = true;
is_css = false;
@ -401,22 +441,36 @@ retry:
lots of web servers serve css with an incorrect content type
*/
if (file && status == RETROK
#ifndef ENABLE_THREADS
&& (dt & RETROKF) &&
((dt & TEXTCSS) || css_allowed)
#else
&& (thread_ctx[index].dt & RETROKF) &&
((thread_ctx[index].dt & TEXTCSS) || css_allowed))
((thread_ctx[index].dt & TEXTCSS) || css_allowed)
#endif
)
{
descend = true;
is_css = true;
}
#ifndef ENABLE_THREADS
if (redirected)
#else
if (thread_ctx[index].redirected)
#endif
{
/* We have been redirected, possibly to another host, or
different path, or wherever. Check whether we really
want to follow it. */
if (descend)
{
#ifndef ENABLE_THREADS
if (!descend_redirect_p (redirected, url_parsed, depth,
#else
if (!descend_redirect_p (thread_ctx[index].redirected,
thread_ctx[index].url_parsed, depth,
#endif
start_url_parsed, blacklist, i))
descend = false;
else
@ -425,15 +479,29 @@ retry:
string_set_add (blacklist, url);
}
#ifndef ENABLE_THREADS
xfree (url);
url = redirected;
#else
xfree (thread_ctx[index].url);
url = thread_ctx[index].redirected;
#endif
}
else
{
#ifndef ENABLE_THREADS
xfree (url);
url = xstrdup (url_parsed->url);
#else
xfree (thread_ctx[index].url);
url = xstrdup (thread_ctx[index].url_parsed->url);
#endif
}
#ifndef ENABLE_THREADS
url_free(url_parsed);
#else
url_free(thread_ctx[index].url_parsed);
#endif
}
if (opt.spider)
@ -549,7 +617,7 @@ retry:
logputs (LOG_VERBOSE, "\n");
register_delete_file (file);
}
#if 0
#ifndef ENABLE_THREADS
xfree (url);
xfree_null (referer);
xfree_null (file);

4
src/retr.c
View File

@ -37,9 +37,9 @@ as that of the covered work. */
#include <errno.h>
#include <string.h>
#include <assert.h>
#ifdef ENABLE_METALINK
#include <pthread.h>
#include <semaphore.h>
#ifdef ENABLE_METALINK
#include <metalink/metalink_parser.h>
#include <metalink/metalink_types.h>
@ -63,11 +63,13 @@ as that of the covered work. */
#include "metalink.h"
#include "multi.h"
#ifdef ENABLE_METALINK
static pthread_mutex_t pconn_mutex = PTHREAD_MUTEX_INITIALIZER;
#define PCONN_LOCK() pthread_mutex_lock (&pconn_mutex)
#define PCONN_UNLOCK() pthread_mutex_unlock (&pconn_mutex)
#endif
/* Total size of downloaded files. Used to enforce quota. */
SUM_SIZE_INT total_downloaded_bytes;