github-repos/wget
1
0
Fork 0
mirror of https://github.com/mirror/wget.git synced 2025-03-14 20:00:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add header comment to new files. Comment the GSoC code.

Browse Source
This commit is contained in:
Ilim Ugur 2012-08-16 19:11:42 +03:00
parent a41525e5c2
commit 87a2b8dcb6
4 changed files with 116 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/metalink.c
View File

@ -1,3 +1,33 @@
/* Declarations for HTTP.
Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Wget. If not, see <http://www.gnu.org/licenses/>.
Additional permission under GNU GPL version 3 section 7
If you modify this program, or any covered work, by linking or
combining it with the OpenSSL project's OpenSSL library (or a
modified version of that library), containing parts covered by the
terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
grants you additional permission to convey the resulting work.
Corresponding Source for a non-source form of such a combination
shall include the source code for the parts of OpenSSL used as well
as that of the covered work. */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -50,11 +80,14 @@ lower_hex_case (unsigned char *hash, int length)
hash[i] += 32;
}
/* Verifies file hash by comparing the file hash(es) found by gnulib functions
and hash(es) provided by metalink file. Returns;
-1 if hashes that were compared turned out to be different.
0 if all pairs of hashes compared turned out to be the same.
1 if due to some error, comparisons could not be made. */
/* Verifies file hash by comparing the file hashes found by gnulib functions
and hashes provided by metalink file. Works by comparing strongest supported
hash type available in the metalink file.
Returns;
-1 if hashes that were compared turned out to be different.
0 if all pairs of hashes compared turned out to be the same.
1 if due to some error, comparisons could not be made. */
int
verify_file_hash (const char *filename, metalink_checksum_t **checksums)
{

59
src/multi.c
View File

@ -1,3 +1,33 @@
/* Declarations for HTTP.
Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
Foundation, Inc.
This file is part of GNU Wget.
GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Wget. If not, see <http://www.gnu.org/licenses/>.
Additional permission under GNU GPL version 3 section 7
If you modify this program, or any covered work, by linking or
combining it with the OpenSSL project's OpenSSL library (or a
modified version of that library), containing parts covered by the
terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
grants you additional permission to convey the resulting work.
Corresponding Source for a non-source form of such a combination
shall include the source code for the parts of OpenSSL used as well
as that of the covered work. */
#include "wget.h"
#include <stdio.h>
@ -13,6 +43,7 @@
static struct range *ranges;
char *temp, **files;
/* Allocate space for temporary file names. */
void
init_temp_files()
{
@ -31,6 +62,7 @@ init_temp_files()
}
}
/* Assign names to temporary files to be used. */
void
name_temp_files()
{
@ -44,6 +76,8 @@ name_temp_files()
}
}
/* Merge the temporary files in which the chunks are stored to form the
resulting file(output). */
void
merge_temp_files(const char *output)
{
@ -67,6 +101,7 @@ merge_temp_files(const char *output)
free(buf);
}
/* Delete the temporary files used. */
void
delete_temp_files()
{
@ -76,6 +111,7 @@ delete_temp_files()
unlink(files[j++]);
}
/* Clean the space allocated for temporary files data. */
void
clean_temp_files()
{
@ -86,13 +122,21 @@ clean_temp_files()
free(files);
}
/* Allocate ranges array to store the ranges data. */
void
init_ranges()
{
if(!(ranges = malloc (opt.jobs * (sizeof *ranges))))
logprintf (LOG_VERBOSE, "Space for ranges data could not be allocated.\n");
{
logprintf (LOG_VERBOSE, "Space for ranges data could not be allocated.\n");
exit(1);
}
}
/* Assign values to the ranges.
Also allocates the resources array each struct range must have.
Returns the number of ranges to which values are assigned. */
int
fill_ranges_data(int num_of_resources, long long int file_size,
long int chunk_size)
@ -115,6 +159,7 @@ fill_ranges_data(int num_of_resources, long long int file_size,
return i;
}
/* Free the resources array of each range allocated by fill_ranges_data(). */
void
clean_range_res_data(int num_of_resources)
{
@ -123,6 +168,7 @@ clean_range_res_data(int num_of_resources)
free (ranges[i].resources);
}
/* Free the ranges array that is used for storing ranges' data. */
void
clean_ranges()
{
@ -130,6 +176,8 @@ clean_ranges()
ranges = NULL;
}
/* Assign 'last minute' data to struct s_thread_ctx instances regarding their
usage and range information. Then create a thread using that instance. */
int
spawn_thread (struct s_thread_ctx *thread_ctx, int index, int resource)
{
@ -144,12 +192,18 @@ spawn_thread (struct s_thread_ctx *thread_ctx, int index, int resource)
thread_ctx[index].range = ranges + index;
(thread_ctx[index].range)->is_assigned = 1;
(thread_ctx[index].range)->resources[resource] = true;
thread_ctx[index].used = 1;
thread_ctx[index].terminated = 0;
return pthread_create (&thread, NULL, segmented_retrieve_url, &thread_ctx[index]);
}
/* Collects the first thread to terminate and updates struct s_thread_ctx
instance's data regarding its 'business' (i.e. being used by a thread).
Returns the index of the struct s_thread_ctx instance that was used in the
terminating thread. */
int
collect_thread (sem_t *retr_sem, struct s_thread_ctx *thread_ctx)
{
@ -163,12 +217,13 @@ collect_thread (sem_t *retr_sem, struct s_thread_ctx *thread_ctx)
{
url_free (thread_ctx[k].url_parsed);
thread_ctx[k].used = 0;
thread_ctx[k].terminated = 0;
(thread_ctx[k].range)->is_assigned = 0;
return k;
}
}
/* The function which is being called by pthread_create in spawn_thread(). It
is used to call retrieve_url(), which requires many arguments. */
static void *
segmented_retrieve_url (void *arg)
{

4
src/multi.h
View File

@ -1,5 +1,5 @@
/* Declarations for Concurrency Related Data and Functions.
Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
/* Declarations for HTTP.
Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software
Foundation, Inc.
This file is part of GNU Wget.

25
src/retr.c
View File

@ -1090,12 +1090,14 @@ retrieve_from_file (const char *file, bool html, int *count)
while (file->resources[num_of_resources])
++num_of_resources;
/* If chunk_size is too small, set it equal to MIN_CHUNK_SIZE. */
chunk_size = (file->size) / opt.jobs;
if(chunk_size < MIN_CHUNK_SIZE)
chunk_size = MIN_CHUNK_SIZE;
j = fill_ranges_data(num_of_resources, file->size, chunk_size);
/* If chunk_size was set to MIN_CHUNK_SIZE, opt.jobs should be corrected. */
if(j < opt.jobs)
opt.jobs = j;
@ -1104,6 +1106,8 @@ retrieve_from_file (const char *file, bool html, int *count)
sem_init (&retr_sem, 0, 0);
j = ranges_covered = 0;
/* Assign values to thread_ctx[] elements and spawn threads that will
conduct the download. */
for (r = 0; r < opt.jobs; ++r)
{
resource = file->resources[j];
@ -1123,6 +1127,7 @@ retrieve_from_file (const char *file, bool html, int *count)
ret = spawn_thread (thread_ctx, r, j);
if (ret)
{
/* If thread creation is unsuccessful */
char *error = url_error (thread_ctx[r].url, thread_ctx[r].url_err);
logprintf (LOG_NOTQUIET, "%s: %s.\n", thread_ctx[r].url, error);
xfree (error);
@ -1135,14 +1140,15 @@ retrieve_from_file (const char *file, bool html, int *count)
++j;
}
/* Until all the ranges are covered, collect threads. */
while (ranges_covered < opt.jobs)
{
r = collect_thread (&retr_sem, thread_ctx);
++ranges_covered;
status = thread_ctx[r].status;
/* Check return status of thread for errors. */
/* Check return status of thread for errors. */
if (IS_IO_ERROR(status))
{
/* The error is of type WGET_EXIT_IO_FAIL given in exits.c.
@ -1161,10 +1167,16 @@ retrieve_from_file (const char *file, bool html, int *count)
if(get_exit_status() != error_severity)
(thread_ctx[r].range)->status_least_severe = status;
PCONN_UNLOCK ();
/* Look for resource from which downloading this range is not
tried. */
for (j = 0; j < num_of_resources; ++j)
if (!((thread_ctx[r].range)->resources)[j])
break;
/* If there is such a resource, then update the range values
to try that not-tried resource and spawn thread.
If all the resources are exhausted, stop collecting the
threads, as the download failed. */
if (j < num_of_resources)
{
thread_ctx[r].url = file->resources[j]->url;
@ -1178,6 +1190,7 @@ retrieve_from_file (const char *file, bool html, int *count)
ret = spawn_thread (thread_ctx, r, j);
if (ret)
{
/* If thread creation is unsuccessful */
char *error = url_error (thread_ctx[r].url, thread_ctx[r].url_err);
logprintf (LOG_NOTQUIET, "%s: %s.\n", thread_ctx[r].url, error);
xfree (error);
@ -1195,13 +1208,13 @@ retrieve_from_file (const char *file, bool html, int *count)
sem_destroy(&retr_sem);
/* Check the download status. If conditions are suitable, retry. */
if (status != RETROK)
{
logprintf (LOG_VERBOSE, "Downloading %s failed. Chunk %d could not be downloaded from any of the URLs listed in metalink file.\n", file->name, r);
/* Unlike downloads with invalid hash values, failed download
should only be retried if the error causing failure is not
an IO error. */
/* Failed downloads should only be retried if the error causing
the failure is not an IO error. */
if (!(IS_IO_ERROR((thread_ctx[r].range)->status_least_severe)))
{
if(retries < opt.n_retries)
@ -1215,7 +1228,7 @@ retrieve_from_file (const char *file, bool html, int *count)
else
{
int res;
/* Form the actual file to be downloaded and verify hash. */
merge_temp_files(file->name);
res = verify_file_hash(file->name, file->checksums);
if(!res)