github-repos/make
1
0
Fork 0
mirror of https://github.com/mirror/make.git synced 2025-01-14 06:10:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support jobserver capability on Windows systems.

Browse Source 
Implementation contributed by Troy Runkel <Troy.Runkel@mathworks.com>
This commit is contained in:
Paul Smith 2011-11-14 00:42:49 +00:00
parent 934f51d166
commit 63888b91f6
6 changed files with 302 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
ChangeLog
View File

@ -7,6 +7,34 @@
* filedef.h (FILE_TIMESTAMP_STAT_MODTIME): Ditto.
Patch provided by Troy Runkel <Troy.Runkel@mathworks.com>
2011-10-11 Troy Runkel <Troy.Runkel@mathworks.com>
* config.h.W32: Enable job server support for Windows.
* main.c [WINDOWS32]: Include sub_proc.h
(main): Create a named semaphore to implement the job server.
(clean_jobserver): Free the job server semaphore when make is finished.
* job.c [WINDOWS32]: Define WAIT_NOHANG
(reap_children): Support non-blocking wait for child processes.
(free_child): Release job server semaphore when child process finished.
(job_noop): Don't define function on Windows.
(set_child_handler_action_flags): Don't define function on Windows.
(new_job): Wait for job server semaphore or child process termination.
(exec_command): Pass new parameters to process_wait_for_any.
* w32/include/sub_proc.h [WINDOWS32]: New/updated EXTERN_DECL entries.
* w32/subproc/sub_proc.c [WINDOWS32]: Added job server implementation.
(open_jobserver_semaphore): Open existing job server semaphore by name.
(create_jobserver_semaphore): Create new job server named semaphore.
(free_jobserver_semaphore): Close existing job server semaphore.
(acquire_jobserver_semaphore): Decrement job server semaphore count.
(release_jobserver_semaphore): Increment job server semaphore count.
(has_jobserver_semaphore): Returns whether job server semaphore exists.
(get_jobserver_semaphore_name): Returns name of job server semaphore.
(wait_for_semaphore_or_child_process): Wait for either the job server
semaphore to become signalled or a child process to terminate.
(process_wait_for_any_private): Support for non-blocking wait for child.
(process_wait_for_any): Added support for non-blocking wait for child.
(process_file_io): Pass new parameters to process_wait_for_any_private.
2011-09-18 Paul Smith <psmith@gnu.org>
* main.c (main): If we're re-exec'ing and we're the master make,

5
NEWS
View File

@ -22,8 +22,11 @@ http://sv.gnu.org/bugs/index.php?group=make&report_id=111&fix_release_id=101&set
multiple consecutive backslash/newlines do not condense into one space.
* In recipes, a recipe prefix following a backslash-newlines is removed.
* New feature: The "job server" capability is now supported on Windows.
Implementation contributed by Troy Runkel <Troy.Runkel@mathworks.com>
* New feature: "!=" shell assignment operator as an alternative to the
$(shell ...) function. Implemented for portability of BSD makefiles.
$(shell ...) function. Implemented for compatibility with BSD makefiles.
WARNING: Backward-incompatibility!
Variables ending in "!" previously defined as "variable!= value" will now be
interpreted as shell assignment. Change your assignment to add whitespace

57
job.c
View File

@ -108,6 +108,7 @@ static void vmsWaitForChildren (int *);
# include "sub_proc.h"
# include "w32err.h"
# include "pathstuff.h"
# define WAIT_NOHANG 1
#endif /* WINDOWS32 */
#ifdef __EMX__
@ -528,9 +529,9 @@ reap_children (int block, int err)
{
#ifndef WINDOWS32
WAIT_T status;
#endif
/* Initially, assume we have some. */
int reap_more = 1;
#endif
#ifdef WAIT_NOHANG
# define REAP_MORE reap_more
@ -699,6 +700,7 @@ reap_children (int block, int err)
HANDLE hPID;
int werr;
HANDLE hcTID, hcPID;
DWORD dwWaitStatus = 0;
exit_code = 0;
exit_sig = 0;
coredump = 0;
@ -722,7 +724,7 @@ reap_children (int block, int err)
}
/* wait for anything to finish */
hPID = process_wait_for_any();
hPID = process_wait_for_any(block, &dwWaitStatus);
if (hPID)
{
@ -744,6 +746,18 @@ reap_children (int block, int err)
coredump = 0;
}
else if (dwWaitStatus == WAIT_FAILED)
{
/* The WaitForMultipleObjects() failed miserably. Punt. */
pfatal_with_name ("WaitForMultipleObjects");
}
else if (dwWaitStatus == WAIT_TIMEOUT)
{
/* No child processes are finished. Give up waiting. */
reap_more = 0;
break;
}
pid = (pid_t) hPID;
}
#endif /* WINDOWS32 */
@ -926,6 +940,19 @@ free_child (struct child *child)
/* If we're using the jobserver and this child is not the only outstanding
job, put a token back into the pipe for it. */
#ifdef WINDOWS32
if (has_jobserver_semaphore() && jobserver_tokens > 1)
{
if (! release_jobserver_semaphore())
{
DWORD err = GetLastError();
fatal (NILF,_("release jobserver semaphore: (Error %d: %s)"),
err, map_windows32_error_to_string(err));
}
DB (DB_JOBS, (_("Released token for child %p (%s).\n"), child, child->file->name));
}
#else
if (job_fds[1] >= 0 && jobserver_tokens > 1)
{
char token = '+';
@ -940,6 +967,7 @@ free_child (struct child *child)
DB (DB_JOBS, (_("Released token for child %p (%s).\n"),
child, child->file->name));
}
#endif
--jobserver_tokens;
@ -991,7 +1019,7 @@ unblock_sigs (void)
}
#endif
#ifdef MAKE_JOBSERVER
#if defined(MAKE_JOBSERVER) && !defined(WINDOWS32)
RETSIGTYPE
job_noop (int sig UNUSED)
{
@ -1740,7 +1768,11 @@ new_job (struct file *file)
just once). Also more thought needs to go into the entire algorithm;
this is where the old parallel job code waits, so... */
#ifdef WINDOWS32
else if (has_jobserver_semaphore())
#else
else if (job_fds[0] >= 0)
#endif
while (1)
{
char token;
@ -1754,6 +1786,7 @@ new_job (struct file *file)
if (!jobserver_tokens)
break;
#ifndef WINDOWS32
/* Read a token. As long as there's no token available we'll block.
We enable interruptible system calls before the read(2) so that if
we get a SIGCHLD while we're waiting, we'll return with EINTR and
@ -1782,6 +1815,7 @@ new_job (struct file *file)
DB (DB_JOBS, ("Duplicate the job FD\n"));
job_rfd = dup (job_fds[0]);
}
#endif
/* Reap anything that's currently waiting. */
reap_children (0, 0);
@ -1800,11 +1834,24 @@ new_job (struct file *file)
if (!children)
fatal (NILF, "INTERNAL: no children as we go to sleep on read\n");
#ifdef WINDOWS32
/* On Windows we simply wait for the jobserver semaphore to become
* signalled or one of our child processes to terminate.
*/
got_token = wait_for_semaphore_or_child_process();
if (got_token < 0)
{
DWORD err = GetLastError();
fatal (NILF,_("semaphore or child process wait: (Error %d: %s)"),
err, map_windows32_error_to_string(err));
}
#else
/* Set interruptible system calls, and read() for a job token. */
set_child_handler_action_flags (1, waiting_jobs != NULL);
got_token = read (job_rfd, &token, 1);
saved_errno = errno;
set_child_handler_action_flags (0, waiting_jobs != NULL);
#endif
/* If we got one, we're done here. */
if (got_token == 1)
@ -1814,6 +1861,7 @@ new_job (struct file *file)
break;
}
#ifndef WINDOWS32
/* If the error _wasn't_ expected (EINTR or EBADF), punt. Otherwise,
go back and reap_children(), and try again. */
errno = saved_errno;
@ -1821,6 +1869,7 @@ new_job (struct file *file)
pfatal_with_name (_("read jobs pipe"));
if (errno == EBADF)
DB (DB_JOBS, ("Read returned EBADF.\n"));
#endif
}
#endif
@ -2150,7 +2199,7 @@ exec_command (char **argv, char **envp)
}
/* wait and reap last child */
hWaitPID = process_wait_for_any();
hWaitPID = process_wait_for_any(1, 0);
while (hWaitPID)
{
/* was an error found on this process? */

81
main.c
View File

@ -32,9 +32,10 @@ this program. If not, see <http://www.gnu.org/licenses/>. */
# include <proto/dos.h>
#endif
#ifdef WINDOWS32
#include <windows.h>
#include <io.h>
#include "pathstuff.h"
# include <windows.h>
# include <io.h>
# include "pathstuff.h"
# include "sub_proc.h"
#endif
#ifdef __EMX__
# include <sys/types.h>
@ -1087,7 +1088,7 @@ main (int argc, char **argv, char **envp)
program = strrchr (argv[0], '\\');
if (program)
{
argv0_len = strlen(program);
argv0_len = strlen (program);
if (argv0_len > 4 && streq (&program[argv0_len - 4], ".exe"))
/* Remove .exe extension */
program[argv0_len - 4] = '\0';
@ -1135,8 +1136,8 @@ main (int argc, char **argv, char **envp)
define_variable_cname (".SHELLFLAGS", "-c", o_default, 0);
/* Set up .FEATURES
We must do this in multiple calls because define_variable_cname() is
a macro and some compilers (MSVC) don't like conditionals in macros. */
Use a separate variable because define_variable_cname() is a macro and
some compilers (MSVC) don't like conditionals in macros. */
{
const char *features = "target-specific order-only second-expansion"
" else-if shortest-stem undefine oneshell"
@ -1170,9 +1171,9 @@ main (int argc, char **argv, char **envp)
while (*ep != '\0' && *ep != '=')
++ep;
#ifdef WINDOWS32
if (!unix_path && strneq(envp[i], "PATH=", 5))
if (!unix_path && strneq (envp[i], "PATH=", 5))
unix_path = ep+1;
else if (!strnicmp(envp[i], "Path=", 5)) {
else if (!strnicmp (envp[i], "Path=", 5)) {
do_not_define = 1; /* it gets defined after loop exits */
if (!windows32_path)
windows32_path = ep+1;
@ -1723,12 +1724,22 @@ main (int argc, char **argv, char **envp)
cp = jobserver_fds->list[0];
#ifdef WINDOWS32
if (! open_jobserver_semaphore(cp))
{
DWORD err = GetLastError();
fatal (NILF,_("internal error: unable to open jobserver semaphore `%s': (Error %d: %s)"),
cp, err, map_windows32_error_to_string(err));
}
DB (DB_JOBS, (_("Jobserver client (semaphore %s)\n"), cp));
#else
if (sscanf (cp, "%d,%d", &job_fds[0], &job_fds[1]) != 2)
fatal (NILF,
_("internal error: invalid --jobserver-fds string `%s'"), cp);
DB (DB_JOBS,
(_("Jobserver client (fds %d,%d)\n"), job_fds[0], job_fds[1]));
#endif
/* The combination of a pipe + !job_slots means we're using the
jobserver. If !job_slots and we don't have a pipe, we can start
@ -1740,11 +1751,11 @@ main (int argc, char **argv, char **envp)
error (NILF,
_("warning: -jN forced in submake: disabling jobserver mode."));
#ifndef WINDOWS32
/* Create a duplicate pipe, that will be closed in the SIGCHLD
handler. If this fails with EBADF, the parent has closed the pipe
on us because it didn't think we were a submake. If so, print a
warning then default to -j1. */
else if ((job_rfd = dup (job_fds[0])) < 0)
{
if (errno != EBADF)
@ -1754,11 +1765,16 @@ main (int argc, char **argv, char **envp)
_("warning: jobserver unavailable: using -j1. Add `+' to parent make rule."));
job_slots = 1;
}
#endif
if (job_slots > 0)
{
#ifdef WINDOWS32
free_jobserver_semaphore ();
#else
close (job_fds[0]);
close (job_fds[1]);
#endif
job_fds[0] = job_fds[1] = -1;
free (jobserver_fds->list);
free (jobserver_fds);
@ -1774,8 +1790,27 @@ main (int argc, char **argv, char **envp)
char *cp;
char c = '+';
#ifdef WINDOWS32
/* sub_proc.c cannot wait for more than MAXIMUM_WAIT_OBJECTS objects
* and one of them is the job-server semaphore object. Limit the
* number of available job slots to (MAXIMUM_WAIT_OBJECTS - 1). */
if (job_slots >= MAXIMUM_WAIT_OBJECTS)
{
job_slots = MAXIMUM_WAIT_OBJECTS - 1;
DB (DB_JOBS, (_("Jobserver slots limited to %d\n"), job_slots));
}
if (! create_jobserver_semaphore(job_slots - 1))
{
DWORD err = GetLastError();
fatal (NILF,_("creating jobserver semaphore: (Error %d: %s)"),
err, map_windows32_error_to_string(err));
}
#else
if (pipe (job_fds) < 0 || (job_rfd = dup (job_fds[0])) < 0)
pfatal_with_name (_("creating jobs pipe"));
#endif
/* Every make assumes that it always has one job it can run. For the
submakes it's the token they were given by their parent. For the
@ -1784,6 +1819,10 @@ main (int argc, char **argv, char **envp)
master_job_slots = job_slots;
#ifdef WINDOWS32
/* We're using the jobserver so set job_slots to 0. */
job_slots = 0;
#else
while (--job_slots)
{
int r;
@ -1792,11 +1831,17 @@ main (int argc, char **argv, char **envp)
if (r != 1)
pfatal_with_name (_("init jobserver pipe"));
}
#endif
/* Fill in the jobserver_fds struct for our children. */
#ifdef WINDOWS32
cp = xmalloc (MAX_PATH + 1);
strcpy (cp, get_jobserver_semaphore_name());
#else
cp = xmalloc ((sizeof ("1024")*2)+1);
sprintf (cp, "%d,%d", job_fds[0], job_fds[1]);
#endif
jobserver_fds = (struct stringlist *)
xmalloc (sizeof (struct stringlist));
@ -3109,7 +3154,11 @@ clean_jobserver (int status)
have written all our tokens so do that now. If tokens are left
after any other error code, that's bad. */
#ifdef WINDOWS32
if (has_jobserver_semaphore() && jobserver_tokens)
#else
if (job_fds[0] != -1 && jobserver_tokens)
#endif
{
if (status != 2)
error (NILF,
@ -3119,11 +3168,16 @@ clean_jobserver (int status)
/* Don't write back the "free" token */
while (--jobserver_tokens)
{
#ifdef WINDOWS32
if (! release_jobserver_semaphore())
perror_with_name ("release_jobserver_semaphore", "");
#else
int r;
EINTRLOOP (r, write (job_fds[1], &token, 1));
if (r != 1)
perror_with_name ("write", "");
#endif
}
}
@ -3135,18 +3189,27 @@ clean_jobserver (int status)
/* We didn't write one for ourself, so start at 1. */
unsigned int tcnt = 1;
#ifdef WINDOWS32
while (acquire_jobserver_semaphore())
++tcnt;
#else
/* Close the write side, so the read() won't hang. */
close (job_fds[1]);
while (read (job_fds[0], &token, 1) == 1)
++tcnt;
#endif
if (tcnt != master_job_slots)
error (NILF,
"INTERNAL: Exiting with %u jobserver tokens available; should be %u!",
tcnt, master_job_slots);
#ifdef WINDOWS32
free_jobserver_semaphore();
#else
close (job_fds[0]);
#endif
/* Clean out jobserver_fds so we don't pass this information to any
sub-makes. Also reset job_slots since it will be put on the command

12
w32/include/sub_proc.h
View File

@ -40,7 +40,7 @@ EXTERN_DECL(long process_pipe_io, (HANDLE proc, char *stdin_data,
int stdin_data_len));
EXTERN_DECL(long process_file_io, (HANDLE proc));
EXTERN_DECL(void process_cleanup, (HANDLE proc));
EXTERN_DECL(HANDLE process_wait_for_any, (VOID_DECL));
EXTERN_DECL(HANDLE process_wait_for_any, (int block, DWORD* pdwWaitStatus));
EXTERN_DECL(void process_register, (HANDLE proc));
EXTERN_DECL(HANDLE process_easy, (char** argv, char** env));
EXTERN_DECL(BOOL process_kill, (HANDLE proc, int signal));
@ -57,4 +57,14 @@ EXTERN_DECL(int process_outcnt, (HANDLE proc));
EXTERN_DECL(int process_errcnt, (HANDLE proc));
EXTERN_DECL(void process_pipes, (HANDLE proc, int pipes[3]));
/* jobserver routines */
EXTERN_DECL(int open_jobserver_semaphore, (char* name));
EXTERN_DECL(int create_jobserver_semaphore, (int tokens));
EXTERN_DECL(void free_jobserver_semaphore, (VOID_DECL));
EXTERN_DECL(int acquire_jobserver_semaphore, (VOID_DECL));
EXTERN_DECL(int release_jobserver_semaphore, (VOID_DECL));
EXTERN_DECL(int has_jobserver_semaphore, (VOID_DECL));
EXTERN_DECL(char* get_jobserver_semaphore_name, (VOID_DECL));
EXTERN_DECL(int wait_for_semaphore_or_child_process, (VOID_DECL));
#endif

142
w32/subproc/sub_proc.c
View File

@ -58,6 +58,126 @@ static sub_process *proc_array[MAXIMUM_WAIT_OBJECTS];
static int proc_index = 0;
static int fake_exits_pending = 0;
/* Windows jobserver implementation variables */
static char jobserver_semaphore_name[MAX_PATH + 1];
static HANDLE jobserver_semaphore = NULL;
/* Open existing jobserver semaphore */
int open_jobserver_semaphore(char* name)
{
jobserver_semaphore = OpenSemaphore(
SEMAPHORE_ALL_ACCESS, // Semaphore access setting
FALSE, // Child processes DON'T inherit
name); // Semaphore name
if (jobserver_semaphore == NULL)
return 0;
return 1;
}
/* Create new jobserver semaphore */
int create_jobserver_semaphore(int tokens)
{
sprintf(jobserver_semaphore_name, "gmake_semaphore_%d", _getpid());
jobserver_semaphore = CreateSemaphore(
NULL, // Use default security descriptor
tokens, // Initial count
tokens, // Maximum count
jobserver_semaphore_name); // Semaphore name
if (jobserver_semaphore == NULL)
return 0;
return 1;
}
/* Close jobserver semaphore */
void free_jobserver_semaphore()
{
if (jobserver_semaphore != NULL)
{
CloseHandle(jobserver_semaphore);
jobserver_semaphore = NULL;
}
}
/* Decrement semaphore count */
int acquire_jobserver_semaphore()
{
DWORD dwEvent = WaitForSingleObject(
jobserver_semaphore, // Handle to semaphore
0); // DON'T wait on semaphore
return (dwEvent == WAIT_OBJECT_0);
}
/* Increment semaphore count */
int release_jobserver_semaphore()
{
BOOL bResult = ReleaseSemaphore(
jobserver_semaphore, // handle to semaphore
1, // increase count by one
NULL); // not interested in previous count
return (bResult);
}
int has_jobserver_semaphore()
{
return (jobserver_semaphore != NULL);
}
char* get_jobserver_semaphore_name()
{
return (jobserver_semaphore_name);
}
/* Wait for either the jobserver semaphore to become signalled or one of our
* child processes to terminate.
*/
int wait_for_semaphore_or_child_process()
{
HANDLE handles[MAXIMUM_WAIT_OBJECTS];
DWORD dwHandleCount = 1;
DWORD dwEvent;
int i;
/* Add jobserver semaphore to first slot. */
handles[0] = jobserver_semaphore;
/* Build array of handles to wait for */
for (i = 0; i < proc_index; i++)
{
/* Don't wait on child processes that have already finished */
if (fake_exits_pending && proc_array[i]->exit_code)
continue;
handles[dwHandleCount++] = (HANDLE) proc_array[i]->pid;
}
dwEvent = WaitForMultipleObjects(
dwHandleCount, // number of objects in array
handles, // array of objects
FALSE, // wait for any object
INFINITE); // wait until object is signalled
switch(dwEvent)
{
case WAIT_FAILED:
return -1;
case WAIT_OBJECT_0:
/* Indicate that the semaphore was signalled */
return 1;
default:
/* Assume that one or more of the child processes terminated. */
return 0;
}
}
/*
* When a process has been waited for, adjust the wait state
* array so that we don't wait for it again
@ -87,7 +207,7 @@ process_adjust_wait_state(sub_process* pproc)
* Waits for any of the registered child processes to finish.
*/
static sub_process *
process_wait_for_any_private(void)
process_wait_for_any_private(int block, DWORD* pdwWaitStatus)
{
HANDLE handles[MAXIMUM_WAIT_OBJECTS];
DWORD retval, which;
@ -106,7 +226,7 @@ process_wait_for_any_private(void)
/* wait for someone to exit */
if (!fake_exits_pending) {
retval = WaitForMultipleObjects(proc_index, handles, FALSE, INFINITE);
retval = WaitForMultipleObjects(proc_index, handles, FALSE, (block ? INFINITE : 0));
which = retval - WAIT_OBJECT_0;
} else {
fake_exits_pending--;
@ -114,13 +234,19 @@ process_wait_for_any_private(void)
which = i;
}
/* If the pointer is not NULL, set the wait status result variable. */
if (pdwWaitStatus)
*pdwWaitStatus = retval;
/* return pointer to process */
if (retval != WAIT_FAILED) {
if ((retval == WAIT_TIMEOUT) || (retval == WAIT_FAILED)) {
return NULL;
}
else {
sub_process* pproc = proc_array[which];
process_adjust_wait_state(pproc);
return pproc;
} else
return NULL;
}
}
/*
@ -179,9 +305,9 @@ process_used_slots(void)
*/
HANDLE
process_wait_for_any(void)
process_wait_for_any(int block, DWORD* pdwWaitStatus)
{
sub_process* pproc = process_wait_for_any_private();
sub_process* pproc = process_wait_for_any_private(block, pdwWaitStatus);
if (!pproc)
return NULL;
@ -865,7 +991,7 @@ process_file_io(
DWORD ierr;
if (proc == NULL)
pproc = process_wait_for_any_private();
pproc = process_wait_for_any_private(1, 0);
else
pproc = (sub_process *)proc;