Many bug fixes etc.

- Apply a fix for the "thundering herd" problem when using "-j -l".
  This also fixes bug #4693.
- Fix bug #7257: allow functions as ifdef arguments
- Fix bug #4518: make sure we print all double-colon rules with -p.
- Upgrade to autconf 2.58/automake 1.8/gettext 0.13.1
- Various doc cleanups, etc.

ChangeLog

@@ -1,5 +1,31 @@
+2004-01-21  Paul D. Smith  <psmith@gnu.org>
+
+	* job.c (load_too_high): Implement an algorithm to control the
+	"thundering herd" problem when using -l to control job creation
+	via the load average.  The system only recomputes the load once a
+	second but we can start many jobs in a second.  To solve this we
+	keep track of the number of jobs started in the last second and
+	apply a weight to try to guess what a correct load would be.
+	The algorithm was provided by Thomas Riedl <thomas.riedl@siemens.com>.
+	Also fixes bug #4693.
+	(reap_children): Decrease the job count for this second.
+	(start_job_command): Increase the job count for this second.
+
+	* read.c (conditional_line): Expand the text after ifn?def before
+	checking to see if it's a single word.  Fixes bug #7257.
+
+2004-01-09  Paul D. Smith  <psmith@gnu.org>
+
+	* file.c (print_file): Recurse to print all targets in
+	double-colon rules.  Fixes bug #4518, reported (with patch) by
+	Andrew Chatham <chatham@google.com>.
+
 2004-01-07  Paul D. Smith  <psmith@gnu.org>
 
+	* acinclude.m4: Remove make_FUNC_SETVBUF_REVERSED.
+	* configure.in: Change make_FUNC_SETVBUF_REVERSED to
+	AC_FUNC_SETVBUF_REVERSED.
+
 	* doc/make.texi (Target-specific): Fix Savannah bug #1772.
 	(MAKE Variable): Fix Savannah bug #4898.
 
@@ -12,6 +38,12 @@
 	Original fix provided in Savannah patch #2349, by Benoit
 	Poulot-Cazajous <Benoit.Poulot-Cazajous@jaluna.com>.
 
+2003-11-22  Paul D. Smith  <psmith@gnu.org>
+
+	* README.W32.template (Outputs): Clarification on -j with
+	BATCH_MODE_ONLY_SEHLL suggested by Jonathan R. Grant
+	<jg-make@jguk.org>.
+
 2003-11-02  Paul D. Smith  <psmith@gnu.org>
 
 	* function.c (func_if): Strip all the trailing whitespace from the

Makefile.am

@@ -1,6 +1,6 @@
 # This is a -*-Makefile-*-, or close enough
 
-AUTOMAKE_OPTIONS = 1.7.6 dist-bzip2 check-news ansi2knr
+AUTOMAKE_OPTIONS = 1.8 dist-bzip2 check-news ansi2knr
 ACLOCAL_AMFLAGS	=  -I config
 
 SUBDIRS =	glob config po doc
@@ -61,13 +61,6 @@ html:
 
 localedir =	$(datadir)/locale
 
-# We need this due to a bug in gettext 0.12.1 Makefile.in.in: without this I
-# can't run distcheck because this file is created but not removed during
-# distclean.
-
-distclean-local:
-	test "$(srcdir)" = . || rm -f po/stamp-po
-
 # --------------- Local INSTALL Section
 
 # If necessary, change the gid of the app and turn on the setgid flag.

NEWS

@@ -1,16 +1,15 @@
 GNU make NEWS                                               -*-indented-text-*-
   History of user-visible changes.
-  17 April 2003
+  21 January 2004
 
-Copyright (C) 2002,2003  Free Software Foundation, Inc.
+Copyright (C) 2002,2003,2004  Free Software Foundation, Inc.
 See the end for copying conditions.
 
 All changes mentioned here are more fully described in the GNU make
 manual, which is contained in this distribution as the file doc/make.texi.
 
 Please send GNU make bug reports to <bug-make@gnu.org>.
-See the README file and the GNU make manual for details on sending bug
+See the README file and the GNU make manual for details on reporting bugs.
-reports.
 
 Version 3.81rc1
 
@@ -26,10 +25,16 @@ Version 3.81rc1
 * In a recursive $(call ...) context, any extra arguments from the outer
   call are now masked in the context of the inner call.
 
+* Implemented a solution for the "thundering herd" problem with "-j -l".
+  This version of GNU make uses an algorithm suggested by Thomas Riedl
+  <thomas.riedl@siemens.com> to track the number of jobs started in the
+  last second and adjust GNU make's view of the system's load average
+  accordingly.
+
 * Enhancements for POSIX compatibility:
    - Only touch targets (under -t) if they have at least one command.
 
-* Updated to autoconf 2.57, automake 1.7.6, and gettext 0.12.1.  Users
+* Updated to autoconf 2.58, automake 1.8, and gettext 0.13.1.  Users
   should not be impacted.
 
 

README.W32.template

@@ -70,7 +70,7 @@ GNU make and sh.exe:
 
 GNU make and brain-dead shells (BATCH_MODE_ONLY_SHELL):
 
-	Some versions of Bourne shell does not behave well when invoked
+	Some versions of Bourne shell do not behave well when invoked
 	as 'sh -c' from CreateProcess().  The main problem is they seem
 	to have a hard time handling quoted strings correctly. This can
 	be circumvented by writing commands to be executed to a batch
@@ -83,7 +83,9 @@ GNU make and brain-dead shells (BATCH_MODE_ONLY_SHELL):
 
 	A native Windows32 system with no Bourne shell will also run
 	in batch mode.  All command lines will be put into batch files
-	and executed via $(COMSPEC) (%COMSPEC%).
+	and executed via $(COMSPEC) (%COMSPEC%).  Note that parallel
+        builds (-j) require a working Bourne shell; they will not work
+        with COM.
 
 GNU make and Cygnus GNU Windows32 tools:
 

acinclude.m4

@@ -111,54 +111,3 @@ AC_DEFUN(AC_STRUCT_ST_MTIM_NSEC,
   fi
  ]
 )
-
-
-dnl ---------------------------------------------------------------------------
-dnl This will be in the next version of autoconf; take this out then!
-
-# make_FUNC_SETVBUF_REVERSED
-# ------------------------
-AC_DEFUN([make_FUNC_SETVBUF_REVERSED],
-[AC_REQUIRE([AC_C_PROTOTYPES])dnl
-AC_CACHE_CHECK(whether setvbuf arguments are reversed,
-  ac_cv_func_setvbuf_reversed,
-  [ac_cv_func_setvbuf_reversed=no
-   AC_LINK_IFELSE(
-     [AC_LANG_PROGRAM(
-	[[#include <stdio.h>
-#	  if PROTOTYPES
-	   int (setvbuf) (FILE *, int, char *, size_t);
-#	  endif]],
-	[[char buf; return setvbuf (stdout, _IOLBF, &buf, 1);]])],
-     [AC_LINK_IFELSE(
-	[AC_LANG_PROGRAM(
-	   [[#include <stdio.h>
-#	     if PROTOTYPES
-	      int (setvbuf) (FILE *, int, char *, size_t);
-#	     endif]],
-	   [[char buf; return setvbuf (stdout, &buf, _IOLBF, 1);]])],
-	[# It compiles and links either way, so it must not be declared
-	 # with a prototype and most likely this is a K&R C compiler.
-	 # Try running it.
-	 AC_RUN_IFELSE(
-	   [AC_LANG_PROGRAM(
-	      [[#include <stdio.h>]],
-	      [[/* This call has the arguments reversed.
-		   A reversed system may check and see that the address of buf
-		   is not _IOLBF, _IONBF, or _IOFBF, and return nonzero.  */
-		char buf;
-		if (setvbuf (stdout, _IOLBF, &buf, 1) != 0)
-		  exit (1);
-		putchar ('\r');
-		exit (0); /* Non-reversed systems SEGV here.  */]])],
-	   ac_cv_func_setvbuf_reversed=yes,
-	   rm -f core core.* *.core,
-	   [[: # Assume setvbuf is not reversed when cross-compiling.]])]
-	ac_cv_func_setvbuf_reversed=yes)])])
-if test $ac_cv_func_setvbuf_reversed = yes; then
-  AC_DEFINE(SETVBUF_REVERSED, 1,
-            [Define to 1 if the `setvbuf' function takes the buffering type as
-             its second argument and the buffer pointer as the third, as on
-             System V before release 3.])
-fi
-])# make_FUNC_SETVBUF_REVERSED

config/.cvsignore

@@ -1,4 +1,5 @@
 *.m4
 config.*
+mkinstalldirs
 
 Makefile Makefile.in

configure.in

@@ -2,8 +2,7 @@
 
 AC_INIT([GNU make],[3.81rc1],[bug-make@gnu.org])
 
-AC_PREREQ(2.57)
+AC_PREREQ(2.58)
-
 AC_REVISION([[$Id$]])
 
 # Autoconf setup
@@ -130,7 +129,7 @@ AC_CHECK_FUNCS(	memcpy memmove strchr strdup mkstemp mktemp fdopen \
 		seteuid setegid setlinebuf setreuid setregid setvbuf pipe \
 		strerror strsignal)
 
-make_FUNC_SETVBUF_REVERSED
+AC_FUNC_SETVBUF_REVERSED
 
 # strcoll() is used by the GNU glob library
 AC_FUNC_STRCOLL

file.c

@@ -751,6 +751,9 @@ print_file (const void *item)
 
   if (f->cmds != 0)
     print_commands (f->cmds);
+
+  if (f->prev)
+    print_file ((const void *) f->prev);
 }
 
 void

job.c

@@ -222,6 +222,10 @@ static struct child *waiting_jobs = 0;
 
 int unixy_shell = 1;
 
+/* Number of jobs started in the current second.  */
+
+unsigned long job_counter = 0;
+
 
 #ifdef WINDOWS32
 /*
@@ -573,6 +577,10 @@ reap_children (int block, int err)
 	      exit_sig = WIFSIGNALED (status) ? WTERMSIG (status) : 0;
 	      coredump = WCOREDUMP (status);
 
+              /* If we have started jobs in this second, remove one.  */
+              if (job_counter)
+                --job_counter;
+
 #ifdef __EMX__
 	      /* the SIGCHLD handler must not be used on OS/2 because, unlike
 		 on UNIX systems, it had to call wait() itself.  Therefore
@@ -1349,6 +1357,9 @@ start_job_command (struct child *child)
 #endif /* WINDOWS32 */
 #endif	/* __MSDOS__ or Amiga or WINDOWS32 */
 
+  /* Bump the number of jobs started in this second.  */
+  ++job_counter;
+
   /* We are the parent side.  Set the state to
      say the commands are running and return.  */
 
@@ -1692,17 +1703,61 @@ job_next_command (struct child *child)
   return 1;
 }
 
+/* Determine if the load average on the system is too high to start a new job.
+   The real system load average is only recomputed once a second.  However, a
+   very parallel make can easily start tens or even hundreds of jobs in a
+   second, which brings the system to its knees for a while until that first
+   batch of jobs clears out.
+
+   To avoid this we use a weighted algorithm to try to account for jobs which
+   have been started since the last second, and guess what the load average
+   would be now if it were computed.
+
+   This algorithm was provided by Thomas Riedl <thomas.riedl@siemens.com>,
+   who writes:
+
+!      calculate something load-oid and add to the observed sys.load,
+!      so that latter can catch up:
+!      - every job started increases jobctr;
+!      - every dying job decreases a positive jobctr;
+!      - the jobctr value gets zeroed every change of seconds,
+!        after its value*weight_b is stored into the 'backlog' value last_sec
+!      - weight_a times the sum of jobctr and last_sec gets
+!        added to the observed sys.load.
+!
+!      The two weights have been tried out on 24 and 48 proc. Sun Solaris-9
+!      machines, using a several-thousand-jobs-mix of cpp, cc, cxx and smallish
+!      sub-shelled commands (rm, echo, sed...) for tests.
+!      lowering the 'direct influence' factor weight_a (e.g. to 0.1)
+!      resulted in significant excession of the load limit, raising it
+!      (e.g. to 0.5) took bad to small, fast-executing jobs and didn't
+!      reach the limit in most test cases.
+!
+!      lowering the 'history influence' weight_b (e.g. to 0.1) resulted in
+!      exceeding the limit for longer-running stuff (compile jobs in
+!      the .5 to 1.5 sec. range),raising it (e.g. to 0.5) overrepresented
+!      small jobs' effects.
+
+ */
+
+#define LOAD_WEIGHT_A           0.25
+#define LOAD_WEIGHT_B           0.25
+
 static int
 load_too_high (void)
 {
 #if defined(__MSDOS__) || defined(VMS) || defined(_AMIGA)
   return 1;
 #else
-  double load;
+  static double last_sec;
+  static time_t last_now;
+  double load, guess;
+  time_t now;
 
   if (max_load_average < 0)
     return 0;
 
+  /* Find the real system load average.  */
   make_access ();
   if (getloadavg (&load, 1) != 1)
     {
@@ -1722,9 +1777,27 @@ load_too_high (void)
     }
   user_access ();
 
-  DB (DB_JOBS, ("Current system load = %f (max requested = %f)\n",
+  /* If we're in a new second zero the counter and correct the backlog
-                load, max_load_average));
+     value.  Only keep the backlog for one extra second; after that it's 0.  */
-  return load >= max_load_average;
+  now = time (NULL);
+  if (last_now < now)
+    {
+      if (last_now == now - 1)
+        last_sec = LOAD_WEIGHT_B * job_counter;
+      else
+        last_sec = 0.0;
+
+      job_counter = 0;
+      last_now = now;
+    }
+
+  /* Try to guess what the load would be right now.  */
+  guess = load + (LOAD_WEIGHT_A * (job_counter + last_sec));
+
+  DB (DB_JOBS, ("Estimated system load = %f (actual = %f) (max requested = %f)\n",
+                guess, load, max_load_average));
+
+  return guess >= max_load_average;
 #endif
 }
 

maintMakefile

@@ -1,10 +1,11 @@
 # Maintainer-only makefile segment.  This contains things that are relevant
 # only if you have the full copy of the GNU make sources from the CVS
 # tree, not a dist copy.
-#
+
+# We like mondo-warnings!
+AM_CFLAGS += -Wall -W
 
 # Find the glob source files... this might be dangerous, but we're maintainers!
-#
 globsrc := $(wildcard glob/*.c)
 globhdr := $(wildcard glob/*.h)
 

po/.cvsignore

@@ -1,6 +1,6 @@
 *.gmo *.mo *.pot *.po
 
-Makefile Makefile.in Makefile.in.in
+Makefile Makefile.in Makefile.in.in Makevars.template
-Rules-quot
+Rules-quot stamp-po
 *.sed *.sin *.header
 POTFILES

read.c

@@ -137,7 +137,6 @@ static void record_files PARAMS ((struct nameseq *filenames, char *pattern, char
 			int have_sysv_atvar,
                         const struct floc *flocp, int set_default));
 static void record_target_var PARAMS ((struct nameseq *filenames, char *defn,
-                                       int two_colon,
                                        enum variable_origin origin,
                                        int enabled,
                                        const struct floc *flocp));
@@ -505,7 +504,7 @@ eval (struct ebuffer *ebuf, int set_default)
 
   while (1)
     {
-      int linelen;
+      unsigned int linelen;
       char *line;
       int len;
       char *p;
@@ -1053,16 +1052,18 @@ eval (struct ebuffer *ebuf, int set_default)
         v_origin = o_file;
         exported = 0;
         if (wtype == w_static)
-          if (word1eq ("override"))
+          {
-            {
+            if (word1eq ("override"))
-              v_origin = o_override;
+              {
-              wtype = get_next_mword (p+len, NULL, &p, &len);
+                v_origin = o_override;
-            }
+                wtype = get_next_mword (p+len, NULL, &p, &len);
-          else if (word1eq ("export"))
+              }
-            {
+            else if (word1eq ("export"))
-              exported = 1;
+              {
-              wtype = get_next_mword (p+len, NULL, &p, &len);
+                exported = 1;
-            }
+                wtype = get_next_mword (p+len, NULL, &p, &len);
+              }
+          }
 
         if (wtype != w_eol)
           wtype = get_next_mword (p+len, NULL, NULL, NULL);
@@ -1079,8 +1080,7 @@ eval (struct ebuffer *ebuf, int set_default)
                                         semip, strlen (semip)+1);
                 p = variable_buffer + l;
               }
-            record_target_var (filenames, p, two_colon, v_origin, exported,
+            record_target_var (filenames, p, v_origin, exported, fstart);
-                               fstart);
             filenames = 0;
             continue;
           }
@@ -1457,17 +1457,20 @@ conditional_line (char *line, const struct floc *flocp)
       /* "Ifdef" or "ifndef".  */
       char *var;
       struct variable *v;
-      register char *p = end_of_token (line);
+      register char *p;
-      i = p - line;
+
+      /* Expand the thing we're looking up, so we can use indirect and
+         constructed variable names.  */
+      var = allocated_variable_expand (line);
+
+      /* Make sure there's only one variable name to test.  */
+      p = end_of_token (var);
+      i = p - var;
       p = next_token (p);
       if (*p != '\0')
 	return -1;
 
-      /* Expand the thing we're looking up, so we can use indirect and
+      var[i] = '\0';
-         constructed variable names.  */
-      line[i] = '\0';
-      var = allocated_variable_expand (line);
-
       v = lookup_variable (var, strlen (var));
       conditionals->ignoring[conditionals->if_cmds - 1]
 	= (v != 0 && *v->value != '\0') == notdef;
@@ -1651,7 +1654,7 @@ uniquize_deps (struct dep *chain)
    variable value list.  */
 
 static void
-record_target_var (struct nameseq *filenames, char *defn, int two_colon,
+record_target_var (struct nameseq *filenames, char *defn,
                    enum variable_origin origin, int exported,
                    const struct floc *flocp)
 {

tests/ChangeLog

@@ -1,3 +1,9 @@
+2004-01-21  Paul D. Smith  <psmith@gnu.org>
+
+	* scripts/features/conditionals: Test arguments to ifn?def which
+	contain whitespace (such as a function that is evaluated).  Bug
+	#7257.
+
 2004-01-07  Paul D. Smith  <psmith@gnu.org>
 
 	* scripts/features/order_only: Test order-only prerequisites in

tests/scripts/features/conditionals

@@ -85,14 +85,21 @@ ifdef $(F)oo
 DEF2 = yes
 endif
 
-all:; @echo DEF=$(DEF) DEF2=$(DEF2)
+
+DEF3 = no
+FUNC = $1
+ifdef $(call FUNC,DEF)3
+  DEF3 = yes
+endif
+
+all:; @echo DEF=$(DEF) DEF2=$(DEF2) DEF3=$(DEF3)
 
 EOF
 
 close(MAKEFILE)
 
 &run_make_with_options($makefile2,"",&get_logfile,0);
-$answer = "DEF=yes DEF2=yes\n";
+$answer = "DEF=yes DEF2=yes DEF3=yes\n";
 &compare_output($answer,&get_logfile(1));