Add new feature: != shell assignment for portability with BSD make.

Feature submitted by David Wheeler.

AUTHORS

@@ -61,6 +61,7 @@ Other contributors:
   Carl Staelin (Princeton University)
   Ian Stewartson (Data Logic Limited)
   Ramon Garcia Fernandez <ramon.garcia.f@gmail.com>
+  David A. Wheeler <dwheeler@dwheeler.com>
 
 With suggestions/comments/bug reports from a cast of ... well ...
 hundreds, anyway :)

ChangeLog

@@ -1,3 +1,20 @@
+2011-04-17  David A. Wheeler  <dwheeler@dwheeler.com>
+
+	* doc/make.texi (Reading Makefiles): Document "!=".
+	(Setting): Ditto.
+	(Features): Ditto.
+	* variable.h (enum variable_flavor): New type "f_shell".
+	* variable.c (shell_result): Send a string to the shell and store
+	the output.
+	(do_variable_definition): Handle f_shell variables: expand the
+	value, then send it to the shell and store the result.
+	(parse_variable_definition): Parse "!=" shell assignments.
+	* read.c (get_next_mword): Treat "!=" as a varassign word.
+	* function.c (fold_newlines): If trim_newlines is set remove all
+	trailing newlines; otherwise remove only the last newline.
+	(func_shell_base): Move the guts of the shell function here.
+	(func_shell): Call func_shell_base().
+
 2011-02-21  Paul Smith  <psmith@gnu.org>
 
 	* strcache.c (various): Increase performance based on comments

NEWS

@@ -1,6 +1,6 @@
 GNU make NEWS                                               -*-indented-text-*-
   History of user-visible changes.
-  29 August 2010
+  17 April 2011
 
 See the end of this file for copyrights and conditions.
 
@@ -22,6 +22,13 @@ http://sv.gnu.org/bugs/index.php?group=make&report_id=111&fix_release_id=101&set
       multiple consecutive backslash/newlines do not condense into one space.
     * In recipes, a recipe prefix following a backslash-newlines is removed.
 
+* New feature: "!=" shell assignment operator as an alternative to the
+  $(shell ...) function.  Implemented for portability of BSD makefiles.
+  WARNING: Backward-incompatibility!
+  Variables ending in "!" previously defined as "variable!= value" will now be
+  interpreted as shell assignment.  Change your assignment to add whitespace
+  between the "!" and "=": "variable! = value"
+
 * New command line option: --trace enables tracing of targets.  When enabled
   the recipe to be invoked is printed even if it would otherwise be suppressed
   by .SILENT or a "@" prefix character.  Also before each recipe is run the

doc/make.texi

@@ -1305,7 +1305,7 @@ specified by the existing contents of @file{mfile}.
 Sometimes it is useful to have a makefile that is mostly just like
 another makefile.  You can often use the @samp{include} directive to
 include one in the other, and add more targets or variable definitions.
-However, it is illegal for two makefiles to give different recipes for
+However, it is invalid for two makefiles to give different recipes for
 the same target.  But there is another way.
 
 @cindex match-anything rule, used to override
@@ -1379,6 +1379,7 @@ chapters.
 @cindex =, expansion
 @cindex ?=, expansion
 @cindex +=, expansion
+@cindex !=, expansion
 @cindex define, expansion
 
 Variable definitions are parsed as follows:
@@ -1388,6 +1389,7 @@ Variable definitions are parsed as follows:
 @var{immediate} ?= @var{deferred}
 @var{immediate} := @var{immediate}
 @var{immediate} += @var{deferred} or @var{immediate}
+@var{immediate} != @var{immediate}
 
 define @var{immediate}
   @var{deferred}
@@ -1408,12 +1410,21 @@ endef
 define @var{immediate} +=
   @var{deferred} or @var{immediate}
 endef
+
+define @var{immediate} !=
+  @var{immediate}
+endef
 @end example
 
 For the append operator, @samp{+=}, the right-hand side is considered
 immediate if the variable was previously set as a simple variable
 (@samp{:=}), and deferred otherwise.
 
+For the shell assignment operator, @samp{!=}, the right-hand side is
+evaluated immediately and handed to the shell.  The result is stored in the
+variable named on the left, and that variable becomes a simple variable
+(and will thus be re-evaluated on each reference).
+
 @subheading Conditional Directives
 @cindex ifdef, expansion
 @cindex ifeq, expansion
@@ -5402,6 +5413,7 @@ Several variables have constant initial values.
 @cindex =
 @cindex :=
 @cindex ?=
+@cindex !=
 
 To set a variable from the makefile, write a line starting with the
 variable name followed by @samp{=} or @samp{:=}.  Whatever follows the
@@ -5457,6 +5469,33 @@ FOO = bar
 endif
 @end example
 
+The shell assignment operator @samp{!=} can be used to execute a
+program and set a variable to its output.  This operator first
+evaluates the right-hand side, then passes that result to the shell
+for execution.  If the result of the execution ends in a newline, that
+one newline is removed; all other newlines are replaced by spaces.
+The resulting string is then placed into the named
+recursively-expanded variable.  For example:
+
+@example
+hash != printf '\043'
+file_list != find . -name '*.c'
+@end example
+
+If the result of the execution could produce a @code{$}, and you don't
+intend what follows that to be interpreted as a make variable or
+function reference, then you must replace every @code{$} with
+@code{$$} as part of the execution.  Alternatively, you can set a
+simply expanded variable to the result of running a program using the
+@code{shell} function call.  @xref{Shell Function, , The @code{shell}
+Function}.  For example:
+
+@example
+hash := $(shell printf '\043')
+var := $(shell find . -name "*.c")
+@end example
+
+
 @node Appending, Override Directive, Setting, Using Variables
 @section Appending More Text to Variables
 @cindex +=
@@ -5977,7 +6016,7 @@ prog: a.o b.o
 
 Due to the @code{private} modifier, @code{a.o} and @code{b.o} will not
 inherit the @code{EXTRA_CFLAGS} variable assignment from the
-@code{progs} target.
+@code{prog} target.
 
 @node Special Variables,  , Suppressing Inheritance, Using Variables
 @comment  node-name,  next,  previous,  up
@@ -6073,7 +6112,7 @@ foo
 @end example
 
 Note that assigning more than one target name to @code{.DEFAULT_GOAL} is
-illegal and will result in an error.
+invalid and will result in an error.
 
 @vindex MAKE_RESTARTS @r{(number of times @code{make} has restarted)}
 @item MAKE_RESTARTS
@@ -10444,6 +10483,11 @@ nonexistent file comes from SunOS 4 @code{make}.  (But note that SunOS 4
 @code{make} does not allow multiple makefiles to be specified in one
 @code{-include} directive.)  The same feature appears with the name
 @code{sinclude} in SGI @code{make} and perhaps others.
+
+@item
+The @code{!=} shell assignment operator exists in many BSD of
+@code{make} and is purposefully implemented here to behave identically
+to those implementations.
 @end itemize
 
 The remaining features are inventions new in GNU @code{make}:

function.c

@@ -1393,14 +1393,14 @@ func_value (char *o, char **argv, const char *funcname UNUSED)
 }
 
 /*
-  \r  is replaced on UNIX as well. Is this desirable?
+  \r is replaced on UNIX as well. Is this desirable?
  */
 static void
-fold_newlines (char *buffer, unsigned int *length)
+fold_newlines (char *buffer, unsigned int *length, int trim_newlines)
 {
   char *dst = buffer;
   char *src = buffer;
-  char *last_nonnl = buffer -1;
+  char *last_nonnl = buffer - 1;
   src[*length] = 0;
   for (; *src != '\0'; ++src)
     {
@@ -1416,6 +1416,10 @@ fold_newlines (char *buffer, unsigned int *length)
 	  *dst++ = *src;
 	}
     }
+
+  if (!trim_newlines && (last_nonnl < (dst - 2)))
+    last_nonnl = dst - 2;
+
   *(++last_nonnl) = '\0';
   *length = last_nonnl - buffer;
 }
@@ -1578,12 +1582,20 @@ msdos_openpipe (int* pipedes, int *pidp, char *text)
 #ifdef VMS
 
 /* VMS can't do $(shell ...)  */
+
+char *
+func_shell_base (char *o, char **argv, int trim_newlines)
+{
+  fprintf (stderr, "This platform does not support shell\n");
+  die (EXIT_FAILURE);
+}
+
 #define func_shell 0
 
 #else
 #ifndef _AMIGA
-static char *
+char *
-func_shell (char *o, char **argv, const char *funcname UNUSED)
+func_shell_base (char *o, char **argv, int trim_newlines)
 {
   char *batch_filename = NULL;
 
@@ -1762,7 +1774,7 @@ func_shell (char *o, char **argv, const char *funcname UNUSED)
 	{
 	  /* The child finished normally.  Replace all newlines in its output
 	     with spaces, and put that in the variable output buffer.  */
-	  fold_newlines (buffer, &i);
+	  fold_newlines (buffer, &i, trim_newlines);
 	  o = variable_buffer_output (o, buffer, i);
 	}
 
@@ -1776,8 +1788,8 @@ func_shell (char *o, char **argv, const char *funcname UNUSED)
 
 /* Do the Amiga version of func_shell.  */
 
-static char *
+char *
-func_shell (char *o, char **argv, const char *funcname)
+func_shell_base (char *o, char **argv, int trim_newlines)
 {
   /* Amiga can't fork nor spawn, but I can start a program with
      redirection of my choice.  However, this means that we
@@ -1854,12 +1866,18 @@ func_shell (char *o, char **argv, const char *funcname)
 
   Close (child_stdout);
 
-  fold_newlines (buffer, &i);
+  fold_newlines (buffer, &i, trim_newlines);
   o = variable_buffer_output (o, buffer, i);
   free (buffer);
   return o;
 }
 #endif  /* _AMIGA */
+
+char *
+func_shell (char *o, char **argv, const char *funcname UNUSED)
+{
+  return func_shell_base (o, argv, 1);
+}
 #endif  /* !VMS */
 
 #ifdef EXPERIMENTAL

read.c

@@ -2463,7 +2463,7 @@ readline (struct ebuffer *ebuf)
      w_colon        A colon
      w_dcolon       A double-colon
      w_semicolon    A semicolon
-     w_varassign    A variable assignment operator (=, :=, +=, or ?=)
+     w_varassign    A variable assignment operator (=, :=, +=, ?=, or !=)
 
    Note that this function is only used when reading certain parts of the
    makefile.  Don't use it where special rules hold sway (RHS of a variable,
@@ -2514,6 +2514,7 @@ get_next_mword (char *buffer, char *delim, char **startp, unsigned int *length)
 
     case '+':
     case '?':
+    case '!':
       if (*p == '=')
         {
           ++p;
@@ -2533,7 +2534,7 @@ get_next_mword (char *buffer, char *delim, char **startp, unsigned int *length)
 
   /* This is some non-operator word.  A word consists of the longest
      string of characters that doesn't contain whitespace, one of [:=#],
-     or [?+]=, or one of the chars in the DELIM string.  */
+     or [?+!]=, or one of the chars in the DELIM string.  */
 
   /* We start out assuming a static word; if we see a variable we'll
      adjust our assumptions then.  */

tests/ChangeLog

@@ -1,3 +1,7 @@
+2011-04-17  David A. Wheeler  <dwheeler@dwheeler.com>
+
+	* scripts/features/shell_assignment: Regression for "!=" feature
+
 2010-11-06  Paul Smith  <psmith@gnu.org>
 
 	* scripts/features/targetvars: Fix known-good output for BS/NL changes.

tests/scripts/features/shell_assignment

@@ -0,0 +1,65 @@
+#                                                                    -*-perl-*-
+
+$description = "Test BSD-style shell assignments (VAR != VAL) for variables.";
+
+$details = "";
+
+# TEST 0: Basic shell assignment (!=).
+
+run_make_test('
+.POSIX:
+
+demo1!=printf \'  1   2 3\n4\n\n5 \n \n 6\n\n\n\n\'
+demo2 != printf \'7 8\n \'
+demo3 != printf \'$$(demo2)\'
+demo4 != printf \' 2 3 \n\'
+demo5 != printf \' 2 3 \n\n\'
+all: ; @echo "<$(demo1)> <$(demo2)> <$(demo3)> <$(demo4)> <${demo5}>"
+',
+              '', "<  1   2 3 4  5     6   > <7 8  > <7 8  > < 2 3 > < 2 3  >\n");
+
+# TEST 1: Handle '#' the same way as BSD make
+
+run_make_test('
+foo1!=echo bar#baz
+hash != printf \'\043\'
+foo2!= echo "bar$(hash)baz"
+
+all: ; @echo "<$(foo1)> <$(hash)> <$(foo2)>"
+',
+              '', "<bar> <#> <bar#baz>\n");
+
+# TEST 2: shell assignment variables (from !=) should be recursive.
+# Note that variables are re-evaluated later, so the shell can output
+# a value like $(XYZZY) as part of !=.  The $(XYZZY) will be EVALUATED
+# when the value containing it is evaluated.  On the negative side, this
+# means if you don't want this, you need to escape dollar signs as $$.
+# On the positive side, it means that shell programs can output macros
+# that are then evaluated as they are traditionally evaluated.. and that
+# you can use traditional macro evaluation semantics to implement !=.
+
+run_make_test('
+XYZZY = fiddle-dee-dee
+dollar = $$
+VAR3 != printf \'%s\' \'$(dollar)(XYZZY)\'
+
+all: ; @echo "<$(VAR3)>"
+',
+              '', "<fiddle-dee-dee>\n");
+
+
+# TEST 3: Overrides invoke shell anyway; they just don't store the result
+# in a way that is visible.
+
+run_make_test('
+
+override != echo abc > ,abc ; cat ,abc
+
+all: ; @echo "<$(override)>" ; cat ,abc
+',
+              'override=xyz', "<xyz>\nabc\n");
+
+unlink(',abc');
+
+
+1;

variable.c

@@ -1111,6 +1111,29 @@ set_special_var (struct variable *var)
   return var;
 }
 
+/* Given a string, shell-execute it and return a malloc'ed string of the
+ * result. This removes only ONE newline (if any) at the end, for maximum
+ * compatibility with the *BSD makes.  If it fails, returns NULL. */
+
+char *
+shell_result (const char *p)
+{
+  char *buf;
+  unsigned int len;
+  char *args[2];
+  char *result;
+
+  install_variable_buffer (&buf, &len);
+
+  args[0] = (char *) p;
+  args[1] = NULL;
+  variable_buffer_output (func_shell_base (variable_buffer, args, 0), "\0", 1);
+  result = strdup (variable_buffer);
+
+  restore_variable_buffer (buf, len);
+  return result;
+}
+
 /* Given a variable, a value, and a flavor, define the variable.
    See the try_variable_definition() function for details on the parameters. */
 
@@ -1140,6 +1163,16 @@ do_variable_definition (const struct floc *flocp, const char *varname,
          target-specific variable.  */
       p = alloc_value = allocated_variable_expand (value);
       break;
+    case f_shell:
+      {
+        /* A shell definition "var != value".  Expand value, pass it to
+           the shell, and store the result in recursively-expanded var. */
+        char *q = allocated_variable_expand (value);
+        p = alloc_value = shell_result (q);
+        free (q);
+        flavor = f_recursive;
+        break;
+      }
     case f_conditional:
       /* A conditional variable definition "var ?= value".
          The value is set IFF the variable is not defined yet. */
@@ -1432,7 +1465,7 @@ parse_variable_definition (const char *p, enum variable_flavor *flavor)
 	  return (char *)p;
 	}
 
-      /* Match assignment variants (:=, +=, ?=)  */
+      /* Match assignment variants (:=, +=, ?=, !=)  */
       if (*p == '=')
         {
           switch (c)
@@ -1446,6 +1479,9 @@ parse_variable_definition (const char *p, enum variable_flavor *flavor)
               case '?':
                 *flavor = f_conditional;
                 break;
+              case '!':
+                *flavor = f_shell;
+                break;
               default:
                 /* If we skipped whitespace, non-assignments means no var.  */
                 if (wspace)

variable.h

@@ -38,7 +38,8 @@ enum variable_flavor
     f_simple,           /* Simple definition (:=) */
     f_recursive,        /* Recursive definition (=) */
     f_append,           /* Appending definition (+=) */
-    f_conditional       /* Conditional definition (?=) */
+    f_conditional,      /* Conditional definition (?=) */
+    f_shell             /* Shell assignment (!=) */
   };
 
 /* Structure that represents one variable definition.
@@ -134,6 +135,8 @@ char *patsubst_expand_pat (char *o, const char *text, const char *pattern,
                            const char *replace, const char *pattern_percent,
                            const char *replace_percent);
 char *patsubst_expand (char *o, const char *text, char *pattern, char *replace);
+char *func_shell_base (char *o, char **argv, int trim_newlines);
+
 
 /* expand.c */
 char *recursively_expand_for_file (struct variable *v, struct file *file);