Fix Savannah bug # 1332: handle backslash-newline pairs in command scripts

according to POSIX rules.

ChangeLog

@@ -1,5 +1,17 @@
 2005-06-25  Paul D. Smith  <psmith@gnu.org>
 
+	* job.c (construct_command_argv_internal): Sanitize handling of
+	backslash/newline pairs according to POSIX: that is, keep the
+	backslash-newline in the command script, but remove a following
+	TAB character, if present.  In the fast path, make sure that the
+	behavior matches what the shell would do both inside and outside
+	of quotes.  In the slow path, quote the backslash and put a
+	literal newline in the string.
+	Fixes Savannah bug #1332.
+	* doc/make.texi (Execution): Document the new behavior and give
+	some examples.
+	* NEWS: Make a note of the new behavior.
+
 	* make.h [WINDOWS32]: #include <direct.h>.
 	Fixes Savannah bug #13478.
 
@@ -8,29 +20,25 @@
 	the symlink we can get as the mtime of the file and don't fail.
 	Fixes Savannah bug #13280.
 
-	Fix Savannah bug #1454.
-
 	* read.c (find_char_unquote): Accept a new argument IGNOREVARS.
 	If it's set, then don't	stop on STOPCHARs or BLANKs if they're
 	inside a variable reference.  Make this function static as it's
 	only used here.
 	(eval): Call find_char_unquote() with IGNOREVARS set when we're
 	parsing an unexpanded line looking for semicolons.
+	Fixes Savannah bug #1454.
 	* misc.c (remove_comments): Move this to read.c and make it static
 	as it's only used there.  Call find_char_unquote() with new arg.
 	* make.h: Remove prototypes for find_char_unquote() and
 	remove_comments() since they're static now.
 
-	Implement the MAKE_RESTARTS variable, and disable -B if it's >0.
-	Fixes Savannah bug #7566.
-
-	* doc/make.texi (Special Variables): Document MAKE_RESTARTS.
-	* NEWS: Mention MAKE_RESTARTS.
 	* main.c (main): If we see MAKE_RESTARTS in the environment, unset
 	its export flag and obtain its value.  When we need to re-exec,
 	increment the value and add it into the environment.
-	(always_make_set): New variable.  Change the -B option to set this
+	* doc/make.texi (Special Variables): Document MAKE_RESTARTS.
-	one instead.
+	* NEWS: Mention MAKE_RESTARTS.
+	* main.c (always_make_set): New variable.  Change the -B option to
+	set this one instead.
 	(main): When checking makefiles, only set always_make_flag if
 	always_make_set is set AND the restarts flag is 0.  When building
 	normal targets, set it IFF always_make_set is set.
@@ -38,6 +46,7 @@
 	files to NEW before we check makefiles if we've never restarted
 	before.  If we have restarted, set what-if files to NEW _after_ we
 	check makefiles.
+	Fixes Savannah bug #7566:
 
 2005-06-17  Paul D. Smith  <psmith@gnu.org>
 

NEWS

@@ -28,6 +28,11 @@ Version 3.81beta3
   double-quote any "$" in your filenames; instead of "foo: boo$$bar" you
   now must write "foo: foo$$$$bar".
 
+* WARNING: Backward-incompatibility!
+  In order to comply with POSIX, the way in which GNU make processes
+  backslash-newline sequences in command strings has changed.  See the
+  GNU make manual section "Shell Execution" for details.
+
 * New command-line option: -L (--check-symlink-times).  On systems that
   support symbolic links, if this option is given then GNU make will
   use the most recent modification time of any symbolic links that are

doc/make.texi

@@ -3565,17 +3565,59 @@ foo : bar/lose
 @cindex @code{\} (backslash), in commands
 @cindex quoting newline, in commands
 @cindex newline, quoting, in commands
-If you would like to split a single shell command into multiple lines of
+A shell command can be split into multiple lines of text by placing a
-text, you must use a backslash at the end of all but the last subline.
+backslash before each newline.  Such a sequence of lines is provided
-Such a sequence of lines is combined into a single line, by deleting the
+to the shell as a single command script.  The backslash and newline
-backslash-newline sequences, before passing it to the shell.  Thus, the
+are preserved in the shell command.  If the first character on the
-following is equivalent to the preceding example:
+line after a backslash-newline is a tab, the tab will @emph{not} be
+included in the shell command.  So, this makefile:
 
 @example
 @group
-foo : bar/lose
+all :
-        cd bar;  \
+        @@echo no\
-        gobble lose > ../foo
+space
+        @@echo no\
+        space
+@end group
+@end example
+
+consists of two separate shell commands where the output is:
+
+@example
+@group
+nospace
+nospace
+@end group
+@end example
+
+As a more complex example, this makefile:
+
+@example
+@group
+all : ; @@echo 'hello \
+        world' ; echo "hello \
+    world"
+@end group
+@end example
+
+will run one shell with a command script of:
+
+@example
+@group
+echo 'hello \
+world' ; echo "hello \
+    world"
+@end group
+@end example
+
+which, according to shell quoting rules, will yield the following output:
+
+@example
+@group
+hello \
+world
+hello     world
 @end group
 @end example
 

job.c

@@ -2363,12 +2363,10 @@ construct_command_argv_internal (char *line, char **restp, char *shell,
   instring = word_has_equals = seen_nonequals = last_argument_was_empty = 0;
   for (p = line; *p != '\0'; ++p)
     {
-      if (ap > end)
+      assert (ap <= end);
-	abort ();
 
       if (instring)
 	{
-	string_char:
 	  /* Inside a string, just copy any char except a closing quote
 	     or a backslash-newline combination.  */
 	  if (*p == instring)
@@ -2378,7 +2376,21 @@ construct_command_argv_internal (char *line, char **restp, char *shell,
 		last_argument_was_empty = 1;
 	    }
 	  else if (*p == '\\' && p[1] == '\n')
-	    goto swallow_escaped_newline;
+            {
+              /* Backslash-newline is handled differently depending on what
+                 kind of string we're in: inside single-quoted strings you
+                 keep them; in double-quoted strings they disappear.  */
+              if (instring == '"')
+                ++p;
+              else
+                {
+                  *(ap++) = *(p++);
+                  *(ap++) = *p;
+                }
+              /* If there's a TAB here, skip it.  */
+              if (p[1] == '\t')
+                ++p;
+            }
 	  else if (*p == '\n' && restp != NULL)
 	    {
 	      /* End of the command line.  */
@@ -2418,37 +2430,21 @@ construct_command_argv_internal (char *line, char **restp, char *shell,
 	    break;
 
 	  case '\\':
-	    /* Backslash-newline combinations are eaten.  */
+	    /* Backslash-newline has special case handling, ref POSIX.
+               We're in the fastpath, so emulate what the shell would do.  */
 	    if (p[1] == '\n')
 	      {
-	      swallow_escaped_newline:
+		/* Throw out the backslash and newline.  */
+                ++p;
 
-		/* Eat the backslash, the newline, and following whitespace,
+		/* If there is a tab after a backslash-newline, remove it.  */
-		   replacing it all with a single space.  */
+		if (p[1] == '\t')
-		p += 2;
+                  ++p;
 
-		/* If there is a tab after a backslash-newline,
+                /* If there's nothing in this argument yet, skip any
-		   remove it from the source line which will be echoed,
+                   whitespace before the start of the next word.  */
-		   since it was most likely used to line
+                if (ap == new_argv[i])
-		   up the continued line with the previous one.  */
+                  p = next_token (p + 1) - 1;
-		if (*p == '\t')
-                  /* Note these overlap and strcpy() is undefined for
-                     overlapping objects in ANSI C.  The strlen() _IS_ right,
-                     since we need to copy the nul byte too.  */
-		  bcopy (p + 1, p, strlen (p));
-
-		if (instring)
-		  goto string_char;
-		else
-		  {
-		    if (ap != new_argv[i])
-		      /* Treat this as a space, ending the arg.
-			 But if it's at the beginning of the arg, it should
-			 just get eaten, rather than becoming an empty arg. */
-		      goto end_of_arg;
-		    else
-		      p = next_token (p) - 1;
-		  }
 	      }
 	    else if (p[1] != '\0')
               {
@@ -2502,7 +2498,6 @@ construct_command_argv_internal (char *line, char **restp, char *shell,
 
 	  case ' ':
 	  case '\t':
-	  end_of_arg:
 	    /* We have the end of an argument.
 	       Terminate the text of the argument.  */
 	    *ap++ = '\0';
@@ -2538,9 +2533,7 @@ construct_command_argv_internal (char *line, char **restp, char *shell,
 	      }
 
 	    /* Ignore multiple whitespace chars.  */
-	    p = next_token (p);
+	    p = next_token (p) - 1;
-	    /* Next iteration should examine the first nonwhite char.  */
-	    --p;
 	    break;
 
 	  default:
@@ -2672,23 +2665,15 @@ construct_command_argv_internal (char *line, char **restp, char *shell,
 	  }
 	else if (*p == '\\' && p[1] == '\n')
 	  {
-	    /* Eat the backslash, the newline, and following whitespace,
+	    /* POSIX says we keep the backslash-newline, but throw out the
-	       replacing it all with a single space (which is escaped
+               next char if it's a TAB.  */
-	       from the shell).  */
+            *(ap++) = '\\';
-	    p += 2;
+            *(ap++) = *(p++);
+            *(ap++) = *p;
 
-	    /* If there is a tab after a backslash-newline,
+	    if (p[1] == '\t')
-	       remove it from the source line which will be echoed,
+	      ++p;
-	       since it was most likely used to line
-	       up the continued line with the previous one.  */
-	    if (*p == '\t')
-	      bcopy (p + 1, p, strlen (p));
 
-	    p = next_token (p);
-	    --p;
-            if (unixy_shell && !batch_mode_shell)
-              *ap++ = '\\';
-	    *ap++ = ' ';
 	    continue;
 	  }
 

tests/ChangeLog

@@ -1,5 +1,10 @@
 2005-06-25  Paul D. Smith  <psmith@gnu.org>
 
+	* scripts/misc/general3: Implement comprehensive testing of
+	backslash-newline behavior in command scripts: various types of
+	quoting, fast path / slow path, etc.
+	Tests fix for Savannah bug #1332.
+
 	* scripts/options/symlinks: Test symlinks to non-existent files.
 	Tests fix for Savannah bug #13280.
 

tests/scripts/misc/general3

@@ -53,4 +53,261 @@ all: ; @:
 ',
               '', 'true; true');
 
+# TEST 4
+
+# Check that backslashes in command scripts are handled according to POSIX.
+# Checks Savannah bug # 1332.
+
+# Test the fastpath / no quotes
+run_make_test('
+all:
+	@echo foo\
+bar
+	@echo foo\
+	bar
+	@echo foo\
+    bar
+	@echo foo\
+	    bar
+	@echo foo \
+bar
+	@echo foo \
+	bar
+	@echo foo \
+    bar
+	@echo foo \
+	    bar
+',
+              '', 'foobar
+foobar
+foo bar
+foo bar
+foo bar
+foo bar
+foo bar
+foo bar');
+
+# Test the fastpath / single quotes
+run_make_test("
+all:
+	\@echo 'foo\\
+bar'
+	\@echo 'foo\\
+	bar'
+	\@echo 'foo\\
+    bar'
+	\@echo 'foo\\
+	    bar'
+	\@echo 'foo \\
+bar'
+	\@echo 'foo \\
+	bar'
+	\@echo 'foo \\
+    bar'
+	\@echo 'foo \\
+	    bar'
+",
+              '', 'foo\
+bar
+foo\
+bar
+foo\
+    bar
+foo\
+    bar
+foo \
+bar
+foo \
+bar
+foo \
+    bar
+foo \
+    bar');
+
+# Test the fastpath / double quotes
+run_make_test('
+all:
+	@echo "foo\
+bar"
+	@echo "foo\
+	bar"
+	@echo "foo\
+    bar"
+	@echo "foo\
+	    bar"
+	@echo "foo \
+bar"
+	@echo "foo \
+	bar"
+	@echo "foo \
+    bar"
+	@echo "foo \
+	    bar"
+',
+              '', 'foobar
+foobar
+foo    bar
+foo    bar
+foo bar
+foo bar
+foo     bar
+foo     bar');
+
+# Test the slow path / no quotes
+run_make_test('
+all:
+	@echo hi; echo foo\
+bar
+	@echo hi; echo foo\
+	bar
+	@echo hi; echo foo\
+ bar
+	@echo hi; echo foo\
+	 bar
+	@echo hi; echo foo \
+bar
+	@echo hi; echo foo \
+	bar
+	@echo hi; echo foo \
+ bar
+	@echo hi; echo foo \
+	 bar
+',
+              '', 'hi
+foobar
+hi
+foobar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar');
+
+# Test the slow path / no quotes.  This time we put the slow path
+# determination _after_ the backslash-newline handling.
+run_make_test('
+all:
+	@echo foo\
+bar; echo hi
+	@echo foo\
+	bar; echo hi
+	@echo foo\
+ bar; echo hi
+	@echo foo\
+	 bar; echo hi
+	@echo foo \
+bar; echo hi
+	@echo foo \
+	bar; echo hi
+	@echo foo \
+ bar; echo hi
+	@echo foo \
+	 bar; echo hi
+',
+              '', 'foobar
+hi
+foobar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar
+hi
+foo bar
+hi');
+
+# Test the slow path / single quotes
+run_make_test("
+all:
+	\@echo hi; echo 'foo\\
+bar'
+	\@echo hi; echo 'foo\\
+	bar'
+	\@echo hi; echo 'foo\\
+    bar'
+	\@echo hi; echo 'foo\\
+	    bar'
+	\@echo hi; echo 'foo \\
+bar'
+	\@echo hi; echo 'foo \\
+	bar'
+	\@echo hi; echo 'foo \\
+    bar'
+	\@echo hi; echo 'foo \\
+	    bar'
+",
+              '', 'hi
+foo\
+bar
+hi
+foo\
+bar
+hi
+foo\
+    bar
+hi
+foo\
+    bar
+hi
+foo \
+bar
+hi
+foo \
+bar
+hi
+foo \
+    bar
+hi
+foo \
+    bar');
+
+# Test the slow path / double quotes
+run_make_test('
+all:
+	@echo hi; echo "foo\
+bar"
+	@echo hi; echo "foo\
+	bar"
+	@echo hi; echo "foo\
+    bar"
+	@echo hi; echo "foo\
+	    bar"
+	@echo hi; echo "foo \
+bar"
+	@echo hi; echo "foo \
+	bar"
+	@echo hi; echo "foo \
+    bar"
+	@echo hi; echo "foo \
+	    bar"
+',
+              '', 'hi
+foobar
+hi
+foobar
+hi
+foo    bar
+hi
+foo    bar
+hi
+foo bar
+hi
+foo bar
+hi
+foo     bar
+hi
+foo     bar');
+
 1;