From f987d181c40270359f4d89fbe3daa1966634bac2 Mon Sep 17 00:00:00 2001
From: Paul Smith <psmith@gnu.org>
Date: Sat, 22 Oct 2022 17:23:17 -0400
Subject: [PATCH] Collect failure details when the regression tests fail

* README.in: Add a section on running regression tests.
* Makefile.am (check-regression): Capture the test run output, and
on failure collect configure and test results into a tar file.
---
 .gitignore       |  4 ++++
 Makefile.am      | 24 ++++++++++++++++++++----
 README.in        | 13 +++++++++++++
 tests/.gitignore |  1 +
 4 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5c558ea4..e93c46b9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,6 +60,10 @@ WinRel/
 GccDebug/
 GccRel/
 
+# Test artifacts
+makeerror-*.tar.gz
+test-suite.log
+
 # Distribution artifacts
 .dep_segment
 .check-git-HEAD
diff --git a/Makefile.am b/Makefile.am
index 5a474613..5912a396 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -137,8 +137,8 @@ src/gmk-default.h: $(top_srcdir)/src/gmk-default.scm
 dist-hook:
 	(cd $(top_srcdir); \
 	 sub=`find tests/scripts -follow \( -name .git -o -name .deps -o -name work -o -name .gitignore -o -name \*.orig -o -name \*.rej -o -name \*~ -o -name \*.out -o -name Makefile \) -prune -o -type f -print`; \
-	 tar chf - $$sub) \
-	| (cd $(distdir); tar xfBp -)
+	 $(AMTAR) chf - $$sub) \
+	| (cd $(distdir); $(AMTAR) xfBp -)
 
 
 # --------------- Local CHECK Section
@@ -162,9 +162,19 @@ MAKETESTFLAGS =
 
 .PHONY: check-regression
 
-GMK_OUTDIR=..
+GMK_OUTDIR = ..
+
+testlog = test-suite.log
+testresult = tests/.test-result
+errorfile = makeerror-$(host_triplet).tar.gz
+
+testfiles = $(testlog) $(testresult) $(errorfile)
+MOSTLYCLEANFILES = $(testfiles)
+
+errordetails = config.status config.log src/config.h $(testlog) tests/work
 
 check-regression: tests/config-flags.pm
+	@rm -f $(testfiles)
 	@if test -f '$(top_srcdir)/tests/run_make_tests'; then \
 	  ulimit -n 128; \
 	  if $(PERL) -v >/dev/null 2>&1; then \
@@ -177,7 +187,13 @@ check-regression: tests/config-flags.pm
 		   done; fi ;; \
 	    esac; \
 	    echo "cd tests && $(PERL) $(PERLFLAGS) ./run_make_tests.pl -srcdir $(abs_top_srcdir) -make $(GMK_OUTDIR)/make$(EXEEXT) $(MAKETESTFLAGS)"; \
-	    cd tests && $(PERL) $(PERLFLAGS) ./run_make_tests.pl -srcdir '$(abs_top_srcdir)' -make '$(GMK_OUTDIR)/make$(EXEEXT)' $(MAKETESTFLAGS); \
+	    (cd tests && $(PERL) $(PERLFLAGS) ./run_make_tests.pl -srcdir '$(abs_top_srcdir)' -make '$(GMK_OUTDIR)/make$(EXEEXT)' $(MAKETESTFLAGS); echo $$? >.test-result) 2>&1 | tee $(testlog); \
+	    er=$$(cat $(testresult)); if test "$$er" -ne 0; then \
+	      $(AMTAR) chf - $(errordetails) | eval GZIP= gzip $(GZIP_ENV) -c >$(errorfile); \
+	      echo '*** Testing failed!  Details saved in: $(errorfile)'; \
+	      echo '*** Please report to <$(PACKAGE_BUGREPORT)>'; echo; \
+	      exit $$er; \
+	    fi; \
 	  else \
 	    echo "Can't find a working Perl ($(PERL)); the test suite requires Perl."; \
 	  fi; \
diff --git a/README.in b/README.in
index 4ef14a6a..f144caf2 100644
--- a/README.in
+++ b/README.in
@@ -66,6 +66,19 @@ You can find most information concerning the development of GNU Make at
 this site.
 
 
+Regression Tests
+----------------
+
+GNU Make contains a suite of regression tests.  To run them use "make check"
+after building GNU Make.  If they fail a tar package will be created
+containing useful information, which can be emailed (as an attachment) to
+the <bug-make@gnu.org> mailing list.
+
+Please note that since these tests rely on known-good-output comparisons,
+they can show spurious failures on some systems (particularly non-POSIX systems
+such as Windows).
+
+
 Bug Reporting
 -------------
 
diff --git a/tests/.gitignore b/tests/.gitignore
index a30a6895..05501046 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1,2 +1,3 @@
+.test-result
 config-flags.pm
 work