fix several problems with stdout/stderr

- subprocess calls should always send stderr outwards (no capture)

- add option to capture or not capture stdout so $(shell) will be captured but
recipes will go outwards (no capture)

- set stdout to flush on every print() to match GNU Make which uses setbuf()
(which isn't available to python) because otherwise stdout can race between
pymake process and subprocess stdout

- start tests on $(SHELLFLAGS) which is what led to finding these problems with
stdout

Author: linuxlizard

pymake/functions.py

@@ -84,9 +84,9 @@ def eval(self, symbol_table):
         if self.fmt:
             t = self.token_list[0]
             filename, linenumber = t.get_pos()
-            print(self.fmt.format(filename, linenumber, msg), file=self.fh)
+            print(self.fmt.format(filename, linenumber, msg), file=self.fh, flush=True)
         else:
-            print("%s" % msg, file=self.fh)
+            print("%s" % msg, file=self.fh, flush=True)
 
         return ""
 

pymake/pymake.py

@@ -502,14 +502,14 @@ def check_prefixes(s):
         s, ignore_failure, silent = check_prefixes(s)
 
         if not silent and not args.silent:
-            print(s)
+            print(s,flush=True)
 
         if args.dry_run:
-            print(s)
+            print(s, flush=True)
             continue
 
         exit_code = 0
-        ret = shell.execute(s, symtable)
+        ret = shell.execute(s, symtable, capture=False)
 
         # 
         # !!! Run a Sub-Make !!!
@@ -535,7 +535,8 @@ def check_prefixes(s):
             ret.stdout = ""
 
         exit_code = ret.exit_code
-        print(ret.stdout,end="")
+#        print(ret.stderr,end="")
+#        print(ret.stdout,end="")
         if exit_code != 0:
             print("make:", ret.stderr, file=sys.stderr, end="")
             print("make: *** [%r: %s] Error %d %s" % (recipe.get_pos(), rule.target, exit_code, "(ignored)" if ignore_failure else ""), file=sys.stderr)

pymake/shell.py

@@ -30,7 +30,7 @@ def __init__(self):
         self.is_submake = False
 
 
-def execute(cmd_str, symbol_table, use_default_shell=True):
+def execute(cmd_str, symbol_table, use_default_shell=True, capture=True):
     """execute a string with the shell, returning a bunch of useful info"""
 
     # capture a timestamp so we can match shell debug messages
@@ -79,7 +79,7 @@ def execute(cmd_str, symbol_table, use_default_shell=True):
     if shell:
         cmd.extend(shell_list)
     if shellflags:
-        cmd.append(shellflags)
+        cmd.extend([f for f in shellflags.split()])
     cmd.append(cmd_str)
 
     env = symbol_table.get_exports()
@@ -115,21 +115,36 @@ def execute(cmd_str, symbol_table, use_default_shell=True):
 #        outfile.write(" ".join(cmd))
 #        outfile.write("\n\n\n")
 
+    # definitely need to capture stdout when we're running a sub-make because
+    # that's how we determine the shell arguments to the actual sub-make
+    if cmd_str.startswith(submake.getname()):
+        capture = True
+
+    logger.debug("cmd=>>>%s<<<", cmd)
+
     try:
-        p = subprocess.run(cmd, 
-                shell=False,
-                stdout=subprocess.PIPE, 
-                stderr=subprocess.PIPE,
-                universal_newlines=True,
-                check=False, # we'll check returncode ourselves
-                env=env
-            )
+        if capture:
+            p = subprocess.run(cmd, 
+                    shell=False,
+                    stdout=subprocess.PIPE,
+                    universal_newlines=True,
+                    check=False, # we'll check returncode ourselves
+                    env=env
+                )
+        else:
+            p = subprocess.run(cmd, 
+                    shell=False,
+                    universal_newlines=True,
+                    check=False, # we'll check returncode ourselves
+                    env=env
+                )
+
         logger.debug("shell ts=%f exit status=%r", ts, p.returncode)
 #        if p.returncode != 0:
 #            breakpoint()
         return_status.exit_code = p.returncode
         return_status.stdout = p.stdout
-        return_status.stderr = p.stderr
+#        return_status.stderr = p.stderr
     except OSError as err:
         logger.error("shell ts=%f error=\"%s\"", ts, err)
         return_status.exit_code = 127
@@ -165,9 +180,10 @@ def execute_tokens(token_list, symbol_table):
     symbol_table.allow_recursion()
 
     # GNU Make returns one whitespace separated string, no CR/LF
-    # "all other newlines are replaced by spaces." gnu_make.pdf
+    # "If the result of the execution ends in a newline, that one newline is
+    # removed; all other newlines are replaced by spaces."  GNU Make PDF
     exe_result.stdout = exe_result.stdout.strip().replace("\n", " ")
-    exe_result.stderr = exe_result.stderr.strip().replace("\n", " ")
+#    exe_result.stderr = exe_result.stderr.strip().replace("\n", " ")
 
     # save shell status
     pos = token_list[0].get_pos()
@@ -188,7 +204,7 @@ def execute_tokens(token_list, symbol_table):
         # otherwise report stderr (if any)
         if exe_result.stderr:
             logger.error("command at %r failed with exit_code=%d", pos, exe_result.exit_code)
-            error_message(pos, exe_result.stderr)
+#            error_message(pos, exe_result.stderr)
         else:
             logger.error("command at %r failed with exit_code=%d (but stderr empty)", pos, exe_result.exit_code)
 

tests/test_export.py

@@ -12,13 +12,13 @@
 #  The subprocess.run_test() will raise error on non-zero exit.
 
 def run_test(makefile, expect, extra_args=None, extra_env=None):
-    output = run.pymake_string(makefile, extra_args, extra_env)
+    output = run.gnumake_string(makefile, extra_args, extra_env)
     if _debug:
-        print("pymake output=",output)
+        print("gnumake output=>>>",output, "<<<")
     run.verify(output,expect)
-    output = run.gnumake_string(makefile, extra_args, extra_env)
+    output = run.pymake_string(makefile, extra_args, extra_env)
     if _debug:
-        print("gnumake output=",output)
+        print("pymake output=>>>",output, "<<<")
     run.verify(output,expect)
 
 def test1():

tests/test_shellflags.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+import run
+
+def test_simple():
+    makefile="""
+$(info .SHELLFLAGS=$(.SHELLFLAGS))
+@:;@:
+"""
+    a = run.gnumake_string(makefile)
+
+    b = run.pymake_string(makefile)
+
+    assert a==b
+
+def test_x_flag():
+    makefile="""
+.SHELLFLAGS+=-x
+$(info .SHELLFLAGS=$(.SHELLFLAGS))
+@:;@:
+"""
+    a = run.gnumake_string(makefile)
+
+    b = run.pymake_string(makefile)
+
+    assert a==b
+