write build output to a log file

Summary:
Update the getdeps builders to save all command output to a log file in the
build directory.  This makes it easier to go back after the fact and look at
the config options selected by the build run.

Reviewed By: wez

Differential Revision: D18777059

fbshipit-source-id: 86d3ffa48ec992fe90c59a3835e18f08310ed88c

build/fbcode_builder/getdeps/builder.py

@@ -59,7 +59,8 @@ class BuilderBase(object):
         if cmd_prefix:
             cmd = cmd_prefix + cmd
 
-        run_cmd(cmd=cmd, env=env, cwd=cwd or self.build_dir)
+        log_file = os.path.join(self.build_dir, "getdeps_build.log")
+        run_cmd(cmd=cmd, env=env, cwd=cwd or self.build_dir, log_file=log_file)
 
     def build(self, install_dirs, reconfigure):
         print("Building %s..." % self.manifest.name)

build/fbcode_builder/getdeps/runcmd.py

@@ -6,7 +6,9 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
+import select
 import subprocess
+import sys
 
 from .envfuncs import Env
 from .platform import is_windows
@@ -44,8 +46,21 @@ def _print_env_diff(env):
             print("+ %s=%s \\" % (k, shellquote(env[k])))
 
 
-def run_cmd(cmd, env=None, cwd=None, allow_fail=False):
-    print("---")
+def run_cmd(cmd, env=None, cwd=None, allow_fail=False, log_file=None):
+    if log_file is not None:
+        with open(log_file, "a") as log:
+
+            def log_function(msg):
+                log.write(msg)
+                sys.stdout.write(msg)
+
+            _run_cmd(cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function)
+    else:
+        _run_cmd(cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=sys.stdout.write)
+
+
+def _run_cmd(cmd, env, cwd, allow_fail, log_fn):
+    log_fn("---\n")
     try:
         cmd_str = " \\\n+      ".join(shellquote(arg) for arg in cmd)
     except TypeError:
@@ -65,21 +80,75 @@ def run_cmd(cmd, env=None, cwd=None, allow_fail=False):
         env = dict(env.items())
 
     if cwd:
-        print("+ cd %s && \\" % shellquote(cwd))
+        log_fn("+ cd %s && \\\n" % shellquote(cwd))
         # Our long path escape sequence may confuse cmd.exe, so if the cwd
         # is short enough, strip that off.
         if is_windows() and (len(cwd) < 250) and cwd.startswith("\\\\?\\"):
             cwd = cwd[4:]
 
-    print("+ %s" % cmd_str)
-
-    if allow_fail:
-        return subprocess.call(cmd, env=env, cwd=cwd)
+    log_fn("+ %s\n" % cmd_str)
 
     try:
-        return subprocess.check_call(cmd, env=env, cwd=cwd)
+        p = subprocess.Popen(
+            cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        )
     except (TypeError, ValueError, OSError) as exc:
+        log_fn("error running `%s`: %s" % (cmd_str, exc))
         raise RunCommandError(
             "%s while running `%s` with env=%r\nos.environ=%r"
             % (str(exc), cmd_str, env, os.environ)
         )
+
+    _pipe_output(p, log_fn)
+
+    p.wait()
+    if p.returncode != 0 and not allow_fail:
+        raise subprocess.CalledProcessError(p.returncode, cmd)
+
+    return p.returncode
+
+
+if hasattr(select, "poll"):
+
+    def _pipe_output(p, log_fn):
+        """Read output from p.stdout and call log_fn() with each chunk of data as it
+        becomes available."""
+        # Perform non-blocking reads
+        import fcntl
+
+        fcntl.fcntl(p.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
+        poll = select.poll()
+        poll.register(p.stdout.fileno(), select.POLLIN)
+
+        buffer_size = 4096
+        while True:
+            poll.poll()
+            data = p.stdout.read(buffer_size)
+            if not data:
+                break
+            # log_fn() accepts arguments as str (binary in Python 2, unicode in
+            # Python 3).  In Python 3 the subprocess output will be plain bytes,
+            # and need to be decoded.
+            if not isinstance(data, str):
+                data = data.decode("utf-8", errors="surrogateescape")
+            log_fn(data)
+
+
+else:
+
+    def _pipe_output(p, log_fn):
+        """Read output from p.stdout and call log_fn() with each chunk of data as it
+        becomes available."""
+        # Perform blocking reads.  Use a smaller buffer size to avoid blocking
+        # for very long when data is available.
+        buffer_size = 64
+        while True:
+            data = p.stdout.read(buffer_size)
+            if not data:
+                break
+            # log_fn() accepts arguments as str (binary in Python 2, unicode in
+            # Python 3).  In Python 3 the subprocess output will be plain bytes,
+            # and need to be decoded.
+            if not isinstance(data, str):
+                data = data.decode("utf-8", errors="surrogateescape")
+            log_fn(data)