mirror of
https://github.com/facebook/sapling.git
synced 2024-10-06 14:58:03 +03:00
distutils_rust: retry on flacky MSVC errors
Summary:
Sometimes the Windows build hit strange C1056 error that looks like a race:
nghttp2\lib\nghttp2_http.c : fatal error C1056: cannot update the time date stamp field in 'C:\cygwin\data\sandcastle\boxes\trunk-hg-fbcode-windows\fbcode\eden\scm\build\cargo-target\release\build\libnghttp2-sys-047b2e8066895d56\out\i\lib\nghttp2\lib\nghttp2_http.o'; error code %u
exit code: 2
This does not always happen. But when it happens, it provides noisy test
signal. Let's detect this error and retry accordingly.
Relevant log:
```
...
Compiling eden_config v0.1.0 (scm\lib\thrift-types\eden_config)
Compiling hgclient_conf v0.1.0 (scm\lib\thrift-types\hgclient_conf)
Compiling fb303 v0.1.0 (scm\lib\thrift-types\fb303)
Compiling eden v0.1.0 (scm\lib\thrift-types\eden)
error: failed to run custom build command for `libnghttp2-sys v0.1.4+1.41.0`
Caused by:
process didn't exit successfully: `scm\build\cargo-target\release\build\libnghttp2-sys-e674116aab730b74\build-script-build` (exit code: 1)
--- stdout
TARGET = Some("x86_64-pc-windows-msvc")
OPT_LEVEL = Some("3")
HOST = Some("x86_64-pc-windows-msvc")
CC_x86_64-pc-windows-msvc = None
CC_x86_64_pc_windows_msvc = None
HOST_CC = None
CC = None
CFLAGS_x86_64-pc-windows-msvc = None
CFLAGS_x86_64_pc_windows_msvc = None
HOST_CFLAGS = None
CFLAGS = None
CRATE_CC_NO_DEFAULTS = None
CARGO_CFG_TARGET_FEATURE = Some("fxsr,sse,sse2")
DEBUG = Some("true")
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_buf.o" "-c" "nghttp2/lib/nghttp2_buf.c"
nghttp2_buf.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_callbacks.o" "-c" "nghttp2/lib/nghttp2_callbacks.c"
nghttp2_callbacks.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_debug.o" "-c" "nghttp2/lib/nghttp2_debug.c"
nghttp2_debug.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_frame.o" "-c" "nghttp2/lib/nghttp2_frame.c"
nghttp2_frame.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_hd.o" "-c" "nghttp2/lib/nghttp2_hd.c"
nghttp2_hd.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_hd_huffman.o" "-c" "nghttp2/lib/nghttp2_hd_huffman.c"
nghttp2_hd_huffman.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_hd_huffman_data.o" "-c" "nghttp2/lib/nghttp2_hd_huffman_data.c"
nghttp2_hd_huffman_data.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_helper.o" "-c" "nghttp2/lib/nghttp2_helper.c"
nghttp2_helper.c
exit code: 0
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_http.o" "-c" "nghttp2/lib/nghttp2_http.c"
nghttp2_http.c
scm\build\cargo-target\release\build\libnghttp2-sys-047b2e8066895d56\out\i\lib\nghttp2\lib\nghttp2_http.o'; error code %u
exit code: 2
running: "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_map.o" "-c" "nghttp2/lib/nghttp2_map.c"
nghttp2_map.c
exit code: 0
--- stderr
error occurred: Command "scm\\build\\cargo-target\\release\\build\\libnghttp2-sys-047b2e8066895d56\\out\\i\\lib\\nghttp2/lib/nghttp2_http.o" "-c" "nghttp2/lib/nghttp2_http.c" with args "cl.exe" did not execute successfully (status code exit code: 2).
warning: build failed, waiting for other jobs to finish...
warning: Error finalizing incremental compilation session directory `\\?\scm\build\cargo-target\release\incremental\revlogindex-h6o98uu5olhe\s-g3wf4w0oih-155kb8p-working`: Access is denied. (os error 5)
warning: `revlogindex` (lib) generated 1 warning
error: build failed
```
Reviewed By: DurhamG
Differential Revision: D32195973
fbshipit-source-id: 859d53e6f57e6fdf2a6ead15c5abb61c5480a0a6
This commit is contained in:
Jun Wu
Facebook GitHub Bot
parent
dd6ae0b17c
commit
e6d9a68865
@ -16,12 +16,13 @@ import distutils.core
|
||||
import distutils.errors
|
||||
import distutils.util
|
||||
import errno
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import threading
|
||||
|
||||
|
||||
# This manifest is merged with the default .exe manifest to allow
|
||||
@ -287,7 +288,7 @@ replace-with = "vendored-sources"
|
||||
env = os.environ.copy()
|
||||
env["LIB_DIRS"] = os.path.abspath(self.build_temp)
|
||||
|
||||
rc = subprocess.call(cmd, env=env)
|
||||
rc = _callretry(cmd, env=env)
|
||||
if rc:
|
||||
raise distutils.errors.CompileError(
|
||||
"compilation of Rust target '%s' failed" % target.name
|
||||
@ -426,6 +427,54 @@ class InstallRustExt(distutils.command.install_scripts.install_scripts):
|
||||
self.outfiles = self.copy_tree(self.build_dir, self.install_dir)
|
||||
|
||||
|
||||
def _tee(stream, *outputs):
|
||||
while True:
|
||||
buf = stream.read(1)
|
||||
if not buf:
|
||||
break
|
||||
for o in outputs:
|
||||
o.write(buf)
|
||||
if b"\n" in buf or b"\r" in buf:
|
||||
o.flush()
|
||||
for o in outputs:
|
||||
o.flush()
|
||||
|
||||
|
||||
def _callretry(cmd, env=None, retry=3):
|
||||
if sys.stderr.isatty():
|
||||
env = (env or os.environ).copy()
|
||||
env["CARGO_TERM_COLOR"] = "always"
|
||||
env["CARGO_TERM_PROGRESS_WHEN"] = "always"
|
||||
env["CARGO_TERM_PROGRESS_WIDTH"] = env.get("COLUMNS") or "80"
|
||||
return _callattempt(cmd, env=env, retry=retry)
|
||||
|
||||
|
||||
def _callattempt(cmd, env=None, retry=3):
|
||||
"""like subprocess.call but retry on flaky failures on Windows"""
|
||||
sio = io.BytesIO()
|
||||
proc = subprocess.Popen(cmd, env=env, stderr=subprocess.PIPE)
|
||||
t = threading.Thread(target=_tee, args=(proc.stderr, sys.stderr.buffer, sio))
|
||||
t.start()
|
||||
returncode = proc.wait()
|
||||
if retry > 0 and returncode != 0:
|
||||
stderr = sio.getvalue()
|
||||
if _isflaky(stderr):
|
||||
distutils.log.warn("retrying %r on flaky error" % (cmd,))
|
||||
return _callattempt(cmd, env=env, retry=retry - 1)
|
||||
return returncode
|
||||
|
||||
|
||||
def _isflaky(stderr):
|
||||
if b"fatal error C1056" in stderr:
|
||||
# full error:
|
||||
# fatal error C1056: cannot update the time date stamp field in
|
||||
# '...\eden\scm\build\cargo-target\release\build\libnghttp2-sys-047b2e8066895d56
|
||||
# \out\i\lib\nghttp2\lib\nghttp2_http.o'; error code %u
|
||||
# exit code: 2
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
distutils.dist.Distribution.rust_ext_modules = ()
|
||||
distutils.dist.Distribution.rust_ext_binaries = ()
|
||||
distutils.command.build.build.sub_commands.append(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user