daml/nix/bazel-retry-cache.patch
Moritz Kiefer e40f62f88a
Revert "Switch to Bazel 5 (#12935)" (#12974)
This reverts commit 4c0118df4d.

This breaks passing -p to bazel test for Haskell tests, e.g.,

bazel test //compiler/damlc/tests:integration-v1dev --test_arg -p
--test_arg InterfaceEq

which breaks with something horrifying like

moritz@adjunction ~/daml (main)> bazel test //compiler/damlc/tests:integration-v1dev --test_arg -p --test_arg InterfaceEq
[dev-env] Building tools.bazel...
[dev-env] Built tools.bazel in /nix/store/m7gzlmr0pqjpl01ihgvazxgfs3sfwl61-bazel and linked to /home/moritz/daml/dev-env/var/gc-roots/bazel
[dev-env] Building tools.find...
[dev-env] Built tools.find in /nix/store/645v3545lcbx77wq7355rgdrgbhn5wx7-findutils-4.8.0 and linked to /home/moritz/daml/dev-env/var/gc-roots/find
INFO: Invocation ID: 034b3e45-851f-472e-ab71-b7f718829582
DEBUG: /home/moritz/.cache/bazel/_bazel_moritz/bb4e4404f889dc1b816f246b08c0d9ea/external/rules_haskell/haskell/private/versions.bzl:60:10: WARNING: bazel version is too recent. Supported versions range from 4.0.0 to 4.2.1, but found: 5.0.0- (@non-git)
/nix/store/dadkhf8vch2i2kvig962ilfr5j3chshr-go-1.17.6
/nix/store/pzh24n543i6jqa01hdmgqknlyf294bn1-bazel-nixpkgs-posix-toolchain
/nix/store/2hfwndk47wpvaib06qyhcdp83b423xvh-jq-1.6-bin
/nix/store/hjggs9s82qh7r5j8sgapn389hf24wdx8-bazel-nixpkgs-cc-toolchain
/nix/store/yxgg3bn4v288sc00kf09svrwz2r461c9-ghc-native-bignum-9.0.2
/nix/store/2hwx0jhcdsx3wfvmb50ih19jkh2ra4jh-glibc-locales-2.33-108
/nix/store/8wpmx049z8m0ffhy3jyi41qb6pbxwvy8-bazel-nixpkgs-java-runtime
ERROR: file 'external/bazel_tools/src/tools/launcher/_objs/launcher/dummy.pic.o' is generated by these conflicting actions:
Label: @bazel_tools//src/tools/launcher:launcher
RuleClass: cc_binary rule
Configuration: 869f6bbd14d2ac66dcacb164312d88fa059b017e3c17513cb5051b23879ac7f9, 8c138cc79f95329f01b0d325511ad0ea89ca5126e421e67e5239706d0f247c2c
Mnemonic: CppCompile
Action key: 3d79fe1470dcb842d5944c98dfe7a62715db6d86fdb12d3ff60af3bdf41b7996
Progress message: Compiling src/tools/launcher/dummy.cc
PrimaryInput: File:[/home/moritz/.cache/bazel/_bazel_moritz/bb4e4404f889dc1b816f246b08c0d9ea/external/bazel_tools[source]]src/tools/launcher/dummy.cc
PrimaryOutput: File:[[<execution_root>]bazel-out/k8-opt/bin]external/bazel_tools/src/tools/launcher/_objs/launcher/dummy.pic.o
Owner information: ConfiguredTargetKey{label=@bazel_tools//src/tools/launcher:launcher, config=BuildConfigurationValue.Key[869f6bbd14d2ac66dcacb164312d88fa059b017e3c17513cb5051b23879ac7f9]}, ConfiguredTargetKey{label=@bazel_tools//src/tools/launcher:launcher, config=BuildConfigurationValue.Key[8c138cc79f95329f01b0d325511ad0ea89ca5126e421e67e5239706d0f247c2c]}
MandatoryInputs: are equal
Outputs: are equal
ERROR: file 'external/bazel_tools/src/tools/launcher/_objs/launcher/dummy.o' is generated by these conflicting actions:
Label: @bazel_tools//src/tools/launcher:launcher
RuleClass: cc_binary rule
Configuration: 869f6bbd14d2ac66dcacb164312d88fa059b017e3c17513cb5051b23879ac7f9, 8c138cc79f95329f01b0d325511ad0ea89ca5126e421e67e5239706d0f247c2c
Mnemonic: CppCompile
Action key: 9f46e824944add9e9951ef51ddb6cb4b744bc97f90b5749132179d1d1699dfa1
Progress message: Compiling src/tools/launcher/dummy.cc
PrimaryInput: File:[/home/moritz/.cache/bazel/_bazel_moritz/bb4e4404f889dc1b816f246b08c0d9ea/external/bazel_tools[source]]src/tools/launcher/dummy.cc
PrimaryOutput: File:[[<execution_root>]bazel-out/k8-opt/bin]external/bazel_tools/src/tools/launcher/_objs/launcher/dummy.o
Owner information: ConfiguredTargetKey{label=@bazel_tools//src/tools/launcher:launcher, config=BuildConfigurationValue.Key[869f6bbd14d2ac66dcacb164312d88fa059b017e3c17513cb5051b23879ac7f9]}, ConfiguredTargetKey{label=@bazel_tools//src/tools/launcher:launcher, config=BuildConfigurationValue.Key[8c138cc79f95329f01b0d325511ad0ea89ca5126e421e67e5239706d0f247c2c]}
MandatoryInputs: are equal
Outputs: are equal
ERROR: com.google.devtools.build.lib.skyframe.ArtifactConflictFinder$ConflictException: com.google.devtools.build.lib.actions.MutableActionGraph$ActionConflictException: for external/bazel_tools/src/tools/launcher/_objs/launcher/dummy.o, previous action: action 'Compiling src/tools/launcher/dummy.cc', attempted action: action 'Compiling src/tools/launcher/dummy.cc'

changelog_begin
changelog_end
2022-02-16 18:24:00 +00:00

563 lines
26 KiB
Diff

diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java
index 57741a8f28..6673149a20 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java
@@ -56,15 +56,16 @@ public final class RemoteCacheClientFactory {
RemoteOptions options,
@Nullable Credentials creds,
Path workingDirectory,
- DigestUtil digestUtil)
+ DigestUtil digestUtil,
+ RemoteRetrier retrier)
throws IOException {
Preconditions.checkNotNull(workingDirectory, "workingDirectory");
if (isHttpCache(options) && isDiskCache(options)) {
return createDiskAndHttpCache(
- workingDirectory, options.diskCache, options, creds, digestUtil);
+ workingDirectory, options.diskCache, options, creds, digestUtil, retrier);
}
if (isHttpCache(options)) {
- return createHttp(options, creds, digestUtil);
+ return createHttp(options, creds, digestUtil, retrier);
}
if (isDiskCache(options)) {
return createDiskCache(
@@ -80,7 +81,7 @@ public final class RemoteCacheClientFactory {
}
private static RemoteCacheClient createHttp(
- RemoteOptions options, Credentials creds, DigestUtil digestUtil) {
+ RemoteOptions options, Credentials creds, DigestUtil digestUtil, RemoteRetrier retrier) {
Preconditions.checkNotNull(options.remoteCache, "remoteCache");
try {
@@ -99,6 +100,7 @@ public final class RemoteCacheClientFactory {
options.remoteVerifyDownloads,
ImmutableList.copyOf(options.remoteHeaders),
digestUtil,
+ retrier,
creds);
} else {
throw new Exception("Remote cache proxy unsupported: " + options.remoteProxy);
@@ -111,6 +113,7 @@ public final class RemoteCacheClientFactory {
options.remoteVerifyDownloads,
ImmutableList.copyOf(options.remoteHeaders),
digestUtil,
+ retrier,
creds);
}
} catch (Exception e) {
@@ -137,7 +140,8 @@ public final class RemoteCacheClientFactory {
PathFragment diskCachePath,
RemoteOptions options,
Credentials cred,
- DigestUtil digestUtil)
+ DigestUtil digestUtil,
+ RemoteRetrier retrier)
throws IOException {
Path cacheDir =
workingDirectory.getRelative(Preconditions.checkNotNull(diskCachePath, "diskCachePath"));
@@ -145,7 +149,7 @@ public final class RemoteCacheClientFactory {
cacheDir.createDirectoryAndParents();
}
- RemoteCacheClient httpCache = createHttp(options, cred, digestUtil);
+ RemoteCacheClient httpCache = createHttp(options, cred, digestUtil, retrier);
return createDiskAndRemoteClient(
workingDirectory,
diskCachePath,
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 350e1afa51..db81481b60 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -59,9 +59,11 @@ import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
import com.google.devtools.build.lib.exec.SpawnStrategyRegistry;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.remote.RemoteServerCapabilities.ServerCapabilitiesRequirement;
+import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
import com.google.devtools.build.lib.remote.common.RemoteExecutionClient;
import com.google.devtools.build.lib.remote.downloader.GrpcRemoteDownloader;
+import com.google.devtools.build.lib.remote.http.HttpException;
import com.google.devtools.build.lib.remote.logging.LoggingInterceptor;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
@@ -95,6 +97,7 @@ import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
import io.reactivex.rxjava3.plugins.RxJavaPlugins;
import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -213,7 +216,31 @@ public final class RemoteModule extends BlazeModule {
remoteOptions,
creds,
Preconditions.checkNotNull(env.getWorkingDirectory(), "workingDirectory"),
- digestUtil);
+ digestUtil,
+ new RemoteRetrier(
+ remoteOptions,
+ (e) -> {
+ boolean retry = false;
+ if (e instanceof ClosedChannelException) {
+ retry = true;
+ } else if (e instanceof HttpException) {
+ retry = true;
+ } else if (e instanceof IOException) {
+ String msg = e.getMessage().toLowerCase();
+ if (msg.contains("connection reset by peer")) {
+ retry = true;
+ }
+ }
+ if (retry) {
+ System.err.println("RETRYING: " + e.toString());
+ } else if (!(e instanceof CacheNotFoundException)) {
+ System.err.println("NOT RETRYING: " + e.toString());
+ }
+ return retry;
+ },
+ retryScheduler,
+ Retrier.ALLOW_ALL_CALLS)
+ );
} catch (IOException e) {
handleInitFailure(env, e, Code.CACHE_INIT_FAILURE);
return;
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/BUILD b/src/main/java/com/google/devtools/build/lib/remote/http/BUILD
index 9ce71c7c52..5c7f2d0728 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/BUILD
@@ -20,6 +20,7 @@ java_library(
deps = [
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
"//src/main/java/com/google/devtools/build/lib/remote/common",
+ "//src/main/java/com/google/devtools/build/lib/remote:Retrier",
"//src/main/java/com/google/devtools/build/lib/remote/util",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//third_party:auth",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java b/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java
index a2e4abf9d8..93843a91dc 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java
@@ -25,12 +25,18 @@ final class DownloadCommand {
private final boolean casDownload;
private final Digest digest;
private final OutputStream out;
+ private final long offset;
- DownloadCommand(URI uri, boolean casDownload, Digest digest, OutputStream out) {
+ DownloadCommand(URI uri, boolean casDownload, Digest digest, OutputStream out, long offset) {
this.uri = Preconditions.checkNotNull(uri);
this.casDownload = casDownload;
this.digest = Preconditions.checkNotNull(digest);
this.out = Preconditions.checkNotNull(out);
+ this.offset = offset;
+ }
+
+ DownloadCommand(URI uri, boolean casDownload, Digest digest, OutputStream out) {
+ this(uri, casDownload, digest, out, 0);
}
public URI uri() {
@@ -48,4 +54,6 @@ final class DownloadCommand {
public OutputStream out() {
return out;
}
+
+ public long offset() { return offset; }
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java
index 1efecd3bb1..3f360dda14 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java
@@ -25,6 +25,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
+import com.google.devtools.build.lib.remote.RemoteRetrier;
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
@@ -84,6 +85,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
@@ -132,6 +134,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
private final boolean useTls;
private final boolean verifyDownloads;
private final DigestUtil digestUtil;
+ private final RemoteRetrier retrier;
private final Object closeLock = new Object();
@@ -153,6 +156,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
boolean verifyDownloads,
ImmutableList<Entry<String, String>> extraHttpHeaders,
DigestUtil digestUtil,
+ RemoteRetrier retrier,
@Nullable final Credentials creds)
throws Exception {
return new HttpCacheClient(
@@ -164,6 +168,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
verifyDownloads,
extraHttpHeaders,
digestUtil,
+ retrier,
creds,
null);
}
@@ -176,6 +181,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
boolean verifyDownloads,
ImmutableList<Entry<String, String>> extraHttpHeaders,
DigestUtil digestUtil,
+ RemoteRetrier retrier,
@Nullable final Credentials creds)
throws Exception {
@@ -189,6 +195,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
verifyDownloads,
extraHttpHeaders,
digestUtil,
+ retrier,
creds,
domainSocketAddress);
} else if (Epoll.isAvailable()) {
@@ -201,6 +208,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
verifyDownloads,
extraHttpHeaders,
digestUtil,
+ retrier,
creds,
domainSocketAddress);
} else {
@@ -217,6 +225,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
boolean verifyDownloads,
ImmutableList<Entry<String, String>> extraHttpHeaders,
DigestUtil digestUtil,
+ RemoteRetrier retrier,
@Nullable final Credentials creds,
@Nullable SocketAddress socketAddress)
throws Exception {
@@ -285,6 +294,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
this.extraHttpHeaders = extraHttpHeaders;
this.verifyDownloads = verifyDownloads;
this.digestUtil = digestUtil;
+ this.retrier = retrier;
}
@SuppressWarnings("FutureReturnValueIgnored")
@@ -461,6 +471,7 @@ public final class HttpCacheClient implements RemoteCacheClient {
@SuppressWarnings("FutureReturnValueIgnored")
private ListenableFuture<Void> get(Digest digest, final OutputStream out, boolean casDownload) {
final AtomicBoolean dataWritten = new AtomicBoolean();
+ AtomicLong bytesDownloaded = new AtomicLong();
OutputStream wrappedOut =
new OutputStream() {
// OutputStream.close() does nothing, which is what we want to ensure that the
@@ -470,12 +481,14 @@ public final class HttpCacheClient implements RemoteCacheClient {
@Override
public void write(byte[] b, int offset, int length) throws IOException {
dataWritten.set(true);
+ bytesDownloaded.addAndGet(length);
out.write(b, offset, length);
}
@Override
public void write(int b) throws IOException {
dataWritten.set(true);
+ bytesDownloaded.incrementAndGet();
out.write(b);
}
@@ -484,57 +497,59 @@ public final class HttpCacheClient implements RemoteCacheClient {
out.flush();
}
};
- DownloadCommand downloadCmd = new DownloadCommand(uri, casDownload, digest, wrappedOut);
- SettableFuture<Void> outerF = SettableFuture.create();
- acquireDownloadChannel()
- .addListener(
- (Future<Channel> channelPromise) -> {
- if (!channelPromise.isSuccess()) {
- outerF.setException(channelPromise.cause());
- return;
- }
+ return retrier.executeAsync(() -> {
+ DownloadCommand downloadCmd = new DownloadCommand(uri, casDownload, digest, wrappedOut, bytesDownloaded.get());
+ SettableFuture<Void> outerF = SettableFuture.create();
+ acquireDownloadChannel()
+ .addListener(
+ (Future<Channel> channelPromise) -> {
+ if (!channelPromise.isSuccess()) {
+ outerF.setException(channelPromise.cause());
+ return;
+ }
- Channel ch = channelPromise.getNow();
- ch.writeAndFlush(downloadCmd)
- .addListener(
- (f) -> {
- try {
- if (f.isSuccess()) {
- outerF.set(null);
- } else {
- Throwable cause = f.cause();
- // cause can be of type HttpException, because Netty uses
- // Unsafe.throwException to
- // re-throw a checked exception that hasn't been declared in the method
- // signature.
- if (cause instanceof HttpException) {
- HttpResponse response = ((HttpException) cause).response();
- if (!dataWritten.get() && authTokenExpired(response)) {
- // The error is due to an auth token having expired. Let's try
- // again.
- try {
- refreshCredentials();
- getAfterCredentialRefresh(downloadCmd, outerF);
+ Channel ch = channelPromise.getNow();
+ ch.writeAndFlush(downloadCmd)
+ .addListener(
+ (f) -> {
+ try {
+ if (f.isSuccess()) {
+ outerF.set(null);
+ } else {
+ Throwable cause = f.cause();
+ // cause can be of type HttpException, because Netty uses
+ // Unsafe.throwException to
+ // re-throw a checked exception that hasn't been declared in the method
+ // signature.
+ if (cause instanceof HttpException) {
+ HttpResponse response = ((HttpException) cause).response();
+ if (!dataWritten.get() && authTokenExpired(response)) {
+ // The error is due to an auth token having expired. Let's try
+ // again.
+ try {
+ refreshCredentials();
+ getAfterCredentialRefresh(downloadCmd, outerF);
+ return;
+ } catch (IOException e) {
+ cause.addSuppressed(e);
+ } catch (RuntimeException e) {
+ logger.atWarning().withCause(e).log("Unexpected exception");
+ cause.addSuppressed(e);
+ }
+ } else if (cacheMiss(response.status())) {
+ outerF.setException(new CacheNotFoundException(digest));
return;
- } catch (IOException e) {
- cause.addSuppressed(e);
- } catch (RuntimeException e) {
- logger.atWarning().withCause(e).log("Unexpected exception");
- cause.addSuppressed(e);
}
- } else if (cacheMiss(response.status())) {
- outerF.setException(new CacheNotFoundException(digest));
- return;
}
+ outerF.setException(cause);
}
- outerF.setException(cause);
+ } finally {
+ releaseDownloadChannel(ch);
}
- } finally {
- releaseDownloadChannel(ch);
- }
- });
- });
- return outerF;
+ });
+ });
+ return outerF;
+ });
}
@SuppressWarnings("FutureReturnValueIgnored")
@@ -673,20 +688,21 @@ public final class HttpCacheClient implements RemoteCacheClient {
@Override
public ListenableFuture<Void> uploadFile(
RemoteActionExecutionContext context, Digest digest, Path file) {
- try {
- return uploadAsync(
- digest.getHash(), digest.getSizeBytes(), file.getInputStream(), /* casUpload= */ true);
- } catch (IOException e) {
- // Can be thrown from file.getInputStream.
- return Futures.immediateFailedFuture(e);
- }
+ return retrier.executeAsync(() -> {
+ try {
+ return uploadAsync(digest.getHash(), digest.getSizeBytes(), file.getInputStream(), /* casUpload= */ true);
+ } catch (IOException e) {
+ // Can be thrown from file.getInputStream.
+ return Futures.immediateFailedFuture(e);
+ }
+ });
}
@Override
public ListenableFuture<Void> uploadBlob(
RemoteActionExecutionContext context, Digest digest, ByteString data) {
- return uploadAsync(
- digest.getHash(), digest.getSizeBytes(), data.newInput(), /* casUpload= */ true);
+ return retrier.executeAsync(() -> uploadAsync(
+ digest.getHash(), digest.getSizeBytes(), data.newInput(), /* casUpload= */ true));
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java
index 50d83d138a..f38dad965f 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java
@@ -20,24 +20,18 @@ import com.google.common.collect.ImmutableList;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPromise;
-import io.netty.handler.codec.http.DefaultFullHttpRequest;
-import io.netty.handler.codec.http.HttpContent;
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.HttpHeaderValues;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpObject;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http.HttpUtil;
-import io.netty.handler.codec.http.HttpVersion;
-import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.handler.codec.http.*;
import io.netty.handler.timeout.ReadTimeoutException;
import io.netty.util.internal.StringUtil;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/** ChannelHandler for downloads. */
final class HttpDownloadHandler extends AbstractHttpHandler<HttpObject> {
@@ -51,6 +45,10 @@ final class HttpDownloadHandler extends AbstractHttpHandler<HttpObject> {
private long contentLength = -1;
/** the path header in the http request */
private String path;
+ /** the offset at which to download */
+ private long offset;
+ /** the bytes to skip in a full or chunked response */
+ private OptionalInt skipBytes;
public HttpDownloadHandler(
Credentials credentials, ImmutableList<Entry<String, String>> extraHttpHeaders) {
@@ -93,7 +91,25 @@ final class HttpDownloadHandler extends AbstractHttpHandler<HttpObject> {
if (contentLengthSet) {
contentLength = HttpUtil.getContentLength(response);
}
- downloadSucceeded = response.status().equals(HttpResponseStatus.OK);
+ boolean full_content = response.status().equals(HttpResponseStatus.OK);
+ boolean partial_content = response.status().equals(HttpResponseStatus.PARTIAL_CONTENT);
+ if (full_content) {
+ if (offset != 0) {
+ // We requested a range but the server replied with a full response.
+ // We need to skip `offset` bytes of the response.
+ if (!skipBytes.isPresent()) {
+ // This is the first chunk, or the full response.
+ skipBytes = OptionalInt.of((int)offset);
+ }
+ }
+ } else if (partial_content) {
+ Optional<HttpException> error = validateContentRangeHeader(response.headers());
+ if (error.isPresent()) {
+ failAndClose(error.get(), ctx);
+ return;
+ }
+ }
+ downloadSucceeded = full_content || partial_content;
if (!downloadSucceeded) {
out = new ByteArrayOutputStream();
}
@@ -105,6 +121,15 @@ final class HttpDownloadHandler extends AbstractHttpHandler<HttpObject> {
ByteBuf content = ((HttpContent) msg).content();
int readableBytes = content.readableBytes();
+ if (skipBytes.isPresent() && skipBytes.getAsInt() > 0) {
+ int skipNow = skipBytes.getAsInt();
+ if (skipNow >= readableBytes) {
+ skipNow = readableBytes;
+ }
+ content.readerIndex(content.readerIndex() + skipNow);
+ skipBytes = OptionalInt.of(skipBytes.getAsInt() - skipNow);
+ readableBytes = readableBytes - skipNow;
+ }
content.readBytes(out, readableBytes);
bytesReceived += readableBytes;
if (msg instanceof LastHttpContent) {
@@ -137,7 +162,9 @@ final class HttpDownloadHandler extends AbstractHttpHandler<HttpObject> {
DownloadCommand cmd = (DownloadCommand) msg;
out = cmd.out();
path = constructPath(cmd.uri(), cmd.digest().getHash(), cmd.casDownload());
- HttpRequest request = buildRequest(path, constructHost(cmd.uri()));
+ offset = cmd.offset();
+ skipBytes = OptionalInt.empty();
+ HttpRequest request = buildRequest(path, constructHost(cmd.uri()), cmd.offset());
addCredentialHeaders(request, cmd.uri());
addExtraRemoteHeaders(request);
addUserAgentHeader(request);
@@ -159,16 +186,36 @@ final class HttpDownloadHandler extends AbstractHttpHandler<HttpObject> {
}
}
- private HttpRequest buildRequest(String path, String host) {
+ private HttpRequest buildRequest(String path, String host, long offset) {
HttpRequest httpRequest =
new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, path);
httpRequest.headers().set(HttpHeaderNames.HOST, host);
httpRequest.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
httpRequest.headers().set(HttpHeaderNames.ACCEPT, "*/*");
httpRequest.headers().set(HttpHeaderNames.ACCEPT_ENCODING, HttpHeaderValues.GZIP);
+ if (offset != 0) {
+ httpRequest.headers().set(HttpHeaderNames.RANGE, String.format("%s=%d-", HttpHeaderValues.BYTES, offset));
+ }
return httpRequest;
}
+ private Optional<HttpException> validateContentRangeHeader(HttpHeaders headers) {
+ if (!headers.contains(HttpHeaderNames.CONTENT_RANGE)) {
+ return Optional.of(new HttpException(response, "Missing 'Content-Range' header", null));
+ }
+ Pattern pattern = Pattern.compile("bytes\\s+(?<start>[0-9]+)-(?<end>[0-9]+)/(?<size>[0-9]*|\\*)");
+ Matcher matcher = pattern.matcher(response.headers().get(HttpHeaderNames.CONTENT_RANGE));
+ if (!matcher.matches()) {
+ return Optional.of(new HttpException(response, "Unexpected 'Content-Range' header", null));
+ }
+ long start = Long.valueOf(matcher.group("start"));
+ if (start != offset) {
+ return Optional.of(new HttpException(
+ response, String.format("Unexpected 'Content-Range' start: Expected %d but got %d", offset, start), null));
+ }
+ return Optional.empty();
+ }
+
private void succeedAndReset(ChannelHandlerContext ctx) {
// All resets must happen *before* completing the user promise. Otherwise there is a race
// condition, where this handler can be reused even though it is closed. In addition, if reset
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java
index 89fde56046..6a2bfd5a50 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java
@@ -18,7 +18,7 @@ import io.netty.handler.codec.http.HttpResponse;
import java.io.IOException;
/** An exception that propagates the http status. */
-final class HttpException extends IOException {
+public final class HttpException extends IOException {
private final HttpResponse response;
HttpException(HttpResponse response, String message, Throwable cause) {