mirror of
https://github.com/digital-asset/daml.git
synced 2024-11-10 10:46:11 +03:00
246 lines
9.8 KiB
Diff
246 lines
9.8 KiB
Diff
From dede0820edd88faa7ee263f691db6bcf338245fa Mon Sep 17 00:00:00 2001
|
|
From: Andreas Herrmann <andreas.herrmann@tweag.io>
|
|
Date: Thu, 21 Feb 2019 14:58:28 +0100
|
|
Subject: [PATCH] Implement combined disk and HTTP cache
|
|
|
|
|
|
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
|
|
index 34a5d9bebd..f2d84fd1d5 100644
|
|
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
|
|
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
|
|
@@ -143,11 +143,6 @@ public final class RemoteModule extends BlazeModule {
|
|
|
|
boolean enableRestCache = SimpleBlobStoreFactory.isRestUrlOptions(remoteOptions);
|
|
boolean enableDiskCache = SimpleBlobStoreFactory.isDiskCache(remoteOptions);
|
|
- if (enableRestCache && enableDiskCache) {
|
|
- throw new AbruptExitException(
|
|
- "Cannot enable HTTP-based and local disk cache simultaneously",
|
|
- ExitCode.COMMAND_LINE_ERROR);
|
|
- }
|
|
boolean enableBlobStoreCache = enableRestCache || enableDiskCache;
|
|
boolean enableGrpcCache = GrpcRemoteCache.isRemoteCacheOptions(remoteOptions);
|
|
if (enableBlobStoreCache && !Strings.isNullOrEmpty(remoteOptions.remoteExecutor)) {
|
|
diff --git a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java
|
|
index ef8ffe22f7..04fcdb6d52 100644
|
|
--- a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java
|
|
+++ b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java
|
|
@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
|
|
import com.google.auth.Credentials;
|
|
import com.google.devtools.build.lib.remote.blobstore.OnDiskBlobStore;
|
|
import com.google.devtools.build.lib.remote.blobstore.SimpleBlobStore;
|
|
+import com.google.devtools.build.lib.remote.blobstore.CombinedDiskHttpBlobStore;
|
|
import com.google.devtools.build.lib.remote.blobstore.http.HttpBlobStore;
|
|
import com.google.devtools.build.lib.vfs.Path;
|
|
import com.google.devtools.build.lib.vfs.PathFragment;
|
|
@@ -66,9 +67,22 @@ public final class SimpleBlobStoreFactory {
|
|
return new OnDiskBlobStore(cacheDir);
|
|
}
|
|
|
|
+ public static SimpleBlobStore createCombinedCache(Path workingDirectory, PathFragment diskCachePath, RemoteOptions options, Credentials cred)
|
|
+ throws IOException {
|
|
+ Path cacheDir = workingDirectory.getRelative(checkNotNull(diskCachePath));
|
|
+ if (!cacheDir.exists()) {
|
|
+ cacheDir.createDirectoryAndParents();
|
|
+ }
|
|
+ return new CombinedDiskHttpBlobStore(cacheDir, createRest(options, cred));
|
|
+ }
|
|
+
|
|
public static SimpleBlobStore create(
|
|
RemoteOptions options, @Nullable Credentials creds, @Nullable Path workingDirectory)
|
|
throws IOException {
|
|
+
|
|
+ if (isRestUrlOptions(options) && isDiskCache(options)) {
|
|
+ return createCombinedCache(workingDirectory, options.diskCache, options, creds);
|
|
+ }
|
|
if (isRestUrlOptions(options)) {
|
|
return createRest(options, creds);
|
|
}
|
|
diff --git a/src/main/java/com/google/devtools/build/lib/remote/blobstore/CombinedDiskHttpBlobStore.java b/src/main/java/com/google/devtools/build/lib/remote/blobstore/CombinedDiskHttpBlobStore.java
|
|
new file mode 100644
|
|
index 0000000000..a8564201f5
|
|
--- /dev/null
|
|
+++ b/src/main/java/com/google/devtools/build/lib/remote/blobstore/CombinedDiskHttpBlobStore.java
|
|
@@ -0,0 +1,148 @@
|
|
+// Copyright 2017 The Bazel Authors. All rights reserved.
|
|
+//
|
|
+// Licensed under the Apache License, Version 2.0 (the "License");
|
|
+// you may not use this file except in compliance with the License.
|
|
+// You may obtain a copy of the License at
|
|
+//
|
|
+// http://www.apache.org/licenses/LICENSE-2.0
|
|
+//
|
|
+// Unless required by applicable law or agreed to in writing, software
|
|
+// distributed under the License is distributed on an "AS IS" BASIS,
|
|
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
+// See the License for the specific language governing permissions and
|
|
+// limitations under the License.
|
|
+package com.google.devtools.build.lib.remote.blobstore;
|
|
+
|
|
+import com.google.common.io.ByteStreams;
|
|
+import com.google.common.util.concurrent.ListenableFuture;
|
|
+import com.google.common.util.concurrent.SettableFuture;
|
|
+import com.google.common.util.concurrent.Futures;
|
|
+import com.google.common.util.concurrent.MoreExecutors;
|
|
+import java.io.ByteArrayInputStream;
|
|
+import java.io.ByteArrayOutputStream;
|
|
+import java.io.IOException;
|
|
+import java.io.InputStream;
|
|
+import java.io.OutputStream;
|
|
+import java.util.logging.Level;
|
|
+import java.util.logging.Logger;
|
|
+import java.util.UUID;
|
|
+
|
|
+import com.google.devtools.build.lib.vfs.Path;
|
|
+
|
|
+/** A {@link SimpleBlobStore} implementation combining two blob stores.
|
|
+ * A local disk blob store and a remote http blob store.
|
|
+ * If a blob isn't found in the first store, the second store is used, and the
|
|
+ * blob added to the first. Put puts the blob on both stores.
|
|
+ */
|
|
+public final class CombinedDiskHttpBlobStore extends OnDiskBlobStore {
|
|
+ private static final Logger logger = Logger.getLogger(CombinedDiskHttpBlobStore.class.getName());
|
|
+ private final SimpleBlobStore bsHttp;
|
|
+
|
|
+ public CombinedDiskHttpBlobStore(Path root, SimpleBlobStore bsHttp) {
|
|
+ super(root);
|
|
+ this.bsHttp = bsHttp;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public boolean containsKey(String key) {
|
|
+ // HTTP cache does not support containsKey.
|
|
+ // Don't support it here either for predictable semantics.
|
|
+ throw new UnsupportedOperationException("HTTP Caching does not use this method.");
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public ListenableFuture<Boolean> get(String key, OutputStream out) {
|
|
+ boolean use_bsDisk = super.containsKey(key);
|
|
+
|
|
+ if (use_bsDisk) {
|
|
+ return super.get(key, out);
|
|
+ } else {
|
|
+ // Write a temporary file first, and then rename, to avoid data corruption in case of a crash.
|
|
+ Path temp = toPath(UUID.randomUUID().toString());
|
|
+
|
|
+ OutputStream tempOut;
|
|
+ try {
|
|
+ tempOut = temp.getOutputStream();
|
|
+ } catch (IOException e) {
|
|
+ return Futures.immediateFailedFuture(e);
|
|
+ }
|
|
+ ListenableFuture<Boolean> chained = Futures.transformAsync(
|
|
+ bsHttp.get(key, tempOut),
|
|
+ (found) -> {
|
|
+ if (!found) {
|
|
+ return Futures.immediateFuture(false);
|
|
+ } else {
|
|
+ Path target = toPath(key);
|
|
+ // The following note and line is taken from OnDiskBlobStore.java
|
|
+ // TODO(ulfjack): Fsync temp here before we rename it to avoid data loss in the case of machine
|
|
+ // crashes (the OS may reorder the writes and the rename).
|
|
+ temp.renameTo(target);
|
|
+
|
|
+ SettableFuture<Boolean> f = SettableFuture.create();
|
|
+ try (InputStream in = target.getInputStream()) {
|
|
+ ByteStreams.copy(in, out);
|
|
+ f.set(true);
|
|
+ } catch (IOException e) {
|
|
+ f.setException(e);
|
|
+ }
|
|
+ return f;
|
|
+ }
|
|
+ },
|
|
+ MoreExecutors.directExecutor()
|
|
+ );
|
|
+ chained.addListener(
|
|
+ () -> {
|
|
+ try {
|
|
+ tempOut.close();
|
|
+ } catch (IOException e) {
|
|
+ // not sure what to do here, we either are here because of another exception being thrown,
|
|
+ // or we have successfully used the file we are trying (and failing) to close
|
|
+ logger.log(Level.WARNING, "Failed to close temporary file on get", e);
|
|
+ }
|
|
+ },
|
|
+ MoreExecutors.directExecutor());
|
|
+ return chained;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public boolean getActionResult(String key, OutputStream out)
|
|
+ throws IOException, InterruptedException {
|
|
+ if (super.getActionResult(key, out)) {
|
|
+ return true;
|
|
+ }
|
|
+
|
|
+ try (ByteArrayOutputStream tmpOut = new ByteArrayOutputStream()) {
|
|
+ if (bsHttp.getActionResult(key, tmpOut)) {
|
|
+ byte[] tmp = tmpOut.toByteArray();
|
|
+ super.putActionResult(key, tmp);
|
|
+ ByteStreams.copy(new ByteArrayInputStream(tmp), out);
|
|
+ return true;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return false;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public void put(String key, long length, InputStream in) throws IOException, InterruptedException {
|
|
+ super.put(key, length, in);
|
|
+ try (InputStream inFile = toPath(key).getInputStream()) {
|
|
+ bsHttp.put(key, length, inFile);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public void putActionResult(String key, byte[] in)
|
|
+ throws IOException, InterruptedException {
|
|
+ super.putActionResult(key, in);
|
|
+ bsHttp.putActionResult(key, in);
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public void close() {
|
|
+ super.close();
|
|
+ bsHttp.close();
|
|
+ }
|
|
+
|
|
+}
|
|
diff --git a/src/main/java/com/google/devtools/build/lib/remote/blobstore/OnDiskBlobStore.java b/src/main/java/com/google/devtools/build/lib/remote/blobstore/OnDiskBlobStore.java
|
|
index 1358cda163..82c89a8a2a 100644
|
|
--- a/src/main/java/com/google/devtools/build/lib/remote/blobstore/OnDiskBlobStore.java
|
|
+++ b/src/main/java/com/google/devtools/build/lib/remote/blobstore/OnDiskBlobStore.java
|
|
@@ -26,7 +26,7 @@ import java.io.OutputStream;
|
|
import java.util.UUID;
|
|
|
|
/** A on-disk store for the remote action cache. */
|
|
-public final class OnDiskBlobStore implements SimpleBlobStore {
|
|
+public class OnDiskBlobStore implements SimpleBlobStore {
|
|
private final Path root;
|
|
static final String ACTION_KEY_PREFIX = "ac_";
|
|
|
|
@@ -63,7 +63,7 @@ public final class OnDiskBlobStore implements SimpleBlobStore {
|
|
}
|
|
|
|
@Override
|
|
- public void put(String key, long length, InputStream in) throws IOException {
|
|
+ public void put(String key, long length, InputStream in) throws IOException, InterruptedException {
|
|
Path target = toPath(key);
|
|
if (target.exists()) {
|
|
return;
|
|
@@ -87,7 +87,7 @@ public final class OnDiskBlobStore implements SimpleBlobStore {
|
|
@Override
|
|
public void close() {}
|
|
|
|
- private Path toPath(String key) {
|
|
+ protected Path toPath(String key) {
|
|
return root.getChild(key);
|
|
}
|
|
}
|
|
--
|
|
2.16.5
|
|
|