diff --git a/bazel_tools/scala.bzl b/bazel_tools/scala.bzl
index dfc41551ef..103bd55718 100644
--- a/bazel_tools/scala.bzl
+++ b/bazel_tools/scala.bzl
@@ -239,71 +239,107 @@ def _create_scala_source_jar(**kwargs):
srcs = kwargs["srcs"],
)
+def _build_nosrc_jar(ctx):
+ # this ensures the file is not empty
+ manifest_path = ctx.actions.declare_file("%s_MANIFEST.MF" % ctx.label.name)
+ ctx.actions.write(manifest_path, "Manifest-Version: 1.0")
+ resources = "META-INF/MANIFEST.MF=%s\n" % manifest_path.path
+
+ zipper_arg_path = ctx.actions.declare_file("%s_zipper_args" % ctx.label.name)
+ ctx.actions.write(zipper_arg_path, resources)
+ cmd = """
+rm -f {jar_output}
+{zipper} c {jar_output} @{path}
+"""
+
+ cmd = cmd.format(
+ path = zipper_arg_path.path,
+ jar_output = ctx.outputs.out.path,
+ zipper = ctx.executable._zipper.path,
+ )
+
+ outs = [ctx.outputs.out]
+ inputs = [manifest_path]
+
+ ctx.actions.run_shell(
+ inputs = inputs,
+ tools = [ctx.executable._zipper, zipper_arg_path],
+ outputs = outs,
+ command = cmd,
+ progress_message = "scala %s" % ctx.label,
+ arguments = [],
+ )
+
def _scaladoc_jar_impl(ctx):
+ # Detect an actual scala source file rather than a srcjar or other label
srcFiles = [
src.path
for src in ctx.files.srcs
+ if src.is_source
]
- # The following plugin handling is lifted from a private library of 'rules_scala'.
- # https://github.com/bazelbuild/rules_scala/blob/1cffc5fcae1f553a7619b98bf7d6456d65081665/scala/private/rule_impls.bzl#L130
- pluginPaths = []
- for p in ctx.attr.plugins:
- if hasattr(p, "path"):
- pluginPaths.append(p)
- elif hasattr(p, "scala"):
- pluginPaths.extend([j.class_jar for j in p.scala.outputs.jars])
- elif hasattr(p, "java"):
- pluginPaths.extend([j.class_jar for j in p.java.outputs.jars])
- # support http_file pointed at a jar. http_jar uses ijar,
- # which breaks scala macros
+ if srcFiles != []:
+ # The following plugin handling is lifted from a private library of 'rules_scala'.
+ # https://github.com/bazelbuild/rules_scala/blob/1cffc5fcae1f553a7619b98bf7d6456d65081665/scala/private/rule_impls.bzl#L130
+ pluginPaths = []
+ for p in ctx.attr.plugins:
+ if hasattr(p, "path"):
+ pluginPaths.append(p)
+ elif hasattr(p, "scala"):
+ pluginPaths.extend([j.class_jar for j in p.scala.outputs.jars])
+ elif hasattr(p, "java"):
+ pluginPaths.extend([j.class_jar for j in p.java.outputs.jars])
+ # support http_file pointed at a jar. http_jar uses ijar,
+ # which breaks scala macros
- elif hasattr(p, "files"):
- pluginPaths.extend([f for f in p.files if "-sources.jar" not in f.basename])
+ elif hasattr(p, "files"):
+ pluginPaths.extend([f for f in p.files if "-sources.jar" not in f.basename])
- transitive_deps = [dep[JavaInfo].transitive_deps for dep in ctx.attr.deps]
- classpath = depset([], transitive = transitive_deps).to_list()
+ transitive_deps = [dep[JavaInfo].transitive_deps for dep in ctx.attr.deps]
+ classpath = depset([], transitive = transitive_deps).to_list()
- outdir = ctx.actions.declare_directory(ctx.label.name + "_tmpdir")
+ outdir = ctx.actions.declare_directory(ctx.label.name + "_tmpdir")
- args = ctx.actions.args()
- args.add_all(["-d", outdir.path])
- args.add("-classpath")
- args.add_joined(classpath, join_with = ":")
- args.add_joined(pluginPaths, join_with = ",", format_joined = "-Xplugin:%s")
- args.add_all(common_scalacopts)
- args.add_all(srcFiles)
+ args = ctx.actions.args()
+ args.add_all(["-d", outdir.path])
+ args.add("-classpath")
+ args.add_joined(classpath, join_with = ":")
+ args.add_joined(pluginPaths, join_with = ",", format_joined = "-Xplugin:%s")
+ args.add_all(common_scalacopts)
+ args.add_all(srcFiles)
- ctx.actions.run(
- executable = ctx.executable._scaladoc,
- inputs = ctx.files.srcs + classpath + pluginPaths,
- outputs = [outdir],
- arguments = [args],
- mnemonic = "ScaladocGen",
- )
+ ctx.actions.run(
+ executable = ctx.executable._scaladoc,
+ inputs = ctx.files.srcs + classpath + pluginPaths,
+ outputs = [outdir],
+ arguments = [args],
+ mnemonic = "ScaladocGen",
+ )
- # since we only have the output directory of the scaladoc generation we need to find
- # all the files below sources_out and add them to the zipper args file
- zipper_args_file = ctx.actions.declare_file(ctx.label.name + ".zipper_args")
- ctx.actions.run_shell(
- mnemonic = "ScaladocFindOutputFiles",
- outputs = [zipper_args_file],
- inputs = [outdir],
- command = "find -L {src_path} -type f | sed -E 's#^{src_path}/(.*)$#\\1={src_path}/\\1#' | sort > {args_file}".format(
- src_path = outdir.path,
- args_file = zipper_args_file.path,
- ),
- progress_message = "find_scaladoc_output_files %s" % zipper_args_file.path,
- use_default_shell_env = True,
- )
+ # since we only have the output directory of the scaladoc generation we need to find
+ # all the files below sources_out and add them to the zipper args file
+ zipper_args_file = ctx.actions.declare_file(ctx.label.name + ".zipper_args")
+ ctx.actions.run_shell(
+ mnemonic = "ScaladocFindOutputFiles",
+ outputs = [zipper_args_file],
+ inputs = [outdir],
+ command = "find -L {src_path} -type f | sed -E 's#^{src_path}/(.*)$#\\1={src_path}/\\1#' | sort > {args_file}".format(
+ src_path = outdir.path,
+ args_file = zipper_args_file.path,
+ ),
+ progress_message = "find_scaladoc_output_files %s" % zipper_args_file.path,
+ use_default_shell_env = True,
+ )
- ctx.actions.run(
- executable = ctx.executable._zipper,
- inputs = ctx.files.srcs + classpath + [outdir, zipper_args_file],
- outputs = [ctx.outputs.out],
- arguments = ["c", ctx.outputs.out.path, "@" + zipper_args_file.path],
- mnemonic = "ScaladocJar",
- )
+ ctx.actions.run(
+ executable = ctx.executable._zipper,
+ inputs = ctx.files.srcs + classpath + [outdir, zipper_args_file],
+ outputs = [ctx.outputs.out],
+ arguments = ["c", ctx.outputs.out.path, "@" + zipper_args_file.path],
+ mnemonic = "ScaladocJar",
+ )
+ else:
+ _build_nosrc_jar(ctx)
scaladoc_jar = rule(
implementation = _scaladoc_jar_impl,
@@ -339,17 +375,8 @@ Arguments:
"""
def _create_scaladoc_jar(**kwargs):
- # Try to not create empty scaladoc jars and limit execution to Linux and MacOS
- # Detect an actual scala source file rather than a srcjar or other label
-
- create_scaladoc = False
- if len(kwargs["srcs"]) > 0 and is_windows == False:
- for src in kwargs["srcs"]:
- if src.endswith(".scala"):
- create_scaladoc = True
- break
-
- if create_scaladoc:
+ # Limit execution to Linux and MacOS
+ if is_windows == False:
plugins = []
if "plugins" in kwargs:
plugins = kwargs["plugins"]
diff --git a/ci/build-unix.yml b/ci/build-unix.yml
index c57a96b82f..5492809859 100644
--- a/ci/build-unix.yml
+++ b/ci/build-unix.yml
@@ -52,6 +52,12 @@ steps:
env:
# to connect to bintray
JFROG_CONFIG_CONTENT: $(JFROG_CONFIG_CONTENT)
+ # For signing artifacts to be uploaded to Maven Central.
+ GPG_KEY: $(gpg-code-signing)
+ # Configuration the Sonatype Open Source Repository Hosting
+ MAVEN_USER: $(MAVEN_USER)
+ MAVEN_PASSWORD: $(MAVEN_PASSWORD)
+ MAVEN_URL: "https://oss.sonatype.org"
name: release
- bash: |
set -euo pipefail
diff --git a/dev-env/bin/gpg b/dev-env/bin/gpg
new file mode 100755
index 0000000000..a361940806
--- /dev/null
+++ b/dev-env/bin/gpg
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Meant to be linked to from `dev-env/bin`, symlink should be named after the
+# tool. Execute a Nix tool from a derivation that creates a `result` directory.
+
+DADE_CURRENT_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$DADE_CURRENT_SCRIPT_DIR/../lib/dade-common"
+base=$(basename $0)
+execTool $base out $base "$@"
diff --git a/docs/source/app-dev/bindings-java/code-snippets/pom.xml b/docs/source/app-dev/bindings-java/code-snippets/pom.xml
index 304bf8239a..aa427d5408 100644
--- a/docs/source/app-dev/bindings-java/code-snippets/pom.xml
+++ b/docs/source/app-dev/bindings-java/code-snippets/pom.xml
@@ -16,27 +16,4 @@
-
-
-
-
-
- false
-
- bintray-digitalassetsdk-DigitalAssetSDK
- bintray
- https://digitalassetsdk.bintray.com/DigitalAssetSDK
-
-
-
-
-
- false
-
- bintray-digitalassetsdk-DigitalAssetSDK
- bintray
- https://digitalassetsdk.bintray.com/DigitalAssetSDK
-
-
-
diff --git a/docs/source/app-dev/bindings-java/codegen.rst b/docs/source/app-dev/bindings-java/codegen.rst
index e793a1e6e7..34fe9d595a 100644
--- a/docs/source/app-dev/bindings-java/codegen.rst
+++ b/docs/source/app-dev/bindings-java/codegen.rst
@@ -94,7 +94,7 @@ The following snippet is an excerpt from the ``pom.xml`` that is part of the :re
.. literalinclude:: ../../getting-started/quickstart/template-root/pom.xml
:language: xml
- :lines: 68-100,116-117
+ :lines: 47-79,95-96
:dedent: 12
diff --git a/docs/source/app-dev/bindings-java/index.rst b/docs/source/app-dev/bindings-java/index.rst
index 07c05b5d32..dab55e3011 100644
--- a/docs/source/app-dev/bindings-java/index.rst
+++ b/docs/source/app-dev/bindings-java/index.rst
@@ -154,21 +154,14 @@ To use the Java bindings library, add the following dependencies to your project
:end-before:
:dedent: 4
-Replace ``x.y.z`` for both dependencies with the version that you want to use. You can find the available versions at
-`https://digitalassetsdk.bintray.com/DigitalAssetSDK/com/daml/ledger/`.
-
-You also have to add the DAML Bintray Repository to your ``pom.xml``:
-
-.. literalinclude:: ./code-snippets/pom.xml
- :language: xml
- :start-after:
- :end-before:
- :dedent: 4
+Replace ``x.y.z`` for both dependencies with the version that you want to use. You can find the available versions by checking
+the `Maven Central Repository `__.
+.. note::
+ As of DAML SDK release 0.13.1, the Java Bindings libraries are available via the public Maven Central repository. Earlier releases are available from the `DAML Bintray repository `__.
You can also take a look at the ``pom.xml`` file from the :ref:`quickstart project `.
-
.. _ledger-api-java-binding-connecting:
Connecting to the ledger
diff --git a/docs/source/getting-started/quickstart/template-root/pom.xml b/docs/source/getting-started/quickstart/template-root/pom.xml
index 85fc388ec8..f4d2430cf9 100644
--- a/docs/source/getting-started/quickstart/template-root/pom.xml
+++ b/docs/source/getting-started/quickstart/template-root/pom.xml
@@ -42,27 +42,6 @@
-
-
-
- false
-
- bintray-digitalassetsdk-DigitalAssetSDK
- bintray
- https://digitalassetsdk.bintray.com/DigitalAssetSDK
-
-
-
-
-
- false
-
- bintray-digitalassetsdk-DigitalAssetSDK
- bintray
- https://digitalassetsdk.bintray.com/DigitalAssetSDK
-
-
-
diff --git a/docs/source/support/release-notes.rst b/docs/source/support/release-notes.rst
index b3b22eec37..fe03149677 100644
--- a/docs/source/support/release-notes.rst
+++ b/docs/source/support/release-notes.rst
@@ -14,6 +14,14 @@ Language
- Add an instance for ``IsParties (Optional Party)``, allowing ``Optional`` values to be used in ``signatory``, ``observer`` and ``maintainer`` clauses.
+Java Bindings
+~~~~~~~~~~~~~
+
+- Release the Java Bindings to the public Maven Central repository. To move to using the Maven Central repository, remove
+ the ``...`` and ``...`` blocks from Maven POM files
+ that use version 0.12.26 (or later) of the Java Bindings.
+ See `#1205 `__.
+
.. _release-0-13-0:
0.13.0 - 2019-06-17
diff --git a/language-support/codegen-main/BUILD.bazel b/language-support/codegen-main/BUILD.bazel
index 73d59298ca..dc539b7b24 100644
--- a/language-support/codegen-main/BUILD.bazel
+++ b/language-support/codegen-main/BUILD.bazel
@@ -4,12 +4,15 @@
load(
"//bazel_tools:scala.bzl",
"da_scala_binary",
+ "scala_source_jar",
+ "scaladoc_jar",
)
load(
"@com_github_johnynek_bazel_jar_jar//:jar_jar.bzl",
"jar_jar",
)
load("//bazel_tools:pom_file.bzl", "pom_file")
+load("@os_info//:os_info.bzl", "is_windows")
da_scala_binary(
name = "codegen-main",
@@ -36,3 +39,16 @@ pom_file(
target = ":shaded_binary",
visibility = ["//visibility:public"],
)
+
+# Create empty Scaladoc JAR for uploading to Maven Central
+scaladoc_jar(
+ name = "shaded_binary_scaladoc",
+ srcs = [],
+ deps = [],
+) if is_windows == False else None
+
+# Create empty Sources JAR for uploading to Maven Central
+scala_source_jar(
+ name = "shaded_binary_src",
+ srcs = [],
+)
diff --git a/language-support/java/codegen/BUILD.bazel b/language-support/java/codegen/BUILD.bazel
index 13db73787f..94ea47462c 100644
--- a/language-support/java/codegen/BUILD.bazel
+++ b/language-support/java/codegen/BUILD.bazel
@@ -6,6 +6,8 @@ load(
"da_scala_binary",
"da_scala_library",
"da_scala_test",
+ "scala_source_jar",
+ "scaladoc_jar",
)
load(
"//rules_daml:daml.bzl",
@@ -21,6 +23,7 @@ load(
"jar_jar",
)
load("//bazel_tools:pom_file.bzl", "pom_file")
+load("@os_info//:os_info.bzl", "is_windows")
da_scala_binary(
name = "codegen",
@@ -105,6 +108,19 @@ pom_file(
visibility = ["//visibility:public"],
)
+# Create empty Scaladoc JAR for uploading to Maven Central
+scaladoc_jar(
+ name = "shaded_binary_scaladoc",
+ srcs = [],
+ deps = [],
+) if is_windows == False else None
+
+# Create empty Sources JAR for uploading to Maven Central
+scala_source_jar(
+ name = "shaded_binary_src",
+ srcs = [],
+)
+
daml_lf_target_versions = [
"1.0",
"1.1",
diff --git a/ledger-api/grpc-definitions/BUILD.bazel b/ledger-api/grpc-definitions/BUILD.bazel
index 6c54751051..900ccfb759 100644
--- a/ledger-api/grpc-definitions/BUILD.bazel
+++ b/ledger-api/grpc-definitions/BUILD.bazel
@@ -6,6 +6,12 @@ load("//bazel_tools:pkg.bzl", "pkg_tar")
load("//bazel_tools:proto.bzl", "proto_gen")
load("//bazel_tools:pom_file.bzl", "pom_file")
load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+load(
+ "//bazel_tools:scala.bzl",
+ "scala_source_jar",
+ "scaladoc_jar",
+)
+load("@os_info//:os_info.bzl", "is_windows")
ledger_api_proto_source_root = "ledger-api/grpc-definitions"
@@ -259,6 +265,19 @@ pom_file(
target = ":ledger-api-scalapb",
)
+# Create empty Scaladoc JAR for uploading to Maven Central
+scaladoc_jar(
+ name = "ledger-api-scalapb_scaladoc",
+ srcs = [],
+ deps = [],
+) if is_windows == False else None
+
+# Create empty Sources JAR for uploading to Maven Central
+scala_source_jar(
+ name = "ledger-api-scalapb_src",
+ srcs = [],
+)
+
proto_gen(
name = "ledger-api-docs",
srcs = [":protos"],
diff --git a/nix/packages.nix b/nix/packages.nix
index 810c01d268..0ee9bac90a 100644
--- a/nix/packages.nix
+++ b/nix/packages.nix
@@ -229,6 +229,10 @@ in rec {
base64 = pkgs.coreutils;
sha1sum = pkgs.coreutils;
xmlstarlet = pkgs.xmlstarlet;
+
+ # Cryptography tooling
+ gnupg = pkgs.gnupg;
+ gpg = gnupg;
# Packaging tools
patchelf = bazel_dependencies.patchelf;
diff --git a/release/BUILD.bazel b/release/BUILD.bazel
index 57ca0a6b7f..e900a25658 100644
--- a/release/BUILD.bazel
+++ b/release/BUILD.bazel
@@ -8,17 +8,25 @@ da_haskell_binary(
srcs = glob(["src/**/*.hs"]),
hazel_deps = [
"aeson",
+ "async",
"ansi-terminal",
"base",
+ "base64-bytestring",
"bytestring",
"conduit",
"conduit-extra",
"containers",
+ "connection",
+ "cryptohash",
"directory",
"exceptions",
"extra",
"fast-logger",
"filepath",
+ "http-client",
+ "http-client-tls",
+ "http-conduit",
+ "http-types",
"lifted-async",
"lifted-base",
"monad-control",
@@ -27,9 +35,12 @@ da_haskell_binary(
"path",
"path-io",
"process",
+ "retry",
"safe",
"safe-exceptions",
+ "time",
"text",
+ "temporary",
"transformers",
"unliftio-core",
"unordered-containers",
diff --git a/release/artifacts.yaml b/release/artifacts.yaml
index c3ec57214d..9f0e822684 100644
--- a/release/artifacts.yaml
+++ b/release/artifacts.yaml
@@ -56,15 +56,19 @@
type: jar-scala
- target: //ledger-api/grpc-definitions:ledger-api-protos-tarball
type: targz
+ mavenUpload: True
location:
groupId: com.digitalasset
artifactId: ledger-api-protos
- target: //ledger-api/rs-grpc-bridge:rs-grpc-bridge
type: jar-lib
+ mavenUpload: True
- target: //language-support/java/bindings:bindings-java
type: jar-lib
+ mavenUpload: True
- target: //language-support/java/bindings-rxjava:bindings-rxjava
type: jar-lib
+ mavenUpload: True
- target: //docs:quickstart-java
type: targz
location:
@@ -78,11 +82,13 @@
- target: //extractor:extractor-binary
type: jar-deploy
- target: //ledger-api/grpc-definitions:ledger-api-scalapb
- type: jar
+ type: jar-scala
+ mavenUpload: True
- target: //ledger-api/testing-utils:testing-utils
type: jar-scala
- target: //language-support/scala/bindings:bindings
type: jar-scala
+ mavenUpload: True
- target: //ledger-api/rs-grpc-akka:rs-grpc-akka
type: jar-scala
- target: //ledger/ledger-api-akka:ledger-api-akka
@@ -107,14 +113,17 @@
type: jar
- target: //language-support/scala/bindings-akka:bindings-akka
type: jar-scala
+ mavenUpload: True
- target: //language-support/java/codegen:shaded_binary
- type: jar
+ type: jar-scala
+ mavenUpload: True
- target: //navigator/backend:navigator-binary
type: jar-deploy
- target: //language-support/codegen-common:codegen-common
- type: jar
+ type: jar-scala
- target: //language-support/codegen-main:shaded_binary
- type: jar
+ type: jar-scala
+ mavenUpload: True
- target: //ledger/participant-state:participant-state-v1
type: jar-scala
- target: //ledger/participant-state:participant-state
diff --git a/release/src/Main.hs b/release/src/Main.hs
index f6f5071883..cbef1c0ff9 100644
--- a/release/src/Main.hs
+++ b/release/src/Main.hs
@@ -5,9 +5,9 @@
{-# LANGUAGE TemplateHaskell #-}
module Main (main) where
+import Control.Monad.Extra
import Control.Monad.IO.Class
import Control.Monad.Logger
-import Data.Traversable
import Data.Yaml
import Path
import Path.IO
@@ -18,6 +18,7 @@ import System.Process
import Options
import Types
+import Upload
import Util
main :: IO ()
@@ -54,10 +55,19 @@ main = do
then do
$logInfo "Make release"
releaseToBintray upload releaseDir (map (\(a, (_, outp)) -> (a, outp)) files)
+
+ -- Uploading to Maven Central
+ mavenUploadConfig <- mavenConfigFromEnv
+
+ let mavenUploadArtifacts = filter (\a -> getMavenUpload $ artMavenUpload a) artifacts
+ uploadArtifacts <- concatMapM (artifactCoords optsAllArtifacts) mavenUploadArtifacts
+ uploadToMavenCentral mavenUploadConfig releaseDir uploadArtifacts
+
-- set variables for next steps in Azure pipelines
liftIO . putStrLn $ "##vso[task.setvariable variable=has_released;isOutput=true]true"
liftIO . putStrLn . T.unpack $ "##vso[task.setvariable variable=release_tag]" # renderVersion sdkVersion
else $logInfo "Make dry run of release"
+
where
runLog Options{..} m0 = do
let m = filterLogger (\_ ll -> ll >= optsLogLevel) m0
diff --git a/release/src/Options.hs b/release/src/Options.hs
index f1f077e14e..e7dc08adb2 100644
--- a/release/src/Options.hs
+++ b/release/src/Options.hs
@@ -30,7 +30,7 @@ data Options = Options
optsParser :: Parser Options
optsParser = Options
<$> strOption (long "artifacts" <> help "Path to yaml file listing the artifacts to be released")
- <*> (PerformUpload <$> switch (long "upload" <> help "upload artifacts to bintray. If false, we don't upload artifacts to artifactory or bintray even when the last commit is a release commit."))
+ <*> (PerformUpload <$> switch (long "upload" <> help "upload artifacts to bintray and Maven Central. If false, we don't upload artifacts to Maven Central or bintray even when the last commit is a release commit."))
<*> option str (long "release-dir" <> help "specify full path to release directory")
<*> option (Just <$> str) (long "slack-release-message" <> help "if present will write out what to write in slack. if there are no releases the file will be empty" <> value Nothing)
<*> switch (long "full-logging" <> help "full logging detail")
diff --git a/release/src/Types.hs b/release/src/Types.hs
index 0c2caa5055..d6818c94bb 100644
--- a/release/src/Types.hs
+++ b/release/src/Types.hs
@@ -6,11 +6,16 @@
module Types (
AllArtifacts(..),
ArtifactId,
+ ArtifactType,
CIException(..),
Classifier,
BintrayPackage(..),
GitRev,
GroupId,
+ MavenAllowUnsecureTls(..),
+ MavenCoords(..),
+ MavenUpload(..),
+ MavenUploadConfig(..),
MonadCI,
OS(..),
PerformUpload(..),
@@ -35,7 +40,8 @@ import Control.Monad.IO.Class (MonadIO, liftIO)
import Control.Monad.IO.Unlift (MonadUnliftIO)
import Control.Monad.Logger
import Control.Monad.Trans.Control (MonadBaseControl)
-import Data.Aeson
+import Data.Aeson
+import Data.Maybe
import Data.Text (Text)
import qualified Data.Text as T
import Data.Typeable (Typeable)
@@ -62,10 +68,23 @@ type TextVersion = Text
type GroupId = [Text]
type ArtifactId = Text
type Classifier = Text
+type ArtifactType = Text
+
+-- Fully qualified coordinates for a Maven artifact.
+data MavenCoords = MavenCoords
+ { groupId :: !GroupId
+ , artifactId :: !ArtifactId
+ , version :: !TextVersion
+ , classifier :: Maybe ArtifactType
+ , artifactType :: !ArtifactType
+ } deriving Show
newtype PlatformDependent = PlatformDependent{getPlatformDependent :: Bool}
deriving (Eq, Show, FromJSON)
+newtype MavenUpload = MavenUpload { getMavenUpload :: Bool }
+ deriving (Eq, Show, FromJSON)
+
-- | If this is True, we produce all artifacts even platform independent artifacts on MacOS.
-- This is useful for testing purposes.
newtype AllArtifacts = AllArtifacts Bool
@@ -147,3 +166,24 @@ parseVersion (T.strip -> txt) = do
renderVersion :: Version -> Text
renderVersion (Version maj min_ patch) = T.intercalate "." [tshow maj, tshow min_, tshow patch]
+
+newtype MavenAllowUnsecureTls = MavenAllowUnsecureTls { getAllowUnsecureTls :: Bool }
+ deriving (Eq, Show, FromJSON)
+
+data MavenUploadConfig = MavenUploadConfig
+ { mucUrl :: !Text
+ , mucUser :: !Text
+ , mucPassword :: !Text
+ , mucAllowUnsecureTls :: !MavenAllowUnsecureTls
+-- ^^ For testing with an Artifactory (or similar) instance using a self-signed SSL certificate.
+-- This flag should NEVER be set in production.
+ , mucSigningKey :: String
+} deriving (Eq, Show)
+
+instance FromJSON MavenUploadConfig where
+ parseJSON = withObject "MavenUploadConfig" $ \o -> MavenUploadConfig
+ <$> o .: "url"
+ <*> o .: "user"
+ <*> o .: "password"
+ <*> (fromMaybe (MavenAllowUnsecureTls False) <$> o .:? "allowUnsecureTls")
+ <*> o .: "signingKey"
diff --git a/release/src/Upload.hs b/release/src/Upload.hs
new file mode 100644
index 0000000000..d60eb5d4f4
--- /dev/null
+++ b/release/src/Upload.hs
@@ -0,0 +1,494 @@
+-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE StrictData #-}
+{-# LANGUAGE TemplateHaskell #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+
+module Upload (
+ uploadToMavenCentral,
+ mavenConfigFromEnv,
+) where
+
+import qualified Control.Concurrent.Async.Lifted.Safe as Async
+import qualified Control.Exception.Safe as E
+import Control.Monad (when)
+import Control.Monad.Logger
+import Control.Monad.IO.Class
+import "cryptohash" Crypto.Hash (Digest, MD5(..), SHA1(..), digestToHexByteString, hash)
+import Control.Retry
+import Data.Aeson
+import Data.Foldable (for_)
+import qualified Data.ByteString as BS
+import qualified Data.ByteString.Lazy as BSL
+import qualified Data.ByteString.Base64 as Base64
+import qualified Data.ByteString.Char8 as C8
+import qualified Data.List as List
+import Data.Text (Text)
+import qualified Data.Text as T
+import Data.Text.Encoding (encodeUtf8)
+import Network.Connection (TLSSettings(..))
+import Network.HTTP.Client
+import Network.HTTP.Client.TLS (mkManagerSettings, tlsManagerSettings)
+import Network.HTTP.Simple (setRequestBasicAuth, setRequestBodyFile, setRequestBodyLBS, setRequestHeader, setRequestMethod, setRequestPath)
+import Network.HTTP.Types.Status
+import Path
+import System.Environment
+import System.IO.Temp
+
+import Types
+import Util
+
+--
+-- Upload the artifacts to Maven Central
+--
+-- The artifacts are first uploaded to a staging repository on the Sonatype Open Source Repository Hosting platform
+-- where the repository contents is verified to conform to the Maven Central standards before being released to
+-- the public repository.
+--
+-- Digitalasset has been assigned the 'com.daml' and 'com.digitalasset' namespaces (group IDs for Maven repos and artifacts
+-- need to be uploaded to the staging repository corresponding with their group ID. The staging repository for each group ID
+-- is handled separately, hence there are several 'duplicated' REST calls.
+--
+-- Further information:
+--
+-- Staging requirements: https://central.sonatype.org/pages/requirements.html
+-- Staging REST API: https://oss.sonatype.org/nexus-staging-plugin/default/docs/index.html
+--
+uploadToMavenCentral :: (MonadCI m) => MavenUploadConfig -> Path Abs Dir -> [(MavenCoords, Path Rel File)] -> m ()
+uploadToMavenCentral MavenUploadConfig{..} releaseDir artifacts = do
+
+ -- Note: TLS verification settings switchable by MavenUpload settings
+ let managerSettings = if getAllowUnsecureTls mucAllowUnsecureTls then noVerifyTlsManagerSettings else tlsManagerSettings
+
+ -- Create HTTP Connection manager with 2min response timeout as the OSSRH can be slow...
+ manager <- liftIO $ newManager managerSettings { managerResponseTimeout = responseTimeoutMicro (120 * 1000 * 1000) }
+
+ parsedUrlRequest <- parseUrlThrow $ T.unpack mucUrl -- Note: Will throw exception on non-2XX responses
+ let baseRequest = setRequestMethod "PUT" $ setRequestBasicAuth (encodeUtf8 mucUser) (encodeUtf8 mucPassword) parsedUrlRequest
+
+ decodedSigningKey <- decodeSigningKey mucSigningKey
+ -- Security Note: Using the withSystemTempDirectory function to always cleanup the private key data from the filesystems.
+ withSystemTempDirectory "gnupg" $ \gnupgTempDir -> do
+
+ -- Write the secret key used for signing into a temporary file and use 'gpg' command line tool to import into
+ -- GPG's internal file tree.
+ secretKeyImportFile <- liftIO $ emptyTempFile gnupgTempDir "gpg-private-key.asc"
+ _ <- liftIO $ BS.writeFile secretKeyImportFile decodedSigningKey
+
+ loggedProcess_ "gpg" [ "--homedir", T.pack gnupgTempDir, "--no-tty", "--quiet", "--import", T.pack secretKeyImportFile ]
+
+ --
+ -- Prepare the remote staging repositories
+ --
+ (comDamlStagingRepoId, comDigitalAssetRepoId) <- prepareStagingRepo baseRequest manager
+
+ --
+ -- Upload the artifacts; each with:
+ -- 1. PGP signature
+ -- 2. SHA1 checksum
+ -- 3. MD5 checksum
+
+ for_ artifacts $ \(coords@MavenCoords{..}, file) -> do
+ let absFile = releaseDir > file -- (T.intercalate "/" (groupId <> [artifactId]))
+
+ sigTempFile <- liftIO $ emptySystemTempFile $ T.unpack $ artifactId <> maybe "" ("-" <>) classifier <> "-" <> artifactType <> ".asc"
+ -- The "--batch" and "--yes" flags are used to prevent gpg waiting on stdin.
+ loggedProcess_ "gpg" [ "--homedir", T.pack gnupgTempDir, "-ab", "-o", T.pack sigTempFile, "--batch", "--yes", T.pack (fromAbsFile absFile) ]
+
+ let artUploadPath = uploadPath coords comDamlStagingRepoId comDigitalAssetRepoId
+ (md5Hash, sha1Hash) <- chksumFileContents absFile
+
+ $logInfo ("(Uploading " <> artUploadPath <> " from " <> tshow absFile <> ")")
+
+ let request
+ = setRequestHeader "Content-Type" [ encodeUtf8 $ getContentType artifactType ]
+ $ setRequestPath (encodeUtf8 artUploadPath)
+ $ setRequestBodyFile (fromAbsFile absFile) baseRequest
+
+ let pgpSigRequest
+ = setRequestHeader "Content-Type" [ "text/plain" ]
+ $ setRequestPath (encodeUtf8 $ artUploadPath <> ".asc")
+ $ setRequestBodyFile sigTempFile baseRequest
+
+ let sha1CksumRequest
+ = setRequestHeader "Content-Type" [ "text/plain" ]
+ $ setRequestPath (encodeUtf8 $ artUploadPath <> ".sha1")
+ $ setRequestBodyLBS sha1Hash baseRequest
+
+ let md5CksumRequest
+ = setRequestHeader "Content-Type" [ "text/plain" ]
+ $ setRequestPath (encodeUtf8 $ artUploadPath <> ".md5")
+ $ setRequestBodyLBS md5Hash baseRequest
+
+ (_, _, _, _) <- Async.runConcurrently $ (,,,)
+ <$> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody request manager))
+ <*> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody pgpSigRequest manager))
+ <*> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody sha1CksumRequest manager))
+ <*> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody md5CksumRequest manager))
+
+ pure ()
+
+ $logInfo "Finished uploading artifacts"
+
+ -- Now 'finish' the staging and release to Maven Central
+ publishStagingRepo baseRequest manager comDamlStagingRepoId comDigitalAssetRepoId
+
+prepareStagingRepo :: (MonadCI m) => Request -> Manager -> m (Text, Text)
+prepareStagingRepo baseRequest manager = do
+
+ --
+ -- Note in Profile IDs
+ --
+ -- Currently the profile IDs are hardcoded. The IDs are fixed to the "namespaces" ('com.daml' and 'com.digitialasset')
+ -- attached to the Digitalasset accounts on the the Sonatype OSSRH.
+ --
+
+ --
+ -- Open the staging repository profile for uploads.
+ --
+ -- Opening the staging repositories explicitly instead of implicitly (by simply uploading the artifacts)
+ -- allows for better managability (i.e. independant of the current state of the remote repositories which
+ -- could be still open due to failures).
+ --
+
+ let startComDamlStagingRepoRequest
+ = setRequestMethod "POST"
+ $ setRequestPath "/service/local/staging/profiles/b6148ff96bfaaa/start" -- Profile key could be requested
+ $ setRequestHeader "content-type" [ "application/json" ]
+ $ setRequestHeader "accept" [ "application/json" ]
+ $ setRequestBodyLBS (BSL.fromStrict (encodeUtf8 "{\"data\":{\"description\":\"\"}}")) baseRequest
+
+ let startComDigitalassetStagingRepoRequest
+ = setRequestMethod "POST"
+ $ setRequestPath "/service/local/staging/profiles/b614bfdbd6b51f/start" -- Profile key could be requested
+ $ setRequestHeader "content-type" [ "application/json" ]
+ $ setRequestHeader "accept" [ "application/json" ]
+ $ setRequestBodyLBS (BSL.fromStrict (encodeUtf8 "{\"data\":{\"description\":\"\"}}")) baseRequest
+
+ (startComDamlStagingReosResponse, startComDigitalassetStagingRepoResponse) <- Async.runConcurrently $ (,)
+ <$> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpLbs startComDamlStagingRepoRequest manager))
+ <*> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpLbs startComDigitalassetStagingRepoRequest manager))
+ comDamlStagingRepoInfo <- decodeStagingPromoteResponse startComDamlStagingReosResponse
+ comDigitalassetStagingRepoInfo <- decodeStagingPromoteResponse startComDigitalassetStagingRepoResponse
+
+ return (stagedRepositoryId $ _data comDamlStagingRepoInfo, stagedRepositoryId $ _data comDigitalassetStagingRepoInfo)
+
+publishStagingRepo :: (MonadCI m) => Request -> Manager -> Text -> Text -> m ()
+publishStagingRepo baseRequest manager comDamlRepoId comDigitalassetRepoId = do
+
+ --
+ -- "Close" the staging profiles which initiates the running of the rules that check the uploaded artifacts
+ -- for compliance with the Maven Central requirements.
+ -- If all the rules pass then the status of the staging repository and profile will become "closed", if anything fails
+ -- then the status will remain set to "open".
+ --
+
+ let finishComDamlStagingRepoRequest
+ = setRequestMethod "POST"
+ $ setRequestPath "/service/local/staging/profiles/b6148ff96bfaaa/finish" -- Profile key could be requested
+ $ setRequestHeader "content-type" [ "application/json" ]
+ $ setRequestBodyLBS (textToLazyByteString $ "{\"data\":{\"stagedRepositoryId\":\"" <> comDamlRepoId <> "\",\"description\":\"\"}}") baseRequest
+
+ let finishComDigitalassetStagingRepoRequest
+ = setRequestMethod "POST"
+ $ setRequestPath "/service/local/staging/profiles/b614bfdbd6b51f/finish" -- Profile key could be requested
+ $ setRequestHeader "content-type" [ "application/json" ]
+ $ setRequestBodyLBS (textToLazyByteString $ "{\"data\":{\"stagedRepositoryId\":\"" <> comDigitalassetRepoId <> "\",\"description\":\"\"}}") baseRequest
+
+ (_, _) <- Async.runConcurrently $ (,)
+ <$> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody finishComDamlStagingRepoRequest manager))
+ <*> Async.Concurrently (recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody finishComDigitalassetStagingRepoRequest manager))
+
+ let comDamlStatusReposRequest
+ = setRequestMethod "GET"
+ $ setRequestPath (encodeUtf8 ("/service/local/staging/repository/" <> comDamlRepoId))
+ $ setRequestHeader "accept" [ "application/json" ] baseRequest
+
+ let comDigitalassetStatusReposRequest
+ = setRequestMethod "GET"
+ $ setRequestPath (encodeUtf8 ("/service/local/staging/repository/" <> comDigitalassetRepoId))
+ $ setRequestHeader "accept" [ "application/json" ] baseRequest
+
+ --
+ -- Poll until the staging repositories are closed or the staging repositories cease to be "transitioning" to a new state
+ --
+ (comDamlNotClosed, comDigitalassetNotClosed) <- Async.runConcurrently $ (,)
+ <$> Async.Concurrently (recovering checkStatusRetryPolicy [ httpResponseHandler, checkRepoStatusHandler ] (\_ -> handleStatusRequest comDamlStatusReposRequest manager))
+ <*> Async.Concurrently (recovering checkStatusRetryPolicy [ httpResponseHandler, checkRepoStatusHandler ] (\_ -> handleStatusRequest comDigitalassetStatusReposRequest manager))
+
+ --
+ -- Drop" (delete) both staging repositories if one or more fails the checks (and are not in the "closed" state)
+ --
+ when (comDamlNotClosed || comDigitalassetNotClosed) $ do
+ when comDamlNotClosed $ do logStagingRepositoryActivity baseRequest manager comDamlRepoId
+ when comDigitalassetNotClosed $ do logStagingRepositoryActivity baseRequest manager comDigitalassetRepoId
+ dropStagingRepositories baseRequest manager [ comDamlRepoId, comDigitalassetRepoId ]
+ throwIO $ RepoFailedToClose $ [ comDamlRepoId | comDamlNotClosed ] <> [ comDigitalassetRepoId | comDigitalassetNotClosed ]
+
+ --
+ -- Now the final step of releasing the staged artifacts into the wild...
+ --
+ let releaseStagingReposRequest
+ = setRequestMethod "POST"
+ $ setRequestPath "/staging/bulk/promote"
+ $ setRequestHeader "content-type" [ "application/json" ]
+ $ setRequestBodyLBS (textToLazyByteString $ "{\"data\":{\"stagedRepositoryIds\":[\"" <> comDamlRepoId <> "\",\"" <> comDigitalassetRepoId <> "\",\"description\":\"\",\"autoDropAfterRelease\":true}}") baseRequest
+
+ _ <- recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody releaseStagingReposRequest manager)
+
+ $logWarn "Published to Maven Central"
+
+ pure ()
+
+-- Print out a log of the repository activity which includes details of which verification rule failed.
+-- The output is not prettified as it should only be print in rare(ish) error cases.
+logStagingRepositoryActivity :: (MonadCI m) => Request -> Manager -> Text -> m ()
+logStagingRepositoryActivity baseRequest manager repoId = do
+
+ let repoActivityRequest
+ = setRequestMethod "GET"
+ $ setRequestPath (encodeUtf8 ("/service/local/staging/repository/" <> repoId <> "/activity"))
+ $ setRequestHeader "accept" [ "application/json" ] baseRequest
+
+ activityResponse <- recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpLbs repoActivityRequest manager)
+ repoActivity <- decodeRepoActivityResponse activityResponse
+
+ $logWarn ("Failed to process staging repository \"" <> repoId <> "\". \n" <> (T.intercalate "\n " $ map tshow repoActivity))
+
+ return ()
+
+dropStagingRepositories :: (MonadCI m) => Request -> Manager -> [Text] -> m ()
+dropStagingRepositories baseRequest manager repoIdList = do
+ --
+ -- Note: This is a "Bulk Drop" request used by the Nexus UI and not a Staging REST API.
+ --
+ let dropReposJson = "{\"data\":{\"description\":\"\",\"stagedRepositoryIds\":" <> tshow repoIdList <> "}}"
+ let dropReposRequest
+ = setRequestMethod "POST"
+ $ setRequestPath "/service/local/staging/bulk/drop"
+ $ setRequestHeader "content-type" [ "application/json" ]
+ $ setRequestHeader "accept" [ "application/json" ]
+ $ setRequestBodyLBS (BSL.fromStrict (encodeUtf8 dropReposJson)) baseRequest
+
+ _ <- recovering uploadRetryPolicy [ httpResponseHandler ] (\_ -> liftIO $ httpNoBody dropReposRequest manager)
+
+ return ()
+
+decodeSigningKey :: (MonadCI m) => String -> m BS.ByteString
+decodeSigningKey signingKey = case Base64.decode $ C8.pack signingKey of
+ Left err -> throwIO $ CannotDecodeSigningKey err
+ Right decodedData -> return decodedData
+
+-- Note: Upload path is NOT documented in the REST API Guide.
+uploadPath :: MavenCoords -> Text -> Text -> Text
+uploadPath MavenCoords{..} comDamlStagingRepoId comDigitalassetRepoId = do
+ let stagingRepoId = if ["com", "daml"] `List.isPrefixOf` groupId then comDamlStagingRepoId else comDigitalassetRepoId
+ T.intercalate "/" ("/service/local/staging/deployByRepositoryId" : [stagingRepoId] <> groupId <> [artifactId, version, artifactId]) <> "-" <> version <> maybe "" ("-" <>) classifier <> "." <> artifactType
+
+getContentType :: ArtifactType -> Text
+getContentType t =
+ case t of
+ "jar" -> "application/java-archive"
+ "pom" -> "application/xml"
+ _ -> "application/octet-stream"
+
+noVerifyTlsSettings :: TLSSettings
+noVerifyTlsSettings = TLSSettingsSimple
+ { settingDisableCertificateValidation = True
+ , settingDisableSession = True
+ , settingUseServerName = False
+ }
+
+noVerifyTlsManagerSettings :: ManagerSettings
+noVerifyTlsManagerSettings = mkManagerSettings noVerifyTlsSettings Nothing
+
+chksumFileContents :: (MonadIO m) => Path Abs File -> m (BSL.ByteString, BSL.ByteString)
+chksumFileContents file = do
+ contents <- liftIO $ BS.readFile $ fromAbsFile file
+ return (BSL.fromStrict (digestToHexByteString (hash contents :: Digest MD5)), BSL.fromStrict (digestToHexByteString (hash contents :: Digest SHA1)))
+
+mavenConfigFromEnv :: (MonadIO m, E.MonadThrow m) => m MavenUploadConfig
+mavenConfigFromEnv = do
+ url <- liftIO $ getEnv "MAVEN_URL"
+ user <- liftIO $ getEnv "MAVEN_USER"
+ password <- liftIO $ getEnv "MAVEN_PASSWORD"
+ mbAllowUnsecureTls <- liftIO $ lookupEnv "MAVEN_UNSECURE_TLS"
+ signingKey <- liftIO $ getEnv "GPG_KEY"
+ pure MavenUploadConfig
+ { mucUrl = T.pack url
+ , mucUser = T.pack user
+ , mucPassword = T.pack password
+ , mucAllowUnsecureTls = MavenAllowUnsecureTls $ mbAllowUnsecureTls == Just "True"
+ , mucSigningKey = signingKey
+ }
+
+textToLazyByteString :: Text -> BSL.ByteString
+textToLazyByteString text = BSL.fromStrict $ encodeUtf8 text
+
+--
+-- HTTP Response Handlers
+--
+
+httpResponseHandler :: (MonadIO m, MonadLogger m) => RetryStatus -> E.Handler m Bool
+httpResponseHandler status = logRetries shouldRetry logRetry status
+
+checkRepoStatusHandler :: (MonadIO m, MonadLogger m) => RetryStatus -> E.Handler m Bool
+checkRepoStatusHandler status = logRetries shouldStatusRetry logStatusRetry status
+
+shouldRetry :: (MonadIO m) => HttpException -> m Bool
+shouldRetry e = case e of
+ HttpExceptionRequest _ ConnectionTimeout -> return True
+ -- Don't retry POST requests if the response timeouts as the request might of been processed
+ HttpExceptionRequest request ResponseTimeout -> return (method request == "POST")
+ HttpExceptionRequest _ (StatusCodeException rsp _) ->
+ case statusCode (responseStatus rsp) of
+ 408 {- requestTimeout -} -> return True
+ 502 {- badGateway -} -> return True
+ 503 {- serviceUnavailable -} -> return True
+ _ -> return False
+ _ -> return False
+
+shouldStatusRetry :: (MonadIO m) => RepoNotClosed -> m Bool
+shouldStatusRetry _ = return True
+
+-- | For use with 'logRetries'.
+logRetry :: (MonadIO m, MonadLogger m, E.Exception e) => Bool -> e -> RetryStatus -> m ()
+logRetry shouldRetry err status = do
+ $logWarn (tshow err <> ". " <> " " <> tshow status <> " - " <> nextMsg)
+ return ()
+ where
+ nextMsg = if shouldRetry then "Retrying." else "Aborting after " <> (tshow $ rsCumulativeDelay status) <> "µs total delay."
+
+logStatusRetry :: (MonadIO m, MonadLogger m, E.Exception e) => Bool -> e -> RetryStatus -> m ()
+logStatusRetry shouldRetry _ status =
+ if shouldRetry
+ then
+ $logDebug ("Staging repository is still processing close request. Checked after " <> tshow (rsCumulativeDelay status) <> "µs")
+ else
+ $logDebug ("Aborting staging repository check after " <> (tshow $ rsCumulativeDelay status) <> "µs.")
+
+uploadRetryPolicy :: RetryPolicy
+uploadRetryPolicy = limitRetriesByCumulativeDelay (60 * 1000 * 1000) (exponentialBackoff (200 * 100))
+
+-- The status of the staging repository can take a number of minutes to change it's
+-- status to closed.
+checkStatusRetryPolicy :: RetryPolicy
+checkStatusRetryPolicy = limitRetriesByCumulativeDelay (5 * 60 * 1000 * 1000) (constantDelay (15 * 1000 * 1000))
+
+handleStatusRequest :: (MonadIO m) => Request -> Manager -> m Bool
+handleStatusRequest request manager = do
+ statusResponse <- liftIO $ httpLbs request manager
+ repoStatus <- liftIO $ decodeRepoStatus $ responseBody statusResponse
+ if transitioning repoStatus
+ then
+ throwIO RepoNotClosed
+ else
+ return $ status repoStatus == "open"
+
+--
+-- Data Transfer Objects for the Nexus Staging REST API.
+-- Note that fields from the REST response that are not used do not need
+-- to be defined as Aeson will simply ignore them.
+-- See https://oss.sonatype.org/nexus-staging-plugin/default/docs/index.html
+--
+data RepoStatusResponse = RepoStatusResponse
+ { repositoryId :: Text
+ , status :: Text
+ , transitioning :: Bool
+ } deriving Show
+
+data StagingPromote = StagingPromote { stagedRepositoryId :: Text }
+data StagingPromoteResponse = StagingPromoteResponse { _data :: StagingPromote }
+
+data NameValue = NameValue
+ { name :: Text
+ , value :: Text
+ }
+instance Show NameValue where
+ show NameValue{..} = T.unpack $ " " <> name <> ": " <> value
+
+data RepoActivityEvent = RepoActivityEvent
+ { name :: Text
+ , properties :: [NameValue]
+ }
+instance Show RepoActivityEvent where
+ show RepoActivityEvent{..} = do
+ T.unpack $ name <> intercalatedValues
+ where
+ intercalatedValues = T.intercalate "\n " ([""] <> map tshow properties <> [""])
+
+data RepoActivityDetails = RepoActivityDetails
+ { name :: Text
+ , events :: [RepoActivityEvent]
+ }
+instance Show RepoActivityDetails where
+ show RepoActivityDetails{..} = do
+ T.unpack $ name <> intercalatedValues
+ where
+ intercalatedValues = T.intercalate "\n " ([""] <> map tshow events <> [""])
+
+-- 'Manual' parsing of required fields as the API uses the Haskell reserved keyword 'type'
+instance FromJSON RepoStatusResponse where
+ parseJSON (Object o) = RepoStatusResponse <$> o .: "repositoryId" <*> o .: "type" <*> o .: "transitioning"
+ parseJSON _ = fail "Expected an Object"
+
+instance FromJSON StagingPromote where
+ parseJSON = withObject "StagingPromote" $ \o -> StagingPromote
+ <$> o .: "stagedRepositoryId"
+
+-- 'Manual' parsing of required fields as the API uses the Haskell reserved keyword 'data'
+instance FromJSON StagingPromoteResponse where
+ parseJSON (Object o) = StagingPromoteResponse <$> o .: "data"
+ parseJSON _ = fail "Expected an Object"
+
+instance FromJSON RepoActivityDetails where
+ parseJSON = withObject "RepoActivityDetails" $ \o -> RepoActivityDetails
+ <$> o .: "name"
+ <*> o .: "events"
+
+instance FromJSON RepoActivityEvent where
+ parseJSON = withObject "RepoActivityEvent" $ \o -> RepoActivityEvent
+ <$> o .: "name"
+ <*> o .: "properties"
+
+instance FromJSON NameValue where
+ parseJSON = withObject "NameValue" $ \o -> NameValue
+ <$> o .: "name"
+ <*> o .: "value"
+
+decodeRepoStatus :: (MonadIO m) => BSL.ByteString -> m RepoStatusResponse
+decodeRepoStatus jsonString = case (eitherDecode jsonString :: Either String RepoStatusResponse) of
+ Left err -> throwIO $ ParseJsonException err
+ Right r -> return r
+
+decodeStagingPromoteResponse :: (MonadIO m) => Response BSL.ByteString -> m StagingPromoteResponse
+decodeStagingPromoteResponse response = case (eitherDecode $ responseBody response :: Either String StagingPromoteResponse) of
+ Left err -> throwIO $ ParseJsonException err
+ Right r -> return r
+
+decodeRepoActivityResponse :: (MonadIO m) => Response BSL.ByteString -> m [RepoActivityDetails]
+decodeRepoActivityResponse response = case (eitherDecode $ responseBody response :: Either String [RepoActivityDetails]) of
+ Left err -> throwIO $ ParseJsonException err
+ Right r -> return r
+
+--
+-- Error definitions
+--
+data UploadFailure
+ = ParseJsonException String
+ | CannotDecodeSigningKey String
+ | RepoFailedToClose [Text]
+
+instance E.Exception UploadFailure
+instance Show UploadFailure where
+ show (ParseJsonException msg) = "Cannot parse JSON data: " <> msg
+ show (CannotDecodeSigningKey msg) = "Cannot Base64 decode signing key: " <> msg
+ show (RepoFailedToClose repoIds) = "The staging repositories " <> show repoIds <> " failed to close"
+
+data RepoNotClosed
+ = RepoNotClosed
+ deriving Show
+
+instance E.Exception RepoNotClosed
diff --git a/release/src/Util.hs b/release/src/Util.hs
index b0731bccc7..c006592aa6 100644
--- a/release/src/Util.hs
+++ b/release/src/Util.hs
@@ -16,13 +16,16 @@ module Util (
Artifact(..),
ArtifactLocation(..),
+ BazelLocations(..),
BazelTarget(..),
artifactFiles,
+ artifactCoords,
copyToReleaseDir,
buildTargets,
getBazelLocations,
- resolvePomData
+ resolvePomData,
+ loggedProcess_,
) where
@@ -66,14 +69,14 @@ data JarType
= Plain
-- ^ Plain java or scala library, without source jar.
| Lib
- -- ^ A java library jar, with a source jar.
+ -- ^ A java library jar, with source and javadoc jars.
| Deploy
-- ^ Deploy jar, e.g. a fat jar containing transitive deps.
| Proto
-- ^ A java protobuf library (*-speed.jar).
| Scala
- -- ^ A scala library jar, with a source jar. Use when source jar
- -- is desired, otherwise use 'Plain'.
+ -- ^ A scala library jar, with source and scaladoc jars. Use when
+ -- source or scaladoc is desired, otherwise use 'Plain'.
deriving (Eq, Show)
instance FromJSON ReleaseType where
@@ -94,6 +97,8 @@ data Artifact c = Artifact
, artPlatformDependent :: !PlatformDependent
, artBintrayPackage :: !BintrayPackage
-- ^ Defaults to sdk-components if not specified
+ , artMavenUpload :: MavenUpload
+ -- ^ Defaults to False if not specified
, artMetadata :: !c
} deriving Show
@@ -103,6 +108,7 @@ instance FromJSON (Artifact (Maybe ArtifactLocation)) where
<*> o .: "type"
<*> (fromMaybe (PlatformDependent False) <$> o .:? "platformDependent")
<*> (fromMaybe PkgSdkComponents <$> o .:? "bintrayPackage")
+ <*> (fromMaybe (MavenUpload False) <$> o .:? "mavenUpload")
<*> o .:? "location"
data ArtifactLocation = ArtifactLocation
@@ -129,7 +135,8 @@ buildTargets art@Artifact{..} =
in [jarTarget, pomTar] <>
[BazelTarget ("//" <> directory <> ":" <> srcJar) | Just srcJar <- pure (sourceJarName art)] <>
[BazelTarget ("//" <> directory <> ":" <> srcJar) | Just srcJar <- pure (scalaSourceJarName art)] <>
- [BazelTarget ("//" <> directory <> ":" <> javadocJar) | Just javadocJar <- pure (javadocJarName art)]
+ [BazelTarget ("//" <> directory <> ":" <> javadocJar) | Just javadocJar <- pure (javadocJarName art)] <>
+ [BazelTarget ("//" <> directory <> ":" <> scaladocJar) | Just scaladocJar <- pure (scaladocJarName art)]
Zip -> [artTarget]
TarGz -> [artTarget]
@@ -191,9 +198,9 @@ splitBazelTarget (BazelTarget t) =
_ -> error ("Malformed bazel target: " <> show t)
mainExt :: ReleaseType -> Text
-mainExt Zip = ".zip"
-mainExt TarGz = ".tar.gz"
-mainExt Jar{} = ".jar"
+mainExt Zip = "zip"
+mainExt TarGz = "tar.gz"
+mainExt Jar{} = "jar"
mainFileName :: ReleaseType -> Text -> Text
mainFileName releaseType name =
@@ -217,6 +224,11 @@ scalaSourceJarName Artifact{..}
| Jar Scala <- artReleaseType = Just $ snd (splitBazelTarget artTarget) <> "_src.jar"
| otherwise = Nothing
+scaladocJarName :: Artifact a -> Maybe Text
+scaladocJarName Artifact{..}
+ | Jar Scala <- artReleaseType = Just $ snd (splitBazelTarget artTarget) <> "_scaladoc.jar"
+ | otherwise = Nothing
+
javadocJarName :: Artifact a -> Maybe Text
javadocJarName Artifact{..}
| Jar Lib <- artReleaseType = Just $ snd (splitBazelTarget artTarget) <> "_javadoc.jar"
@@ -233,32 +245,61 @@ artifactFiles allArtifacts art@Artifact{..} = do
directory <- parseRelDir $ unpack directory
mainArtifactIn <- parseRelFile $ unpack $ mainFileName artReleaseType name
- mainArtifactOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # mainExt artReleaseType))
+ mainArtifactOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "." # mainExt artReleaseType))
pomFileIn <- parseRelFile (unpack (name <> "_pom.xml"))
pomFileOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion #".pom"))
mbSourceJarIn <- traverse (parseRelFile . unpack) (sourceJarName art)
- sourceJarOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "-sources" # mainExt artReleaseType))
+ sourceJarOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "-sources" # "." # mainExt artReleaseType))
mbScalaSourceJarIn <- traverse (parseRelFile . unpack) (scalaSourceJarName art)
- scalaSourceJarOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "-sources" # mainExt artReleaseType))
+ scalaSourceJarOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "-sources" # "." # mainExt artReleaseType))
mbJavadocJarIn <- traverse (parseRelFile . unpack) (javadocJarName art)
- javadocJarOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "-javadoc" # mainExt artReleaseType))
+ javadocJarOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "-javadoc" # "." # mainExt artReleaseType))
+
+ mbScaladocJarIn <- traverse (parseRelFile . unpack) (scaladocJarName art)
+ scaladocJarOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "-javadoc" # "." # mainExt artReleaseType))
+
+ let shouldReleasePlatInd = shouldRelease allArtifacts (PlatformDependent False)
pure $
[(directory > mainArtifactIn, outDir > mainArtifactOut) | shouldRelease allArtifacts artPlatformDependent] <>
- [(directory > pomFileIn, outDir > pomFileOut) | isJar artReleaseType, shouldRelease allArtifacts (PlatformDependent False)] <>
- [(directory > sourceJarIn, outDir > sourceJarOut) | shouldRelease allArtifacts (PlatformDependent False), Just sourceJarIn <- pure mbSourceJarIn] <>
- [(directory > scalaSourceJarIn, outDir > scalaSourceJarOut) | shouldRelease allArtifacts (PlatformDependent False), Just scalaSourceJarIn <- pure mbScalaSourceJarIn] <>
- [(directory > javadocJarIn, outDir > javadocJarOut) | shouldRelease allArtifacts (PlatformDependent False), Just javadocJarIn <- pure mbJavadocJarIn]
+ [(directory > pomFileIn, outDir > pomFileOut) | isJar artReleaseType, shouldReleasePlatInd] <>
+ [(directory > sourceJarIn, outDir > sourceJarOut) | shouldReleasePlatInd, Just sourceJarIn <- pure mbSourceJarIn] <>
+ [(directory > scalaSourceJarIn, outDir > scalaSourceJarOut) | shouldReleasePlatInd, Just scalaSourceJarIn <- pure mbScalaSourceJarIn] <>
+ [(directory > javadocJarIn, outDir > javadocJarOut) | shouldReleasePlatInd, Just javadocJarIn <- pure mbJavadocJarIn] <>
+ [(directory > scaladocJarIn, outDir > scaladocJarOut) | shouldReleasePlatInd, Just scaladocJarIn <- pure mbScaladocJarIn]
+ -- ^ Note that the Scaladoc is specified with the "javadoc" classifier.
+
+-- | Given an artifact, produce a list of pairs of an input file and the Maven coordinates
+artifactCoords :: E.MonadThrow m => AllArtifacts -> Artifact PomData -> m [(MavenCoords, Path Rel File)]
+artifactCoords allArtifacts Artifact{..} = do
+ let PomData{..} = artMetadata
+ let jarClassifier = if getPlatformDependent artPlatformDependent then Just osName else Nothing
+ outDir <- parseRelDir $ unpack $
+ T.intercalate "/" pomGroupId #"/"# pomArtifactId #"/"# pomVersion #"/"
+ let ostxt = if getPlatformDependent artPlatformDependent then "-" <> osName else ""
+
+ mainArtifactFile <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "." # mainExt artReleaseType))
+ pomFile <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion #".pom"))
+ sourcesFile <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion #"-sources.jar"))
+ javadocFile <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion #"-javadoc.jar"))
+
+ let mavenCoords classifier artifactType =
+ MavenCoords { groupId = pomGroupId, artifactId = pomArtifactId, version = pomVersion, classifier, artifactType }
+ let shouldReleasePlatInd = shouldRelease allArtifacts (PlatformDependent False)
+
+ pure $ [ (mavenCoords jarClassifier $ mainExt artReleaseType, outDir > mainArtifactFile) | shouldReleasePlatInd] <>
+ [ (mavenCoords Nothing "pom", outDir > pomFile) | isJar artReleaseType, shouldReleasePlatInd] <>
+ [ (mavenCoords (Just "sources") "jar", outDir > sourcesFile) | isJar artReleaseType, shouldReleasePlatInd] <>
+ [ (mavenCoords (Just "javadoc") "jar", outDir > javadocFile) | isJar artReleaseType, shouldReleasePlatInd]
shouldRelease :: AllArtifacts -> PlatformDependent -> Bool
shouldRelease (AllArtifacts allArtifacts) (PlatformDependent platformDependent) =
allArtifacts || platformDependent || osName == "linux"
-
copyToReleaseDir :: (MonadLogger m, MonadIO m) => BazelLocations -> Path Abs Dir -> Path Rel File -> Path Rel File -> m ()
copyToReleaseDir BazelLocations{..} releaseDir inp out = do
binExists <- doesFileExist (bazelBin > inp)