ledger-api: Use proto_jars, and publish Protobuf sources separately from the Scala classes. [KVL-714] (#8091)

* ledger-api: Use `proto_jars`.

CHANGELOG_BEGIN
- [Ledger API] The Scala JARs containing the gRPC definitions no longer
  contain the *.proto files used to generate the ScalaPB-based classes.
CHANGELOG_END

* Create a source JAR for *.proto files in `proto_jars`.

* ledger-api: Publish the protobuf sources as "ledger-api-proto".

CHANGELOG_BEGIN
- [Ledger API] The *.proto files containing the gRPC definitions are now
  provided by a new Maven Central artifact, with the group "com.daml"
  and the artifact name "ledger-api-proto".
CHANGELOG_END

* release: We don't need the "main-jar" option.

* Bazel: Proto JARs will always have a Maven artifact suffix.

* Bazel: Simplify Protobuf source file TAR and JAR targets.

* Bazel: Extract out Protobuf functions.

bazel_tools/proto.bzl

@@ -1,9 +1,11 @@
 # Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+load("//bazel_tools:java.bzl", "da_java_library")
 load("//bazel_tools:javadoc_library.bzl", "javadoc_library")
 load("//bazel_tools:pkg.bzl", "pkg_empty_zip")
 load("//bazel_tools:pom_file.bzl", "pom_file")
+load("//bazel_tools:scala.bzl", "scala_source_jar", "scaladoc_jar")
 load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
 load("@os_info//:os_info.bzl", "is_windows")
 load("@rules_pkg//:pkg.bzl", "pkg_tar")
@@ -166,16 +168,41 @@ proto_gen = rule(
 def _is_windows(ctx):
     return ctx.configuration.host_path_separator == ";"
 
-def maven_tags(group, artifact_prefix, artifact_suffix):
-    if group and artifact_prefix and artifact_suffix:
-        return ["maven_coordinates=%s:%s-%s:__VERSION__" % (group, artifact_prefix, artifact_suffix)]
+def _maven_tags(group, artifact_prefix, artifact_suffix):
+    if group and artifact_prefix:
+        artifact = artifact_prefix + "-" + artifact_suffix
+        return ["maven_coordinates=%s:%s:__VERSION__" % (group, artifact)]
     else:
         return []
 
+def _proto_scala_srcs(name, grpc):
+    return [":%s" % name] + ([
+        "@com_github_googleapis_googleapis//google/rpc:code_proto",
+        "@com_github_googleapis_googleapis//google/rpc:status_proto",
+        "@com_github_grpc_grpc//src/proto/grpc/health/v1:health_proto_descriptor",
+    ] if grpc else [])
+
+def _proto_scala_deps(grpc, proto_deps):
+    return [
+        "@maven//:com_google_protobuf_protobuf_java",
+        "@maven//:com_thesamet_scalapb_lenses_2_12",
+        "@maven//:com_thesamet_scalapb_scalapb_runtime_2_12",
+    ] + ([
+        "@maven//:com_thesamet_scalapb_scalapb_runtime_grpc_2_12",
+        "@maven//:io_grpc_grpc_api",
+        "@maven//:io_grpc_grpc_core",
+        "@maven//:io_grpc_grpc_protobuf",
+        "@maven//:io_grpc_grpc_stub",
+    ] if grpc else []) + [
+        "%s_scala" % label
+        for label in proto_deps
+    ]
+
 def proto_jars(
         name,
         srcs,
         strip_import_prefix = "",
+        grpc = False,
         deps = [],
         proto_deps = [],
         java_deps = [],
@@ -183,16 +210,35 @@ def proto_jars(
         javadoc_root_packages = [],
         maven_group = None,
         maven_artifact_prefix = None,
-        maven_java_artifact_suffix = "java-proto"):
+        maven_artifact_proto_suffix = "proto",
+        maven_artifact_java_suffix = "java-proto",
+        maven_artifact_scala_suffix = "scala-proto"):
     # Tarball containing the *.proto files.
     pkg_tar(
-        name = "%s_src" % name,
+        name = "%s_tar" % name,
         srcs = srcs,
         extension = "tar.gz",
         strip_prefix = strip_import_prefix,
         visibility = ["//visibility:public"],
     )
 
+    # JAR and source JAR containing the *.proto files.
+    da_java_library(
+        name = "%s_jar" % name,
+        srcs = None,
+        deps = None,
+        resources = srcs,
+        resource_strip_prefix = "%s/%s/" % (native.package_name(), strip_import_prefix),
+        tags = _maven_tags(maven_group, maven_artifact_prefix, maven_artifact_proto_suffix),
+        visibility = ["//visibility:public"],
+    )
+
+    # An empty Javadoc JAR for uploading the source proto JAR to Maven Central.
+    pkg_empty_zip(
+        name = "%s_jar_javadoc" % name,
+        out = "%s_jar_javadoc.jar" % name,
+    )
+
     # Compiled protobufs. Used in subsequent targets.
     proto_library(
         name = name,
@@ -202,17 +248,18 @@ def proto_jars(
         deps = deps + proto_deps,
     )
 
-    # JAR containing the generated Java bindings.
+    # JAR and source JAR containing the generated Java bindings.
     native.java_proto_library(
         name = "%s_java" % name,
-        tags = maven_tags(maven_group, maven_artifact_prefix, maven_java_artifact_suffix),
+        tags = _maven_tags(maven_group, maven_artifact_prefix, maven_artifact_java_suffix),
         visibility = ["//visibility:public"],
-        deps = [":%s" % name] + java_deps,
+        deps = [":%s" % name],
     )
 
     if maven_group and maven_artifact_prefix:
         pom_file(
             name = "%s_java_pom" % name,
+            tags = _maven_tags(maven_group, maven_artifact_prefix, maven_artifact_java_suffix),
             target = ":%s_java" % name,
             visibility = ["//visibility:public"],
         )
@@ -226,9 +273,7 @@ def proto_jars(
             deps = ["@maven//:com_google_protobuf_protobuf_java"],
         ) if not is_windows else None
     else:
-        # Create an empty Javadoc JAR for uploading proto JARs to Maven Central.
-        # We don't need to create an empty JAR file for sources, because `java_proto_library`
-        # creates a source JAR automatically.
+        # An empty Javadoc JAR for uploading the compiled proto JAR to Maven Central.
         pkg_empty_zip(
             name = "%s_java_javadoc" % name,
             out = "%s_java_javadoc.jar" % name,
@@ -237,21 +282,42 @@ def proto_jars(
     # JAR containing the generated Scala bindings.
     proto_gen(
         name = "%s_scala_sources" % name,
-        srcs = [":%s" % name],
+        srcs = _proto_scala_srcs(name, grpc),
         plugin_exec = "//scala-protoc-plugins/scalapb:protoc-gen-scalapb",
         plugin_name = "scalapb",
+        plugin_options = ["grpc"] if grpc else [],
         visibility = ["//visibility:public"],
         deps = deps + proto_deps,
     )
 
+    all_scala_deps = _proto_scala_deps(grpc, proto_deps)
+
     scala_library(
         name = "%s_scala" % name,
         srcs = [":%s_scala_sources" % name],
+        tags = _maven_tags(maven_group, maven_artifact_prefix, maven_artifact_scala_suffix),
         unused_dependency_checker_mode = "error",
         visibility = ["//visibility:public"],
-        deps = [
-            "@maven//:com_google_protobuf_protobuf_java",
-            "@maven//:com_thesamet_scalapb_lenses_2_12",
-            "@maven//:com_thesamet_scalapb_scalapb_runtime_2_12",
-        ] + ["%s_scala" % label for label in proto_deps] + scala_deps,
+        deps = all_scala_deps,
+        exports = all_scala_deps,
     )
+
+    scala_source_jar(
+        name = "%s_scala_src" % name,
+        srcs = [":%s_scala_sources" % name],
+    )
+
+    scaladoc_jar(
+        name = "%s_scala_scaladoc" % name,
+        srcs = [":%s_scala_sources" % name],
+        tags = ["scaladoc"],
+        deps = [],
+        visibility = ["//visibility:public"],
+    ) if is_windows == False else None
+
+    if maven_group and maven_artifact_prefix:
+        pom_file(
+            name = "%s_scala_pom" % name,
+            target = ":%s_scala" % name,
+            visibility = ["//visibility:public"],
+        )

daml-lf/archive/BUILD.bazel

@@ -45,7 +45,7 @@ load("@rules_pkg//:pkg.bzl", "pkg_tar")
             ],
             maven_artifact_prefix = "daml-lf-%s-archive" % version,
             maven_group = "com.daml",
-            strip_import_prefix = "src/main/protobuf/",
+            strip_import_prefix = "src/main/protobuf",
         ),
     ]
     for version in LF_VERSIONS

language-support/java/bindings-rxjava/BUILD.bazel

@@ -63,7 +63,7 @@ da_scala_library(
         ":bindings-rxjava",
         "//daml-lf/data",
         "//language-support/java/bindings:bindings-java",
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "//ledger-api/rs-grpc-bridge",
         "//ledger/ledger-api-auth",
         "//ledger/ledger-api-common",
@@ -97,7 +97,7 @@ da_scala_test_suite(
         ":bindings-java-tests-lib",
         ":bindings-rxjava",
         "//language-support/java/bindings:bindings-java",
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "//ledger/ledger-api-auth",
         "//ledger/ledger-api-common",
         "//libs-scala/grpc-utils",

language-support/java/bindings/BUILD.bazel

@@ -11,7 +11,7 @@ load("//bazel_tools:java.bzl", "da_java_library")
 
 proto_gen(
     name = "ledger-api-java",
-    srcs = ["//ledger-api/grpc-definitions:protos"],
+    srcs = ["//ledger-api/grpc-definitions:ledger_api_proto"],
     plugin_name = "java",
     visibility = [
         "//visibility:public",
@@ -51,7 +51,7 @@ java_library(
 
 proto_gen(
     name = "ledger-api-java-grpc",
-    srcs = ["//ledger-api/grpc-definitions:protos"],
+    srcs = ["//ledger-api/grpc-definitions:ledger_api_proto"],
     plugin_exec = "@io_grpc_grpc_java//compiler:grpc_java_plugin",
     plugin_name = "java-grpc",
     visibility = [

language-support/scala/bindings/BUILD.bazel

@@ -20,13 +20,13 @@ da_scala_library(
     ],
     exports = [
         "//daml-lf/data",
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "@maven//:io_grpc_grpc_core",
         "@maven//:org_scalaz_scalaz_core_2_12",
     ],
     deps = [
         "//daml-lf/data",
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "@maven//:io_grpc_grpc_core",
         "@maven//:org_scalaz_scalaz_core_2_12",
     ],

ledger-api/grpc-definitions/BUILD.bazel

@@ -1,27 +1,25 @@
 # Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("//bazel_tools:haskell.bzl", "da_haskell_library")
-load("//bazel_tools:proto.bzl", "proto_gen")
-load("//bazel_tools:pom_file.bzl", "pom_file")
-load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+load("//bazel_tools:proto.bzl", "proto_gen", "proto_jars")
 load(
     "//bazel_tools:scala.bzl",
     "scala_source_jar",
     "scaladoc_jar",
 )
 load("@os_info//:os_info.bzl", "is_windows")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
 
 ledger_api_proto_source_root = "ledger-api/grpc-definitions"
 
-proto_library(
-    name = "protos",
+proto_jars(
+    name = "ledger_api_proto",
     srcs = glob(["**/*.proto"]),
-    strip_import_prefix = "/" + ledger_api_proto_source_root,
-    visibility = [
-        "//visibility:public",
-    ],
+    grpc = True,
+    maven_artifact_prefix = "ledger-api",
+    maven_artifact_scala_suffix = "scalapb",
+    maven_group = "com.daml",
     deps = [
         "@com_github_googleapis_googleapis//google/rpc:status_proto",
         "@com_google_protobuf//:any_proto",
@@ -33,40 +31,6 @@ proto_library(
     ],
 )
 
-pkg_tar(
-    name = "ledger-api-protos",
-    srcs = glob(["**/*.proto"]) + ["README.md"],
-    extension = "tar.gz",
-    strip_prefix = "./",
-    visibility = [
-        "//visibility:public",
-    ],
-)
-
-proto_gen(
-    name = "ledger-api-scalapb-sources",
-    srcs = [
-        ":protos",
-        "@com_github_googleapis_googleapis//google/rpc:code_proto",
-        "@com_github_googleapis_googleapis//google/rpc:status_proto",
-        "@com_github_grpc_grpc//src/proto/grpc/health/v1:health_proto_descriptor",
-    ],
-    plugin_exec = "//scala-protoc-plugins/scalapb:protoc-gen-scalapb",
-    plugin_name = "scalapb",
-    plugin_options = ["grpc"],
-    visibility = [
-        "//visibility:public",
-    ],
-    deps = [
-        "@com_google_protobuf//:any_proto",
-        "@com_google_protobuf//:descriptor_proto",
-        "@com_google_protobuf//:duration_proto",
-        "@com_google_protobuf//:empty_proto",
-        "@com_google_protobuf//:timestamp_proto",
-        "@com_google_protobuf//:wrappers_proto",
-    ],
-)
-
 google_protobuf_src = "external/com_google_protobuf/src"
 
 genrule(
@@ -272,62 +236,9 @@ da_haskell_library(
     visibility = ["//visibility:public"],
 )
 
-# ScalaPB Ledger API plus protobuf sources for customized compilation
-scala_library(
-    name = "ledger-api-scalapb",
-    srcs = [":ledger-api-scalapb-sources"],
-    resource_strip_prefix = ledger_api_proto_source_root,
-    resources = glob(["**/*.proto"]),
-    tags = [
-        "maven_coordinates=com.daml:ledger-api-scalapb:__VERSION__",
-    ],
-    unused_dependency_checker_mode = "error",
-    visibility = ["//visibility:public"],
-    exports = [
-        "@maven//:com_google_protobuf_protobuf_java",
-        "@maven//:com_thesamet_scalapb_lenses_2_12",
-        "@maven//:com_thesamet_scalapb_scalapb_runtime_2_12",
-        "@maven//:com_thesamet_scalapb_scalapb_runtime_grpc_2_12",
-        "@maven//:io_grpc_grpc_api",
-        "@maven//:io_grpc_grpc_core",
-        "@maven//:io_grpc_grpc_protobuf",
-        "@maven//:io_grpc_grpc_stub",
-    ],
-    deps = [
-        "@maven//:com_google_protobuf_protobuf_java",
-        "@maven//:com_thesamet_scalapb_lenses_2_12",
-        "@maven//:com_thesamet_scalapb_scalapb_runtime_2_12",
-        "@maven//:com_thesamet_scalapb_scalapb_runtime_grpc_2_12",
-        "@maven//:io_grpc_grpc_api",
-        "@maven//:io_grpc_grpc_core",
-        "@maven//:io_grpc_grpc_protobuf",
-        "@maven//:io_grpc_grpc_stub",
-    ],
-)
-
-pom_file(
-    name = "ledger-api-scalapb_pom",
-    target = ":ledger-api-scalapb",
-)
-
-# Create empty Scaladoc JAR for uploading to Maven Central
-scaladoc_jar(
-    name = "ledger-api-scalapb_scaladoc",
-    srcs = [],
-    tags = ["scaladoc"],
-    deps = [],
-) if is_windows == False else None
-
-# Provide protobuf sources
-scala_source_jar(
-    name = "ledger-api-scalapb_src",
-    srcs = glob(["**/*.proto"]),
-    strip_upto = ledger_api_proto_source_root,
-)
-
 proto_gen(
     name = "ledger-api-docs",
-    srcs = [":protos"],
+    srcs = [":ledger_api_proto"],
     plugin_exec = "@com_github_pseudomuto_protoc_gen_doc//cmd/protoc-gen-doc:protoc-gen-doc",
     plugin_name = "doc",
     plugin_options = [

ledger-api/testing-utils/BUILD.bazel

@@ -18,7 +18,7 @@ da_scala_library(
         "@maven//:org_slf4j_slf4j_api",
     ],
     deps = [
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "//ledger-api/rs-grpc-akka",
         "//ledger-api/rs-grpc-bridge",
         "//libs-scala/concurrent",

ledger/ledger-api-akka/BUILD.bazel

@@ -7,7 +7,7 @@ load("//bazel_tools:proto.bzl", "proto_gen")
 proto_gen(
     name = "ledger-api-akka-srcs",
     srcs = [
-        "//ledger-api/grpc-definitions:protos",
+        "//ledger-api/grpc-definitions:ledger_api_proto",
         "@com_github_grpc_grpc//src/proto/grpc/health/v1:health_proto_descriptor",
     ],
     plugin_exec = "//scala-protoc-plugins/scala-akka:protoc-gen-scala-akka",
@@ -31,7 +31,7 @@ da_scala_library(
         "//visibility:public",
     ],
     deps = [
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "//ledger-api/rs-grpc-akka",
         "//ledger-api/rs-grpc-bridge",
         "@maven//:com_typesafe_akka_akka_actor_2_12",

ledger/ledger-api-auth/BUILD.bazel

@@ -19,7 +19,7 @@ da_scala_library(
     runtime_deps = [],
     deps = [
         "//daml-lf/data",
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "//ledger-service/jwt",
         "//ledger/ledger-api-common",
         "//libs-scala/concurrent",

ledger/participant-state-index/BUILD.bazel

@@ -20,7 +20,7 @@ da_scala_library(
         "//daml-lf/data",
         "//daml-lf/language",
         "//daml-lf/transaction",
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "//ledger/ledger-api-domain",
         "//ledger/ledger-api-health",
         "//ledger/participant-state",

ledger/participant-state/kvutils/BUILD.bazel

@@ -166,7 +166,7 @@ proto_jars(
         "//daml-lf/transaction:value_proto",
         "//ledger/participant-state/protobuf:ledger_configuration_proto",
     ],
-    strip_import_prefix = "src/main/protobuf/",
+    strip_import_prefix = "src/main/protobuf",
     deps = [
         "@com_google_protobuf//:duration_proto",
         "@com_google_protobuf//:empty_proto",

ledger/test-common/BUILD.bazel

@@ -25,7 +25,7 @@ da_scala_library(
         "//visibility:public",
     ],
     deps = [
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "//libs-scala/timer-utils",
         "@maven//:ch_qos_logback_logback_classic",
         "@maven//:ch_qos_logback_logback_core",

libs-scala/grpc-utils/BUILD.bazel

@@ -14,7 +14,7 @@ da_scala_library(
         "//visibility:public",
     ],
     deps = [
-        "//ledger-api/grpc-definitions:ledger-api-scalapb",
+        "//ledger-api/grpc-definitions:ledger_api_proto_scala",
         "@maven//:io_grpc_grpc_api",
     ],
 )

release/artifacts.yaml

@@ -57,7 +57,9 @@
   type: jar-scala
 - target: //language-support/scala/codegen:codegen
   type: jar-scala
-- target: //ledger-api/grpc-definitions:ledger-api-scalapb
+- target: //ledger-api/grpc-definitions:ledger_api_proto_jar
+  type: jar-lib
+- target: //ledger-api/grpc-definitions:ledger_api_proto_scala
   type: jar-scala
 - target: //ledger-api/rs-grpc-akka:rs-grpc-akka
   type: jar-scala

release/util.bzl

@@ -139,11 +139,14 @@ protos_zip = rule(
         "daml_lf_tarballs": attr.label_list(
             allow_files = True,
             default = [
-                Label("//daml-lf/archive:daml_lf_{}_archive_proto_src.tar.gz".format(version))
+                Label("//daml-lf/archive:daml_lf_{}_archive_proto_tar.tar.gz".format(version))
                 for version in LF_VERSIONS
             ],
         ),
-        "ledger_api_tarball": attr.label(allow_single_file = True, default = Label("//ledger-api/grpc-definitions:ledger-api-protos.tar.gz")),
+        "ledger_api_tarball": attr.label(
+            allow_single_file = True,
+            default = Label("//ledger-api/grpc-definitions:ledger_api_proto_tar.tar.gz"),
+        ),
         "zipper": attr.label(
             default = Label("@bazel_tools//tools/zip:zipper"),
             cfg = "host",