diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f9734b8f55..b925dfd029 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,13 +90,6 @@ jobs: ~/.cache key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }} restore-keys: ${{ runner.os }}-sbt- - - name: Cache Local Maven Repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- # Build Artifacts - name: Enable Release Mode @@ -115,7 +108,7 @@ jobs: working-directory: repo run: | sleep 1 - sbt --no-colors runner/assembly + sbt --no-colors engine-runner/assembly - name: Build the Project Manager Uberjar working-directory: repo run: | @@ -198,8 +191,8 @@ jobs: - name: Build Base Java Extensions shell: bash run: | - cd std-bits - mvn --no-transfer-progress package + sleep 1 + sbt --no-colors std-bits/package # The way artifacts are uploaded currently does not preserve the # executable bits for Unix. However putting artifacts into a ZIP would diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 20801bc869..f22243e40c 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -98,13 +98,6 @@ jobs: ~/.cache key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }} restore-keys: ${{ runner.os }}-sbt- - - name: Cache Local Maven Repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- # Compile - name: Bootstrap Enso project @@ -125,8 +118,8 @@ jobs: - name: Build Base Java Extensions shell: bash run: | - cd std-bits - mvn --no-transfer-progress package + sleep 1 + sbt --no-colors std-bits/package - name: Build the Launcher run: | sleep 1 @@ -156,7 +149,7 @@ jobs: - name: Build the Runner & Runtime Uberjars run: | sleep 1 - sbt --no-colors runner/assembly + sbt --no-colors engine-runner/assembly - name: Build the Project Manager Uberjar run: | sleep 1 diff --git a/.prettierignore b/.prettierignore index 0b95e22384..de5e9ac76f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,4 +2,8 @@ target/ .github/PULL_REQUEST_TEMPLATE.md .github/ISSUE_TEMPLATE - +distribution/launcher/THIRD-PARTY +distribution/engine/THIRD-PARTY +tools/legal-review +distribution/std-lib/Base/third-party-licenses +distribution/std-lib/Base/THIRD-PARTY diff --git a/build.sbt b/build.sbt index 35a427cea6..81139496b3 100644 --- a/build.sbt +++ b/build.sbt @@ -1,17 +1,11 @@ import java.io.File -import com.typesafe.sbt.SbtLicenseReport.autoImportImpl.{ - licenseReportNotes, - licenseReportStyleRules -} -import scala.sys.process._ import org.enso.build.BenchTasks._ import org.enso.build.WithDebugCommand import sbt.Keys.{libraryDependencies, scalacOptions} import sbt.addCompilerPlugin import sbtassembly.AssemblyPlugin.defaultUniversalScript import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType} -import com.typesafe.sbt.license.DepModuleInfo // ============================================================================ // === Global Configuration =================================================== @@ -33,17 +27,44 @@ val ensoVersion = "0.1.0" // Note [Engine And Launcher Version] organization in ThisBuild := "org.enso" scalaVersion in ThisBuild := scalacVersion -val licenseSettings = Seq( - licenseConfigurations := Set("compile"), - licenseReportStyleRules := Some( - "table, th, td {border: 1px solid black;}" + +lazy val gatherLicenses = + taskKey[Unit]("Gathers licensing information for relevant dependencies") +gatherLicenses := GatherLicenses.run.value +GatherLicenses.distributions := Seq( + Distribution( + "launcher", + file("distribution/launcher/THIRD-PARTY"), + Distribution.sbtProjects(launcher) ), - licenseReportNotes := { - case DepModuleInfo(group, _, _) if group == "org.enso" => - "Internal library" - } -) -val coursierCache = file("~/.cache/coursier/v1") + Distribution( + "engine", + file("distribution/engine/THIRD-PARTY"), + Distribution.sbtProjects( + runtime, + `engine-runner`, + `project-manager`, + `language-server` + ) + ), + Distribution( + "std-lib-Base", + file("distribution/std-lib/Base/THIRD-PARTY"), + Distribution.sbtProjects(`std-bits`) + ) + ) +GatherLicenses.licenseConfigurations := Set("compile") +GatherLicenses.configurationRoot := file("tools/legal-review") + +lazy val openLegalReviewReport = + taskKey[Unit]( + "Gathers licensing information for relevant dependencies and opens the " + + "report in review mode in the browser." + ) +openLegalReviewReport := { + gatherLicenses.value + GatherLicenses.runReportServer() +} Global / onChangedBuildSource := ReloadOnSourceChanges @@ -142,7 +163,7 @@ lazy val enso = (project in file(".")) `logging-service`, `akka-native`, `version-output`, - runner, + `engine-runner`, runtime, searcher, launcher, @@ -297,6 +318,10 @@ val splainOptions = Seq( "-P:splain:tree:true" ) +// === std-lib ================================================================ + +val icuVersion = "67.1" + // === ZIO ==================================================================== val zioVersion = "1.0.1" @@ -347,7 +372,6 @@ lazy val logger = crossProject(JVMPlatform, JSPlatform) libraryDependencies ++= scalaCompiler ) .jsSettings(jsSettings) - .settings(licenseSettings) lazy val flexer = crossProject(JVMPlatform, JSPlatform) .withoutSuffixFor(JVMPlatform) @@ -364,7 +388,6 @@ lazy val flexer = crossProject(JVMPlatform, JSPlatform) ) ) .jsSettings(jsSettings) - .settings(licenseSettings) lazy val `syntax-definition` = crossProject(JVMPlatform, JSPlatform) .withoutSuffixFor(JVMPlatform) @@ -383,7 +406,6 @@ lazy val `syntax-definition` = crossProject(JVMPlatform, JSPlatform) ) ) .jsSettings(jsSettings) - .settings(licenseSettings) lazy val syntax = crossProject(JVMPlatform, JSPlatform) .withoutSuffixFor(JVMPlatform) @@ -433,7 +455,6 @@ lazy val syntax = crossProject(JVMPlatform, JSPlatform) testFrameworks := List(new TestFramework("org.scalatest.tools.Framework")), Compile / fullOptJS / artifactPath := file("target/scala-parser.js") ) - .settings(licenseSettings) lazy val `parser-service` = (project in file("lib/scala/parser-service")) .dependsOn(syntax.jvm) @@ -441,7 +462,6 @@ lazy val `parser-service` = (project in file("lib/scala/parser-service")) libraryDependencies ++= akka, mainClass := Some("org.enso.ParserServiceMain") ) - .settings(licenseSettings) lazy val `text-buffer` = project .in(file("lib/scala/text-buffer")) @@ -453,7 +473,6 @@ lazy val `text-buffer` = project "org.scalacheck" %% "scalacheck" % scalacheckVersion % Test ) ) - .settings(licenseSettings) lazy val graph = (project in file("lib/scala/graph/")) .dependsOn(logger.jvm) @@ -481,7 +500,6 @@ lazy val graph = (project in file("lib/scala/graph/")) ), scalacOptions ++= splainOptions ) - .settings(licenseSettings) lazy val pkg = (project in file("lib/scala/pkg")) .settings( @@ -494,7 +512,6 @@ lazy val pkg = (project in file("lib/scala/pkg")) "commons-io" % "commons-io" % commonsIoVersion ) ) - .settings(licenseSettings) lazy val `akka-native` = project .in(file("lib/scala/akka-native")) @@ -507,7 +524,6 @@ lazy val `akka-native` = project // Note [Native Image Workaround for GraalVM 20.2] libraryDependencies += "org.graalvm.nativeimage" % "svm" % graalVersion % "provided" ) - .settings(licenseSettings) lazy val `logging-service` = project .in(file("lib/scala/logging-service")) @@ -531,7 +547,6 @@ lazy val `logging-service` = project else (Compile / unmanagedSourceDirectories) += (Compile / sourceDirectory).value / "java-unix" ) - .settings(licenseSettings) .dependsOn(`akka-native`) lazy val cli = project @@ -545,7 +560,6 @@ lazy val cli = project ), parallelExecution in Test := false ) - .settings(licenseSettings) lazy val `version-output` = (project in file("lib/scala/version-output")) .settings( @@ -564,7 +578,6 @@ lazy val `version-output` = (project in file("lib/scala/version-output")) ) }.taskValue ) - .settings(licenseSettings) lazy val `project-manager` = (project in file("lib/scala/project-manager")) .settings( @@ -629,7 +642,6 @@ lazy val `project-manager` = (project in file("lib/scala/project-manager")) .dependsOn(runtime / assembly) .value ) - .settings(licenseSettings) .dependsOn(`version-output`) .dependsOn(pkg) .dependsOn(`language-server`) @@ -670,7 +682,6 @@ lazy val `json-rpc-server` = project "org.scalatest" %% "scalatest" % scalatestVersion % Test ) ) - .settings(licenseSettings) lazy val `json-rpc-server-test` = project .in(file("lib/scala/json-rpc-server-test")) @@ -683,7 +694,6 @@ lazy val `json-rpc-server-test` = project "org.scalatest" %% "scalatest" % scalatestVersion ) ) - .settings(licenseSettings) .dependsOn(`json-rpc-server`) lazy val testkit = project @@ -720,7 +730,6 @@ lazy val `core-definition` = (project in file("lib/scala/core-definition")) ), scalacOptions ++= splainOptions ) - .settings(licenseSettings) .dependsOn(graph) .dependsOn(syntax.jvm) @@ -741,7 +750,6 @@ lazy val searcher = project fork in Benchmark := true ) .dependsOn(testkit % Test) - .settings(licenseSettings) .dependsOn(`polyglot-api`) lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl")) @@ -749,7 +757,6 @@ lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl")) version := "0.1", libraryDependencies += "com.google.auto.service" % "auto-service" % "1.0-rc7" ) - .settings(licenseSettings) // ============================================================================ // === Sub-Projects =========================================================== @@ -794,7 +801,6 @@ lazy val `polyglot-api` = project GenerateFlatbuffers.flatcVersion := flatbuffersVersion, sourceGenerators in Compile += GenerateFlatbuffers.task ) - .settings(licenseSettings) .dependsOn(pkg) .dependsOn(`text-buffer`) @@ -830,7 +836,6 @@ lazy val `language-server` = (project in file("engine/language-server")) new TestFramework("org.scalameter.ScalaMeterFramework") ) ) - .settings(licenseSettings) .dependsOn(`polyglot-api`) .dependsOn(`json-rpc-server`) .dependsOn(`json-rpc-server-test` % Test) @@ -935,7 +940,9 @@ lazy val runtime = (project in file("engine/runtime")) .value ) .settings( - (Test / compile) := (Test / compile).dependsOn(StdBits.preparePackage).value + (Test / compile) := (Test / compile) + .dependsOn(`std-bits` / Compile / packageBin) + .value ) .settings( logBuffered := false, @@ -966,7 +973,6 @@ lazy val runtime = (project in file("engine/runtime")) case _ => MergeStrategy.first } ) - .settings(licenseSettings) .dependsOn(pkg) .dependsOn(`interpreter-dsl`) .dependsOn(syntax.jvm) @@ -990,7 +996,7 @@ lazy val runtime = (project in file("engine/runtime")) * recompilation but still convince the IDE that it is a .jar dependency. */ -lazy val runner = project +lazy val `engine-runner` = project .in(file("engine/runner")) .settings( javaOptions ++= { @@ -1056,7 +1062,6 @@ lazy val runner = project .dependsOn(runtime / assembly) .value ) - .settings(licenseSettings) .dependsOn(`version-output`) .dependsOn(pkg) .dependsOn(`language-server`) @@ -1113,12 +1118,36 @@ lazy val launcher = project .value, parallelExecution in Test := false ) - .settings(licenseSettings) .dependsOn(cli) .dependsOn(`version-output`) .dependsOn(pkg) .dependsOn(`logging-service`) +val `std-lib-root` = file("distribution/std-lib/") +val `std-lib-polyglot-root` = `std-lib-root` / "Base" / "polyglot" / "java" + +lazy val `std-bits` = project + .in(file("std-bits")) + .settings( + autoScalaLibrary := false, + Compile / packageBin / artifactPath := + `std-lib-polyglot-root` / "std-bits.jar", + libraryDependencies ++= Seq( + "com.ibm.icu" % "icu4j" % icuVersion + ), + Compile / packageBin := Def.task { + val result = (Compile / packageBin).value + StdBits + .copyDependencies( + `std-lib-polyglot-root`, + "std-bits.jar", + ignoreScalaLibrary = true + ) + .value + result + }.value + ) + /* Note [HTTPS in the Launcher] * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * The launcher uses Apache HttpClient for making web requests. It does not use diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 6adafb3e2d..ac96a722df 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -285,7 +285,7 @@ In order to build a fat jar with the CLI component, run the `assembly` task inside the `runner` subproject: ```bash -sbt "runner/assembly" +sbt "engine-runner/assembly" ``` This will produce an executable `runner.jar` fat jar and a `runtime.jar` fat jar diff --git a/docs/distribution/README.md b/docs/distribution/README.md index 91cc28e18d..214e3621f0 100644 --- a/docs/distribution/README.md +++ b/docs/distribution/README.md @@ -21,7 +21,7 @@ dependencies, and Enso projects for use by our users. tool for launching various components and managing Enso versions. - [**Launcher CLI:**](./launcher-cli.md) Explanation of the command-line interface of the launcher. -- [**Licences:**](licences.md) Information on gathering license information of +- [**Licenses:**](licenses.md) Information on gathering license information of dependencies included in the distribution. - [**Fallback Launcher Release Infrastructure:**](fallback-launcher-release-infrastructure.md) Explanation of the fallback infrastructure that can be enabled to keep diff --git a/docs/distribution/licences.md b/docs/distribution/licences.md deleted file mode 100644 index a391ba41ed..0000000000 --- a/docs/distribution/licences.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -layout: developer-doc -title: Licenses -category: distribution -tags: [distribution, licenses] -order: 6 ---- - -# Licenses - -When distributing Enso, we include code from many dependencies that are used -within it. We need to ensure that we comply with the licenses of the -dependencies that we distribute with Enso. - - - -- [Gathering Used Dependencies](#gathering-used-dependencies) - - [SBT](#sbt) - - [Rust](#rust) -- [Preparing the Distribution](#preparing-the-distribution) - - [Launcher](#launcher) - - [Engine Components](#engine-components) - - - -## Gathering Used Dependencies - -As a first step, we need to gather a list of which dependencies are used in the -distributed artifacts. - -### SBT - -We can use the plugin `sbt-license-report` to gather a list of used dependencies -and their licences. To use it, run `enso/dumpLicenseReport` in the SBT shell. -This will gather dependency information for all the subprojects. For each -subproject, `license-reports` directory will be created in its `target` -directory, containing the reports in multiple formats. - -It is important to note that the report for the root `enso` project will not -contain dependencies of the subprojects. Instead, to list the relevant -dependencies, you need to look at the reports for the subprojects that are -actually built into distributable artifacts. For now these are: `runtime`, -`runner`, `project-manager` and `launcher`. - -#### `sbt-license-report` Configuration - -Settings for the plugin are defined in the `licenseSettings` variable in -[`build.sbt`](../../build.sbt). The settings have to be applied to each project -by adding `.settings(licenseSettings)` to the project definition (defining these -settings at the top level or in the `ThisBuild` configuration yielded no -effects, so this workaround is required). - -The most relevant setting is `licenseConfigurations` which defines which `ivy` -configurations are considered to search for dependencies. - -Currently it is set to only consider `compile` dependencies, as dependencies for -`provided`, `test` or `benchmark` are not distributed. - -### Rust - -We do not distribute any Rust-based artifacts in this repository. - -> The actionables for this section are: -> -> - When the parser is rewritten to Rust and is distributed within the -> artifacts, this section should be revisited to describe a scheme of -> gathering dependencies used in the Rust projects. - -## Preparing the Distribution - -When a new dependency is added, its transitive dependencies have to be analysed -as described in the previous section. Various action has to be taken depending -on the particular licences. - -For most dependencies under the Apache, MIT or BSD licences, a copy of the -licence has to be included within the distribution and if the dependency -includes a `NOTICE` file, the contents of this file have to be reproduced within -the distribution by adding them to an aggregate `NOTICE` file. To find these -`NOTICE` files it may be necessary to walk through the `JAR`s containing the -dependencies or visit project websites of each dependency. - -### Launcher - -As the launcher is distributed as a native binary executable, the licences and -notices have to be included separately. - -The licences should be put in the -[`distribution/launcher/components-licences`](../../distribution/launcher/components-licences) -directory. The notices should be gathered in -[`distribution/launcher/NOTICE`](../../distribution/launcher/NOTICE). These -files are included by the CI build within the built artifacts. - -### Engine Components - -#### Standard Library - -The third-party licenses for Java extensions of the standard library are -gathered in the `third-party-licenses` directory in the `Base` library. The -gathering process is automatic, triggered by the `package` goal of the -associated Maven configuration file. - -> The actionables for this section are: -> -> - The engine components as distributed as a JAR archive that everyone can -> open. At least some of the dependencies contain their licences within that -> archive. It should be checked if this is enough or if the licences and -> notices should be replicated in the distributed packages anyway. - -### Bundles - -Beside the launcher and engine components, the distributed bundles also contain -a distribution of GraalVM CE. This distribution however contains its own licence -and notices within itself, so no further action should be necessary in that -regard. diff --git a/docs/distribution/licenses.md b/docs/distribution/licenses.md new file mode 100644 index 0000000000..6031bea8d9 --- /dev/null +++ b/docs/distribution/licenses.md @@ -0,0 +1,254 @@ +--- +layout: developer-doc +title: Licenses +category: distribution +tags: [distribution, licenses] +order: 6 +--- + +# Licenses + +When distributing Enso, we include code from many dependencies that are used +within it. We need to ensure that we comply with the licenses of the +dependencies that we distribute with Enso. + + + +- [Gathering Used Dependencies](#gathering-used-dependencies) + - [SBT](#sbt) + - [Rust](#rust) +- [Preparing the Distribution](#preparing-the-distribution) + - [Review](#review) +- [Standard Library](#standard-library) +- [Bundles](#bundles) + + + +## Gathering Used Dependencies + +As a first step, we need to gather a list of which dependencies are used in the +distributed artifacts. + +### SBT + +We use a `GatherLicenses` task that uses the `sbt-license-report` and other +sources to gather copyright information related to the used dependencies. + +To configure the task, `GatherLicenses.distributions` should be set with +sequence of distributions. Each distribution describes one component that is +distributed separately and should include all references to all projects that +are included as part of its distribution. Currently, we have the `launcher` +distribution that consists of one `launcher` component and the `engine` +distribution which includes `runtime`, `engine-runner` and `project-manager`. + +Another relevant setting is `GatherLicenses.licenseConfigurations` which defines +which `ivy` configurations are considered to search for dependencies. Currently +it is set to only consider `compile` dependencies, as dependencies for +`provided`, `test` or `benchmark` are not distributed and there are no +`assembly`-specific dependencies. + +`GatherLicenses.configurationRoot` specifies where the review tool will look for +the files specifying review state and `GatherLicenses.distributionRoot` +specifies where the final notice packages should be generated. + +To run the automated license gathering task, run `enso/gatherLicenses` in SBT. +This will create a report and packages which are described in the +[next section](#preparing-the-distribution). + +### Rust + +We do not distribute any Rust-based artifacts in this repository. + +> The actionables for this section are: +> +> - When the parser is rewritten to Rust and is distributed within the +> artifacts, this section should be revisited to describe a scheme of +> gathering dependencies used in the Rust projects. +> - It would be good to re-use the SBT task as much as possible, possibly by +> creating a frontend for it using `cargo-license`. + +## Preparing the Distribution + +When a new dependency is added, the `enso/gatherLicenses` should be re-run to +generate the updated report. + +The report can be opened in review mode by launching a server located in +`tools/legal-review-helper` by running `npm start` in that directory. +Alternatively, `enso/openLegalReviewReport` can be used instead to automatically +open the report in review-mode after generating it (but it requires `npm` to be +visible on the system PATH in SBT). + +The report will show what license the dependencies use and include any copyright +notices and files found within each dependency. + +Each copyright notice and file should be reviewed to decide if it should be kept +or ignored. If a notice is automatically detected in a wrong way, it should be +ignored and a fixed one should be added manually. The review process is +described in detail in the [next section](#review). + +Each new type of license has to be reviewed to ensure that it is compatible with +our distribution and usage scheme. Licenses are reviewed per-distribution, as +for example the binary distribution of the launcher may impose different +requirements than distribution of the engine as JARs. + +After the review is done, the `enso/gatherLicenses` should be re-run again to +generate the updated packages that are included in the distribution. Before a PR +is merged, it should be ensure that there are no warnings in the generation. The +packages are located in separate subdirectories of the `distribution` directory +for each artifact. + +> This may possibly be automated in the next PR that includes the current legal +> review. The generating task could write a file indicating if there were any +> warnings and containing a hash of the dependency list. The build job on CI +> could then run a task `verifyLegalReview` which would check if there are no +> warnings and if the review is up to date (by comparing the hash of the +> dependency list at the time of the review with the current one). + +> TODO [RW] currently the auto-generated notice packages are not included in the +> built artifacts. That is because the legal review settings have not yet been +> prepared. Once that is done, the CI should be modified accordingly. This will +> be updated in the next PR. + +### Review + +The review can be performed manually by modifying the settings inside of the +`tools/legal-review` directory or it can be partially automated. + +#### Review Process + +1. Open the review in edit mode using the helper script. + - You can type `enso / openLegalReviewReport` if you have `npm` in your PATH + as visible from SBT. + - Or you can just run `npm start` (and `npm install` if needed) in the + `tools/legal-review-helper` directory. +2. For each package listed in the review for a given distribution: + 1. Review licenses + - Make sure that the component's license is accepted - that we know that + its license type is compatible with our distribution scheme. + - When a license is accepted, a file should be added in the + `reviewed-licenses` directory, with name as indicated in the report. The + file should contain a single line that is the path (relative to + repository root) to the default license file for that license type which + should be included in the distribution. + - The license may have been already accepted if it is the same license + as earlier dependencies for the same artifact. + - Check if any license-like files have been automatically found in the + attached files. If an attached file contains (case-insensitive) + 'license' or 'licence' in its name, the review tool will compare it with + the default license file. + - To trigger this comparison, the license must have been already + reviewed when the report was being generated, so you may consider + re-running the report after reviewing the license types to get this + information. + - If an attached file is exactly the same as the license file, it can be + safely ignored. + - If an attached file differs from the default license file, it should + be carefully checked. + - Most of the time, that file should be marked as kept and the default + license ignored. + - To ignore the default license, create an empty file `custom-license` + inside the directory belonging to the relevant package. + 2. Review which files to include + - You can click on a filename to display its contents. + - We want to include any NOTICE files that contain copyright notices or + credits. + - False-positives (unrelated files) or duplicates may be ignored. + 3. Review copyright notices + - You may click on a copyright line to display context (surrounding text) + and in which files it was found. + - We want to include most of the notices, as it is better to include + duplicates rather than skip something important. + - But we need to ignore false-positives, for example code that contains + the word 'copyright' in it and was falsely classified. + - You may click 'Keep' to add the displayed copyright line to the + copyright notice or if there is exactly one context associated with the + line, you can click 'Keep as context' to add this whole context to the + notice. + 4. Add missing information + - You can manually add additional copyright notices by adding them to a + file `copyright-add` inside the directory belonging to the relevant + package. + - You can manually add additional files by adding them into a subdirectory + called `files-add` located in the directory belonging to the relevant + package. +3. Add any additional information: + - You can add additional files by adding them into a subdirectory called + `files-add` in the root directory of distribution configuration. + - You can create a custom notice header that will replace the default one, by + creating a file called `notice-header`. +4. After you are done, re-run `enso/gatherLicenses` to generate the updated + packages. + - Ensure that there are no more warnings, and if there are any go back to fix + the issues. + +The updates performed using the web script are remembered locally, so they will +not show up after the refresh. If you ever need to open the edit mode after +closing its window, you should re-generate the report using +`enso/gatherLicenses` or just open it using `enso/openLegalReviewReport` which +will refresh it automatically. + +#### Review Configuration + +The review state is driven by configuration files located in +`tools/legal-review`. This directory contains separate subdirectories for each +artifact. + +The subdirectory for each artifact may contain the following entries: + +- `notice-header` - contains the header that will start the main generated + NOTICE +- `files-add` - directory that may contain additional files that should be added + to the notice package +- `reviewed-licenses` - directory that may contain files for reviewed licenses; + the files should be named with the normalized license name and they should + contain a path to that license's file (the path should be relative to the + repository root) +- and for each dependency, a subdirectory named as its `packageName` with + following entries: + - `files-add` - directory that may contain additional files that should be + added to the subdirectory for this package + - `files-keep` - a file containing names of files found in the package sources + that should be included in the package + - `files-ignore` - a file containing names of files found in the package + sources that should not be included + - `custom-license` - a file that indicates that the dependency should not + point to the default license, but it should contain a custom one within its + files + - `copyright-keep` - copyright lines that should be included in the notice + summary for the package + - `copyright-keep-context` - copyright lines that should be included + (alongside with their context) in the notice summary for the package + - `copyright-ignore` - copyright lines that should not be included in the + notice summary for the package + - `copyright-add` - a single file whose contents will be added to the notice + summary for the package + +Manually adding files and copyright, modifying the notice header and marking +licenses as reviewed has to be done manually. But deciding if a file or +copyright notice should be kept or ignored can be done much quicker using the +GUI launched by `enso/openLegalReviewReport`. The GUI is a very simple one - it +assumes that the report is up to date and uses the server to modify the +configuration. The configuration changes are not refreshed automatically - +instead if the webpage is refreshed after modifications it may contain stale +information - to get up-to-date information, `enso/openLegalReviewReport` or +`enso/gatherLicenses` has to be re-run. + +## Standard Library + +The dependencies of standard library are built using Maven, so they have to be +handled separately. Currently there are not many of them so this is handled +manually. If that becomes a problem, they could be attached to the frontend of +the `GatherLicenses` task. + +The third-party licenses for Java extensions of the standard library are +gathered in the `third-party-licenses` directory in the `Base` library. The +gathering process is partially-automatic, triggered by the `package` goal of the +associated Maven configuration file. However when another dependency is added to +the standard library, its licenses should be reviewed before merging the PR. + +## Bundles + +Beside the launcher and engine components, the distributed bundles also contain +a distribution of GraalVM CE. This distribution however contains its own licence +and notices within itself, so no further action should be necessary in that +regard. diff --git a/project/Distribution.scala b/project/Distribution.scala new file mode 100644 index 0000000000..f4330289d0 --- /dev/null +++ b/project/Distribution.scala @@ -0,0 +1,72 @@ +import com.typesafe.sbt.SbtLicenseReport.autoImportImpl.{ + licenseOverrides, + licenseSelection +} +import com.typesafe.sbt.license +import sbt.Keys.{ivyModule, streams, update, updateClassifiers} +import sbt.{File, Project} +import src.main.scala.licenses.{ + DistributionDescription, + SBTDistributionComponent +} + +import scala.language.experimental.macros +import scala.reflect.macros.blackbox + +object Distribution { + + /** + * Creates a [[DistributionDescription]]. + */ + def apply( + name: String, + packageDestination: File, + sbtComponents: Seq[SBTDistributionComponent] + ): DistributionDescription = + DistributionDescription(name, packageDestination, sbtComponents) + + /** + * A macro that creates [[SBTDistributionComponent]] descriptions from a list + * of project references. + */ + def sbtProjects(projects: Project*): Seq[SBTDistributionComponent] = + macro sbtProjectsImpl + + /** + * Implementation of the [[sbtProjects]] macro. + * + * It triggers execution of the tasks that are used to get information from + * SBT on each project. + */ + def sbtProjectsImpl(c: blackbox.Context)( + projects: c.Expr[Project]* + ): c.Expr[Seq[SBTDistributionComponent]] = { + import c.universe._ + val gathered = { + projects.map(p => + reify { + val deliberatelyTriggerAndIgnore = (p.splice / update).value + + val ivyMod = (p.splice / ivyModule).value + val overrides = (p.splice / licenseOverrides).value.lift + val report = license.LicenseReport.makeReport( + ivyMod, + GatherLicenses.licenseConfigurations.value, + (p.splice / licenseSelection).value, + overrides, + (p.splice / streams).value.log + ) + SBTDistributionComponent( + p.splice.id, + report, + ivyMod, + (p.splice / updateClassifiers).value + ) + } + ) + } + c.Expr[Seq[SBTDistributionComponent]]( + Apply(reify(Seq).tree, gathered.map(_.tree).toList) + ) + } +} diff --git a/project/GatherLicenses.scala b/project/GatherLicenses.scala new file mode 100644 index 0000000000..064a657249 --- /dev/null +++ b/project/GatherLicenses.scala @@ -0,0 +1,136 @@ +import sbt.Keys._ +import sbt._ +import src.main.scala.licenses.backend.{ + CombinedBackend, + GatherCopyrights, + GatherNotices, + GithubHeuristic +} +import src.main.scala.licenses.frontend.SbtLicenses +import src.main.scala.licenses.report.{ + PackageNotices, + Report, + Review, + WithWarnings +} +import src.main.scala.licenses.{DependencySummary, DistributionDescription} + +import scala.sys.process._ + +/** + * The task and configuration for automatically gathering license information. + */ +object GatherLicenses { + val distributions = taskKey[Seq[DistributionDescription]]( + "Defines descriptions of distributions." + ) + val configurationRoot = settingKey[File]("Path to review configuration.") + val licenseConfigurations = + settingKey[Set[String]]("The ivy configurations we consider in the review.") + + /** + * The task that performs the whole license gathering process. + */ + lazy val run = Def.task { + val log = state.value.log + val targetRoot = target.value + log.info( + "Gathering license files and copyright notices. " + + "This task may take a long time." + ) + + val configRoot = configurationRoot.value + + val reports = distributions.value.map { distribution => + log.info(s"Processing the ${distribution.artifactName} distribution") + val projectNames = distribution.sbtComponents.map(_.name) + log.info( + s"It consists of the following sbt project roots:" + + s" ${projectNames.mkString(", ")}" + ) + val (sbtInfo, sbtWarnings) = + SbtLicenses.analyze(distribution.sbtComponents, log) + sbtWarnings.foreach(log.warn(_)) + + val allInfo = sbtInfo // TODO [RW] add Rust frontend result here (#1187) + + log.info(s"${allInfo.size} unique dependencies discovered") + val defaultBackend = CombinedBackend(GatherNotices, GatherCopyrights) + + val processed = allInfo.map { dependency => + log.debug( + s"Processing ${dependency.moduleInfo} (${dependency.license}) -> " + + s"${dependency.url}" + ) + val defaultAttachments = defaultBackend.run(dependency.sources) + val attachments = + if (defaultAttachments.nonEmpty) defaultAttachments + else GithubHeuristic(dependency, log).run() + (dependency, attachments) + } + + val summary = DependencySummary(processed) + val WithWarnings(processedSummary, summaryWarnings) = + Review(configRoot / distribution.artifactName, summary).run() + val allWarnings = sbtWarnings ++ summaryWarnings + val reportDestination = + targetRoot / s"${distribution.artifactName}-report.html" + + sbtWarnings.foreach(log.warn(_)) + if (summaryWarnings.size > 10) + log.warn( + s"There are too many warnings (${summaryWarnings.size}) to " + + s"display. Please inspect the generated report." + ) + else allWarnings.foreach(log.warn(_)) + + Report.writeHTML( + distribution, + processedSummary, + allWarnings, + reportDestination + ) + log.info( + s"Written the report for ${distribution.artifactName} to " + + s"`${reportDestination}`." + ) + val packagePath = distribution.packageDestination + PackageNotices.create(distribution, processedSummary, packagePath) + log.info(s"Re-generated distribution notices at `$packagePath`.") + if (summaryWarnings.nonEmpty) { + // TODO [RW] A separate task should be added to verify that the package + // has been built without warnings that would report these warnings as + // errors for the final distribution; this task should be run on CI to + // verify the report is correct and up-to-date + log.warn( + "The distribution notices were regenerated, but there are " + + "not-reviewed issues within the report. The notices are probably " + + "incomplete." + ) + } + + (distribution, processedSummary) + } + + log.warn( + "Finished gathering license information. " + + "This is an automated process, make sure that its output is reviewed " + + "by a human to ensure that all licensing requirements are met." + ) + + reports + } + + /** + * Launches a server that allows to easily review the generated report. + * + * Requires `npm` to be on the system PATH. + */ + def runReportServer(): Unit = { + Seq("npm", "install").! + Process(Seq("npm", "start"), file("tools/legal-review-helper")) + .run(connectInput = true) + .exitValue() + } + +} diff --git a/project/StdBits.scala b/project/StdBits.scala index 1ca7fdfb4a..151b9d2847 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -1,32 +1,65 @@ -import java.io.IOException - -import sbt.Def - -import scala.sys.process._ +import sbt.Keys._ +import sbt._ +import sbt.io.IO +import sbt.librarymanagement.{ConfigurationFilter, DependencyFilter} object StdBits { - def preparePackage = - Def.task { - val cmd = Seq("mvn", "package", "-f", "std-bits") - val exitCode = - try { - if (Platform.isWindows) { - (Seq("cmd", "/c") ++ cmd).! - } else { - cmd.! - } - } catch { - case e @ (_: RuntimeException | _: IOException) => - throw new RuntimeException( - "Cannot run `mvn`, " + - "make sure that it is installed and present on your PATH.", - e - ) - } - if (exitCode != 0) { - throw new RuntimeException("std-bits build failed.") + /** + * Discovers dependencies of a project and copies them into the destination + * directory. + * + * @param destination location where to put the dependencies + * @param baseJarName name of the JAR generated by the `package` task; + * unexpected (old) files are removed, so this task needs + * to know this file's name to avoid removing it + * @param ignoreScalaLibrary whether to ignore Scala dependencies that are + * added by default be SBT and are not relevant in + * pure-Java projects + */ + def copyDependencies( + destination: File, + baseJarName: String, + ignoreScalaLibrary: Boolean + ): Def.Initialize[Task[Unit]] = + Def.task { + val libraryUpdates = (Compile / update).value + val log = streams.value.log + + val ignoredConfigurations: NameFilter = + if (ignoreScalaLibrary) + new ExactFilter(Configurations.ScalaTool.name) + else NothingFilter + val filter: ConfigurationFilter = + DependencyFilter.configurationFilter(-ignoredConfigurations) + val relevantFiles = libraryUpdates.select(filter) + + val dependencyStore = + streams.value.cacheStoreFactory.make("std-bits-dependencies") + Tracked.diffInputs(dependencyStore, FileInfo.hash)(relevantFiles.toSet) { + report => + val expectedFileNames = report.checked.map(_.getName) + baseJarName + for (existing <- IO.listFiles(destination)) { + if (!expectedFileNames.contains(existing.getName)) { + log.info( + s"Removing outdated std-bits dependency ${existing.getName}." + ) + IO.delete(existing) + } + } + for (changed <- report.modified -- report.removed) { + log.info( + s"Updating changed std-bits dependency ${changed.getName}." + ) + IO.copyFile(changed, destination / changed.getName) + } + for (file <- report.checked) { + val dest = destination / file.getName + if (!dest.exists()) { + log.info(s"Adding missing std-bits dependency ${file.getName}.") + IO.copyFile(file, dest) + } + } } } - } diff --git a/project/src/main/scala/licenses/Attachment.scala b/project/src/main/scala/licenses/Attachment.scala new file mode 100644 index 0000000000..8a2115f3fd --- /dev/null +++ b/project/src/main/scala/licenses/Attachment.scala @@ -0,0 +1,149 @@ +package src.main.scala.licenses + +import java.nio.file.Path + +import sbt.IO + +/** + * Represents a licensing information related to a dependency. + */ +sealed trait Attachment + +/** + * Represents a file attached to the dependency's sources. + * + * This may be a license file, a copyright notice, a credits file etc. + */ +case class AttachedFile( + path: Path, + content: String, + origin: Option[String] = None +) extends Attachment { + + /** + * @inheritdoc + */ + override def toString: String = { + val originRepr = origin.map(o => s", origin = $o") + s"AttachedFile($path, ...$originRepr)" + } + + /** + * Name of the file. + */ + def fileName: String = path.getFileName.toString +} + +/** + * Represents a copyright mention extracted from a file. + * + * The copyright mention may come from comments in the source code, for + * example. + * + * Equal copyright mentions are merged, so a single mention may contain + * multiple contexts and origins if it was a result of merging from different + * mentions. + * + * @param content cleaned content of the line that was extracted + * @param contexts contexts (surrounding lines) that are associated with the mention + * @param origins paths to the files that the mention comes from + */ +case class CopyrightMention( + content: String, + contexts: Seq[String], + origins: Seq[Path] +) extends Attachment { + override def toString: String = s"CopyrightMention('$content')" +} + +object CopyrightMention { + + /** + * Transforms the sequence of copyright mentions by merging ones that have + * equal content. + */ + def mergeByContent(copyrights: Seq[CopyrightMention]): Seq[CopyrightMention] = + copyrights + .groupBy(c => c.content) + .map({ case (_, equal) => mergeEqual(equal) }) + .toSeq + + /** + * Transforms the sequence of copyright mentions by merging ones that have + * equal context. + * + * This may be useful as a single comment block may contain multiple mentions + * of the word copyright, so multiple mentions coming from the same context + * would be redundant. + * + * It only allows to merge mentions that have only one context, as a + * heuristic to avoid too aggressive merging. + */ + def mergeByContext( + copyrights: Seq[CopyrightMention] + ): Seq[CopyrightMention] = { + val (eligible, rest) = copyrights.partition(_.contexts.size == 1) + val merged = eligible.groupBy(c => c.contexts.head).map { + case (_, equal) => + val ref = findBestRepresentative(equal) + ref.copy(origins = equal.flatMap(_.origins).distinct) + } + (merged ++ rest).toSeq + } + + /** + * Returns the best representative to use as content of the merged set of + * copyrights that share a context. + * + * Usually, mentions that start with the word 'copyright' are most + * interesting, if the word appears somewhere in the middle it is usually + * just a continuation of a sentence. So we choose the instance starting with + * the word 'copyright' if such instance exists. Otherwise an arbitrary + * instance is selected. + */ + private def findBestRepresentative( + copyrights: Seq[CopyrightMention] + ): CopyrightMention = { + copyrights + .find(_.content.stripLeading.toLowerCase.startsWith("copyright")) + .getOrElse(copyrights.head) + } + + /** + * Merges multiple copyright mentions that have equal content. + * + * Their contexts and origins are concatenated (ignoring duplicates). + */ + def mergeEqual(copyrights: Seq[CopyrightMention]): CopyrightMention = { + val ref = copyrights.headOption.getOrElse( + throw new IllegalArgumentException("Copyrights must not be empty") + ) + val ok = copyrights.forall(_.content == ref.content) + if (!ok) { + throw new IllegalArgumentException("Copyrights must be equal to merge") + } + CopyrightMention( + ref.content, + copyrights.flatMap(_.contexts).distinct, + copyrights.flatMap(_.origins).distinct + ) + } +} + +object AttachedFile { + + /** + * Reads a file at the given path into an [[AttachedFile]]. + * + * If `relativizeTo` is provided, the path included in the resulting + * [[AttachedFile]] is relative to the one provided. + */ + def read(path: Path, relativizeTo: Option[Path]): AttachedFile = { + val content = IO.read(path.toFile) + val actualPath = relativizeTo match { + case Some(root) => root.relativize(path) + case None => path + } + AttachedFile(actualPath, content) + } +} diff --git a/project/src/main/scala/licenses/DependencyInformation.scala b/project/src/main/scala/licenses/DependencyInformation.scala new file mode 100644 index 0000000000..05861f3b0f --- /dev/null +++ b/project/src/main/scala/licenses/DependencyInformation.scala @@ -0,0 +1,48 @@ +package src.main.scala.licenses + +import java.nio.file.Path + +import com.typesafe.sbt.license.{DepModuleInfo, LicenseInfo} +import src.main.scala.licenses.report.Review + +/** + * Defines a way to access sources of a dependency. + * + * This may involve various actions, such as downloading or extracting files + * into a temporary directory. + */ +trait SourceAccess { + + /** + * Calls the callback with a path to the available sources. + * + * The provided path is only valid during the callbacks invocation and is + * considered invalid as soon as that function completes. + */ + def access[R](withSources: Path => R): R +} + +/** + * Information about a single dependency. + * + * @param moduleInfo description of the module + * @param license information about the module's discovered license + * @param sources access to module's sources + * @param url project URL (if available) + */ +case class DependencyInformation( + moduleInfo: DepModuleInfo, + license: LicenseInfo, + sources: Seq[SourceAccess], + url: Option[String] +) { + + /** + * Normalized name of the package that uniquely identifies the dependency. + */ + def packageName: String = + Review.normalizeName( + moduleInfo.organization + "." + moduleInfo.name + "-" + + moduleInfo.version + ) +} diff --git a/project/src/main/scala/licenses/DependencySummary.scala b/project/src/main/scala/licenses/DependencySummary.scala new file mode 100644 index 0000000000..f6adc5f227 --- /dev/null +++ b/project/src/main/scala/licenses/DependencySummary.scala @@ -0,0 +1,199 @@ +package src.main.scala.licenses + +import java.nio.file.Path + +import sbt.IO +import src.main.scala.licenses.report.WithWarnings + +/** + * Contains a sequence of dependencies and any attachments found. + */ +case class DependencySummary( + dependencies: Seq[(DependencyInformation, Seq[Attachment])] +) + +/** + * Review status of the [[Attachment]]. + */ +sealed trait AttachmentStatus { + + /** + * Determines if the attachment with this status should be included in the + * final package. + */ + def included: Boolean +} +object AttachmentStatus { + + /** + * Indicates that the attachment should be kept. + */ + case object Keep extends AttachmentStatus { + + /** + * @inheritdoc + */ + override def included: Boolean = true + } + + /** + * Indicates that the copyright mention should be kept, but its whole context + * should be used instead of its content. + * + * Only valid for [[CopyrightMention]]. + */ + case object KeepWithContext extends AttachmentStatus { + + /** + * @inheritdoc + */ + override def included: Boolean = true + } + + /** + * Indicates that the attachment should be ignored. + */ + case object Ignore extends AttachmentStatus { + + /** + * @inheritdoc + */ + override def included: Boolean = false + } + + /** + * Indicates that the attachment has been added manually. + */ + case object Added extends AttachmentStatus { + + /** + * @inheritdoc + */ + override def included: Boolean = true + } + + /** + * Indicates that the attachment was not yet reviewed. + */ + case object NotReviewed extends AttachmentStatus { + + /** + * @inheritdoc + */ + override def included: Boolean = false + } +} + +/** + * Gathers information related to a dependency after the review. + * + * @param information original [[DependencyInformation]] + * @param licenseReviewed indicates if the license associated with the + * dependency is marked as reviewed + * @param licensePath may contain a path to the default license file that will + * be used; if empty, `files` should contain an alternative + * license + * @param files list of files attached to the dependency, with their review + * statuses + * @param copyrights list of copyright mentions attached to the dependency, + * with their review statuses + */ +case class ReviewedDependency( + information: DependencyInformation, + licenseReviewed: Boolean, + licensePath: Option[Path], + files: Seq[(AttachedFile, AttachmentStatus)], + copyrights: Seq[(CopyrightMention, AttachmentStatus)] +) + +/** + * Summarizes the dependency review. + * + * The reviewed version of [[DependencySummary]]. + * + * @param dependencies sequence of reviewed dependencies + * @param noticeHeader header to include in the generated NOTICE + * @param additionalFiles additional files that should be added to the root of + * the notice package + */ +case class ReviewedSummary( + dependencies: Seq[ReviewedDependency], + noticeHeader: String, + additionalFiles: Seq[AttachedFile] +) { + + /** + * Returns a license-like file that is among attached files that are included + * (if such file exists). + */ + def includedLicense(dependency: ReviewedDependency): Option[AttachedFile] = + dependency.files + .find { f => + val isIncluded = f._2.included + val name = f._1.path.getFileName.toString.toLowerCase + val isLicense = name.contains("license") || name.contains("licence") + isIncluded && isLicense + } + .map(_._1) +} + +object ReviewedSummary { + + /** + * Returns a list of warnings that indicate missing reviews or other issues. + */ + def warnAboutMissingReviews(summary: ReviewedSummary): WithWarnings[Unit] = { + val warnings = summary.dependencies.flatMap { dep => + val warnings = collection.mutable.Buffer[String]() + val name = dep.information.moduleInfo.toString + if (!dep.licenseReviewed) { + warnings.append( + s"License ${dep.information.license.name} for $name is not reviewed." + ) + } + + val missingFiles = dep.files.filter(_._2 == AttachmentStatus.NotReviewed) + if (missingFiles.nonEmpty) { + warnings.append( + s"${missingFiles.size} files are not reviewed in $name." + ) + } + val missingCopyrights = + dep.copyrights.filter(_._2 == AttachmentStatus.NotReviewed) + if (missingCopyrights.nonEmpty) { + warnings.append( + s"${missingCopyrights.size} copyrights are not reviewed in $name." + ) + } + + val includedInfos = + (dep.files.map(_._2) ++ dep.copyrights.map(_._2)).filter(_.included) + if (includedInfos.isEmpty) { + warnings.append( + s"No files or copyright information are included for $name." + ) + } + + (summary.includedLicense(dep), dep.licensePath) match { + case (Some(kept), Some(reviewedLicense)) => + val licenseContent = IO.read(reviewedLicense.toFile) + if (licenseContent.strip != kept.content) { + warnings.append( + s"A license file was discovered in $name that is different " + + s"from the default license file that is associated with its " + + s"license ${dep.information.license.name}." + ) + } + case (None, None) => + warnings.append( + s"The license for $name is set to but no license-like " + + s"file is found." + ) + case _ => + } + + warnings + } + WithWarnings.justWarnings(warnings) + } +} diff --git a/project/src/main/scala/licenses/DistributionDescription.scala b/project/src/main/scala/licenses/DistributionDescription.scala new file mode 100644 index 0000000000..4e74fe2abf --- /dev/null +++ b/project/src/main/scala/licenses/DistributionDescription.scala @@ -0,0 +1,46 @@ +package src.main.scala.licenses + +import com.typesafe.sbt.license.LicenseReport +import com.typesafe.sbt.license.SbtCompat.IvySbt +import sbt.File +import sbt.librarymanagement.UpdateReport + +/** + * Describes a component included in the distribution managed by SBT. + * + * @param name name of the component + * @param licenseReport license report generated by the `updateLicenses` task + * @param ivyModule `ivyModule` associated with the project + * @param classifiedArtifactsReport result of the `updateClassifiers` task + */ +case class SBTDistributionComponent( + name: String, + licenseReport: LicenseReport, + ivyModule: IvySbt#Module, + classifiedArtifactsReport: UpdateReport +) + +/** + * Describes an artifact consisting of multiple components that is distributed + * independently. + * + * @param artifactName name of the artifact + * @param packageDestination location of the generated notice package + * @param sbtComponents sequence of SBT components that constitute this + * artifact; only root components (components that are + * directly packaged and distributed) are required (i.e. + * if X is distributed and X depends on Y but Y is not + * directly included, it does not have to be on this list + * as it will be automatically discovered) + */ +case class DistributionDescription( + artifactName: String, + packageDestination: File, + sbtComponents: Seq[SBTDistributionComponent] +) { + + /** + * Returns names of root components included in the distribution. + */ + def rootComponentsNames: Seq[String] = sbtComponents.map(_.name) +} diff --git a/project/src/main/scala/licenses/backend/AttachmentGatherer.scala b/project/src/main/scala/licenses/backend/AttachmentGatherer.scala new file mode 100644 index 0000000000..8044e4a87a --- /dev/null +++ b/project/src/main/scala/licenses/backend/AttachmentGatherer.scala @@ -0,0 +1,48 @@ +package src.main.scala.licenses.backend + +import java.nio.file.{Files, Path} +import java.util.stream.Collectors + +import src.main.scala.licenses.{Attachment, SourceAccess} + +import scala.collection.JavaConverters._ + +/** + * Common interface for algorithms that gather attachments based on available + * sources. + */ +trait AttachmentGatherer { + + /** + * Runs the algorithm that will scan the sources located in `root` and return + * any attachments found. + */ + def run(root: Path): Seq[Attachment] + + /** + * A helper function that uses the provided [[SourceAccess]] instances to + * analyze sources provided by them and return any attachments found. + */ + def run(sources: Seq[SourceAccess]): Seq[Attachment] = { + sources.flatMap(_.access(run(_))) + } +} + +object AttachmentGatherer { + + /** + * A helper method that recursively traverses the directory structure at + * `root` and collects results of calling `action` on each encountered entry. + * + * The action is called for all kinds of entries that are encountered. + */ + def walk[R](root: Path)(action: Path => Seq[R]): Seq[R] = { + val stream = Files.walk(root) + try { + val list = stream.collect(Collectors.toList()) + list.asScala.flatMap(action) + } finally { + stream.close() + } + } +} diff --git a/project/src/main/scala/licenses/backend/CombinedBackend.scala b/project/src/main/scala/licenses/backend/CombinedBackend.scala new file mode 100644 index 0000000000..46cf4acc67 --- /dev/null +++ b/project/src/main/scala/licenses/backend/CombinedBackend.scala @@ -0,0 +1,28 @@ +package src.main.scala.licenses.backend +import java.nio.file.Path + +import src.main.scala.licenses.Attachment + +/** + * An [[AttachmentGatherer]] that combines results from running multiple + * algorithms. + */ +class CombinedBackend(backends: Seq[AttachmentGatherer]) + extends AttachmentGatherer { + + /** + * @inheritdoc + */ + override def run(root: Path): Seq[Attachment] = { + backends.flatMap(_.run(root)) + } +} + +object CombinedBackend { + + /** + * Creates a [[CombinedBackend]] from multiple [[AttachmentGatherer]]s. + */ + def apply(backends: AttachmentGatherer*): CombinedBackend = + new CombinedBackend(backends) +} diff --git a/project/src/main/scala/licenses/backend/GatherCopyrights.scala b/project/src/main/scala/licenses/backend/GatherCopyrights.scala new file mode 100644 index 0000000000..c2bdae355d --- /dev/null +++ b/project/src/main/scala/licenses/backend/GatherCopyrights.scala @@ -0,0 +1,126 @@ +package src.main.scala.licenses.backend +import java.nio.file.{Files, Path} + +import sbt.IO +import src.main.scala.licenses.{Attachment, CopyrightMention} + +import scala.util.control.NonFatal + +/** + * The algorithm for gathering any copyright notices inside of source files. + * + * It reads all text files and gathers any lines that contain the word + * copyright - it may introduce some false positives but these can be manually + * reviewed and ignored. + * + * It tries to include context in which the line has appeared, first by trying + * to detect if the line is a part of a longer comment; if the first method + * fails, it just shows 2 lines before and after the selected one. + */ +object GatherCopyrights extends AttachmentGatherer { + + /** + * @inheritdoc + */ + override def run(root: Path): Seq[Attachment] = { + val allCopyrights = AttachmentGatherer.walk(root) { path => + if (Files.isRegularFile(path)) { + val relativePath = root.relativize(path) + try { + val lines = IO.readLines(path.toFile).zipWithIndex.toIndexedSeq + lines + .filter(l => mayBeCopyright(l._1)) + .map { case (str, idx) => (str, findContext(lines)(idx)) } + .map { + case (line, context) => + CopyrightMention(line, Seq(context), Seq(relativePath)) + } + .map(mention => mention.copy(content = cleanup(mention.content))) + } catch { + case NonFatal(e) => + Seq( + CopyrightMention( + "", + Seq(e.toString), + Seq(relativePath) + ) + ) + } + } else { + Seq() + } + } + CopyrightMention.mergeByContext( + CopyrightMention.mergeByContent(allCopyrights) + ) + } + + /** + * Decides if the line may contain a copyright. + */ + private def mayBeCopyright(line: String): Boolean = + line.toLowerCase.contains("copyright") + + /** + * Finds context of the given line. + * + * If the selected line seems to be a part of a block comment, the context + * becomes the whole comment, otherwise it just includes 2 lines before and + * after the selected line. + * + * @param allLines list of all lines available (that will be searched for + * context) + * @param idx index of the selected line in `allLines` + * @return a string representing the found context + */ + private def findContext( + allLines: IndexedSeq[(String, Int)] + )(idx: Int): String = { + val (line, _) = allLines(idx) + val nearbyLines: Seq[String] = if (line.stripLeading().startsWith("*")) { + val start = allLines + .take(idx + 1) + .filter(_._1.contains("/*")) + .map(_._2) + .lastOption + .getOrElse(idx - 2) + val end = allLines + .drop(idx) + .filter(_._1.contains("*/")) + .map(_._2) + .headOption + .getOrElse(idx + 2) + allLines.slice(start, end + 1).map(_._1) + } else if ( + possiblePrefixes.exists(s => line.stripLeading().headOption.contains(s)) + ) { + val maxLinesOnSide = 20 + val prefix = + line.takeWhile(c => c.isWhitespace || possiblePrefixes.contains(c)) + val before = allLines + .take(idx) + .reverse + .takeWhile(_._1.startsWith(prefix)) + .take(maxLinesOnSide) + .reverse + val after = allLines + .drop(idx) + .takeWhile(_._1.startsWith(prefix)) + .take(maxLinesOnSide) + (before ++ after).map(_._1) + } else { + allLines.slice(idx - 2, idx + 3).map(_._1) + } + nearbyLines.mkString("\n") + } + + private val possiblePrefixes = Seq('-', '#', ';', '/') + + /** + * Strips comment-related characters from the prefix. + */ + private def cleanup(string: String): String = { + val charsToIgnore = Seq('*', '-', '#', '/') + string.dropWhile(char => char.isWhitespace || charsToIgnore.contains(char)) + } +} diff --git a/project/src/main/scala/licenses/backend/GatherNotices.scala b/project/src/main/scala/licenses/backend/GatherNotices.scala new file mode 100644 index 0000000000..702d978d4c --- /dev/null +++ b/project/src/main/scala/licenses/backend/GatherNotices.scala @@ -0,0 +1,73 @@ +package src.main.scala.licenses.backend + +import java.nio.file.{Files, Path} + +import src.main.scala.licenses.{AttachedFile, Attachment} + +/** + * The algorithm for gathering any copyright-related files found in the + * sources. + * + * It gathers any files whose name contains one of the keywords from + * `possibleNames`, but filters out ordinary source files. + */ +object GatherNotices extends AttachmentGatherer { + + /** + * @inheritdoc + */ + override def run(root: Path): Seq[Attachment] = { + AttachmentGatherer.walk(root) { path => + if (Files.isRegularFile(path) && mayBeRelevant(path)) { + Seq(AttachedFile.read(path, Some(root))) + } else Seq() + } + } + + /** + * Decides if the path may be relevant and should be included in the result. + */ + def mayBeRelevant(fileName: String): Boolean = { + val extension = { + val lastDot = fileName.lastIndexOf(".") + if (lastDot < 0) None + else Some(fileName.substring(lastDot + 1)) + } + if (extension.exists(ignoredExtensions.contains)) { + false + } else { + val simplifiedName = fileName.filter(_.isLetterOrDigit) + possibleNames.exists(simplifiedName.contains) + } + } + + /** + * Decides if the filename is relevant and should be included in the result. + */ + def mayBeRelevant(path: Path): Boolean = + mayBeRelevant(path.getFileName.toString.toLowerCase) + + /** + * File extensions that are ignored. + * + * Source files are ignored because they are scanned for copyright notices by + * [[GatherCopyrights]], they are not likely to constitute separate copyright + * files. + */ + private val ignoredExtensions = Seq("scala", "java") + + /** + * File name keywords that indicate that a file should be included. + */ + val possibleNames = + Seq( + "notice", + "copyright", + "thirdparty", + "license", + "licence", + "credit", + "copying", + "authors" + ) +} diff --git a/project/src/main/scala/licenses/backend/GithubHeuristic.scala b/project/src/main/scala/licenses/backend/GithubHeuristic.scala new file mode 100644 index 0000000000..5f2500daa9 --- /dev/null +++ b/project/src/main/scala/licenses/backend/GithubHeuristic.scala @@ -0,0 +1,99 @@ +package src.main.scala.licenses.backend + +import java.nio.file.Path + +import sbt.Logger +import sbt.io.syntax.url +import src.main.scala.licenses.{ + AttachedFile, + Attachment, + CopyrightMention, + DependencyInformation +} + +import scala.sys.process._ +import scala.util.control.NonFatal + +/** + * Tries to find copyright mentions in the GitHub project homepage and any + * copyright-related files contained in the repository root. + * + * This is a fallback backend that may be used if the primary backends do not + * find anything. It should not be used otherwise, as the primary backends are + * most likely to yield more precise results - for example, the + * GitHub-published version may be different than the one we use. + */ +case class GithubHeuristic(info: DependencyInformation, log: Logger) { + + /** + * Runs the gathering process and returns any attachments found. + * + * It proceeds only if the project has an URL that seems to point to GitHub. + */ + def run(): Seq[Attachment] = { + info.url match { + case Some(url) if url.contains("github.com") => + tryDownloadingAttachments(url.replace("http://", "https://")) + case _ => Seq() + } + } + + /** + * Downloads the project homepage at `address` and looks for any copyright + * mentions or links that may lead to copyright-related files. + * + * Any found files are fetched and saved into the results. + */ + def tryDownloadingAttachments(address: String): Seq[Attachment] = + try { + val homePage = url(address).cat.!! + val fileRegex = """(.*?)""".r("href", "name") + val matches = fileRegex + .findAllMatchIn(homePage) + .map(m => (m.group("name"), m.group("href"))) + .filter(p => mayBeRelevant(p._1)) + .toList + val files = matches.flatMap { + case (_, href) => + try { + val content = + url("https://github.com" + href.replace("blob", "raw")).cat.!! + Seq( + AttachedFile(Path.of(href), content, origin = Some("github.com")) + ) + } catch { + case NonFatal(error) => + log.warn( + s"Found file $href but cannot download it: $error" + ) + Seq() + } + } + val copyrights = homePage.linesIterator.toList + .filter(_.toLowerCase.contains("copyright")) + .map(line => + CopyrightMention( + line, + Seq(s"Found at $address"), + Seq(Path.of("github.com")) + ) + ) + files ++ copyrights + } catch { + case NonFatal(error) => + log.warn(s"GitHub backend for ${info.packageName} failed with $error") + Seq() + } + + /** + * Decides if the file may be relevant and should be included in the result. + * + * Filenames that contain spaces are ignored because they are usually normal + * links and do not lead to relevant files. + */ + private def mayBeRelevant(name: String): Boolean = { + val normalized = name.strip().toLowerCase() + !normalized.contains(' ') && + GatherNotices.mayBeRelevant(normalized) + } +} diff --git a/project/src/main/scala/licenses/frontend/DependencyFilter.scala b/project/src/main/scala/licenses/frontend/DependencyFilter.scala new file mode 100644 index 0000000000..0ad1dd922e --- /dev/null +++ b/project/src/main/scala/licenses/frontend/DependencyFilter.scala @@ -0,0 +1,19 @@ +package src.main.scala.licenses.frontend + +import src.main.scala.licenses.DependencyInformation + +/** + * Filters out irrelevant dependencies. + * + * Currently, dependencies whose organisation is `org.enso` are ignored, as + * they are owned by us, so they do not require any additional licensing + * notices. + */ +object DependencyFilter { + + /** + * Decides if the dependency should be kept for further processing. + */ + def shouldKeep(dependencyInformation: DependencyInformation): Boolean = + dependencyInformation.moduleInfo.organization != "org.enso" +} diff --git a/project/src/main/scala/licenses/frontend/SbtLicenses.scala b/project/src/main/scala/licenses/frontend/SbtLicenses.scala new file mode 100644 index 0000000000..f3b77cc3e3 --- /dev/null +++ b/project/src/main/scala/licenses/frontend/SbtLicenses.scala @@ -0,0 +1,189 @@ +package src.main.scala.licenses.frontend + +import java.nio.file.Path + +import com.typesafe.sbt.license.SbtCompat.IvySbt +import com.typesafe.sbt.license.{DepLicense, DepModuleInfo, LicenseReport} +import org.apache.ivy.core.report.ResolveReport +import org.apache.ivy.core.resolve.IvyNode +import sbt.Compile +import sbt.internal.util.ManagedLogger +import sbt.io.IO +import sbt.librarymanagement.ConfigRef +import src.main.scala.licenses.{ + DependencyInformation, + SBTDistributionComponent, + SourceAccess +} + +import scala.collection.JavaConverters._ + +/** + * Defines the algorithm for discovering dependency metadata. + */ +object SbtLicenses { + + /** + * Defines configurations that are deemed relevant for dependency discovery. + * + * Currently we only analyse Compile dependencies as these are the ones that + * get packaged. + * + * Provided dependencies are assumed to be already present in the used + * runtime, so we do not distribute them. One exception is the launcher which + * does distribute the provided SubstrateVM dependencies as part of it being + * compiled with the SVM. But that has to be handled independently anyway. + */ + val relevantConfigurations = Seq(Compile) + + /** + * Analyzes the provided [[SBTDistributionComponent]]s collecting their + * unique dependencies and issuing any warnings. + * + * @param components description of SBT components included in the + * distribution + * @param log logger to use when resolving dependencies + * @return a sequence of collected dependency information and a sequence of + * encountered warnings + */ + def analyze( + components: Seq[SBTDistributionComponent], + log: ManagedLogger + ): (Seq[DependencyInformation], Seq[String]) = { + val results: Seq[(Seq[Dependency], Vector[Path], Seq[String])] = + components.map { component => + val report = resolveIvy(component.ivyModule, log) + val ivyDeps = + report.getDependencies.asScala.map(_.asInstanceOf[IvyNode]) + val sourceArtifacts = component.classifiedArtifactsReport + .select((configRef: ConfigRef) => + relevantConfigurations.map(_.name).contains(configRef.name) + ) + .map(_.toPath) + .filter(_.getFileName.toString.endsWith("sources.jar")) + val deps = for { + dep <- component.licenseReport.licenses + depNode = + ivyDeps + .find(ivyDep => safeModuleInfo(ivyDep) == Some(dep.module)) + .getOrElse( + throw new RuntimeException( + s"Could not find Ivy node for resolved module ${dep.module}." + ) + ) + } yield { + val sources = sourceArtifacts.filter( + _.getFileName.toString.startsWith(dep.module.name) + ) + Dependency(dep, depNode, sources) + } + + val warnings = + if (component.licenseReport.licenses.isEmpty) + Seq(s"License report for component ${component.name} is empty.") + else Seq() + + (deps, sourceArtifacts, warnings) + } + + val distinctDependencies = + results.flatMap(_._1).groupBy(_.depLicense.module).map(_._2.head).toSeq + val distinctSources = results.flatMap(_._2).distinct + + val wrappedDeps = + for (dependency <- distinctDependencies) + yield DependencyInformation( + moduleInfo = dependency.depLicense.module, + license = dependency.depLicense.license, + sources = findSources(dependency), + url = tryFindingUrl(dependency) + ) + val relevantDeps = wrappedDeps.filter(DependencyFilter.shouldKeep) + + val missingWarnings = for { + dep <- relevantDeps + if dep.sources.isEmpty + } yield s"Could not find sources for ${dep.moduleInfo}" + val unexpectedWarnings = for { + source <- distinctSources + if !distinctDependencies.exists(_.sourcesJARPaths.contains(source)) + } yield s"Found a source $source that does not belong to any known " + + s"dependencies, perhaps the algorithm needs updating?" + val reportsWarnings = results.flatMap(_._3) + + (relevantDeps, missingWarnings ++ unexpectedWarnings ++ reportsWarnings) + } + + /** + * Uses the [[LicenseReport]] plugin to resolve the dependencies of the Ivy + * module of an SBT component. + * + * Returns the resolved report or throws an exception if any errors were + * encountered. + */ + private def resolveIvy( + ivyModule: IvySbt#Module, + log: ManagedLogger + ): ResolveReport = { + val (report, err) = LicenseReport.resolve(ivyModule, log) + err.foreach(throw _) + report + } + + /** + * Returns a project URL if it is defined for the dependency or None. + */ + private def tryFindingUrl(dependency: Dependency): Option[String] = + Option(dependency.ivyNode.getDescriptor).flatMap(descriptor => + Option(descriptor.getHomePage) + ) + + /** + * Creates a [[SourceAccess]] instance that unpacks the source files from a + * JAR archive into a temporary directory. + * + * It removes the temporary directory after the analysis is finished. + */ + private def createSourceAccessFromJAR(jarPath: Path): SourceAccess = + new SourceAccess { + override def access[R](withSources: Path => R): R = + IO.withTemporaryDirectory { root => + IO.unzip(jarPath.toFile, root) + withSources(root.toPath) + } + } + + /** + * Returns a sequence of [[SourceAccess]] instances that give access to any + * sources JARs that are available with the dependency. + */ + private def findSources(dependency: Dependency): Seq[SourceAccess] = + dependency.sourcesJARPaths.map(createSourceAccessFromJAR) + + /** + * Wraps information related to a dependency. + * + * @param depLicense information on the license + * @param ivyNode Ivy node that can be used to find metadata + * @param sourcesJARPaths paths to JARs containing dependency's sources + */ + case class Dependency( + depLicense: DepLicense, + ivyNode: IvyNode, + sourcesJARPaths: Seq[Path] + ) + + /** + * Returns [[DepModuleInfo]] for an [[IvyNode]] if it is defined, or None. + */ + private def safeModuleInfo(dep: IvyNode): Option[DepModuleInfo] = + for { + moduleId <- Option(dep.getModuleId) + moduleRevision <- Option(dep.getModuleRevision) + revisionId <- Option(moduleRevision.getId) + } yield DepModuleInfo( + moduleId.getOrganisation, + moduleId.getName, + revisionId.getRevision + ) +} diff --git a/project/src/main/scala/licenses/report/HTMLWriter.scala b/project/src/main/scala/licenses/report/HTMLWriter.scala new file mode 100644 index 0000000000..e0111e7c28 --- /dev/null +++ b/project/src/main/scala/licenses/report/HTMLWriter.scala @@ -0,0 +1,268 @@ +package src.main.scala.licenses.report + +import java.io.{BufferedWriter, PrintWriter} +import java.nio.file.Files + +import sbt.File + +/** + * Allows to create very simple HTML reports. + * + * @param bufferedWriter writer to which the generated code will be written + */ +class HTMLWriter(bufferedWriter: BufferedWriter) { + val writer = new PrintWriter(bufferedWriter) + + /** + * Writes the header containing basic styles and scripts for the reports. + * + * @param title the page title + */ + def writeHeader(title: String): Unit = { + val heading = + s""" + | + | + | + | + |$title + | + | + | + |""".stripMargin + writer.println(heading) + } + + /** + * A helper class that allows to manage columns in a table row. + * + * @param columns expected columns count + */ + class RowWriter(columns: Int) { + var added = 0 + + /** + * A helper function that adds a column just containing plain text. + */ + def addColumn(text: String): Unit = { + addColumn(writeText(text)) + } + + /** + * Adds a cell in the next column that will contain anything that is + * written when executing the `writeAction`. + */ + def addColumn(writeAction: => Unit): Unit = { + writer.println("") + writeAction + writer.println("") + added += 1 + } + + /** + * Finishes the current row and ensures that all expected columns have been + * added. + */ + def finish(): Unit = { + if (added < columns) + throw new IllegalStateException("Not enough columns added") + } + } + + /** + * Writes a HTML table based on the provided descriptiom. + * + * @param headers sequence of header names for each column + * @param rows sequence of functions that will be called to create rows of + * the table, each function is provided with a [[RowWriter]] that + * should write as many columns as there are headers + */ + def writeTable(headers: Seq[String], rows: Seq[RowWriter => Unit]): Unit = { + writer.println("") + writer.println("") + for (header <- headers) { + writer.println(s"") + } + writer.println("") + + val columns = headers.length + for (row <- rows) { + writer.println("") + val rw = new RowWriter(columns) + row(rw) + rw.finish() + writer.println("") + } + + writer.println("
$header
") + } + + /** + * Writes an unordered list. + * + * @param elements sequence of functions that will be called to write each of + * the list's elements; everything written inside of each + * function will be part of its list element + */ + def writeList(elements: Seq[() => Unit]): Unit = { + writer.println("") + } + + /** + * Writes a link. + * + * @param text link text + * @param url link target + */ + def writeLink(text: String, url: String): Unit = + writer.println(s"""$text""") + + /** + * Writes a H1 heading. + */ + def writeHeading(heading: String): Unit = + writer.println(s"

$heading

") + + /** + * Writes a H2 heading. + */ + def writeSubHeading(heading: String): Unit = + writer.println(s"

$heading

") + + /** + * Writes a paragraph of styled text. + */ + def writeParagraph(text: String, styles: Style*): Unit = + writer.println(s"""

$text

""") + + /** + * Writes plain (but potentially styled) text. + */ + def writeText(text: String, styles: Style*): Unit = + if (styles.nonEmpty) + writer.println(s"""$text""") + else writer.println(text) + + /** + * Writes a collapsible entry that by default just displays the `title`, but + * expands when clicked to show the `content`. + */ + def writeCollapsible( + title: String, + content: String + ): Unit = { + writer.println(s"""
+ |

$title

+ |
+ |
+                      |$content
+                      |
+ |
""".stripMargin) + } + + /** + * Writes an empty element that can be overridden by an injected script. + * + * Parameters names and values need to be properly escaped to fit for HTML. + */ + def makeInjectionHandler( + className: String, + params: (String, String)* + ): String = { + val mappedParams = params + .map { + case (name, value) => s"""data-$name="$value" """ + } + .mkString(" ") + s""" + |""".stripMargin + } + + /** + * Escapes a string that may contain HTML markup to not be rendered but + * displayed as normal text. + */ + def escape(string: String): String = + string.replace("&", "&").replace("<", "<").replace(">", ">") + + /** + * Finishes writing the document and closes the output. + */ + def close(): Unit = { + writer.println("") + writer.close() + } +} + +object HTMLWriter { + + /** + * Creates an [[HTMLWriter]] that writes its output to a [[File]] (creating + * it or overwriting if it exists). + */ + def toFile(destination: File): HTMLWriter = + new HTMLWriter(Files.newBufferedWriter(destination.toPath)) +} + +/** + * Styles that can be used to make text stand out. + */ +sealed trait Style +object Style { + + /** + * Makes the text bold. + */ + case object Bold extends Style { + override def toString: String = "font-weight:bold" + } + + /** + * Makes the text red. + */ + case object Red extends Style { + override def toString: String = "color:red" + } + + /** + * Makes the text gray. + */ + case object Gray extends Style { + override def toString: String = "color:gray" + } + + /** + * Makes the text green. + */ + case object Green extends Style { + override def toString: String = "color:green" + } + + /** + * Makes the text black. + */ + case object Black extends Style { + override def toString: String = "color:black" + } +} diff --git a/project/src/main/scala/licenses/report/PackageNotices.scala b/project/src/main/scala/licenses/report/PackageNotices.scala new file mode 100644 index 0000000000..ed8db80bd5 --- /dev/null +++ b/project/src/main/scala/licenses/report/PackageNotices.scala @@ -0,0 +1,138 @@ +package src.main.scala.licenses.report + +import java.nio.file.{Files, Path} + +import sbt._ +import src.main.scala.licenses.{ + AttachmentStatus, + CopyrightMention, + DistributionDescription, + ReviewedSummary +} + +import scala.annotation.tailrec + +/** + * Defines writing the notices package for distribution. + */ +object PackageNotices { + + /** + * Creates the notices package based on the `summary` in `destination`. + * + * The `destination` directory is cleaned before creating the new notices. + * + * It copies licences, files and gathers copyright notices that were set to + * be included in the review and generates a summary `NOTICE` file. + */ + def create( + description: DistributionDescription, + summary: ReviewedSummary, + destination: File + ): Unit = { + IO.delete(destination) + IO.createDirectory(destination) + if (IO.listFiles(destination).nonEmpty) { + throw new RuntimeException( + s"Could not clean ${destination}, cannnot continue creating the " + + s"package." + ) + } + + val artifactName = description.artifactName + val mainNotice = new StringBuilder + mainNotice.append(summary.noticeHeader) + + val licensesRoot = destination / "licenses" + val processedLicenses = collection.mutable.Set[Path]() + + for (dependency <- summary.dependencies) { + val name = dependency.information.moduleInfo.name + val licenseName = dependency.information.license.name + mainNotice.append( + s"\n\n'$name', licensed under the $licenseName, " + + s"is distributed with the $artifactName.\n" + ) + dependency.licensePath match { + case Some(path) => + val name = path.getFileName.toString + if (!processedLicenses.contains(path)) { + val destination = licensesRoot / name + IO.copyFile(path.toFile, destination) + if (!destination.exists()) { + throw new RuntimeException(s"Failed to copy the license $path") + } + processedLicenses.add(path) + } + mainNotice.append( + s"The license file can be found at `licenses/$name`.\n" + ) + case None => + mainNotice.append( + s"The license file can be found at along the copyright notices.\n" + ) + } + + val packageName = dependency.information.packageName + val packageRoot = destination / packageName + + val files = dependency.files.filter(_._2.included).map(_._1) + val copyrights = dependency.copyrights.filter(_._2.included) + + if (files.size + copyrights.size > 0) { + mainNotice.append( + s"Copyright notices related to this dependency can be found in the " + + s"directory `$packageName`.\n" + ) + IO.createDirectory(packageRoot) + } + + @tailrec + def findFreeName(name: String, counter: Int = 0): File = { + val actualName = if (counter > 0) s"$name.$counter" else name + val file = packageRoot / actualName + if (Files.exists(file.toPath)) findFreeName(name, counter + 1) + else file + } + + for (attachedFile <- files) { + val file = findFreeName(attachedFile.path.getFileName.toString) + IO.write(file, attachedFile.content) + } + + def renderCopyright( + copyright: CopyrightMention, + status: AttachmentStatus + ): String = + status match { + case AttachmentStatus.Keep => copyright.content + case AttachmentStatus.KeepWithContext => + if (copyright.contexts.size != 1) { + throw new IllegalStateException( + "`KeepWithContext` can only be used for copyrights that " + + "have exactly one context." + ) + } + copyright.contexts.head + case AttachmentStatus.Added => + copyright.contexts.head + case _ => + throw new IllegalStateException( + "Only `included` copyrights should be present at this point." + ) + } + + if (copyrights.nonEmpty) { + val compiledCopyrights = copyrights + .map { + case (m, s) => renderCopyright(m, s) + } + .mkString("\n\n") + val freeName = findFreeName("NOTICES") + IO.write(freeName, compiledCopyrights) + } + } + + IO.write(destination / "NOTICE", mainNotice.toString()) + } +} diff --git a/project/src/main/scala/licenses/report/Report.scala b/project/src/main/scala/licenses/report/Report.scala new file mode 100644 index 0000000000..eef85d1e3b --- /dev/null +++ b/project/src/main/scala/licenses/report/Report.scala @@ -0,0 +1,266 @@ +package src.main.scala.licenses.report + +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.util.Base64 + +import sbt.{File, IO} +import src.main.scala.licenses._ + +/** + * Allows to write a report summarizing current status of the review. + * + * The report lists all dependencies and status of found attachments. + * + * With the legal-review-helper script, the generated report can be used to + * interactively perform substantial parts of the review. + */ +object Report { + + /** + * Writes the report in HTML format. + * + * @param description description of the distribution + * @param summary reviewed summary of findings + * @param warnings sequence of warnings + * @param destination location of the generated HTML file + */ + def writeHTML( + description: DistributionDescription, + summary: ReviewedSummary, + warnings: Seq[String], + destination: File + ): Unit = { + IO.createDirectory(destination.getParentFile) + val writer = HTMLWriter.toFile(destination) + try { + writer.writeHeader(s"Dependency summary for ${description.artifactName}") + writer.writeHeader( + s"Gathered Information on ${description.artifactName} Distribution" + ) + writer.writeParagraph( + s"Root components analyzed for the distribution: " + + s"${description.rootComponentsNames.mkString(", ")}." + ) + + if (warnings.isEmpty) { + writer.writeParagraph("There are no warnings.", Style.Green) + } else { + writer.writeParagraph( + s"There are ${warnings.size} warnings!", + Style.Bold, + Style.Red + ) + } + + writeDependencySummary(writer, summary) + + writer.writeList(warnings.map { warning => () => + writer.writeText(writer.escape(warning)) + }) + + writer.writeCollapsible("NOTICE header", summary.noticeHeader) + if (summary.additionalFiles.nonEmpty) { + writer.writeSubHeading("Additional files that will be added:") + writer.writeList(summary.additionalFiles.map { file => () => + writer.writeText(file.path.toString) + }) + } + + } finally { + writer.close() + } + } + + /** + * Renders [[AttachmentStatus]] as HTML that will show the status name and a + * color associated with it. + */ + private def renderStatus(attachmentStatus: AttachmentStatus): String = { + val style = attachmentStatus match { + case AttachmentStatus.Keep => Style.Black + case AttachmentStatus.KeepWithContext => Style.Black + case AttachmentStatus.Ignore => Style.Gray + case AttachmentStatus.Added => Style.Green + case AttachmentStatus.NotReviewed => Style.Red + } + s"""$attachmentStatus""" + } + + /** + * Renders a message about similarity of the file to a selected license file. + * + * If the filename is license-like, the file is compared with the selected + * license (if any). If the file differs from the selected license and is + * ignored, a warning is displayed. If the file is kept but is identical (so + * it is redundant) the message is also displayed as a minor warning. + */ + private def renderSimilarity( + licensePath: Option[Path], + file: AttachedFile, + status: AttachmentStatus + ): String = { + val name = file.path.getFileName.toString.toLowerCase + if (name.contains("license") || name.contains("licence")) { + licensePath match { + case Some(value) => + val defaultText = IO.read(value.toFile) + if (defaultText.strip() == file.content.strip()) { + val color = if (status.included) Style.Red else Style.Green + s"""100% identical to default license""" + } else + s"""Differs from used license!""" + case None => "" + } + } else "" + } + + /** + * Writes a table containing summary of dependencies and their gathered + * copyright information. + */ + private def writeDependencySummary( + writer: HTMLWriter, + summary: ReviewedSummary + ): Unit = { + val sorted = summary.dependencies.sortBy(dep => + (dep.information.moduleInfo.organization, dep.information.moduleInfo.name) + ) + + writer.writeSubHeading("Summary of all compile dependencies") + writer.writeTable( + headers = Seq( + "Organization", + "Name", + "Version", + "URL", + "License", + "Attached files", + "Possible copyrights" + ), + rows = sorted.map { + case dep @ ReviewedDependency( + information, + licenseReviewed, + licensePath, + files, + copyrights + ) => + (rowWriter: writer.RowWriter) => + val moduleInfo = information.moduleInfo + rowWriter.addColumn(moduleInfo.organization) + rowWriter.addColumn(moduleInfo.name) + rowWriter.addColumn(moduleInfo.version) + rowWriter.addColumn { + information.url match { + case Some(value) => + writer.writeLink(value, value) + case None => writer.writeText("No URL") + } + } + val license = information.license + rowWriter.addColumn { + writer.writeLink(license.name, license.url) + writer.writeText(s"(${license.category.name})
") + licensePath match { + case Some(path) => + writer.writeText( + path.getFileName.toString, + if (licenseReviewed) Style.Green else Style.Red + ) + case None if licenseReviewed => + val licenseFile = summary.includedLicense(dep) + licenseFile match { + case Some(file) => + writer.writeText( + s"Custom license ${file.path.getFileName}", + Style.Green + ) + case None => + writer.writeText( + "Custom license defined but not provided!", + Style.Red + ) + } + case None if !licenseReviewed => + val name = Review.normalizeName(license.name) + writer.writeText( + s"Not reviewed, filename:
$name
", + Style.Red + ) + } + } + rowWriter.addColumn { + if (files.isEmpty) writer.writeText("No attached files.") + else + writer.writeList(files.map { + case (file, status) => + () => + val injection = writer.makeInjectionHandler( + "file-ui", + "package" -> dep.information.packageName, + "filename" -> file.path.toString, + "status" -> status.toString + ) + val origin = file.origin + .map(origin => s" (Found at $origin)") + .getOrElse("") + writer.writeCollapsible( + s"${file.fileName} (${renderStatus(status)})$origin " + + s"${renderSimilarity(licensePath, file, status)}", + injection + + writer.escape(file.content) + ) + }) + } + rowWriter.addColumn { + if (copyrights.isEmpty) { + val bothEmpty = files.isEmpty && copyrights.isEmpty + if (bothEmpty) { + writer.writeText( + "No notices or copyright information found, " + + "this may be a problem.", + Style.Red + ) + } else { + writer.writeText("No copyright information found.") + } + } else + writer.writeList(copyrights.map { + case (mention, status) => + () => + val foundAt = mention.origins match { + case Seq() => "" + case Seq(one) => s"Found at $one" + case Seq(first, rest @ _*) => + s"Found at $first and ${rest.size} other files." + } + + val contexts = if (mention.contexts.nonEmpty) { + mention.contexts + .map(c => "\n" + writer.escape(c) + "\n") + .mkString("
") + } else "" + + val injection = writer.makeInjectionHandler( + "copyright-ui", + "package" -> dep.information.packageName, + "content" -> Base64.getEncoder.encodeToString( + mention.content.getBytes(StandardCharsets.UTF_8) + ), + "contexts" -> mention.contexts.length.toString, + "status" -> status.toString + ) + + val content = writer.escape(mention.content) + val moreInfo = injection + foundAt + contexts + writer.writeCollapsible( + s"$content (${renderStatus(status)})", + moreInfo + ) + }) + } + } + ) + } +} diff --git a/project/src/main/scala/licenses/report/Review.scala b/project/src/main/scala/licenses/report/Review.scala new file mode 100644 index 0000000000..920097c8ab --- /dev/null +++ b/project/src/main/scala/licenses/report/Review.scala @@ -0,0 +1,333 @@ +package src.main.scala.licenses.report + +import java.nio.file.{Files, Path} +import java.time.LocalDate + +import sbt._ +import src.main.scala.licenses._ + +import scala.util.control.NonFatal + +/** + * Reads settings from the `root` to add review statuses to discovered + * attachments and add any additional attachments coming from the settings. + * + * The review settings consist of the following files or directories (all are + * optional): + * + * - `notice-header` - contains the header that will start the main generated + * NOTICE + * - `files-add` - directory that may contain additional files that should be + * added to the package + * - `reviewed-licenses` - directory that may contain files for reviewed + * licenses; the files should be named with the normalized license name and + * they should contain a path to that license's file + * - and for each dependency, a subdirectory named as its `packageName` with + * following entries: + * - `files-add` - directory that may contain additional files that should + * be added to the subdirectory for this package + * - `files-keep` - a file containing names of files found in the package + * sources that should be included in the package + * - `files-ignore` - a file containing names of files found in the package + * sources that should not be included + * - `custom-license` - a file that indicates that the dependency should not + * point to the default license, but it should contain a custom one within + * its files + * - `copyright-keep` - copyright lines that should be included in the + * notice summary for the package + * - `copyright-keep-context` - copyright lines that should be included + * (alongside with their context) in the notice summary for the package + * - `copyright-ignore` - copyright lines that should not be included in the + * notice summary for the package + * - `copyright-add` - a single file whose contents will be added to the + * notice summary for the package + * + * @param root directory that contains files which guide the review process + * @param dependencySummary summary of discovered dependency information + */ +case class Review(root: File, dependencySummary: DependencySummary) { + + /** + * Runs the review process, returning a [[ReviewedDependency]] which includes + * information from the [[DependencySummary]] enriched with review statuses. + */ + def run(): WithWarnings[ReviewedSummary] = + for { + reviews <- dependencySummary.dependencies.map { + case (information, attachments) => + reviewDependency(information, attachments) + }.flip + + header = findHeader() + files = findAdditionalFiles(root / "files-add") + summary = ReviewedSummary(reviews, header, files) + _ <- ReviewedSummary.warnAboutMissingReviews(summary) + existingPackages = dependencySummary.dependencies.map(_._1.packageName) + _ <- warnAboutMissingDependencies(existingPackages) + } yield summary + + /** + * Returns a list of warnings for dependencies whose configuration has been + * detected but which have not been detected. + * + * This may be used to detect dependencies that have been removed after an + * update. + */ + private def warnAboutMissingDependencies( + existingPackageNames: Seq[String] + ): WithWarnings[Unit] = { + val foundConfigurations = listFiles(root).filter(_.isDirectory) + val expectedFileNames = + existingPackageNames ++ Seq("files-add", "reviewed-licenses") + val unexpectedConfigurations = + foundConfigurations.filter(p => !expectedFileNames.contains(p.getName)) + val warnings = unexpectedConfigurations.map(p => + s"Found legal review configuration for package ${p.getName}, " + + s"but no such dependency has been found. Perhaps it has been removed?" + ) + WithWarnings.justWarnings(warnings) + } + + /** + * Finds a header defined in the settings or + */ + private def findHeader(): String = + readFile(root / "notice-header").getOrElse(Review.defaultHeader) + + /** + * Reads files from the provided directory as [[AttachedFile]]. + */ + private def findAdditionalFiles(dir: File): Seq[AttachedFile] = + listFiles(dir).map(f => AttachedFile.read(f.toPath, Some(dir.toPath))) + + /** + * Splits the sequence of attachments into sequences of files and copyrights. + */ + private def splitAttachments( + attachments: Seq[Attachment] + ): (Seq[AttachedFile], Seq[CopyrightMention]) = { + val notices = attachments.collect { case n: AttachedFile => n } + val copyrights = attachments.collect { case c: CopyrightMention => c } + (notices, copyrights) + } + + /** + * Returns only such copyrights that are not included in one of the + * discovered files. + */ + private def removeCopyrightsIncludedInNotices( + copyrights: Seq[CopyrightMention], + notices: Seq[AttachedFile] + ): Seq[CopyrightMention] = { + def shouldKeepCopyright(copyright: CopyrightMention): Boolean = { + val allOriginsAreFresh = copyright.origins.forall { path => + !notices.exists(_.path == path) + } + allOriginsAreFresh + } + copyrights.filter(shouldKeepCopyright) + } + + /** + * Returns the review status of the license and any attachments associated + * with the dependency. + */ + private def reviewDependency( + info: DependencyInformation, + attachments: Seq[Attachment] + ): WithWarnings[ReviewedDependency] = { + val packageRoot = root / info.packageName + val (licenseReviewed, licensePath) = reviewLicense(packageRoot, info) + val (files, copyrights) = splitAttachments(attachments) + val copyrightsDeduplicated = + removeCopyrightsIncludedInNotices(copyrights, files) + + for { + processedFiles <- reviewFiles(packageRoot, files) ++ addFiles(packageRoot) + processedCopyrights <- + reviewCopyrights(packageRoot, copyrightsDeduplicated) ++ + addCopyrights(packageRoot) + } yield ReviewedDependency( + information = info, + licenseReviewed = licenseReviewed, + licensePath = licensePath, + files = processedFiles, + copyrights = processedCopyrights + ) + } + + /** + * Enriches the file attachments with their review status. + */ + private def reviewFiles( + packageRoot: File, + files: Seq[AttachedFile] + ): WithWarnings[Seq[(AttachedFile, AttachmentStatus)]] = { + def keyForFile(file: AttachedFile): String = file.path.toString + val keys = files.map(keyForFile) + for { + ignore <- readExpectedLines("files-ignore", keys, packageRoot) + keep <- readExpectedLines("files-keep", keys, packageRoot) + } yield { + def review(file: AttachedFile): AttachmentStatus = { + val key = keyForFile(file) + if (keep.contains(key)) AttachmentStatus.Keep + else if (ignore.contains(key)) AttachmentStatus.Ignore + else AttachmentStatus.NotReviewed + } + + files.map(f => (f, review(f))) + } + } + + /** + * Returns any additional file attachments that are manually added in the + * review. + */ + private def addFiles( + packageRoot: File + ): Seq[(AttachedFile, AttachmentStatus)] = + findAdditionalFiles(packageRoot / "files-add") + .map((_, AttachmentStatus.Added)) + + /** + * Enriches the copyright attachments with their review status. + */ + private def reviewCopyrights( + packageRoot: File, + copyrights: Seq[CopyrightMention] + ): WithWarnings[Seq[(CopyrightMention, AttachmentStatus)]] = { + def keyForMention(copyrightMention: CopyrightMention): String = + copyrightMention.content.strip + val keys = copyrights.map(keyForMention) + for { + ignore <- readExpectedLines("copyright-ignore", keys, packageRoot) + keep <- readExpectedLines("copyright-keep", keys, packageRoot) + keepContext <- + readExpectedLines("copyright-keep-context", keys, packageRoot) + } yield { + + def review(copyright: CopyrightMention): AttachmentStatus = { + val key = keyForMention(copyright) + if (keepContext.contains(key)) AttachmentStatus.KeepWithContext + else if (keep.contains(key)) AttachmentStatus.Keep + else if (ignore.contains(key)) AttachmentStatus.Ignore + else AttachmentStatus.NotReviewed + } + + copyrights.map(c => (c, review(c))) + } + } + + /** + * Returns any additional copyright attachments that are manually added in + * the review. + */ + private def addCopyrights( + packageRoot: File + ): Seq[(CopyrightMention, AttachmentStatus)] = + readFile(packageRoot / "copyright-add") + .map(text => + ( + CopyrightMention( + content = "", + contexts = Seq(text), + origins = Seq() + ), + AttachmentStatus.Added + ) + ) + .toSeq + + /** + * Checks if the license has been reviewed. + * + * Returns a boolean value indicating if it has been reviewed and a path to + * the license file if a default file is used. + */ + private def reviewLicense( + packageRoot: File, + info: DependencyInformation + ): (Boolean, Option[Path]) = { + if (Files.exists((packageRoot / "custom-license").toPath)) (true, None) + else + readFile( + root / "reviewed-licenses" / Review.normalizeName(info.license.name) + ) + .map(p => (true, Some(Path.of(p.strip())))) + .getOrElse((false, None)) + } + + /** + * Reads the file as lines. + * + * Returns an empty sequence if the file cannot be read. + */ + private def readLines(file: File): Seq[String] = + try { IO.readLines(file).map(_.strip).filter(_.nonEmpty) } + catch { case NonFatal(_) => Seq() } + + /** + * Reads the file as lines and reports any lines that were not expected to be + * found. + */ + private def readExpectedLines( + fileName: String, + expectedLines: Seq[String], + packageRoot: File + ): WithWarnings[Seq[String]] = { + val lines = readLines(packageRoot / fileName) + val unexpectedLines = lines.filter(l => !expectedLines.contains(l)) + val warnings = unexpectedLines.map(l => + s"File $fileName in ${packageRoot.getName} contains entry `$l`, but no " + + s"such entry has been detected. Perhaps it has disappeared after an " + + s"update? Please remove it from the file and make sure that the report " + + s"contains all necessary elements after this change." + ) + WithWarnings(lines, warnings) + } + + /** + * Reads the file as a [[String]]. + * + * Returns None if the file cannot be read. + */ + private def readFile(file: File): Option[String] = + try { Some(IO.read(file)) } + catch { case NonFatal(_) => None } + + /** + * Returns a sequence of files contained in a directory. + * + * If the directory does not exist or otherwise cannot be queried, returns an + * empty sequence. + */ + private def listFiles(dir: File): Seq[File] = { + try { IO.listFiles(dir) } + catch { case NonFatal(_) => Seq() } + } +} + +object Review { + + /** + * Normalizes a name so that it can be used as a filename. + */ + def normalizeName(string: String): String = { + val charsToReplace = " /:;," + charsToReplace.foldLeft(string)((str, char) => str.replace(char, '_')) + } + + /** + * Default NOTICE header. + */ + val defaultHeader: String = { + val yearStart = 2020 + val yearCurrent = LocalDate.now().getYear + val year = + if (yearCurrent > yearStart) s"$yearStart - $yearCurrent" + else s"$yearStart" + s"""Enso + |Copyright $year New Byte Order sp. z o. o.""".stripMargin + } +} diff --git a/project/src/main/scala/licenses/report/WithWarnings.scala b/project/src/main/scala/licenses/report/WithWarnings.scala new file mode 100644 index 0000000000..b7376a0eed --- /dev/null +++ b/project/src/main/scala/licenses/report/WithWarnings.scala @@ -0,0 +1,63 @@ +package src.main.scala.licenses.report + +/** + * A simple monad for storing warnings related to a result. + */ +case class WithWarnings[+A](value: A, warnings: Seq[String]) { + + /** + * Returns a result with a mapped value and the same warnings. + */ + def map[B](f: A => B): WithWarnings[B] = WithWarnings(f(value), warnings) + + /** + * Combines two computations returning warnings, preserving warnings from + * both of them. + */ + def flatMap[B](f: A => WithWarnings[B]): WithWarnings[B] = { + val result = f(value) + WithWarnings(result.value, warnings ++ result.warnings) + } +} + +object WithWarnings { + implicit class SeqWithWarningsSyntax[+A](seq: Seq[WithWarnings[A]]) { + + /** + * Turns a sequence of [[WithWarnings]] instances into a single + * [[WithWarnings]] that contains the sequence of values and combined + * warnings. + */ + def flip: WithWarnings[Seq[A]] = + WithWarnings(seq.map(_.value), seq.flatMap(_.warnings)) + } + + implicit class SeqLikeSyntax[A](seq: WithWarnings[Seq[A]]) { + + /** + * A shorthand syntax to concatenate two sequences wrapped with warnings, + * combining their warnings. + */ + def ++(other: WithWarnings[Seq[A]]): WithWarnings[Seq[A]] = + for { + lhs <- seq + rhs <- other + } yield lhs ++ rhs + + /** + * A shorthand syntax to concatenate an ordinary sequence to a sequence + * with warnings. + */ + def ++(other: Seq[A]): WithWarnings[Seq[A]] = + for { + lhs <- seq + } yield lhs ++ other + } + + /** + * Creates a [[WithWarnings]] containing Unit and a provided sequence of + * warnings. + */ + def justWarnings(warnings: Seq[String]): WithWarnings[Unit] = + WithWarnings((), warnings) +} diff --git a/std-bits/pom.xml b/std-bits/pom.xml deleted file mode 100644 index b4abd5278b..0000000000 --- a/std-bits/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - 4.0.0 - - org.enso - base - 1.0 - jar - - base - - UTF-8 - - - - com.ibm.icu - icu4j - 67.1 - - - - - - - maven-compiler-plugin - 3.8.0 - - 1.8 - 1.8 - - - - org.apache.maven.plugins - maven-jar-plugin - 2.3.1 - - - ${project.build.directory}/../../distribution/std-lib/Base/polyglot/java - - - - - org.codehaus.mojo - license-maven-plugin - 2.0.0 - - - add-third-party-licenses - package - - download-licenses - - - - ${project.build.directory}/../../distribution/std-lib/Base/third-party-licenses - - - - - add-third-party-licenses-list - package - - add-third-party - - - - ${project.build.directory}/../../distribution/std-lib/Base/third-party-licenses - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy - package - - copy - - - - - com.ibm.icu - icu4j - 67.1 - jar - true - icu4j.jar - - - - ${project.build.directory}/../../distribution/std-lib/Base/polyglot/java - - - - - - - - diff --git a/tools/legal-review-helper/index.js b/tools/legal-review-helper/index.js new file mode 100644 index 0000000000..8b0bdebb1f --- /dev/null +++ b/tools/legal-review-helper/index.js @@ -0,0 +1,136 @@ +const reviewRoot = "../../target"; +const settingsRoot = "../../tools/legal-review"; + +const express = require("express"); +const app = express(); +const open = require("open"); +const fs = require("fs"); +const path = require("path"); + +// The home page that lists available reports. +app.get("/", function (req, res) { + let html = "

Report review

"; + const files = fs.readdirSync(reviewRoot); + const reports = files + .map((f) => f.match(/^(.*)-report.html$/)) + .filter((m) => m != null) + .map((m) => m[1]); + if (reports.length == 0) { + html += + "No reports found. " + + 'Run
enso / gatherLicenses
first.'; + } else { + html += "Select report:"; + html += "
    "; + reports.forEach((report) => { + html += '
  • ' + report + "
  • "; + }); + html += "
"; + } + res.send(html); +}); + +// Serves the injection script. +app.use("/static", express.static("static")); + +// Serves contents of the given report, injecting the review-mode script. +app.get("/report/:report", function (req, res) { + const report = req.params["report"]; + console.log("Opening report for ", report); + fs.readFile( + path.join(reviewRoot, report + "-report.html"), + "utf-8", + (err, data) => { + const injection = + '' + + ''; + if (err) { + res.status(400).send(err); + } else { + const injected = data.replace("", injection + ""); + res.send(injected); + } + } + ); +}); + +// Appends a line to the setting file. +function addLine(report, package, file, line) { + const dir = path.join(settingsRoot, report, package); + const location = path.join(dir, file); + console.log("Adding " + line + " to " + location); + fs.mkdirSync(dir, { + recursive: true, + }); + fs.appendFileSync(location, line + "\n"); +} + +// Removes a line from the setting file. +function removeLine(report, package, file, line) { + const location = path.join(settingsRoot, report, package, file); + console.log("Removing " + line + " from " + location); + const lines = fs + .readFileSync(location, "utf-8") + .split(/\r?\n/) + .filter((x) => x.length > 0); + const toRemove = lines.filter((x) => x == line); + const others = lines.filter((x) => x != line); + if (toRemove.length == 0) { + throw ( + "Line " + + line + + " was not present in the file. " + + "Are you sure the report is up to date?" + ); + } else { + var newContent = others.join("\n") + "\n"; + if (others.length == 0) { + newContent = ""; + } + fs.writeFileSync(location, newContent); + } +} + +// Handles the requests to add or remove lines. +app.use(express.urlencoded({ extended: true })); +app.post("/modify/:report", function (req, res) { + const report = req.params["report"]; + const package = req.body["package"]; + const action = req.body["action"]; + const file = req.body["file"]; + const line = req.body["line"]; + + try { + if (action == "add") { + addLine(report, package, file, line); + } else if (action == "remove") { + removeLine(report, package, file, line); + } else { + throw "Unknown action"; + } + res.send("OK"); + } catch (error) { + console.error(error); + res.status(500).send(error); + } +}); + +/* + * Listens on a random free port, opens a browser with the home page and waits + * for a newline to terminate. + */ +const server = app.listen(0, () => { + const port = server.address().port; + console.log("Listening on at ", "http://localhost:" + port + "/"); + open("http://localhost:" + port + "/"); + + console.log("Press ENTER to stop the server."); + process.stdin.on("data", function (chunk) { + if (chunk.indexOf("\n") >= 0) { + console.log("Good bye"); + process.exit(0); + } + }); +}); diff --git a/tools/legal-review-helper/package.json b/tools/legal-review-helper/package.json new file mode 100644 index 0000000000..d15a2c4eac --- /dev/null +++ b/tools/legal-review-helper/package.json @@ -0,0 +1,16 @@ +{ + "name": "legal-review-helper", + "version": "1.0.0", + "description": "Server for reviewing the generated report.", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node index.js" + }, + "author": "Enso Team", + "license": "Apache-2.0", + "dependencies": { + "express": "^4.17.1", + "open": "^7.3.0" + } +} diff --git a/tools/legal-review-helper/static/inject.js b/tools/legal-review-helper/static/inject.js new file mode 100644 index 0000000000..68be2a0180 --- /dev/null +++ b/tools/legal-review-helper/static/inject.js @@ -0,0 +1,134 @@ +/** Sets a status text in bottom left part of the screen. */ +function setStatus(text, color) { + var status = $("#status"); + status.html(text); + if (color === undefined) { + color = "white"; + } + status.css("background-color", color); +} + +/** Creates a handler that will request to add or remove a line from a file. */ +function makeHandler(elem, data, file, action) { + return function (ev) { + data["file"] = file; + data["action"] = action; + $.post("/modify/" + reportName, data, function (response) { + $(elem).html( + 'Modified, if you want to ' + + "change this value, regenerate the report first" + ); + var tab = $(elem).closest("div").parent(); + var title = tab.children("h4"); + tab.accordion("option", "active", false); + var info = "added " + file; + if (action == "remove") { + info = "undone review"; + } + var newTitle = + '' + + title.html() + + "
" + + info; + title.html(newTitle); + title.find("span").css("color", "gray"); + setStatus("Review for " + data["package"] + " sent."); + }).fail(function (err) { + setStatus("Failed to send review: " + JSON.stringify(err), "red"); + }); + setStatus("Sending review..."); + }; +} + +$(function () { + $("body").append( + '
' + + "Loading...
" + ); + var copys = $(".copyright-ui"); + var files = $(".file-ui"); + + copyrightMap = { + Ignore: "copyright-ignore", + KeepWithContext: "copyright-keep-context", + Keep: "copyright-keep", + }; + + copys.each(function (index) { + var package = $(this).data("package"); + var content = atob($(this).data("content")); + var status = $(this).data("status"); + var contexts = parseInt($(this).data("contexts")); + var data = { + line: content, + package: package, + }; + if (status == "NotReviewed") { + var buttons = + '' + + '' + + ''; + $(this).html(buttons); + $(this) + .children(".ignore") + .on("click", makeHandler(this, data, "copyright-ignore", "add")); + $(this) + .children(".keep") + .on("click", makeHandler(this, data, "copyright-keep", "add")); + if (contexts == 1) { + $(this) + .children(".keepctx") + .on( + "click", + makeHandler(this, data, "copyright-keep-context", "add") + ); + } else { + $(this).children(".keepctx").attr("disabled", true); + } + } else if (status != "Added") { + $(this).html(""); + $(this) + .children("button") + .on("click", makeHandler(this, data, copyrightMap[status], "remove")); + } else { + $(this).html(""); + } + }); + + filesMap = { + Ignore: "files-ignore", + Keep: "files-keep", + }; + + files.each(function (index) { + var package = $(this).data("package"); + var filename = $(this).data("filename"); + var status = $(this).data("status"); + var data = { + line: filename, + package: package, + }; + if (status == "NotReviewed") { + var buttons = + '' + + ''; + $(this).html(buttons); + $(this) + .children(".ignore") + .on("click", makeHandler(this, data, "files-ignore", "add")); + $(this) + .children(".keep") + .on("click", makeHandler(this, data, "files-keep", "add")); + } else if (status != "Added") { + $(this).html(""); + $(this) + .children("button") + .on("click", makeHandler(this, data, filesMap[status], "remove")); + } else { + $(this).html(""); + } + }); + + setStatus("Initialized"); +}); diff --git a/tools/legal-review/launcher/com.typesafe.akka.akka-protobuf-v3_2.13-2.6.6/files-add/COPYING.protobuf b/tools/legal-review/launcher/com.typesafe.akka.akka-protobuf-v3_2.13-2.6.6/files-add/COPYING.protobuf new file mode 100644 index 0000000000..705db579c9 --- /dev/null +++ b/tools/legal-review/launcher/com.typesafe.akka.akka-protobuf-v3_2.13-2.6.6/files-add/COPYING.protobuf @@ -0,0 +1,33 @@ +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/tools/legal-review/launcher/commons-io.commons-io-2.7/custom-license b/tools/legal-review/launcher/commons-io.commons-io-2.7/custom-license new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/legal-review/license-texts/APACHE2.0 b/tools/legal-review/license-texts/APACHE2.0 new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/tools/legal-review/license-texts/APACHE2.0 @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.