enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-08-18 01:10:37 +03:00
Code Issues Projects Releases Wiki Activity

Automate License Information Gathering (#1198)

Browse Source
This commit is contained in:
Radosław Waśko 2020-10-09 16:19:58 +02:00 committed by GitHub
parent a2be12c3e9
commit 0a9e2a42ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 3217 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/release.yml vendored
View File

@ -90,13 +90,6 @@ jobs:
~/.cache
key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }}
restore-keys: ${{ runner.os }}-sbt-
- name: Cache Local Maven Repository
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
# Build Artifacts
- name: Enable Release Mode
@ -115,7 +108,7 @@ jobs:
working-directory: repo
run: |
sleep 1
sbt --no-colors runner/assembly
sbt --no-colors engine-runner/assembly
- name: Build the Project Manager Uberjar
working-directory: repo
run: |
@ -198,8 +191,8 @@ jobs:
- name: Build Base Java Extensions
shell: bash
run: |
cd std-bits
mvn --no-transfer-progress package
sleep 1
sbt --no-colors std-bits/package
# The way artifacts are uploaded currently does not preserve the
# executable bits for Unix. However putting artifacts into a ZIP would

13
.github/workflows/scala.yml vendored
View File

@ -98,13 +98,6 @@ jobs:
~/.cache
key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }}
restore-keys: ${{ runner.os }}-sbt-
- name: Cache Local Maven Repository
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
# Compile
- name: Bootstrap Enso project
@ -125,8 +118,8 @@ jobs:
- name: Build Base Java Extensions
shell: bash
run: |
cd std-bits
mvn --no-transfer-progress package
sleep 1
sbt --no-colors std-bits/package
- name: Build the Launcher
run: |
sleep 1
@ -156,7 +149,7 @@ jobs:
- name: Build the Runner & Runtime Uberjars
run: |
sleep 1
sbt --no-colors runner/assembly
sbt --no-colors engine-runner/assembly
- name: Build the Project Manager Uberjar
run: |
sleep 1

6
.prettierignore
View File

@ -2,4 +2,8 @@
target/
.github/PULL_REQUEST_TEMPLATE.md
.github/ISSUE_TEMPLATE
distribution/launcher/THIRD-PARTY
distribution/engine/THIRD-PARTY
tools/legal-review
distribution/std-lib/Base/third-party-licenses
distribution/std-lib/Base/THIRD-PARTY

113
build.sbt
View File

@ -1,17 +1,11 @@
import java.io.File
import com.typesafe.sbt.SbtLicenseReport.autoImportImpl.{
licenseReportNotes,
licenseReportStyleRules
}
import scala.sys.process._
import org.enso.build.BenchTasks._
import org.enso.build.WithDebugCommand
import sbt.Keys.{libraryDependencies, scalacOptions}
import sbt.addCompilerPlugin
import sbtassembly.AssemblyPlugin.defaultUniversalScript
import sbtcrossproject.CrossPlugin.autoImport.{crossProject, CrossType}
import com.typesafe.sbt.license.DepModuleInfo
// ============================================================================
// === Global Configuration ===================================================
@ -33,17 +27,44 @@ val ensoVersion = "0.1.0" // Note [Engine And Launcher Version]
organization in ThisBuild := "org.enso"
scalaVersion in ThisBuild := scalacVersion
val licenseSettings = Seq(
licenseConfigurations := Set("compile"),
licenseReportStyleRules := Some(
"table, th, td {border: 1px solid black;}"
lazy val gatherLicenses =
taskKey[Unit]("Gathers licensing information for relevant dependencies")
gatherLicenses := GatherLicenses.run.value
GatherLicenses.distributions := Seq(
Distribution(
"launcher",
file("distribution/launcher/THIRD-PARTY"),
Distribution.sbtProjects(launcher)
),
licenseReportNotes := {
case DepModuleInfo(group, _, _) if group == "org.enso" =>
"Internal library"
}
)
val coursierCache = file("~/.cache/coursier/v1")
Distribution(
"engine",
file("distribution/engine/THIRD-PARTY"),
Distribution.sbtProjects(
runtime,
`engine-runner`,
`project-manager`,
`language-server`
)
),
Distribution(
"std-lib-Base",
file("distribution/std-lib/Base/THIRD-PARTY"),
Distribution.sbtProjects(`std-bits`)
)
)
GatherLicenses.licenseConfigurations := Set("compile")
GatherLicenses.configurationRoot := file("tools/legal-review")
lazy val openLegalReviewReport =
taskKey[Unit](
"Gathers licensing information for relevant dependencies and opens the " +
"report in review mode in the browser."
)
openLegalReviewReport := {
gatherLicenses.value
GatherLicenses.runReportServer()
}
Global / onChangedBuildSource := ReloadOnSourceChanges
@ -142,7 +163,7 @@ lazy val enso = (project in file("."))
`logging-service`,
`akka-native`,
`version-output`,
runner,
`engine-runner`,
runtime,
searcher,
launcher,
@ -297,6 +318,10 @@ val splainOptions = Seq(
"-P:splain:tree:true"
)
// === std-lib ================================================================
val icuVersion = "67.1"
// === ZIO ====================================================================
val zioVersion = "1.0.1"
@ -347,7 +372,6 @@ lazy val logger = crossProject(JVMPlatform, JSPlatform)
libraryDependencies ++= scalaCompiler
)
.jsSettings(jsSettings)
.settings(licenseSettings)
lazy val flexer = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
@ -364,7 +388,6 @@ lazy val flexer = crossProject(JVMPlatform, JSPlatform)
)
)
.jsSettings(jsSettings)
.settings(licenseSettings)
lazy val `syntax-definition` = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
@ -383,7 +406,6 @@ lazy val `syntax-definition` = crossProject(JVMPlatform, JSPlatform)
)
)
.jsSettings(jsSettings)
.settings(licenseSettings)
lazy val syntax = crossProject(JVMPlatform, JSPlatform)
.withoutSuffixFor(JVMPlatform)
@ -433,7 +455,6 @@ lazy val syntax = crossProject(JVMPlatform, JSPlatform)
testFrameworks := List(new TestFramework("org.scalatest.tools.Framework")),
Compile / fullOptJS / artifactPath := file("target/scala-parser.js")
)
.settings(licenseSettings)
lazy val `parser-service` = (project in file("lib/scala/parser-service"))
.dependsOn(syntax.jvm)
@ -441,7 +462,6 @@ lazy val `parser-service` = (project in file("lib/scala/parser-service"))
libraryDependencies ++= akka,
mainClass := Some("org.enso.ParserServiceMain")
)
.settings(licenseSettings)
lazy val `text-buffer` = project
.in(file("lib/scala/text-buffer"))
@ -453,7 +473,6 @@ lazy val `text-buffer` = project
"org.scalacheck" %% "scalacheck" % scalacheckVersion % Test
)
)
.settings(licenseSettings)
lazy val graph = (project in file("lib/scala/graph/"))
.dependsOn(logger.jvm)
@ -481,7 +500,6 @@ lazy val graph = (project in file("lib/scala/graph/"))
),
scalacOptions ++= splainOptions
)
.settings(licenseSettings)
lazy val pkg = (project in file("lib/scala/pkg"))
.settings(
@ -494,7 +512,6 @@ lazy val pkg = (project in file("lib/scala/pkg"))
"commons-io" % "commons-io" % commonsIoVersion
)
)
.settings(licenseSettings)
lazy val `akka-native` = project
.in(file("lib/scala/akka-native"))
@ -507,7 +524,6 @@ lazy val `akka-native` = project
// Note [Native Image Workaround for GraalVM 20.2]
libraryDependencies += "org.graalvm.nativeimage" % "svm" % graalVersion % "provided"
)
.settings(licenseSettings)
lazy val `logging-service` = project
.in(file("lib/scala/logging-service"))
@ -531,7 +547,6 @@ lazy val `logging-service` = project
else
(Compile / unmanagedSourceDirectories) += (Compile / sourceDirectory).value / "java-unix"
)
.settings(licenseSettings)
.dependsOn(`akka-native`)
lazy val cli = project
@ -545,7 +560,6 @@ lazy val cli = project
),
parallelExecution in Test := false
)
.settings(licenseSettings)
lazy val `version-output` = (project in file("lib/scala/version-output"))
.settings(
@ -564,7 +578,6 @@ lazy val `version-output` = (project in file("lib/scala/version-output"))
)
}.taskValue
)
.settings(licenseSettings)
lazy val `project-manager` = (project in file("lib/scala/project-manager"))
.settings(
@ -629,7 +642,6 @@ lazy val `project-manager` = (project in file("lib/scala/project-manager"))
.dependsOn(runtime / assembly)
.value
)
.settings(licenseSettings)
.dependsOn(`version-output`)
.dependsOn(pkg)
.dependsOn(`language-server`)
@ -670,7 +682,6 @@ lazy val `json-rpc-server` = project
"org.scalatest" %% "scalatest" % scalatestVersion % Test
)
)
.settings(licenseSettings)
lazy val `json-rpc-server-test` = project
.in(file("lib/scala/json-rpc-server-test"))
@ -683,7 +694,6 @@ lazy val `json-rpc-server-test` = project
"org.scalatest" %% "scalatest" % scalatestVersion
)
)
.settings(licenseSettings)
.dependsOn(`json-rpc-server`)
lazy val testkit = project
@ -720,7 +730,6 @@ lazy val `core-definition` = (project in file("lib/scala/core-definition"))
),
scalacOptions ++= splainOptions
)
.settings(licenseSettings)
.dependsOn(graph)
.dependsOn(syntax.jvm)
@ -741,7 +750,6 @@ lazy val searcher = project
fork in Benchmark := true
)
.dependsOn(testkit % Test)
.settings(licenseSettings)
.dependsOn(`polyglot-api`)
lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl"))
@ -749,7 +757,6 @@ lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl"))
version := "0.1",
libraryDependencies += "com.google.auto.service" % "auto-service" % "1.0-rc7"
)
.settings(licenseSettings)
// ============================================================================
// === Sub-Projects ===========================================================
@ -794,7 +801,6 @@ lazy val `polyglot-api` = project
GenerateFlatbuffers.flatcVersion := flatbuffersVersion,
sourceGenerators in Compile += GenerateFlatbuffers.task
)
.settings(licenseSettings)
.dependsOn(pkg)
.dependsOn(`text-buffer`)
@ -830,7 +836,6 @@ lazy val `language-server` = (project in file("engine/language-server"))
new TestFramework("org.scalameter.ScalaMeterFramework")
)
)
.settings(licenseSettings)
.dependsOn(`polyglot-api`)
.dependsOn(`json-rpc-server`)
.dependsOn(`json-rpc-server-test` % Test)
@ -935,7 +940,9 @@ lazy val runtime = (project in file("engine/runtime"))
.value
)
.settings(
(Test / compile) := (Test / compile).dependsOn(StdBits.preparePackage).value
(Test / compile) := (Test / compile)
.dependsOn(`std-bits` / Compile / packageBin)
.value
)
.settings(
logBuffered := false,
@ -966,7 +973,6 @@ lazy val runtime = (project in file("engine/runtime"))
case _ => MergeStrategy.first
}
)
.settings(licenseSettings)
.dependsOn(pkg)
.dependsOn(`interpreter-dsl`)
.dependsOn(syntax.jvm)
@ -990,7 +996,7 @@ lazy val runtime = (project in file("engine/runtime"))
* recompilation but still convince the IDE that it is a .jar dependency.
*/
lazy val runner = project
lazy val `engine-runner` = project
.in(file("engine/runner"))
.settings(
javaOptions ++= {
@ -1056,7 +1062,6 @@ lazy val runner = project
.dependsOn(runtime / assembly)
.value
)
.settings(licenseSettings)
.dependsOn(`version-output`)
.dependsOn(pkg)
.dependsOn(`language-server`)
@ -1113,12 +1118,36 @@ lazy val launcher = project
.value,
parallelExecution in Test := false
)
.settings(licenseSettings)
.dependsOn(cli)
.dependsOn(`version-output`)
.dependsOn(pkg)
.dependsOn(`logging-service`)
val `std-lib-root` = file("distribution/std-lib/")
val `std-lib-polyglot-root` = `std-lib-root` / "Base" / "polyglot" / "java"
lazy val `std-bits` = project
.in(file("std-bits"))
.settings(
autoScalaLibrary := false,
Compile / packageBin / artifactPath :=
`std-lib-polyglot-root` / "std-bits.jar",
libraryDependencies ++= Seq(
"com.ibm.icu" % "icu4j" % icuVersion
),
Compile / packageBin := Def.task {
val result = (Compile / packageBin).value
StdBits
.copyDependencies(
`std-lib-polyglot-root`,
"std-bits.jar",
ignoreScalaLibrary = true
)
.value
result
}.value
)
/* Note [HTTPS in the Launcher]
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* The launcher uses Apache HttpClient for making web requests. It does not use

2
docs/CONTRIBUTING.md
View File

@ -285,7 +285,7 @@ In order to build a fat jar with the CLI component, run the `assembly` task
inside the `runner` subproject:
```bash
sbt "runner/assembly"
sbt "engine-runner/assembly"
```
This will produce an executable `runner.jar` fat jar and a `runtime.jar` fat jar

2
docs/distribution/README.md
View File

@ -21,7 +21,7 @@ dependencies, and Enso projects for use by our users.
tool for launching various components and managing Enso versions.
- [**Launcher CLI:**](./launcher-cli.md) Explanation of the command-line
interface of the launcher.
- [**Licences:**](licences.md) Information on gathering license information of
- [**Licenses:**](licenses.md) Information on gathering license information of
dependencies included in the distribution.
- [**Fallback Launcher Release Infrastructure:**](fallback-launcher-release-infrastructure.md)
Explanation of the fallback infrastructure that can be enabled to keep

114
docs/distribution/licences.md
View File

@ -1,114 +0,0 @@
---
layout: developer-doc
title: Licenses
category: distribution
tags: [distribution, licenses]
order: 6
---
# Licenses
When distributing Enso, we include code from many dependencies that are used
within it. We need to ensure that we comply with the licenses of the
dependencies that we distribute with Enso.
<!-- MarkdownTOC levels="2,3" autolink="true" -->
- [Gathering Used Dependencies](#gathering-used-dependencies)
- [SBT](#sbt)
- [Rust](#rust)
- [Preparing the Distribution](#preparing-the-distribution)
- [Launcher](#launcher)
- [Engine Components](#engine-components)
<!-- /MarkdownTOC -->
## Gathering Used Dependencies
As a first step, we need to gather a list of which dependencies are used in the
distributed artifacts.
### SBT
We can use the plugin `sbt-license-report` to gather a list of used dependencies
and their licences. To use it, run `enso/dumpLicenseReport` in the SBT shell.
This will gather dependency information for all the subprojects. For each
subproject, `license-reports` directory will be created in its `target`
directory, containing the reports in multiple formats.
It is important to note that the report for the root `enso` project will not
contain dependencies of the subprojects. Instead, to list the relevant
dependencies, you need to look at the reports for the subprojects that are
actually built into distributable artifacts. For now these are: `runtime`,
`runner`, `project-manager` and `launcher`.
#### `sbt-license-report` Configuration
Settings for the plugin are defined in the `licenseSettings` variable in
[`build.sbt`](../../build.sbt). The settings have to be applied to each project
by adding `.settings(licenseSettings)` to the project definition (defining these
settings at the top level or in the `ThisBuild` configuration yielded no
effects, so this workaround is required).
The most relevant setting is `licenseConfigurations` which defines which `ivy`
configurations are considered to search for dependencies.
Currently it is set to only consider `compile` dependencies, as dependencies for
`provided`, `test` or `benchmark` are not distributed.
### Rust
We do not distribute any Rust-based artifacts in this repository.
> The actionables for this section are:
>
> - When the parser is rewritten to Rust and is distributed within the
> artifacts, this section should be revisited to describe a scheme of
> gathering dependencies used in the Rust projects.
## Preparing the Distribution
When a new dependency is added, its transitive dependencies have to be analysed
as described in the previous section. Various action has to be taken depending
on the particular licences.
For most dependencies under the Apache, MIT or BSD licences, a copy of the
licence has to be included within the distribution and if the dependency
includes a `NOTICE` file, the contents of this file have to be reproduced within
the distribution by adding them to an aggregate `NOTICE` file. To find these
`NOTICE` files it may be necessary to walk through the `JAR`s containing the
dependencies or visit project websites of each dependency.
### Launcher
As the launcher is distributed as a native binary executable, the licences and
notices have to be included separately.
The licences should be put in the
[`distribution/launcher/components-licences`](../../distribution/launcher/components-licences)
directory. The notices should be gathered in
[`distribution/launcher/NOTICE`](../../distribution/launcher/NOTICE). These
files are included by the CI build within the built artifacts.
### Engine Components
#### Standard Library
The third-party licenses for Java extensions of the standard library are
gathered in the `third-party-licenses` directory in the `Base` library. The
gathering process is automatic, triggered by the `package` goal of the
associated Maven configuration file.
> The actionables for this section are:
>
> - The engine components as distributed as a JAR archive that everyone can
> open. At least some of the dependencies contain their licences within that
> archive. It should be checked if this is enough or if the licences and
> notices should be replicated in the distributed packages anyway.
### Bundles
Beside the launcher and engine components, the distributed bundles also contain
a distribution of GraalVM CE. This distribution however contains its own licence
and notices within itself, so no further action should be necessary in that
regard.

254
docs/distribution/licenses.md Normal file
View File

@ -0,0 +1,254 @@
---
layout: developer-doc
title: Licenses
category: distribution
tags: [distribution, licenses]
order: 6
---
# Licenses
When distributing Enso, we include code from many dependencies that are used
within it. We need to ensure that we comply with the licenses of the
dependencies that we distribute with Enso.
<!-- MarkdownTOC levels="2,3" autolink="true" -->
- [Gathering Used Dependencies](#gathering-used-dependencies)
- [SBT](#sbt)
- [Rust](#rust)
- [Preparing the Distribution](#preparing-the-distribution)
- [Review](#review)
- [Standard Library](#standard-library)
- [Bundles](#bundles)
<!-- /MarkdownTOC -->
## Gathering Used Dependencies
As a first step, we need to gather a list of which dependencies are used in the
distributed artifacts.
### SBT
We use a `GatherLicenses` task that uses the `sbt-license-report` and other
sources to gather copyright information related to the used dependencies.
To configure the task, `GatherLicenses.distributions` should be set with
sequence of distributions. Each distribution describes one component that is
distributed separately and should include all references to all projects that
are included as part of its distribution. Currently, we have the `launcher`
distribution that consists of one `launcher` component and the `engine`
distribution which includes `runtime`, `engine-runner` and `project-manager`.
Another relevant setting is `GatherLicenses.licenseConfigurations` which defines
which `ivy` configurations are considered to search for dependencies. Currently
it is set to only consider `compile` dependencies, as dependencies for
`provided`, `test` or `benchmark` are not distributed and there are no
`assembly`-specific dependencies.
`GatherLicenses.configurationRoot` specifies where the review tool will look for
the files specifying review state and `GatherLicenses.distributionRoot`
specifies where the final notice packages should be generated.
To run the automated license gathering task, run `enso/gatherLicenses` in SBT.
This will create a report and packages which are described in the
[next section](#preparing-the-distribution).
### Rust
We do not distribute any Rust-based artifacts in this repository.
> The actionables for this section are:
>
> - When the parser is rewritten to Rust and is distributed within the
> artifacts, this section should be revisited to describe a scheme of
> gathering dependencies used in the Rust projects.
> - It would be good to re-use the SBT task as much as possible, possibly by
> creating a frontend for it using `cargo-license`.
## Preparing the Distribution
When a new dependency is added, the `enso/gatherLicenses` should be re-run to
generate the updated report.
The report can be opened in review mode by launching a server located in
`tools/legal-review-helper` by running `npm start` in that directory.
Alternatively, `enso/openLegalReviewReport` can be used instead to automatically
open the report in review-mode after generating it (but it requires `npm` to be
visible on the system PATH in SBT).
The report will show what license the dependencies use and include any copyright
notices and files found within each dependency.
Each copyright notice and file should be reviewed to decide if it should be kept
or ignored. If a notice is automatically detected in a wrong way, it should be
ignored and a fixed one should be added manually. The review process is
described in detail in the [next section](#review).
Each new type of license has to be reviewed to ensure that it is compatible with
our distribution and usage scheme. Licenses are reviewed per-distribution, as
for example the binary distribution of the launcher may impose different
requirements than distribution of the engine as JARs.
After the review is done, the `enso/gatherLicenses` should be re-run again to
generate the updated packages that are included in the distribution. Before a PR
is merged, it should be ensure that there are no warnings in the generation. The
packages are located in separate subdirectories of the `distribution` directory
for each artifact.
> This may possibly be automated in the next PR that includes the current legal
> review. The generating task could write a file indicating if there were any
> warnings and containing a hash of the dependency list. The build job on CI
> could then run a task `verifyLegalReview` which would check if there are no
> warnings and if the review is up to date (by comparing the hash of the
> dependency list at the time of the review with the current one).
> TODO [RW] currently the auto-generated notice packages are not included in the
> built artifacts. That is because the legal review settings have not yet been
> prepared. Once that is done, the CI should be modified accordingly. This will
> be updated in the next PR.
### Review
The review can be performed manually by modifying the settings inside of the
`tools/legal-review` directory or it can be partially automated.
#### Review Process
1. Open the review in edit mode using the helper script.
- You can type `enso / openLegalReviewReport` if you have `npm` in your PATH
as visible from SBT.
- Or you can just run `npm start` (and `npm install` if needed) in the
`tools/legal-review-helper` directory.
2. For each package listed in the review for a given distribution:
1. Review licenses
- Make sure that the component's license is accepted - that we know that
its license type is compatible with our distribution scheme.
- When a license is accepted, a file should be added in the
`reviewed-licenses` directory, with name as indicated in the report. The
file should contain a single line that is the path (relative to
repository root) to the default license file for that license type which
should be included in the distribution.
- The license may have been already accepted if it is the same license
as earlier dependencies for the same artifact.
- Check if any license-like files have been automatically found in the
attached files. If an attached file contains (case-insensitive)
'license' or 'licence' in its name, the review tool will compare it with
the default license file.
- To trigger this comparison, the license must have been already
reviewed when the report was being generated, so you may consider
re-running the report after reviewing the license types to get this
information.
- If an attached file is exactly the same as the license file, it can be
safely ignored.
- If an attached file differs from the default license file, it should
be carefully checked.
- Most of the time, that file should be marked as kept and the default
license ignored.
- To ignore the default license, create an empty file `custom-license`
inside the directory belonging to the relevant package.
2. Review which files to include
- You can click on a filename to display its contents.
- We want to include any NOTICE files that contain copyright notices or
credits.
- False-positives (unrelated files) or duplicates may be ignored.
3. Review copyright notices
- You may click on a copyright line to display context (surrounding text)
and in which files it was found.
- We want to include most of the notices, as it is better to include
duplicates rather than skip something important.
- But we need to ignore false-positives, for example code that contains
the word 'copyright' in it and was falsely classified.
- You may click 'Keep' to add the displayed copyright line to the
copyright notice or if there is exactly one context associated with the
line, you can click 'Keep as context' to add this whole context to the
notice.
4. Add missing information
- You can manually add additional copyright notices by adding them to a
file `copyright-add` inside the directory belonging to the relevant
package.
- You can manually add additional files by adding them into a subdirectory
called `files-add` located in the directory belonging to the relevant
package.
3. Add any additional information:
- You can add additional files by adding them into a subdirectory called
`files-add` in the root directory of distribution configuration.
- You can create a custom notice header that will replace the default one, by
creating a file called `notice-header`.
4. After you are done, re-run `enso/gatherLicenses` to generate the updated
packages.
- Ensure that there are no more warnings, and if there are any go back to fix
the issues.
The updates performed using the web script are remembered locally, so they will
not show up after the refresh. If you ever need to open the edit mode after
closing its window, you should re-generate the report using
`enso/gatherLicenses` or just open it using `enso/openLegalReviewReport` which
will refresh it automatically.
#### Review Configuration
The review state is driven by configuration files located in
`tools/legal-review`. This directory contains separate subdirectories for each
artifact.
The subdirectory for each artifact may contain the following entries:
- `notice-header` - contains the header that will start the main generated
NOTICE
- `files-add` - directory that may contain additional files that should be added
to the notice package
- `reviewed-licenses` - directory that may contain files for reviewed licenses;
the files should be named with the normalized license name and they should
contain a path to that license's file (the path should be relative to the
repository root)
- and for each dependency, a subdirectory named as its `packageName` with
following entries:
- `files-add` - directory that may contain additional files that should be
added to the subdirectory for this package
- `files-keep` - a file containing names of files found in the package sources
that should be included in the package
- `files-ignore` - a file containing names of files found in the package
sources that should not be included
- `custom-license` - a file that indicates that the dependency should not
point to the default license, but it should contain a custom one within its
files
- `copyright-keep` - copyright lines that should be included in the notice
summary for the package
- `copyright-keep-context` - copyright lines that should be included
(alongside with their context) in the notice summary for the package
- `copyright-ignore` - copyright lines that should not be included in the
notice summary for the package
- `copyright-add` - a single file whose contents will be added to the notice
summary for the package
Manually adding files and copyright, modifying the notice header and marking
licenses as reviewed has to be done manually. But deciding if a file or
copyright notice should be kept or ignored can be done much quicker using the
GUI launched by `enso/openLegalReviewReport`. The GUI is a very simple one - it
assumes that the report is up to date and uses the server to modify the
configuration. The configuration changes are not refreshed automatically -
instead if the webpage is refreshed after modifications it may contain stale
information - to get up-to-date information, `enso/openLegalReviewReport` or
`enso/gatherLicenses` has to be re-run.
## Standard Library
The dependencies of standard library are built using Maven, so they have to be
handled separately. Currently there are not many of them so this is handled
manually. If that becomes a problem, they could be attached to the frontend of
the `GatherLicenses` task.
The third-party licenses for Java extensions of the standard library are
gathered in the `third-party-licenses` directory in the `Base` library. The
gathering process is partially-automatic, triggered by the `package` goal of the
associated Maven configuration file. However when another dependency is added to
the standard library, its licenses should be reviewed before merging the PR.
## Bundles
Beside the launcher and engine components, the distributed bundles also contain
a distribution of GraalVM CE. This distribution however contains its own licence
and notices within itself, so no further action should be necessary in that
regard.

72
project/Distribution.scala Normal file
View File

@ -0,0 +1,72 @@
import com.typesafe.sbt.SbtLicenseReport.autoImportImpl.{
licenseOverrides,
licenseSelection
}
import com.typesafe.sbt.license
import sbt.Keys.{ivyModule, streams, update, updateClassifiers}
import sbt.{File, Project}
import src.main.scala.licenses.{
DistributionDescription,
SBTDistributionComponent
}
import scala.language.experimental.macros
import scala.reflect.macros.blackbox
object Distribution {
/**
* Creates a [[DistributionDescription]].
*/
def apply(
name: String,
packageDestination: File,
sbtComponents: Seq[SBTDistributionComponent]
): DistributionDescription =
DistributionDescription(name, packageDestination, sbtComponents)
/**
* A macro that creates [[SBTDistributionComponent]] descriptions from a list
* of project references.
*/
def sbtProjects(projects: Project*): Seq[SBTDistributionComponent] =
macro sbtProjectsImpl
/**
* Implementation of the [[sbtProjects]] macro.
*
* It triggers execution of the tasks that are used to get information from
* SBT on each project.
*/
def sbtProjectsImpl(c: blackbox.Context)(
projects: c.Expr[Project]*
): c.Expr[Seq[SBTDistributionComponent]] = {
import c.universe._
val gathered = {
projects.map(p =>
reify {
val deliberatelyTriggerAndIgnore = (p.splice / update).value
val ivyMod = (p.splice / ivyModule).value
val overrides = (p.splice / licenseOverrides).value.lift
val report = license.LicenseReport.makeReport(
ivyMod,
GatherLicenses.licenseConfigurations.value,
(p.splice / licenseSelection).value,
overrides,
(p.splice / streams).value.log
)
SBTDistributionComponent(
p.splice.id,
report,
ivyMod,
(p.splice / updateClassifiers).value
)
}
)
}
c.Expr[Seq[SBTDistributionComponent]](
Apply(reify(Seq).tree, gathered.map(_.tree).toList)
)
}
}

136
project/GatherLicenses.scala Normal file
View File

@ -0,0 +1,136 @@
import sbt.Keys._
import sbt._
import src.main.scala.licenses.backend.{
CombinedBackend,
GatherCopyrights,
GatherNotices,
GithubHeuristic
}
import src.main.scala.licenses.frontend.SbtLicenses
import src.main.scala.licenses.report.{
PackageNotices,
Report,
Review,
WithWarnings
}
import src.main.scala.licenses.{DependencySummary, DistributionDescription}
import scala.sys.process._
/**
* The task and configuration for automatically gathering license information.
*/
object GatherLicenses {
val distributions = taskKey[Seq[DistributionDescription]](
"Defines descriptions of distributions."
)
val configurationRoot = settingKey[File]("Path to review configuration.")
val licenseConfigurations =
settingKey[Set[String]]("The ivy configurations we consider in the review.")
/**
* The task that performs the whole license gathering process.
*/
lazy val run = Def.task {
val log = state.value.log
val targetRoot = target.value
log.info(
"Gathering license files and copyright notices. " +
"This task may take a long time."
)
val configRoot = configurationRoot.value
val reports = distributions.value.map { distribution =>
log.info(s"Processing the ${distribution.artifactName} distribution")
val projectNames = distribution.sbtComponents.map(_.name)
log.info(
s"It consists of the following sbt project roots:" +
s" ${projectNames.mkString(", ")}"
)
val (sbtInfo, sbtWarnings) =
SbtLicenses.analyze(distribution.sbtComponents, log)
sbtWarnings.foreach(log.warn(_))
val allInfo = sbtInfo // TODO [RW] add Rust frontend result here (#1187)
log.info(s"${allInfo.size} unique dependencies discovered")
val defaultBackend = CombinedBackend(GatherNotices, GatherCopyrights)
val processed = allInfo.map { dependency =>
log.debug(
s"Processing ${dependency.moduleInfo} (${dependency.license}) -> " +
s"${dependency.url}"
)
val defaultAttachments = defaultBackend.run(dependency.sources)
val attachments =
if (defaultAttachments.nonEmpty) defaultAttachments
else GithubHeuristic(dependency, log).run()
(dependency, attachments)
}
val summary = DependencySummary(processed)
val WithWarnings(processedSummary, summaryWarnings) =
Review(configRoot / distribution.artifactName, summary).run()
val allWarnings = sbtWarnings ++ summaryWarnings
val reportDestination =
targetRoot / s"${distribution.artifactName}-report.html"
sbtWarnings.foreach(log.warn(_))
if (summaryWarnings.size > 10)
log.warn(
s"There are too many warnings (${summaryWarnings.size}) to " +
s"display. Please inspect the generated report."
)
else allWarnings.foreach(log.warn(_))
Report.writeHTML(
distribution,
processedSummary,
allWarnings,
reportDestination
)
log.info(
s"Written the report for ${distribution.artifactName} to " +
s"`${reportDestination}`."
)
val packagePath = distribution.packageDestination
PackageNotices.create(distribution, processedSummary, packagePath)
log.info(s"Re-generated distribution notices at `$packagePath`.")
if (summaryWarnings.nonEmpty) {
// TODO [RW] A separate task should be added to verify that the package
// has been built without warnings that would report these warnings as
// errors for the final distribution; this task should be run on CI to
// verify the report is correct and up-to-date
log.warn(
"The distribution notices were regenerated, but there are " +
"not-reviewed issues within the report. The notices are probably " +
"incomplete."
)
}
(distribution, processedSummary)
}
log.warn(
"Finished gathering license information. " +
"This is an automated process, make sure that its output is reviewed " +
"by a human to ensure that all licensing requirements are met."
)
reports
}
/**
* Launches a server that allows to easily review the generated report.
*
* Requires `npm` to be on the system PATH.
*/
def runReportServer(): Unit = {
Seq("npm", "install").!
Process(Seq("npm", "start"), file("tools/legal-review-helper"))
.run(connectInput = true)
.exitValue()
}
}

85
project/StdBits.scala
View File

@ -1,32 +1,65 @@
import java.io.IOException
import sbt.Def
import scala.sys.process._
import sbt.Keys._
import sbt._
import sbt.io.IO
import sbt.librarymanagement.{ConfigurationFilter, DependencyFilter}
object StdBits {
def preparePackage =
Def.task {
val cmd = Seq("mvn", "package", "-f", "std-bits")
val exitCode =
try {
if (Platform.isWindows) {
(Seq("cmd", "/c") ++ cmd).!
} else {
cmd.!
}
} catch {
case e @ (_: RuntimeException | _: IOException) =>
throw new RuntimeException(
"Cannot run `mvn`, " +
"make sure that it is installed and present on your PATH.",
e
)
}
if (exitCode != 0) {
throw new RuntimeException("std-bits build failed.")
/**
* Discovers dependencies of a project and copies them into the destination
* directory.
*
* @param destination location where to put the dependencies
* @param baseJarName name of the JAR generated by the `package` task;
* unexpected (old) files are removed, so this task needs
* to know this file's name to avoid removing it
* @param ignoreScalaLibrary whether to ignore Scala dependencies that are
* added by default be SBT and are not relevant in
* pure-Java projects
*/
def copyDependencies(
destination: File,
baseJarName: String,
ignoreScalaLibrary: Boolean
): Def.Initialize[Task[Unit]] =
Def.task {
val libraryUpdates = (Compile / update).value
val log = streams.value.log
val ignoredConfigurations: NameFilter =
if (ignoreScalaLibrary)
new ExactFilter(Configurations.ScalaTool.name)
else NothingFilter
val filter: ConfigurationFilter =
DependencyFilter.configurationFilter(-ignoredConfigurations)
val relevantFiles = libraryUpdates.select(filter)
val dependencyStore =
streams.value.cacheStoreFactory.make("std-bits-dependencies")
Tracked.diffInputs(dependencyStore, FileInfo.hash)(relevantFiles.toSet) {
report =>
val expectedFileNames = report.checked.map(_.getName) + baseJarName
for (existing <- IO.listFiles(destination)) {
if (!expectedFileNames.contains(existing.getName)) {
log.info(
s"Removing outdated std-bits dependency ${existing.getName}."
)
IO.delete(existing)
}
}
for (changed <- report.modified -- report.removed) {
log.info(
s"Updating changed std-bits dependency ${changed.getName}."
)
IO.copyFile(changed, destination / changed.getName)
}
for (file <- report.checked) {
val dest = destination / file.getName
if (!dest.exists()) {
log.info(s"Adding missing std-bits dependency ${file.getName}.")
IO.copyFile(file, dest)
}
}
}
}
}

149
project/src/main/scala/licenses/Attachment.scala Normal file
View File

@ -0,0 +1,149 @@
package src.main.scala.licenses
import java.nio.file.Path
import sbt.IO
/**
* Represents a licensing information related to a dependency.
*/
sealed trait Attachment
/**
* Represents a file attached to the dependency's sources.
*
* This may be a license file, a copyright notice, a credits file etc.
*/
case class AttachedFile(
path: Path,
content: String,
origin: Option[String] = None
) extends Attachment {
/**
* @inheritdoc
*/
override def toString: String = {
val originRepr = origin.map(o => s", origin = $o")
s"AttachedFile($path, ...$originRepr)"
}
/**
* Name of the file.
*/
def fileName: String = path.getFileName.toString
}
/**
* Represents a copyright mention extracted from a file.
*
* The copyright mention may come from comments in the source code, for
* example.
*
* Equal copyright mentions are merged, so a single mention may contain
* multiple contexts and origins if it was a result of merging from different
* mentions.
*
* @param content cleaned content of the line that was extracted
* @param contexts contexts (surrounding lines) that are associated with the mention
* @param origins paths to the files that the mention comes from
*/
case class CopyrightMention(
content: String,
contexts: Seq[String],
origins: Seq[Path]
) extends Attachment {
override def toString: String = s"CopyrightMention('$content')"
}
object CopyrightMention {
/**
* Transforms the sequence of copyright mentions by merging ones that have
* equal content.
*/
def mergeByContent(copyrights: Seq[CopyrightMention]): Seq[CopyrightMention] =
copyrights
.groupBy(c => c.content)
.map({ case (_, equal) => mergeEqual(equal) })
.toSeq
/**
* Transforms the sequence of copyright mentions by merging ones that have
* equal context.
*
* This may be useful as a single comment block may contain multiple mentions
* of the word copyright, so multiple mentions coming from the same context
* would be redundant.
*
* It only allows to merge mentions that have only one context, as a
* heuristic to avoid too aggressive merging.
*/
def mergeByContext(
copyrights: Seq[CopyrightMention]
): Seq[CopyrightMention] = {
val (eligible, rest) = copyrights.partition(_.contexts.size == 1)
val merged = eligible.groupBy(c => c.contexts.head).map {
case (_, equal) =>
val ref = findBestRepresentative(equal)
ref.copy(origins = equal.flatMap(_.origins).distinct)
}
(merged ++ rest).toSeq
}
/**
* Returns the best representative to use as content of the merged set of
* copyrights that share a context.
*
* Usually, mentions that start with the word 'copyright' are most
* interesting, if the word appears somewhere in the middle it is usually
* just a continuation of a sentence. So we choose the instance starting with
* the word 'copyright' if such instance exists. Otherwise an arbitrary
* instance is selected.
*/
private def findBestRepresentative(
copyrights: Seq[CopyrightMention]
): CopyrightMention = {
copyrights
.find(_.content.stripLeading.toLowerCase.startsWith("copyright"))
.getOrElse(copyrights.head)
}
/**
* Merges multiple copyright mentions that have equal content.
*
* Their contexts and origins are concatenated (ignoring duplicates).
*/
def mergeEqual(copyrights: Seq[CopyrightMention]): CopyrightMention = {
val ref = copyrights.headOption.getOrElse(
throw new IllegalArgumentException("Copyrights must not be empty")
)
val ok = copyrights.forall(_.content == ref.content)
if (!ok) {
throw new IllegalArgumentException("Copyrights must be equal to merge")
}
CopyrightMention(
ref.content,
copyrights.flatMap(_.contexts).distinct,
copyrights.flatMap(_.origins).distinct
)
}
}
object AttachedFile {
/**
* Reads a file at the given path into an [[AttachedFile]].
*
* If `relativizeTo` is provided, the path included in the resulting
* [[AttachedFile]] is relative to the one provided.
*/
def read(path: Path, relativizeTo: Option[Path]): AttachedFile = {
val content = IO.read(path.toFile)
val actualPath = relativizeTo match {
case Some(root) => root.relativize(path)
case None => path
}
AttachedFile(actualPath, content)
}
}

48
project/src/main/scala/licenses/DependencyInformation.scala Normal file
View File

@ -0,0 +1,48 @@
package src.main.scala.licenses
import java.nio.file.Path
import com.typesafe.sbt.license.{DepModuleInfo, LicenseInfo}
import src.main.scala.licenses.report.Review
/**
* Defines a way to access sources of a dependency.
*
* This may involve various actions, such as downloading or extracting files
* into a temporary directory.
*/
trait SourceAccess {
/**
* Calls the callback with a path to the available sources.
*
* The provided path is only valid during the callbacks invocation and is
* considered invalid as soon as that function completes.
*/
def access[R](withSources: Path => R): R
}
/**
* Information about a single dependency.
*
* @param moduleInfo description of the module
* @param license information about the module's discovered license
* @param sources access to module's sources
* @param url project URL (if available)
*/
case class DependencyInformation(
moduleInfo: DepModuleInfo,
license: LicenseInfo,
sources: Seq[SourceAccess],
url: Option[String]
) {
/**
* Normalized name of the package that uniquely identifies the dependency.
*/
def packageName: String =
Review.normalizeName(
moduleInfo.organization + "." + moduleInfo.name + "-" +
moduleInfo.version
)
}

199
project/src/main/scala/licenses/DependencySummary.scala Normal file
View File

@ -0,0 +1,199 @@
package src.main.scala.licenses
import java.nio.file.Path
import sbt.IO
import src.main.scala.licenses.report.WithWarnings
/**
* Contains a sequence of dependencies and any attachments found.
*/
case class DependencySummary(
dependencies: Seq[(DependencyInformation, Seq[Attachment])]
)
/**
* Review status of the [[Attachment]].
*/
sealed trait AttachmentStatus {
/**
* Determines if the attachment with this status should be included in the
* final package.
*/
def included: Boolean
}
object AttachmentStatus {
/**
* Indicates that the attachment should be kept.
*/
case object Keep extends AttachmentStatus {
/**
* @inheritdoc
*/
override def included: Boolean = true
}
/**
* Indicates that the copyright mention should be kept, but its whole context
* should be used instead of its content.
*
* Only valid for [[CopyrightMention]].
*/
case object KeepWithContext extends AttachmentStatus {
/**
* @inheritdoc
*/
override def included: Boolean = true
}
/**
* Indicates that the attachment should be ignored.
*/
case object Ignore extends AttachmentStatus {
/**
* @inheritdoc
*/
override def included: Boolean = false
}
/**
* Indicates that the attachment has been added manually.
*/
case object Added extends AttachmentStatus {
/**
* @inheritdoc
*/
override def included: Boolean = true
}
/**
* Indicates that the attachment was not yet reviewed.
*/
case object NotReviewed extends AttachmentStatus {
/**
* @inheritdoc
*/
override def included: Boolean = false
}
}
/**
* Gathers information related to a dependency after the review.
*
* @param information original [[DependencyInformation]]
* @param licenseReviewed indicates if the license associated with the
* dependency is marked as reviewed
* @param licensePath may contain a path to the default license file that will
* be used; if empty, `files` should contain an alternative
* license
* @param files list of files attached to the dependency, with their review
* statuses
* @param copyrights list of copyright mentions attached to the dependency,
* with their review statuses
*/
case class ReviewedDependency(
information: DependencyInformation,
licenseReviewed: Boolean,
licensePath: Option[Path],
files: Seq[(AttachedFile, AttachmentStatus)],
copyrights: Seq[(CopyrightMention, AttachmentStatus)]
)
/**
* Summarizes the dependency review.
*
* The reviewed version of [[DependencySummary]].
*
* @param dependencies sequence of reviewed dependencies
* @param noticeHeader header to include in the generated NOTICE
* @param additionalFiles additional files that should be added to the root of
* the notice package
*/
case class ReviewedSummary(
dependencies: Seq[ReviewedDependency],
noticeHeader: String,
additionalFiles: Seq[AttachedFile]
) {
/**
* Returns a license-like file that is among attached files that are included
* (if such file exists).
*/
def includedLicense(dependency: ReviewedDependency): Option[AttachedFile] =
dependency.files
.find { f =>
val isIncluded = f._2.included
val name = f._1.path.getFileName.toString.toLowerCase
val isLicense = name.contains("license") || name.contains("licence")
isIncluded && isLicense
}
.map(_._1)
}
object ReviewedSummary {
/**
* Returns a list of warnings that indicate missing reviews or other issues.
*/
def warnAboutMissingReviews(summary: ReviewedSummary): WithWarnings[Unit] = {
val warnings = summary.dependencies.flatMap { dep =>
val warnings = collection.mutable.Buffer[String]()
val name = dep.information.moduleInfo.toString
if (!dep.licenseReviewed) {
warnings.append(
s"License ${dep.information.license.name} for $name is not reviewed."
)
}
val missingFiles = dep.files.filter(_._2 == AttachmentStatus.NotReviewed)
if (missingFiles.nonEmpty) {
warnings.append(
s"${missingFiles.size} files are not reviewed in $name."
)
}
val missingCopyrights =
dep.copyrights.filter(_._2 == AttachmentStatus.NotReviewed)
if (missingCopyrights.nonEmpty) {
warnings.append(
s"${missingCopyrights.size} copyrights are not reviewed in $name."
)
}
val includedInfos =
(dep.files.map(_._2) ++ dep.copyrights.map(_._2)).filter(_.included)
if (includedInfos.isEmpty) {
warnings.append(
s"No files or copyright information are included for $name."
)
}
(summary.includedLicense(dep), dep.licensePath) match {
case (Some(kept), Some(reviewedLicense)) =>
val licenseContent = IO.read(reviewedLicense.toFile)
if (licenseContent.strip != kept.content) {
warnings.append(
s"A license file was discovered in $name that is different " +
s"from the default license file that is associated with its " +
s"license ${dep.information.license.name}."
)
}
case (None, None) =>
warnings.append(
s"The license for $name is set to <custom> but no license-like " +
s"file is found."
)
case _ =>
}
warnings
}
WithWarnings.justWarnings(warnings)
}
}

46
project/src/main/scala/licenses/DistributionDescription.scala Normal file
View File

@ -0,0 +1,46 @@
package src.main.scala.licenses
import com.typesafe.sbt.license.LicenseReport
import com.typesafe.sbt.license.SbtCompat.IvySbt
import sbt.File
import sbt.librarymanagement.UpdateReport
/**
* Describes a component included in the distribution managed by SBT.
*
* @param name name of the component
* @param licenseReport license report generated by the `updateLicenses` task
* @param ivyModule `ivyModule` associated with the project
* @param classifiedArtifactsReport result of the `updateClassifiers` task
*/
case class SBTDistributionComponent(
name: String,
licenseReport: LicenseReport,
ivyModule: IvySbt#Module,
classifiedArtifactsReport: UpdateReport
)
/**
* Describes an artifact consisting of multiple components that is distributed
* independently.
*
* @param artifactName name of the artifact
* @param packageDestination location of the generated notice package
* @param sbtComponents sequence of SBT components that constitute this
* artifact; only root components (components that are
* directly packaged and distributed) are required (i.e.
* if X is distributed and X depends on Y but Y is not
* directly included, it does not have to be on this list
* as it will be automatically discovered)
*/
case class DistributionDescription(
artifactName: String,
packageDestination: File,
sbtComponents: Seq[SBTDistributionComponent]
) {
/**
* Returns names of root components included in the distribution.
*/
def rootComponentsNames: Seq[String] = sbtComponents.map(_.name)
}

48
project/src/main/scala/licenses/backend/AttachmentGatherer.scala Normal file
View File

@ -0,0 +1,48 @@
package src.main.scala.licenses.backend
import java.nio.file.{Files, Path}
import java.util.stream.Collectors
import src.main.scala.licenses.{Attachment, SourceAccess}
import scala.collection.JavaConverters._
/**
* Common interface for algorithms that gather attachments based on available
* sources.
*/
trait AttachmentGatherer {
/**
* Runs the algorithm that will scan the sources located in `root` and return
* any attachments found.
*/
def run(root: Path): Seq[Attachment]
/**
* A helper function that uses the provided [[SourceAccess]] instances to
* analyze sources provided by them and return any attachments found.
*/
def run(sources: Seq[SourceAccess]): Seq[Attachment] = {
sources.flatMap(_.access(run(_)))
}
}
object AttachmentGatherer {
/**
* A helper method that recursively traverses the directory structure at
* `root` and collects results of calling `action` on each encountered entry.
*
* The action is called for all kinds of entries that are encountered.
*/
def walk[R](root: Path)(action: Path => Seq[R]): Seq[R] = {
val stream = Files.walk(root)
try {
val list = stream.collect(Collectors.toList())
list.asScala.flatMap(action)
} finally {
stream.close()
}
}
}

28
project/src/main/scala/licenses/backend/CombinedBackend.scala Normal file
View File

@ -0,0 +1,28 @@
package src.main.scala.licenses.backend
import java.nio.file.Path
import src.main.scala.licenses.Attachment
/**
* An [[AttachmentGatherer]] that combines results from running multiple
* algorithms.
*/
class CombinedBackend(backends: Seq[AttachmentGatherer])
extends AttachmentGatherer {
/**
* @inheritdoc
*/
override def run(root: Path): Seq[Attachment] = {
backends.flatMap(_.run(root))
}
}
object CombinedBackend {
/**
* Creates a [[CombinedBackend]] from multiple [[AttachmentGatherer]]s.
*/
def apply(backends: AttachmentGatherer*): CombinedBackend =
new CombinedBackend(backends)
}

126
project/src/main/scala/licenses/backend/GatherCopyrights.scala Normal file
View File

@ -0,0 +1,126 @@
package src.main.scala.licenses.backend
import java.nio.file.{Files, Path}
import sbt.IO
import src.main.scala.licenses.{Attachment, CopyrightMention}
import scala.util.control.NonFatal
/**
* The algorithm for gathering any copyright notices inside of source files.
*
* It reads all text files and gathers any lines that contain the word
* copyright - it may introduce some false positives but these can be manually
* reviewed and ignored.
*
* It tries to include context in which the line has appeared, first by trying
* to detect if the line is a part of a longer comment; if the first method
* fails, it just shows 2 lines before and after the selected one.
*/
object GatherCopyrights extends AttachmentGatherer {
/**
* @inheritdoc
*/
override def run(root: Path): Seq[Attachment] = {
val allCopyrights = AttachmentGatherer.walk(root) { path =>
if (Files.isRegularFile(path)) {
val relativePath = root.relativize(path)
try {
val lines = IO.readLines(path.toFile).zipWithIndex.toIndexedSeq
lines
.filter(l => mayBeCopyright(l._1))
.map { case (str, idx) => (str, findContext(lines)(idx)) }
.map {
case (line, context) =>
CopyrightMention(line, Seq(context), Seq(relativePath))
}
.map(mention => mention.copy(content = cleanup(mention.content)))
} catch {
case NonFatal(e) =>
Seq(
CopyrightMention(
"<some files could not be read>",
Seq(e.toString),
Seq(relativePath)
)
)
}
} else {
Seq()
}
}
CopyrightMention.mergeByContext(
CopyrightMention.mergeByContent(allCopyrights)
)
}
/**
* Decides if the line may contain a copyright.
*/
private def mayBeCopyright(line: String): Boolean =
line.toLowerCase.contains("copyright")
/**
* Finds context of the given line.
*
* If the selected line seems to be a part of a block comment, the context
* becomes the whole comment, otherwise it just includes 2 lines before and
* after the selected line.
*
* @param allLines list of all lines available (that will be searched for
* context)
* @param idx index of the selected line in `allLines`
* @return a string representing the found context
*/
private def findContext(
allLines: IndexedSeq[(String, Int)]
)(idx: Int): String = {
val (line, _) = allLines(idx)
val nearbyLines: Seq[String] = if (line.stripLeading().startsWith("*")) {
val start = allLines
.take(idx + 1)
.filter(_._1.contains("/*"))
.map(_._2)
.lastOption
.getOrElse(idx - 2)
val end = allLines
.drop(idx)
.filter(_._1.contains("*/"))
.map(_._2)
.headOption
.getOrElse(idx + 2)
allLines.slice(start, end + 1).map(_._1)
} else if (
possiblePrefixes.exists(s => line.stripLeading().headOption.contains(s))
) {
val maxLinesOnSide = 20
val prefix =
line.takeWhile(c => c.isWhitespace || possiblePrefixes.contains(c))
val before = allLines
.take(idx)
.reverse
.takeWhile(_._1.startsWith(prefix))
.take(maxLinesOnSide)
.reverse
val after = allLines
.drop(idx)
.takeWhile(_._1.startsWith(prefix))
.take(maxLinesOnSide)
(before ++ after).map(_._1)
} else {
allLines.slice(idx - 2, idx + 3).map(_._1)
}
nearbyLines.mkString("\n")
}
private val possiblePrefixes = Seq('-', '#', ';', '/')
/**
* Strips comment-related characters from the prefix.
*/
private def cleanup(string: String): String = {
val charsToIgnore = Seq('*', '-', '#', '/')
string.dropWhile(char => char.isWhitespace || charsToIgnore.contains(char))
}
}

73
project/src/main/scala/licenses/backend/GatherNotices.scala Normal file
View File

@ -0,0 +1,73 @@
package src.main.scala.licenses.backend
import java.nio.file.{Files, Path}
import src.main.scala.licenses.{AttachedFile, Attachment}
/**
* The algorithm for gathering any copyright-related files found in the
* sources.
*
* It gathers any files whose name contains one of the keywords from
* `possibleNames`, but filters out ordinary source files.
*/
object GatherNotices extends AttachmentGatherer {
/**
* @inheritdoc
*/
override def run(root: Path): Seq[Attachment] = {
AttachmentGatherer.walk(root) { path =>
if (Files.isRegularFile(path) && mayBeRelevant(path)) {
Seq(AttachedFile.read(path, Some(root)))
} else Seq()
}
}
/**
* Decides if the path may be relevant and should be included in the result.
*/
def mayBeRelevant(fileName: String): Boolean = {
val extension = {
val lastDot = fileName.lastIndexOf(".")
if (lastDot < 0) None
else Some(fileName.substring(lastDot + 1))
}
if (extension.exists(ignoredExtensions.contains)) {
false
} else {
val simplifiedName = fileName.filter(_.isLetterOrDigit)
possibleNames.exists(simplifiedName.contains)
}
}
/**
* Decides if the filename is relevant and should be included in the result.
*/
def mayBeRelevant(path: Path): Boolean =
mayBeRelevant(path.getFileName.toString.toLowerCase)
/**
* File extensions that are ignored.
*
* Source files are ignored because they are scanned for copyright notices by
* [[GatherCopyrights]], they are not likely to constitute separate copyright
* files.
*/
private val ignoredExtensions = Seq("scala", "java")
/**
* File name keywords that indicate that a file should be included.
*/
val possibleNames =
Seq(
"notice",
"copyright",
"thirdparty",
"license",
"licence",
"credit",
"copying",
"authors"
)
}

99
project/src/main/scala/licenses/backend/GithubHeuristic.scala Normal file
View File

@ -0,0 +1,99 @@
package src.main.scala.licenses.backend
import java.nio.file.Path
import sbt.Logger
import sbt.io.syntax.url
import src.main.scala.licenses.{
AttachedFile,
Attachment,
CopyrightMention,
DependencyInformation
}
import scala.sys.process._
import scala.util.control.NonFatal
/**
* Tries to find copyright mentions in the GitHub project homepage and any
* copyright-related files contained in the repository root.
*
* This is a fallback backend that may be used if the primary backends do not
* find anything. It should not be used otherwise, as the primary backends are
* most likely to yield more precise results - for example, the
* GitHub-published version may be different than the one we use.
*/
case class GithubHeuristic(info: DependencyInformation, log: Logger) {
/**
* Runs the gathering process and returns any attachments found.
*
* It proceeds only if the project has an URL that seems to point to GitHub.
*/
def run(): Seq[Attachment] = {
info.url match {
case Some(url) if url.contains("github.com") =>
tryDownloadingAttachments(url.replace("http://", "https://"))
case _ => Seq()
}
}
/**
* Downloads the project homepage at `address` and looks for any copyright
* mentions or links that may lead to copyright-related files.
*
* Any found files are fetched and saved into the results.
*/
def tryDownloadingAttachments(address: String): Seq[Attachment] =
try {
val homePage = url(address).cat.!!
val fileRegex = """<a .*? href="(.*?)".*?>(.*?)</a>""".r("href", "name")
val matches = fileRegex
.findAllMatchIn(homePage)
.map(m => (m.group("name"), m.group("href")))
.filter(p => mayBeRelevant(p._1))
.toList
val files = matches.flatMap {
case (_, href) =>
try {
val content =
url("https://github.com" + href.replace("blob", "raw")).cat.!!
Seq(
AttachedFile(Path.of(href), content, origin = Some("github.com"))
)
} catch {
case NonFatal(error) =>
log.warn(
s"Found file $href but cannot download it: $error"
)
Seq()
}
}
val copyrights = homePage.linesIterator.toList
.filter(_.toLowerCase.contains("copyright"))
.map(line =>
CopyrightMention(
line,
Seq(s"Found at $address"),
Seq(Path.of("github.com"))
)
)
files ++ copyrights
} catch {
case NonFatal(error) =>
log.warn(s"GitHub backend for ${info.packageName} failed with $error")
Seq()
}
/**
* Decides if the file may be relevant and should be included in the result.
*
* Filenames that contain spaces are ignored because they are usually normal
* links and do not lead to relevant files.
*/
private def mayBeRelevant(name: String): Boolean = {
val normalized = name.strip().toLowerCase()
!normalized.contains(' ') &&
GatherNotices.mayBeRelevant(normalized)
}
}

19
project/src/main/scala/licenses/frontend/DependencyFilter.scala Normal file
View File

@ -0,0 +1,19 @@
package src.main.scala.licenses.frontend
import src.main.scala.licenses.DependencyInformation
/**
* Filters out irrelevant dependencies.
*
* Currently, dependencies whose organisation is `org.enso` are ignored, as
* they are owned by us, so they do not require any additional licensing
* notices.
*/
object DependencyFilter {
/**
* Decides if the dependency should be kept for further processing.
*/
def shouldKeep(dependencyInformation: DependencyInformation): Boolean =
dependencyInformation.moduleInfo.organization != "org.enso"
}

189
project/src/main/scala/licenses/frontend/SbtLicenses.scala Normal file
View File

@ -0,0 +1,189 @@
package src.main.scala.licenses.frontend
import java.nio.file.Path
import com.typesafe.sbt.license.SbtCompat.IvySbt
import com.typesafe.sbt.license.{DepLicense, DepModuleInfo, LicenseReport}
import org.apache.ivy.core.report.ResolveReport
import org.apache.ivy.core.resolve.IvyNode
import sbt.Compile
import sbt.internal.util.ManagedLogger
import sbt.io.IO
import sbt.librarymanagement.ConfigRef
import src.main.scala.licenses.{
DependencyInformation,
SBTDistributionComponent,
SourceAccess
}
import scala.collection.JavaConverters._
/**
* Defines the algorithm for discovering dependency metadata.
*/
object SbtLicenses {
/**
* Defines configurations that are deemed relevant for dependency discovery.
*
* Currently we only analyse Compile dependencies as these are the ones that
* get packaged.
*
* Provided dependencies are assumed to be already present in the used
* runtime, so we do not distribute them. One exception is the launcher which
* does distribute the provided SubstrateVM dependencies as part of it being
* compiled with the SVM. But that has to be handled independently anyway.
*/
val relevantConfigurations = Seq(Compile)
/**
* Analyzes the provided [[SBTDistributionComponent]]s collecting their
* unique dependencies and issuing any warnings.
*
* @param components description of SBT components included in the
* distribution
* @param log logger to use when resolving dependencies
* @return a sequence of collected dependency information and a sequence of
* encountered warnings
*/
def analyze(
components: Seq[SBTDistributionComponent],
log: ManagedLogger
): (Seq[DependencyInformation], Seq[String]) = {
val results: Seq[(Seq[Dependency], Vector[Path], Seq[String])] =
components.map { component =>
val report = resolveIvy(component.ivyModule, log)
val ivyDeps =
report.getDependencies.asScala.map(_.asInstanceOf[IvyNode])
val sourceArtifacts = component.classifiedArtifactsReport
.select((configRef: ConfigRef) =>
relevantConfigurations.map(_.name).contains(configRef.name)
)
.map(_.toPath)
.filter(_.getFileName.toString.endsWith("sources.jar"))
val deps = for {
dep <- component.licenseReport.licenses
depNode =
ivyDeps
.find(ivyDep => safeModuleInfo(ivyDep) == Some(dep.module))
.getOrElse(
throw new RuntimeException(
s"Could not find Ivy node for resolved module ${dep.module}."
)
)
} yield {
val sources = sourceArtifacts.filter(
_.getFileName.toString.startsWith(dep.module.name)
)
Dependency(dep, depNode, sources)
}
val warnings =
if (component.licenseReport.licenses.isEmpty)
Seq(s"License report for component ${component.name} is empty.")
else Seq()
(deps, sourceArtifacts, warnings)
}
val distinctDependencies =
results.flatMap(_._1).groupBy(_.depLicense.module).map(_._2.head).toSeq
val distinctSources = results.flatMap(_._2).distinct
val wrappedDeps =
for (dependency <- distinctDependencies)
yield DependencyInformation(
moduleInfo = dependency.depLicense.module,
license = dependency.depLicense.license,
sources = findSources(dependency),
url = tryFindingUrl(dependency)
)
val relevantDeps = wrappedDeps.filter(DependencyFilter.shouldKeep)
val missingWarnings = for {
dep <- relevantDeps
if dep.sources.isEmpty
} yield s"Could not find sources for ${dep.moduleInfo}"
val unexpectedWarnings = for {
source <- distinctSources
if !distinctDependencies.exists(_.sourcesJARPaths.contains(source))
} yield s"Found a source $source that does not belong to any known " +
s"dependencies, perhaps the algorithm needs updating?"
val reportsWarnings = results.flatMap(_._3)
(relevantDeps, missingWarnings ++ unexpectedWarnings ++ reportsWarnings)
}
/**
* Uses the [[LicenseReport]] plugin to resolve the dependencies of the Ivy
* module of an SBT component.
*
* Returns the resolved report or throws an exception if any errors were
* encountered.
*/
private def resolveIvy(
ivyModule: IvySbt#Module,
log: ManagedLogger
): ResolveReport = {
val (report, err) = LicenseReport.resolve(ivyModule, log)
err.foreach(throw _)
report
}
/**
* Returns a project URL if it is defined for the dependency or None.
*/
private def tryFindingUrl(dependency: Dependency): Option[String] =
Option(dependency.ivyNode.getDescriptor).flatMap(descriptor =>
Option(descriptor.getHomePage)
)
/**
* Creates a [[SourceAccess]] instance that unpacks the source files from a
* JAR archive into a temporary directory.
*
* It removes the temporary directory after the analysis is finished.
*/
private def createSourceAccessFromJAR(jarPath: Path): SourceAccess =
new SourceAccess {
override def access[R](withSources: Path => R): R =
IO.withTemporaryDirectory { root =>
IO.unzip(jarPath.toFile, root)
withSources(root.toPath)
}
}
/**
* Returns a sequence of [[SourceAccess]] instances that give access to any
* sources JARs that are available with the dependency.
*/
private def findSources(dependency: Dependency): Seq[SourceAccess] =
dependency.sourcesJARPaths.map(createSourceAccessFromJAR)
/**
* Wraps information related to a dependency.
*
* @param depLicense information on the license
* @param ivyNode Ivy node that can be used to find metadata
* @param sourcesJARPaths paths to JARs containing dependency's sources
*/
case class Dependency(
depLicense: DepLicense,
ivyNode: IvyNode,
sourcesJARPaths: Seq[Path]
)
/**
* Returns [[DepModuleInfo]] for an [[IvyNode]] if it is defined, or None.
*/
private def safeModuleInfo(dep: IvyNode): Option[DepModuleInfo] =
for {
moduleId <- Option(dep.getModuleId)
moduleRevision <- Option(dep.getModuleRevision)
revisionId <- Option(moduleRevision.getId)
} yield DepModuleInfo(
moduleId.getOrganisation,
moduleId.getName,
revisionId.getRevision
)
}

268
project/src/main/scala/licenses/report/HTMLWriter.scala Normal file
View File

@ -0,0 +1,268 @@
package src.main.scala.licenses.report
import java.io.{BufferedWriter, PrintWriter}
import java.nio.file.Files
import sbt.File
/**
* Allows to create very simple HTML reports.
*
* @param bufferedWriter writer to which the generated code will be written
*/
class HTMLWriter(bufferedWriter: BufferedWriter) {
val writer = new PrintWriter(bufferedWriter)
/**
* Writes the header containing basic styles and scripts for the reports.
*
* @param title the page title
*/
def writeHeader(title: String): Unit = {
val heading =
s"""<html>
|<head>
|<meta charset="utf-8">
|<script src="https://code.jquery.com/jquery-1.12.4.js"></script>
|<script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
|<title>$title</title>
|<style>
|table, th, td {
| border: solid 1px;
|}
|h4 {
| font-weight: normal;
| display: inline;
|}
|</style>
| <script>
|$$( function() {
| $$( ".accordion" ).accordion({
| active: false,
| collapsible: true
| });
|});
|</script>
|</head>
|<body>""".stripMargin
writer.println(heading)
}
/**
* A helper class that allows to manage columns in a table row.
*
* @param columns expected columns count
*/
class RowWriter(columns: Int) {
var added = 0
/**
* A helper function that adds a column just containing plain text.
*/
def addColumn(text: String): Unit = {
addColumn(writeText(text))
}
/**
* Adds a cell in the next column that will contain anything that is
* written when executing the `writeAction`.
*/
def addColumn(writeAction: => Unit): Unit = {
writer.println("<td>")
writeAction
writer.println("</td>")
added += 1
}
/**
* Finishes the current row and ensures that all expected columns have been
* added.
*/
def finish(): Unit = {
if (added < columns)
throw new IllegalStateException("Not enough columns added")
}
}
/**
* Writes a HTML table based on the provided descriptiom.
*
* @param headers sequence of header names for each column
* @param rows sequence of functions that will be called to create rows of
* the table, each function is provided with a [[RowWriter]] that
* should write as many columns as there are headers
*/
def writeTable(headers: Seq[String], rows: Seq[RowWriter => Unit]): Unit = {
writer.println("<table>")
writer.println("<tr>")
for (header <- headers) {
writer.println(s"<th>$header</th>")
}
writer.println("</tr>")
val columns = headers.length
for (row <- rows) {
writer.println("<tr>")
val rw = new RowWriter(columns)
row(rw)
rw.finish()
writer.println("</tr>")
}
writer.println("</table>")
}
/**
* Writes an unordered list.
*
* @param elements sequence of functions that will be called to write each of
* the list's elements; everything written inside of each
* function will be part of its list element
*/
def writeList(elements: Seq[() => Unit]): Unit = {
writer.println("<ul>")
for (elem <- elements) {
writer.println("<li>")
elem()
writer.println("</li>")
}
writer.println("</ul>")
}
/**
* Writes a link.
*
* @param text link text
* @param url link target
*/
def writeLink(text: String, url: String): Unit =
writer.println(s"""<a href="$url">$text</a>""")
/**
* Writes a H1 heading.
*/
def writeHeading(heading: String): Unit =
writer.println(s"<h1>$heading</h1>")
/**
* Writes a H2 heading.
*/
def writeSubHeading(heading: String): Unit =
writer.println(s"<h2>$heading</h2>")
/**
* Writes a paragraph of styled text.
*/
def writeParagraph(text: String, styles: Style*): Unit =
writer.println(s"""<p style="${styles.mkString(";")}">$text</p>""")
/**
* Writes plain (but potentially styled) text.
*/
def writeText(text: String, styles: Style*): Unit =
if (styles.nonEmpty)
writer.println(s"""<span style="${styles.mkString(";")}">$text</span>""")
else writer.println(text)
/**
* Writes a collapsible entry that by default just displays the `title`, but
* expands when clicked to show the `content`.
*/
def writeCollapsible(
title: String,
content: String
): Unit = {
writer.println(s"""<div class="accordion">
|<h4>$title</h4>
|<div style="display:none">
|<pre>
|$content
|</pre>
|</div></div>""".stripMargin)
}
/**
* Writes an empty element that can be overridden by an injected script.
*
* Parameters names and values need to be properly escaped to fit for HTML.
*/
def makeInjectionHandler(
className: String,
params: (String, String)*
): String = {
val mappedParams = params
.map {
case (name, value) => s"""data-$name="$value" """
}
.mkString(" ")
s"""<span class="$className" $mappedParams></span>
|""".stripMargin
}
/**
* Escapes a string that may contain HTML markup to not be rendered but
* displayed as normal text.
*/
def escape(string: String): String =
string.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
/**
* Finishes writing the document and closes the output.
*/
def close(): Unit = {
writer.println("</body></html>")
writer.close()
}
}
object HTMLWriter {
/**
* Creates an [[HTMLWriter]] that writes its output to a [[File]] (creating
* it or overwriting if it exists).
*/
def toFile(destination: File): HTMLWriter =
new HTMLWriter(Files.newBufferedWriter(destination.toPath))
}
/**
* Styles that can be used to make text stand out.
*/
sealed trait Style
object Style {
/**
* Makes the text bold.
*/
case object Bold extends Style {
override def toString: String = "font-weight:bold"
}
/**
* Makes the text red.
*/
case object Red extends Style {
override def toString: String = "color:red"
}
/**
* Makes the text gray.
*/
case object Gray extends Style {
override def toString: String = "color:gray"
}
/**
* Makes the text green.
*/
case object Green extends Style {
override def toString: String = "color:green"
}
/**
* Makes the text black.
*/
case object Black extends Style {
override def toString: String = "color:black"
}
}

138
project/src/main/scala/licenses/report/PackageNotices.scala Normal file
View File

@ -0,0 +1,138 @@
package src.main.scala.licenses.report
import java.nio.file.{Files, Path}
import sbt._
import src.main.scala.licenses.{
AttachmentStatus,
CopyrightMention,
DistributionDescription,
ReviewedSummary
}
import scala.annotation.tailrec
/**
* Defines writing the notices package for distribution.
*/
object PackageNotices {
/**
* Creates the notices package based on the `summary` in `destination`.
*
* The `destination` directory is cleaned before creating the new notices.
*
* It copies licences, files and gathers copyright notices that were set to
* be included in the review and generates a summary `NOTICE` file.
*/
def create(
description: DistributionDescription,
summary: ReviewedSummary,
destination: File
): Unit = {
IO.delete(destination)
IO.createDirectory(destination)
if (IO.listFiles(destination).nonEmpty) {
throw new RuntimeException(
s"Could not clean ${destination}, cannnot continue creating the " +
s"package."
)
}
val artifactName = description.artifactName
val mainNotice = new StringBuilder
mainNotice.append(summary.noticeHeader)
val licensesRoot = destination / "licenses"
val processedLicenses = collection.mutable.Set[Path]()
for (dependency <- summary.dependencies) {
val name = dependency.information.moduleInfo.name
val licenseName = dependency.information.license.name
mainNotice.append(
s"\n\n'$name', licensed under the $licenseName, " +
s"is distributed with the $artifactName.\n"
)
dependency.licensePath match {
case Some(path) =>
val name = path.getFileName.toString
if (!processedLicenses.contains(path)) {
val destination = licensesRoot / name
IO.copyFile(path.toFile, destination)
if (!destination.exists()) {
throw new RuntimeException(s"Failed to copy the license $path")
}
processedLicenses.add(path)
}
mainNotice.append(
s"The license file can be found at `licenses/$name`.\n"
)
case None =>
mainNotice.append(
s"The license file can be found at along the copyright notices.\n"
)
}
val packageName = dependency.information.packageName
val packageRoot = destination / packageName
val files = dependency.files.filter(_._2.included).map(_._1)
val copyrights = dependency.copyrights.filter(_._2.included)
if (files.size + copyrights.size > 0) {
mainNotice.append(
s"Copyright notices related to this dependency can be found in the " +
s"directory `$packageName`.\n"
)
IO.createDirectory(packageRoot)
}
@tailrec
def findFreeName(name: String, counter: Int = 0): File = {
val actualName = if (counter > 0) s"$name.$counter" else name
val file = packageRoot / actualName
if (Files.exists(file.toPath)) findFreeName(name, counter + 1)
else file
}
for (attachedFile <- files) {
val file = findFreeName(attachedFile.path.getFileName.toString)
IO.write(file, attachedFile.content)
}
def renderCopyright(
copyright: CopyrightMention,
status: AttachmentStatus
): String =
status match {
case AttachmentStatus.Keep => copyright.content
case AttachmentStatus.KeepWithContext =>
if (copyright.contexts.size != 1) {
throw new IllegalStateException(
"`KeepWithContext` can only be used for copyrights that " +
"have exactly one context."
)
}
copyright.contexts.head
case AttachmentStatus.Added =>
copyright.contexts.head
case _ =>
throw new IllegalStateException(
"Only `included` copyrights should be present at this point."
)
}
if (copyrights.nonEmpty) {
val compiledCopyrights = copyrights
.map {
case (m, s) => renderCopyright(m, s)
}
.mkString("\n\n")
val freeName = findFreeName("NOTICES")
IO.write(freeName, compiledCopyrights)
}
}
IO.write(destination / "NOTICE", mainNotice.toString())
}
}

266
project/src/main/scala/licenses/report/Report.scala Normal file
View File

@ -0,0 +1,266 @@
package src.main.scala.licenses.report
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import java.util.Base64
import sbt.{File, IO}
import src.main.scala.licenses._
/**
* Allows to write a report summarizing current status of the review.
*
* The report lists all dependencies and status of found attachments.
*
* With the legal-review-helper script, the generated report can be used to
* interactively perform substantial parts of the review.
*/
object Report {
/**
* Writes the report in HTML format.
*
* @param description description of the distribution
* @param summary reviewed summary of findings
* @param warnings sequence of warnings
* @param destination location of the generated HTML file
*/
def writeHTML(
description: DistributionDescription,
summary: ReviewedSummary,
warnings: Seq[String],
destination: File
): Unit = {
IO.createDirectory(destination.getParentFile)
val writer = HTMLWriter.toFile(destination)
try {
writer.writeHeader(s"Dependency summary for ${description.artifactName}")
writer.writeHeader(
s"Gathered Information on ${description.artifactName} Distribution"
)
writer.writeParagraph(
s"Root components analyzed for the distribution: " +
s"${description.rootComponentsNames.mkString(", ")}."
)
if (warnings.isEmpty) {
writer.writeParagraph("There are no warnings.", Style.Green)
} else {
writer.writeParagraph(
s"There are ${warnings.size} warnings!",
Style.Bold,
Style.Red
)
}
writeDependencySummary(writer, summary)
writer.writeList(warnings.map { warning => () =>
writer.writeText(writer.escape(warning))
})
writer.writeCollapsible("NOTICE header", summary.noticeHeader)
if (summary.additionalFiles.nonEmpty) {
writer.writeSubHeading("Additional files that will be added:")
writer.writeList(summary.additionalFiles.map { file => () =>
writer.writeText(file.path.toString)
})
}
} finally {
writer.close()
}
}
/**
* Renders [[AttachmentStatus]] as HTML that will show the status name and a
* color associated with it.
*/
private def renderStatus(attachmentStatus: AttachmentStatus): String = {
val style = attachmentStatus match {
case AttachmentStatus.Keep => Style.Black
case AttachmentStatus.KeepWithContext => Style.Black
case AttachmentStatus.Ignore => Style.Gray
case AttachmentStatus.Added => Style.Green
case AttachmentStatus.NotReviewed => Style.Red
}
s"""<span style="$style">$attachmentStatus</span>"""
}
/**
* Renders a message about similarity of the file to a selected license file.
*
* If the filename is license-like, the file is compared with the selected
* license (if any). If the file differs from the selected license and is
* ignored, a warning is displayed. If the file is kept but is identical (so
* it is redundant) the message is also displayed as a minor warning.
*/
private def renderSimilarity(
licensePath: Option[Path],
file: AttachedFile,
status: AttachmentStatus
): String = {
val name = file.path.getFileName.toString.toLowerCase
if (name.contains("license") || name.contains("licence")) {
licensePath match {
case Some(value) =>
val defaultText = IO.read(value.toFile)
if (defaultText.strip() == file.content.strip()) {
val color = if (status.included) Style.Red else Style.Green
s"""<span style="$color">100% identical to default license</span>"""
} else
s"""<span style="${Style.Red}">Differs from used license!</span>"""
case None => ""
}
} else ""
}
/**
* Writes a table containing summary of dependencies and their gathered
* copyright information.
*/
private def writeDependencySummary(
writer: HTMLWriter,
summary: ReviewedSummary
): Unit = {
val sorted = summary.dependencies.sortBy(dep =>
(dep.information.moduleInfo.organization, dep.information.moduleInfo.name)
)
writer.writeSubHeading("Summary of all compile dependencies")
writer.writeTable(
headers = Seq(
"Organization",
"Name",
"Version",
"URL",
"License",
"Attached files",
"Possible copyrights"
),
rows = sorted.map {
case dep @ ReviewedDependency(
information,
licenseReviewed,
licensePath,
files,
copyrights
) =>
(rowWriter: writer.RowWriter) =>
val moduleInfo = information.moduleInfo
rowWriter.addColumn(moduleInfo.organization)
rowWriter.addColumn(moduleInfo.name)
rowWriter.addColumn(moduleInfo.version)
rowWriter.addColumn {
information.url match {
case Some(value) =>
writer.writeLink(value, value)
case None => writer.writeText("No URL")
}
}
val license = information.license
rowWriter.addColumn {
writer.writeLink(license.name, license.url)
writer.writeText(s"(${license.category.name}) <br>")
licensePath match {
case Some(path) =>
writer.writeText(
path.getFileName.toString,
if (licenseReviewed) Style.Green else Style.Red
)
case None if licenseReviewed =>
val licenseFile = summary.includedLicense(dep)
licenseFile match {
case Some(file) =>
writer.writeText(
s"Custom license ${file.path.getFileName}",
Style.Green
)
case None =>
writer.writeText(
"Custom license defined but not provided!",
Style.Red
)
}
case None if !licenseReviewed =>
val name = Review.normalizeName(license.name)
writer.writeText(
s"Not reviewed, filename: <pre>$name</pre>",
Style.Red
)
}
}
rowWriter.addColumn {
if (files.isEmpty) writer.writeText("No attached files.")
else
writer.writeList(files.map {
case (file, status) =>
() =>
val injection = writer.makeInjectionHandler(
"file-ui",
"package" -> dep.information.packageName,
"filename" -> file.path.toString,
"status" -> status.toString
)
val origin = file.origin
.map(origin => s" (Found at $origin)")
.getOrElse("")
writer.writeCollapsible(
s"${file.fileName} (${renderStatus(status)})$origin " +
s"${renderSimilarity(licensePath, file, status)}",
injection +
writer.escape(file.content)
)
})
}
rowWriter.addColumn {
if (copyrights.isEmpty) {
val bothEmpty = files.isEmpty && copyrights.isEmpty
if (bothEmpty) {
writer.writeText(
"No notices or copyright information found, " +
"this may be a problem.",
Style.Red
)
} else {
writer.writeText("No copyright information found.")
}
} else
writer.writeList(copyrights.map {
case (mention, status) =>
() =>
val foundAt = mention.origins match {
case Seq() => ""
case Seq(one) => s"Found at $one"
case Seq(first, rest @ _*) =>
s"Found at $first and ${rest.size} other files."
}
val contexts = if (mention.contexts.nonEmpty) {
mention.contexts
.map(c => "\n" + writer.escape(c) + "\n")
.mkString("<hr>")
} else ""
val injection = writer.makeInjectionHandler(
"copyright-ui",
"package" -> dep.information.packageName,
"content" -> Base64.getEncoder.encodeToString(
mention.content.getBytes(StandardCharsets.UTF_8)
),
"contexts" -> mention.contexts.length.toString,
"status" -> status.toString
)
val content = writer.escape(mention.content)
val moreInfo = injection + foundAt + contexts
writer.writeCollapsible(
s"$content (${renderStatus(status)})",
moreInfo
)
})
}
}
)
}
}

333
project/src/main/scala/licenses/report/Review.scala Normal file
View File

@ -0,0 +1,333 @@
package src.main.scala.licenses.report
import java.nio.file.{Files, Path}
import java.time.LocalDate
import sbt._
import src.main.scala.licenses._
import scala.util.control.NonFatal
/**
* Reads settings from the `root` to add review statuses to discovered
* attachments and add any additional attachments coming from the settings.
*
* The review settings consist of the following files or directories (all are
* optional):
*
* - `notice-header` - contains the header that will start the main generated
* NOTICE
* - `files-add` - directory that may contain additional files that should be
* added to the package
* - `reviewed-licenses` - directory that may contain files for reviewed
* licenses; the files should be named with the normalized license name and
* they should contain a path to that license's file
* - and for each dependency, a subdirectory named as its `packageName` with
* following entries:
* - `files-add` - directory that may contain additional files that should
* be added to the subdirectory for this package
* - `files-keep` - a file containing names of files found in the package
* sources that should be included in the package
* - `files-ignore` - a file containing names of files found in the package
* sources that should not be included
* - `custom-license` - a file that indicates that the dependency should not
* point to the default license, but it should contain a custom one within
* its files
* - `copyright-keep` - copyright lines that should be included in the
* notice summary for the package
* - `copyright-keep-context` - copyright lines that should be included
* (alongside with their context) in the notice summary for the package
* - `copyright-ignore` - copyright lines that should not be included in the
* notice summary for the package
* - `copyright-add` - a single file whose contents will be added to the
* notice summary for the package
*
* @param root directory that contains files which guide the review process
* @param dependencySummary summary of discovered dependency information
*/
case class Review(root: File, dependencySummary: DependencySummary) {
/**
* Runs the review process, returning a [[ReviewedDependency]] which includes
* information from the [[DependencySummary]] enriched with review statuses.
*/
def run(): WithWarnings[ReviewedSummary] =
for {
reviews <- dependencySummary.dependencies.map {
case (information, attachments) =>
reviewDependency(information, attachments)
}.flip
header = findHeader()
files = findAdditionalFiles(root / "files-add")
summary = ReviewedSummary(reviews, header, files)
_ <- ReviewedSummary.warnAboutMissingReviews(summary)
existingPackages = dependencySummary.dependencies.map(_._1.packageName)
_ <- warnAboutMissingDependencies(existingPackages)
} yield summary
/**
* Returns a list of warnings for dependencies whose configuration has been
* detected but which have not been detected.
*
* This may be used to detect dependencies that have been removed after an
* update.
*/
private def warnAboutMissingDependencies(
existingPackageNames: Seq[String]
): WithWarnings[Unit] = {
val foundConfigurations = listFiles(root).filter(_.isDirectory)
val expectedFileNames =
existingPackageNames ++ Seq("files-add", "reviewed-licenses")
val unexpectedConfigurations =
foundConfigurations.filter(p => !expectedFileNames.contains(p.getName))
val warnings = unexpectedConfigurations.map(p =>
s"Found legal review configuration for package ${p.getName}, " +
s"but no such dependency has been found. Perhaps it has been removed?"
)
WithWarnings.justWarnings(warnings)
}
/**
* Finds a header defined in the settings or
*/
private def findHeader(): String =
readFile(root / "notice-header").getOrElse(Review.defaultHeader)
/**
* Reads files from the provided directory as [[AttachedFile]].
*/
private def findAdditionalFiles(dir: File): Seq[AttachedFile] =
listFiles(dir).map(f => AttachedFile.read(f.toPath, Some(dir.toPath)))
/**
* Splits the sequence of attachments into sequences of files and copyrights.
*/
private def splitAttachments(
attachments: Seq[Attachment]
): (Seq[AttachedFile], Seq[CopyrightMention]) = {
val notices = attachments.collect { case n: AttachedFile => n }
val copyrights = attachments.collect { case c: CopyrightMention => c }
(notices, copyrights)
}
/**
* Returns only such copyrights that are not included in one of the
* discovered files.
*/
private def removeCopyrightsIncludedInNotices(
copyrights: Seq[CopyrightMention],
notices: Seq[AttachedFile]
): Seq[CopyrightMention] = {
def shouldKeepCopyright(copyright: CopyrightMention): Boolean = {
val allOriginsAreFresh = copyright.origins.forall { path =>
!notices.exists(_.path == path)
}
allOriginsAreFresh
}
copyrights.filter(shouldKeepCopyright)
}
/**
* Returns the review status of the license and any attachments associated
* with the dependency.
*/
private def reviewDependency(
info: DependencyInformation,
attachments: Seq[Attachment]
): WithWarnings[ReviewedDependency] = {
val packageRoot = root / info.packageName
val (licenseReviewed, licensePath) = reviewLicense(packageRoot, info)
val (files, copyrights) = splitAttachments(attachments)
val copyrightsDeduplicated =
removeCopyrightsIncludedInNotices(copyrights, files)
for {
processedFiles <- reviewFiles(packageRoot, files) ++ addFiles(packageRoot)
processedCopyrights <-
reviewCopyrights(packageRoot, copyrightsDeduplicated) ++
addCopyrights(packageRoot)
} yield ReviewedDependency(
information = info,
licenseReviewed = licenseReviewed,
licensePath = licensePath,
files = processedFiles,
copyrights = processedCopyrights
)
}
/**
* Enriches the file attachments with their review status.
*/
private def reviewFiles(
packageRoot: File,
files: Seq[AttachedFile]
): WithWarnings[Seq[(AttachedFile, AttachmentStatus)]] = {
def keyForFile(file: AttachedFile): String = file.path.toString
val keys = files.map(keyForFile)
for {
ignore <- readExpectedLines("files-ignore", keys, packageRoot)
keep <- readExpectedLines("files-keep", keys, packageRoot)
} yield {
def review(file: AttachedFile): AttachmentStatus = {
val key = keyForFile(file)
if (keep.contains(key)) AttachmentStatus.Keep
else if (ignore.contains(key)) AttachmentStatus.Ignore
else AttachmentStatus.NotReviewed
}
files.map(f => (f, review(f)))
}
}
/**
* Returns any additional file attachments that are manually added in the
* review.
*/
private def addFiles(
packageRoot: File
): Seq[(AttachedFile, AttachmentStatus)] =
findAdditionalFiles(packageRoot / "files-add")
.map((_, AttachmentStatus.Added))
/**
* Enriches the copyright attachments with their review status.
*/
private def reviewCopyrights(
packageRoot: File,
copyrights: Seq[CopyrightMention]
): WithWarnings[Seq[(CopyrightMention, AttachmentStatus)]] = {
def keyForMention(copyrightMention: CopyrightMention): String =
copyrightMention.content.strip
val keys = copyrights.map(keyForMention)
for {
ignore <- readExpectedLines("copyright-ignore", keys, packageRoot)
keep <- readExpectedLines("copyright-keep", keys, packageRoot)
keepContext <-
readExpectedLines("copyright-keep-context", keys, packageRoot)
} yield {
def review(copyright: CopyrightMention): AttachmentStatus = {
val key = keyForMention(copyright)
if (keepContext.contains(key)) AttachmentStatus.KeepWithContext
else if (keep.contains(key)) AttachmentStatus.Keep
else if (ignore.contains(key)) AttachmentStatus.Ignore
else AttachmentStatus.NotReviewed
}
copyrights.map(c => (c, review(c)))
}
}
/**
* Returns any additional copyright attachments that are manually added in
* the review.
*/
private def addCopyrights(
packageRoot: File
): Seq[(CopyrightMention, AttachmentStatus)] =
readFile(packageRoot / "copyright-add")
.map(text =>
(
CopyrightMention(
content = "<manually added mentions>",
contexts = Seq(text),
origins = Seq()
),
AttachmentStatus.Added
)
)
.toSeq
/**
* Checks if the license has been reviewed.
*
* Returns a boolean value indicating if it has been reviewed and a path to
* the license file if a default file is used.
*/
private def reviewLicense(
packageRoot: File,
info: DependencyInformation
): (Boolean, Option[Path]) = {
if (Files.exists((packageRoot / "custom-license").toPath)) (true, None)
else
readFile(
root / "reviewed-licenses" / Review.normalizeName(info.license.name)
)
.map(p => (true, Some(Path.of(p.strip()))))
.getOrElse((false, None))
}
/**
* Reads the file as lines.
*
* Returns an empty sequence if the file cannot be read.
*/
private def readLines(file: File): Seq[String] =
try { IO.readLines(file).map(_.strip).filter(_.nonEmpty) }
catch { case NonFatal(_) => Seq() }
/**
* Reads the file as lines and reports any lines that were not expected to be
* found.
*/
private def readExpectedLines(
fileName: String,
expectedLines: Seq[String],
packageRoot: File
): WithWarnings[Seq[String]] = {
val lines = readLines(packageRoot / fileName)
val unexpectedLines = lines.filter(l => !expectedLines.contains(l))
val warnings = unexpectedLines.map(l =>
s"File $fileName in ${packageRoot.getName} contains entry `$l`, but no " +
s"such entry has been detected. Perhaps it has disappeared after an " +
s"update? Please remove it from the file and make sure that the report " +
s"contains all necessary elements after this change."
)
WithWarnings(lines, warnings)
}
/**
* Reads the file as a [[String]].
*
* Returns None if the file cannot be read.
*/
private def readFile(file: File): Option[String] =
try { Some(IO.read(file)) }
catch { case NonFatal(_) => None }
/**
* Returns a sequence of files contained in a directory.
*
* If the directory does not exist or otherwise cannot be queried, returns an
* empty sequence.
*/
private def listFiles(dir: File): Seq[File] = {
try { IO.listFiles(dir) }
catch { case NonFatal(_) => Seq() }
}
}
object Review {
/**
* Normalizes a name so that it can be used as a filename.
*/
def normalizeName(string: String): String = {
val charsToReplace = " /:;,"
charsToReplace.foldLeft(string)((str, char) => str.replace(char, '_'))
}
/**
* Default NOTICE header.
*/
val defaultHeader: String = {
val yearStart = 2020
val yearCurrent = LocalDate.now().getYear
val year =
if (yearCurrent > yearStart) s"$yearStart - $yearCurrent"
else s"$yearStart"
s"""Enso
|Copyright $year New Byte Order sp. z o. o.""".stripMargin
}
}

63
project/src/main/scala/licenses/report/WithWarnings.scala Normal file
View File

@ -0,0 +1,63 @@
package src.main.scala.licenses.report
/**
* A simple monad for storing warnings related to a result.
*/
case class WithWarnings[+A](value: A, warnings: Seq[String]) {
/**
* Returns a result with a mapped value and the same warnings.
*/
def map[B](f: A => B): WithWarnings[B] = WithWarnings(f(value), warnings)
/**
* Combines two computations returning warnings, preserving warnings from
* both of them.
*/
def flatMap[B](f: A => WithWarnings[B]): WithWarnings[B] = {
val result = f(value)
WithWarnings(result.value, warnings ++ result.warnings)
}
}
object WithWarnings {
implicit class SeqWithWarningsSyntax[+A](seq: Seq[WithWarnings[A]]) {
/**
* Turns a sequence of [[WithWarnings]] instances into a single
* [[WithWarnings]] that contains the sequence of values and combined
* warnings.
*/
def flip: WithWarnings[Seq[A]] =
WithWarnings(seq.map(_.value), seq.flatMap(_.warnings))
}
implicit class SeqLikeSyntax[A](seq: WithWarnings[Seq[A]]) {
/**
* A shorthand syntax to concatenate two sequences wrapped with warnings,
* combining their warnings.
*/
def ++(other: WithWarnings[Seq[A]]): WithWarnings[Seq[A]] =
for {
lhs <- seq
rhs <- other
} yield lhs ++ rhs
/**
* A shorthand syntax to concatenate an ordinary sequence to a sequence
* with warnings.
*/
def ++(other: Seq[A]): WithWarnings[Seq[A]] =
for {
lhs <- seq
} yield lhs ++ other
}
/**
* Creates a [[WithWarnings]] containing Unit and a provided sequence of
* warnings.
*/
def justWarnings(warnings: Seq[String]): WithWarnings[Unit] =
WithWarnings((), warnings)
}

106
std-bits/pom.xml
View File

@ -1,106 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.enso</groupId>
<artifactId>base</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<name>base</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>67.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.3.1</version>
<configuration>
<outputDirectory>
${project.build.directory}/../../distribution/std-lib/Base/polyglot/java
</outputDirectory>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
<version>2.0.0</version>
<executions>
<execution>
<id>add-third-party-licenses</id>
<phase>package</phase>
<goals>
<goal>download-licenses</goal>
</goals>
<configuration>
<licensesOutputDirectory>
${project.build.directory}/../../distribution/std-lib/Base/third-party-licenses
</licensesOutputDirectory>
</configuration>
</execution>
<execution>
<id>add-third-party-licenses-list</id>
<phase>package</phase>
<goals>
<goal>add-third-party</goal>
</goals>
<configuration>
<outputDirectory>
${project.build.directory}/../../distribution/std-lib/Base/third-party-licenses
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.10</version>
<executions>
<execution>
<id>copy</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>67.1</version>
<type>jar</type>
<overWrite>true</overWrite>
<destFileName>icu4j.jar</destFileName>
</artifactItem>
</artifactItems>
<outputDirectory>
${project.build.directory}/../../distribution/std-lib/Base/polyglot/java
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

136
tools/legal-review-helper/index.js Normal file
View File

@ -0,0 +1,136 @@
const reviewRoot = "../../target";
const settingsRoot = "../../tools/legal-review";
const express = require("express");
const app = express();
const open = require("open");
const fs = require("fs");
const path = require("path");
// The home page that lists available reports.
app.get("/", function (req, res) {
let html = "<h1>Report review</h1>";
const files = fs.readdirSync(reviewRoot);
const reports = files
.map((f) => f.match(/^(.*)-report.html$/))
.filter((m) => m != null)
.map((m) => m[1]);
if (reports.length == 0) {
html +=
"No reports found. " +
'Run <pre style="display:inline">enso / gatherLicenses</pre> first.';
} else {
html += "Select report:";
html += "<ul>";
reports.forEach((report) => {
html += '<li><a href="/report/' + report + '">' + report + "</a></li>";
});
html += "</ul>";
}
res.send(html);
});
// Serves the injection script.
app.use("/static", express.static("static"));
// Serves contents of the given report, injecting the review-mode script.
app.get("/report/:report", function (req, res) {
const report = req.params["report"];
console.log("Opening report for ", report);
fs.readFile(
path.join(reviewRoot, report + "-report.html"),
"utf-8",
(err, data) => {
const injection =
'<script src="/static/inject.js"></script>' +
'<script>var reportName = "' +
report +
'";</script>';
if (err) {
res.status(400).send(err);
} else {
const injected = data.replace("</head>", injection + "</head>");
res.send(injected);
}
}
);
});
// Appends a line to the setting file.
function addLine(report, package, file, line) {
const dir = path.join(settingsRoot, report, package);
const location = path.join(dir, file);
console.log("Adding " + line + " to " + location);
fs.mkdirSync(dir, {
recursive: true,
});
fs.appendFileSync(location, line + "\n");
}
// Removes a line from the setting file.
function removeLine(report, package, file, line) {
const location = path.join(settingsRoot, report, package, file);
console.log("Removing " + line + " from " + location);
const lines = fs
.readFileSync(location, "utf-8")
.split(/\r?\n/)
.filter((x) => x.length > 0);
const toRemove = lines.filter((x) => x == line);
const others = lines.filter((x) => x != line);
if (toRemove.length == 0) {
throw (
"Line " +
line +
" was not present in the file. " +
"Are you sure the report is up to date?"
);
} else {
var newContent = others.join("\n") + "\n";
if (others.length == 0) {
newContent = "";
}
fs.writeFileSync(location, newContent);
}
}
// Handles the requests to add or remove lines.
app.use(express.urlencoded({ extended: true }));
app.post("/modify/:report", function (req, res) {
const report = req.params["report"];
const package = req.body["package"];
const action = req.body["action"];
const file = req.body["file"];
const line = req.body["line"];
try {
if (action == "add") {
addLine(report, package, file, line);
} else if (action == "remove") {
removeLine(report, package, file, line);
} else {
throw "Unknown action";
}
res.send("OK");
} catch (error) {
console.error(error);
res.status(500).send(error);
}
});
/*
* Listens on a random free port, opens a browser with the home page and waits
* for a newline to terminate.
*/
const server = app.listen(0, () => {
const port = server.address().port;
console.log("Listening on at ", "http://localhost:" + port + "/");
open("http://localhost:" + port + "/");
console.log("Press ENTER to stop the server.");
process.stdin.on("data", function (chunk) {
if (chunk.indexOf("\n") >= 0) {
console.log("Good bye");
process.exit(0);
}
});
});

16
tools/legal-review-helper/package.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "legal-review-helper",
"version": "1.0.0",
"description": "Server for reviewing the generated report.",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node index.js"
},
"author": "Enso Team",
"license": "Apache-2.0",
"dependencies": {
"express": "^4.17.1",
"open": "^7.3.0"
}
}

134
tools/legal-review-helper/static/inject.js Normal file
View File

@ -0,0 +1,134 @@
/** Sets a status text in bottom left part of the screen. */
function setStatus(text, color) {
var status = $("#status");
status.html(text);
if (color === undefined) {
color = "white";
}
status.css("background-color", color);
}
/** Creates a handler that will request to add or remove a line from a file. */
function makeHandler(elem, data, file, action) {
return function (ev) {
data["file"] = file;
data["action"] = action;
$.post("/modify/" + reportName, data, function (response) {
$(elem).html(
'<span style="color:gray">Modified, if you want to ' +
"change this value, regenerate the report first</span>"
);
var tab = $(elem).closest("div").parent();
var title = tab.children("h4");
tab.accordion("option", "active", false);
var info = "added " + file;
if (action == "remove") {
info = "undone review";
}
var newTitle =
'<span style="text-decoration: line-through;">' +
title.html() +
"</span><br>" +
info;
title.html(newTitle);
title.find("span").css("color", "gray");
setStatus("Review for " + data["package"] + " sent.");
}).fail(function (err) {
setStatus("Failed to send review: " + JSON.stringify(err), "red");
});
setStatus("Sending review...");
};
}
$(function () {
$("body").append(
'<div id="status" ' +
'style="position: fixed;left:4pt;bottom:4pt">' +
"Loading...</div>"
);
var copys = $(".copyright-ui");
var files = $(".file-ui");
copyrightMap = {
Ignore: "copyright-ignore",
KeepWithContext: "copyright-keep-context",
Keep: "copyright-keep",
};
copys.each(function (index) {
var package = $(this).data("package");
var content = atob($(this).data("content"));
var status = $(this).data("status");
var contexts = parseInt($(this).data("contexts"));
var data = {
line: content,
package: package,
};
if (status == "NotReviewed") {
var buttons =
'<button class="ignore">Ignore</button>' +
'<button class="keep">Keep</button>' +
'<button class="keepctx">Keep as context</button>';
$(this).html(buttons);
$(this)
.children(".ignore")
.on("click", makeHandler(this, data, "copyright-ignore", "add"));
$(this)
.children(".keep")
.on("click", makeHandler(this, data, "copyright-keep", "add"));
if (contexts == 1) {
$(this)
.children(".keepctx")
.on(
"click",
makeHandler(this, data, "copyright-keep-context", "add")
);
} else {
$(this).children(".keepctx").attr("disabled", true);
}
} else if (status != "Added") {
$(this).html("<button>Undo review</button>");
$(this)
.children("button")
.on("click", makeHandler(this, data, copyrightMap[status], "remove"));
} else {
$(this).html("<button disabled>This notice was added manually</button>");
}
});
filesMap = {
Ignore: "files-ignore",
Keep: "files-keep",
};
files.each(function (index) {
var package = $(this).data("package");
var filename = $(this).data("filename");
var status = $(this).data("status");
var data = {
line: filename,
package: package,
};
if (status == "NotReviewed") {
var buttons =
'<button class="ignore">Ignore</button>' +
'<button class="keep">Keep</button>';
$(this).html(buttons);
$(this)
.children(".ignore")
.on("click", makeHandler(this, data, "files-ignore", "add"));
$(this)
.children(".keep")
.on("click", makeHandler(this, data, "files-keep", "add"));
} else if (status != "Added") {
$(this).html("<button>Undo review</button>");
$(this)
.children("button")
.on("click", makeHandler(this, data, filesMap[status], "remove"));
} else {
$(this).html("<button disabled>This file was added manually</button>");
}
});
setStatus("Initialized");
});

33
tools/legal-review/launcher/com.typesafe.akka.akka-protobuf-v3_2.13-2.6.6/files-add/COPYING.protobuf Normal file
View File

@ -0,0 +1,33 @@
Copyright 2008, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Code generated by the Protocol Buffer compiler is owned by the owner
of the input file used when generating it. This code is not
standalone and requires a support library to be linked with it. This
support library is itself covered by the above license.

0
tools/legal-review/launcher/commons-io.commons-io-2.7/custom-license Normal file
View File

201
tools/legal-review/license-texts/APACHE2.0 Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.