mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-11 04:45:57 +03:00
* add blackduck scan to run on master (#6130) * add blackduck scan * disable go scanning exclude entire language-support/ts directory for node scanning break to multiple lines to make command line params easier to parse * Increase timeout for blackduck binary scan * update blackduck scan config * remove some exclusions, force python3 * exclude GO until path to go executable can be resolved * added readme explanation of why we want this file * fail in case of policy violation * ensure haskell bazel scan completes before running second round scan for bazel jvm and node and other langs * trigger notices file gen to ensure BOM complete * remove trailing end of lines * run with latest detect version and unique code location name changes to wrapper script * Add blackduck to daily compat job * DO NOT MERGE: condition false to disable other jobs for testing * remove parameters not available to cronjob * Revert changes to regular CI pipeline CHANGELOG_BEGIN CHANGELOG_END Signed-off-by: Brian Healey <brian.healey@digitalasset.com> * Do not get branch name from variable * Upgrade com.fasterxml.jackson.core:jackson-databind to 2.12.0 to address security vulnerability * Remove disabling of other jobs, set to branch to be used on prod runs * Apply suggestions from code review Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com> * Address code review comments * Updated NOTICES file * Run bazel build, update NOTICES file * Correct dade-assist * do not have perms to pipe to dev/null * Add md file explaining how to update NOTICES file * Add instructions for running blackduck locally * Add a link to full security-blackduck readme Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com>
This commit is contained in:
Brian Healey
GitHub
parent
4ac7b3f22d
commit
ca294eb14d
33
NOTICES.md
Normal file
33
NOTICES.md
Normal file
@ -0,0 +1,33 @@
|
||||
## Generating NOTICES file
|
||||
|
||||
The notices file is generated after the completion of an automated Blackduck scan of the entire daml repo.
|
||||
|
||||
At present this needs to be updated by running the scan manually and checking in the updated NOTICES file on a PR. In future a PR will be automatically created when a change in the NOTICES file is detected as part of the Blackduck scan within the daily compat job on master.
|
||||
|
||||
To generate the file locally, you should run the Blackduck scan after performing a full Bazel build on the DAML repo
|
||||
|
||||
Full details on running a Blackduck scan can be found @ https://github.com/DACH-NY/security-blackduck/blob/master/README.md
|
||||
|
||||
|
||||
1) Run full Bazel build
|
||||
```bazel build //...```
|
||||
|
||||
2) Create personal Blackduck token and add to environment variable
|
||||
Create a personal Blackduck token by authenticating to the Blackduck site with your DA Google account
|
||||
https://digitalasset.blackducksoftware.com/api/current-user/tokens
|
||||
|
||||
Click Create New Token and give yourself read and write access, giving a memorable name (<username>-<machine> or similar)
|
||||
Copy the contents of this token and define in a local environment variable called BLACKDUCK_HUBDETECT_TOKEN
|
||||
```export BLACKDUCK_HUB_DETECT_TOKEN=<token_you_have_just_created>```
|
||||
|
||||
2) Run Haskell Blackduck scan
|
||||
https://github.com/digital-asset/daml/blob/a17b340b47a711b53a1a5eb141c7835a9fb9bbbe/ci/cron/daily-compat.yml#L227-L234
|
||||
|
||||
3) Run Scan for all remaining languages, waiting for notices file to be generated
|
||||
https://github.com/digital-asset/daml/blob/a17b340b47a711b53a1a5eb141c7835a9fb9bbbe/ci/cron/daily-compat.yml#L241-L257
|
||||
|
||||
4) Remove windows line endings and rename file to NOTICES
|
||||
```tr -d '\015' <*_Black_Duck_Notices_Report.txt | grep -v dach-ny_daml-on-corda >NOTICES```
|
||||
|
||||
5) Create a new PR with the changes and submit for review for merge to master
|
||||
|
||||
13
Pipfile
Normal file
13
Pipfile
Normal file
@ -0,0 +1,13 @@
|
||||
# root Pipfile needed to identify that all python projects in this repo should be scanned as 3.7 for blackduck, and to give pypi url to use
|
||||
|
||||
[[source]]
|
||||
name = "pypi"
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
@ -153,8 +153,8 @@ def install_java_deps():
|
||||
"io.gatling.highcharts:gatling-highcharts:3.3.1",
|
||||
"io.gatling:gatling-http:3.3.1",
|
||||
"io.gatling:gatling-http-client:3.3.1",
|
||||
"com.fasterxml.jackson.core:jackson-core:2.11.2",
|
||||
"com.fasterxml.jackson.core:jackson-databind:2.11.2",
|
||||
"com.fasterxml.jackson.core:jackson-core:2.12.0",
|
||||
"com.fasterxml.jackson.core:jackson-databind:2.12.0",
|
||||
],
|
||||
fetch_sources = True,
|
||||
maven_install_json = "@com_github_digital_asset_daml//:maven_install.json",
|
||||
|
||||
@ -196,3 +196,66 @@ jobs:
|
||||
env:
|
||||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||||
- template: ../daily_tell_slack.yml
|
||||
|
||||
- job: blackduck_scan
|
||||
timeoutInMinutes: 1200
|
||||
pool:
|
||||
name: linux-pool
|
||||
demands: assignment -equals default
|
||||
steps:
|
||||
- checkout: self
|
||||
- bash: ci/dev-env-install.sh
|
||||
displayName: 'Build/Install the Developer Environment'
|
||||
- bash: ci/configure-bazel.sh
|
||||
displayName: 'Configure Bazel'
|
||||
env:
|
||||
IS_FORK: $(System.PullRequest.IsFork)
|
||||
# to upload to the bazel cache
|
||||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
eval "$(dev-env/bin/dade assist)"
|
||||
export LC_ALL=en_US.UTF-8
|
||||
|
||||
bazel build //...
|
||||
# Make sure that Bazel query works
|
||||
bazel query 'deps(//...)'
|
||||
displayName: 'Build'
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||||
ci-build digital-asset_daml master \
|
||||
--logging.level.com.synopsys.integration=DEBUG \
|
||||
--detect.tools=BAZEL \
|
||||
--detect.bazel.target=//... \
|
||||
--detect.bazel.dependency.type=haskell_cabal_library \
|
||||
--detect.notices.report=true \
|
||||
--detect.report.timeout=1500
|
||||
displayName: 'Blackduck Haskell Scan'
|
||||
env:
|
||||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||||
- bash: |
|
||||
set -euo pipefail
|
||||
eval "$(./dev-env/bin/dade-assist)"
|
||||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||||
ci-build digital-asset_daml master \
|
||||
--logging.level.com.synopsys.integration=DEBUG \
|
||||
--detect.npm.include.dev.dependencies=false \
|
||||
--detect.excluded.detector.types=NUGET \
|
||||
--detect.excluded.detector.types=GO_MOD \
|
||||
--detect.yarn.prod.only=true \
|
||||
--detect.python.python3=true \
|
||||
--detect.tools=DETECTOR,BAZEL,DOCKER \
|
||||
--detect.bazel.target=//... \
|
||||
--detect.bazel.dependency.type=maven_install \
|
||||
--detect.detector.search.exclusion.paths=language-support/ts/codegen/tests/ts,language-support/ts,language-support/scala/examples/iou-no-codegen,language-support/scala/examples/quickstart-scala,docs/source/app-dev/bindings-java/code-snippets,docs/source/app-dev/bindings-java/quickstart/template-root,language-support/scala/examples/quickstart-scala,language-support/scala/examples/iou-no-codegen \
|
||||
--detect.cleanup=false \
|
||||
--detect.policy.check.fail.on.severities=MAJOR,CRITICAL,BLOCKER \
|
||||
--detect.notices.report=true \
|
||||
--detect.cleanup.bdio.files=true \
|
||||
--detect.report.timeout=4500
|
||||
displayName: 'Blackduck Scan'
|
||||
env:
|
||||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||||
- template: ../daily_tell_slack.yml
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user