diff --git a/.github/workflows/gui.yml b/.github/workflows/gui.yml index a4a31fa680..36211fa106 100644 --- a/.github/workflows/gui.yml +++ b/.github/workflows/gui.yml @@ -647,6 +647,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./run ide build --wasm-source current-ci-run --backend-source current-ci-run env: + APPLEID: ${{ secrets.APPLE_NOTARIZATION_USERNAME }} + APPLEIDPASS: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }} + CSC_IDENTITY_AUTO_DISCOVERY: "true" + CSC_KEY_PASSWORD: ${{ secrets.APPLE_CODE_SIGNING_CERT_PASSWORD }} + CSC_LINK: ${{ secrets.APPLE_CODE_SIGNING_CERT }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: List files if failed (Windows) run: Get-ChildItem -Force -Recurse diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 39edb6de25..e29783a4bc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -583,6 +583,11 @@ jobs: ./run ide upload --wasm-source current-ci-run --backend-source release --backend-release ${{env.ENSO_RELEASE_ID}} env: + APPLEID: ${{ secrets.APPLE_NOTARIZATION_USERNAME }} + APPLEIDPASS: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }} + CSC_IDENTITY_AUTO_DISCOVERY: "true" + CSC_KEY_PASSWORD: ${{ secrets.APPLE_CODE_SIGNING_CERT_PASSWORD }} + CSC_LINK: ${{ secrets.APPLE_CODE_SIGNING_CERT }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: List files if failed (Windows) run: Get-ChildItem -Force -Recurse diff --git a/Cargo.lock b/Cargo.lock index 3b65032829..1f6e614038 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,16 +116,6 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" -[[package]] -name = "assert-json-diff" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f1c3703dd33532d7f0ca049168930e9099ecac238e23cf932f3a69c42f06da" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "assert_approx_eq" version = "1.1.0" @@ -1592,25 +1582,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" -[[package]] -name = "deadpool" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" -dependencies = [ - "async-trait", - "deadpool-runtime", - "num_cpus", - "retain_mut", - "tokio 1.19.2", -] - -[[package]] -name = "deadpool-runtime" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" - [[package]] name = "debug-scene-component-group" version = "0.1.0" @@ -1858,7 +1829,7 @@ dependencies = [ [[package]] name = "enso-build" version = "0.1.0" -source = "git+https://github.com/enso-org/ci-build?branch=develop#93d7afc1af0ad528444157943c24f9abb5d0b955" +source = "git+https://github.com/enso-org/ci-build?branch=develop#e283a55fba4b43bb3eeec4ce2d3982d20c8748d2" dependencies = [ "anyhow", "async-compression", @@ -1934,7 +1905,7 @@ dependencies = [ [[package]] name = "enso-build-cli" version = "0.1.0" -source = "git+https://github.com/enso-org/ci-build?branch=develop#93d7afc1af0ad528444157943c24f9abb5d0b955" +source = "git+https://github.com/enso-org/ci-build?branch=develop#e283a55fba4b43bb3eeec4ce2d3982d20c8748d2" dependencies = [ "anyhow", "byte-unit", @@ -3317,12 +3288,6 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" -[[package]] -name = "futures-timer" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" - [[package]] name = "futures-util" version = "0.3.21" @@ -3667,27 +3632,6 @@ dependencies = [ "serde", ] -[[package]] -name = "http-types" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" -dependencies = [ - "anyhow", - "async-channel", - "base64 0.13.0", - "futures-lite", - "http", - "infer", - "pin-project-lite 0.2.9", - "rand 0.7.3", - "serde", - "serde_json", - "serde_qs", - "serde_urlencoded", - "url 2.2.2", -] - [[package]] name = "httparse" version = "1.7.1" @@ -3849,10 +3793,10 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5617e92fc2f2501c3e2bc6ce547cad841adba2bae5b921c7e52510beca6d084c" dependencies = [ - "base64 0.10.1", + "base64 0.11.0", "bytes 1.1.0", "http", - "httpdate 0.3.2", + "httpdate 1.0.2", "language-tags 0.3.2", "mime 0.3.16", "percent-encoding 2.1.0", @@ -3862,7 +3806,7 @@ dependencies = [ [[package]] name = "ide-ci" version = "0.1.0" -source = "git+https://github.com/enso-org/ci-build?branch=develop#93d7afc1af0ad528444157943c24f9abb5d0b955" +source = "git+https://github.com/enso-org/ci-build?branch=develop#e283a55fba4b43bb3eeec4ce2d3982d20c8748d2" dependencies = [ "anyhow", "async-compression", @@ -3939,7 +3883,6 @@ dependencies = [ "walkdir", "which", "whoami", - "wiremock", "zip 0.6.2", ] @@ -4148,12 +4091,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "infer" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" - [[package]] name = "instant" version = "0.1.12" @@ -5894,12 +5831,6 @@ dependencies = [ "winreg 0.10.1", ] -[[package]] -name = "retain_mut" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" - [[package]] name = "ring" version = "0.16.20" @@ -6172,17 +6103,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" -dependencies = [ - "percent-encoding 2.1.0", - "serde", - "thiserror", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -7754,27 +7674,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "wiremock" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b12f508bdca434a55d43614d26f02e6b3e98ebeecfbc5a1614e0a0c8bf3e315" -dependencies = [ - "assert-json-diff", - "async-trait", - "deadpool", - "futures 0.3.21", - "futures-timer", - "http-types", - "hyper 0.14.18", - "log 0.4.17", - "once_cell", - "regex", - "serde", - "serde_json", - "tokio 1.19.2", -] - [[package]] name = "ws2_32-sys" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index de682eefe1..0122b9ec8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ members = [ "lib/rust/profiler/data", "integration-test", "tools/language-server/logstat", - "tools/language-server/wstest" + "tools/language-server/wstest", ] # The default memebers are those we want to check and test by default. default-members = ["app/gui", "lib/rust/*"] diff --git a/app/ide-desktop/lib/client/electron-builder-config.ts b/app/ide-desktop/lib/client/electron-builder-config.ts index 5bbab819a8..c5ba5096ac 100644 --- a/app/ide-desktop/lib/client/electron-builder-config.ts +++ b/app/ide-desktop/lib/client/electron-builder-config.ts @@ -10,11 +10,13 @@ */ import path from 'node:path' +import child_process from 'node:child_process' import fs from 'node:fs/promises' import { CliOptions, Configuration, LinuxTargetSpecificOptions, Platform } from 'electron-builder' import builder from 'electron-builder' +import { notarize } from 'electron-notarize' +import signArchivesMacOs from './tasks/signArchivesMacOs.js' -import { require_env } from '../../utils.js' import { project_manager_bundle } from './paths.js' import build from '../../build.json' assert { type: 'json' } import yargs from 'yargs' @@ -147,9 +149,38 @@ const config: Configuration = { }, afterAllArtifactBuild: path.join('tasks', 'computeHashes.cjs'), - // TODO [mwu]: Temporarily disabled, signing should be revised. - // In particular, engine should handle signing of its artifacts. - // afterPack: 'tasks/prepareToSign.js', + afterPack: ctx => { + if (args.platform === Platform.MAC) { + // Make the subtree writable so we can sign the binaries. + // This is needed because GraalVM distribution comes with read-only binaries. + child_process.execFileSync('chmod', ['-R', 'u+w', ctx.appOutDir]) + } + }, + + afterSign: async context => { + // Notarization for macOS. + if (args.platform === Platform.MAC) { + const { packager, appOutDir } = context + const appName = packager.appInfo.productFilename + + // We need to manually re-sign our build artifacts before notarization. + console.log(' • Performing additional signing of dependencies.') + await signArchivesMacOs({ + appOutDir: appOutDir, + productFilename: appName, + entitlements: context.packager.config.mac.entitlements, + identity: 'Developer ID Application: New Byte Order Sp. z o. o. (NM77WTZJFQ)', + }) + + console.log(' • Notarizing.') + notarize({ + appBundleId: packager.platformSpecificBuildOptions.appId, + appPath: `${appOutDir}/${appName}.app`, + appleId: process.env.APPLEID, + appleIdPassword: process.env.APPLEIDPASS, + }) + } + }, publish: null, } diff --git a/app/ide-desktop/lib/client/package.json b/app/ide-desktop/lib/client/package.json index d61a23e63c..0780d93751 100644 --- a/app/ide-desktop/lib/client/package.json +++ b/app/ide-desktop/lib/client/package.json @@ -22,16 +22,17 @@ "mime-types": "^2.1.35", "@electron/remote": "^2.0.8", "electron-is-dev": "^1.2.0", - "yargs": "^15.3.0" + "yargs": "^16.2.0" }, "devDependencies": { "electron": "17.1.0", - "electron-builder": "^22.14.13", + "electron-builder": "^23.3.3", "esbuild": "^0.14.43", "crypto-js": "4.1.1", "electron-notarize": "1.2.1", "enso-copy-plugin": "^1.0.0", - "ts-node": "^10.9.1" + "ts-node": "^10.9.1", + "fast-glob": "^3.2.12" }, "scripts": { "start": "electron ../../../../dist/content -- ", diff --git a/app/ide-desktop/lib/client/tasks/notarize.js b/app/ide-desktop/lib/client/tasks/notarize.js deleted file mode 100644 index 8de26ef0e4..0000000000 --- a/app/ide-desktop/lib/client/tasks/notarize.js +++ /dev/null @@ -1,25 +0,0 @@ -/// This script will trigger the notarisation process for macOS and trigger our pre-processing -/// and signing of the engine. -require('dotenv').config() -const { notarize } = require('electron-notarize') - -exports.default = async function notarizing(context) { - const { electronPlatformName, appOutDir } = context - if (electronPlatformName !== 'darwin') { - return - } - // We need to manually re-sign our build artifacts before notarisation. - // See the script for more information. - console.log(' • Performing additional signing of dependencies.') - await require('./signArchivesMacOs').default() - - // Notarize the application. - const appName = context.packager.appInfo.productFilename - console.log(' • Notarizing.') - return await notarize({ - appBundleId: 'com.enso.ide', - appPath: `${appOutDir}/${appName}.app`, - appleId: process.env.APPLEID, - appleIdPassword: process.env.APPLEIDPASS, - }) -} diff --git a/app/ide-desktop/lib/client/tasks/prepareToSign.js b/app/ide-desktop/lib/client/tasks/prepareToSign.js deleted file mode 100644 index 785a6e48c3..0000000000 --- a/app/ide-desktop/lib/client/tasks/prepareToSign.js +++ /dev/null @@ -1,11 +0,0 @@ -const { beforeSign } = require('./signArchivesMacOs') - -// ================ -// === Callback === -// ================ - -exports.default = async function (context) { - if (context.electronPlatformName === 'darwin') { - beforeSign() - } -} diff --git a/app/ide-desktop/lib/client/tasks/signArchivesMacOs.js b/app/ide-desktop/lib/client/tasks/signArchivesMacOs.js deleted file mode 100644 index 4b75e9e04c..0000000000 --- a/app/ide-desktop/lib/client/tasks/signArchivesMacOs.js +++ /dev/null @@ -1,284 +0,0 @@ -/** - This script signs the content of all archives that we have for macOS. For this to work this needs - to run on macOS with `codesign`, and a JDK installed. `codesign` is needed to sign the files, - while the JDK is needed for correct packing and unpacking of java archives. - - We require this extra step as our dependencies contain files that require us to re-sign jar - contents that cannot be opened as pure zip archives, but require a java toolchain to extract - and re-assemble to preserve manifest information. This functionality is not provided by - `electron-osx-sign` out of the box. - - This code is based on https://github.com/electron/electron-osx-sign/pull/231 but our use-case - is unlikely to be supported by electron-osx-sign as it adds a java toolchain as additional - dependency. - This script should be removed once the engine is signed. -**/ -const fs = require('fs') -const path = require('path') -const child_process = require('child_process') -const { engineVersion } = require('../../../build.json') - -const dist_var_name = 'ENSO_BUILD_IDE' -const dist = - process.env[dist_var_name] ?? - (() => { - throw Error(`Missing ${dist_var_name} environment variable.`) - })() - -// `electron-builder`'s output directory name. -function contentDirName() { - if (process.arch === 'arm64') { - return 'mac-arm64' - } else { - return 'mac' - } -} - -const contentRoot = path.join(dist, 'client', contentDirName(), 'Enso.app', 'Contents') -const resRoot = path.join(contentRoot, 'Resources') - -const ID = '"Developer ID Application: New Byte Order Sp. z o. o. (NM77WTZJFQ)"' -// Placeholder name for temporary archives. -const tmpArchive = 'temporary_archive.zip' - -const GRAALVM = 'graalvm-ce-java11-21.1.0' - -// Helper to execute a command in a given directory and return the output. -const run = (cmd, cwd) => child_process.execSync(cmd, { shell: true, cwd }).toString() - -// Run the signing command. -function sign(targetPath, cwd) { - console.log(`Signing ${targetPath} in ${cwd}`) - const entitlements_path = path.resolve('./', 'entitlements.mac.plist') - return run( - `codesign -vvv --entitlements ${entitlements_path} --force --options=runtime ` + - `--sign ${ID} ${targetPath}`, - cwd - ) -} - -// Create and return an empty directory in the current folder. The directory will be named `.temp`. -// If it already exists all content will be deleted. -function getTmpDir() { - const workingDir = '.temp' - run(`rm -rf ${workingDir}`) - run(`mkdir ${workingDir}`) - return path.resolve(workingDir) -} - -/** - * Sign content of an archive. This function extracts the archive, signs the required files, - * re-packages the archive and replaces the original. - * - * @param {string} archivePath - folder the archive is located in. - * @param {string} archiveName - file name of the archive - * @param {string[]} binPaths - paths of files to be signed. Must be relative to archive root. - */ -function signArchive(archivePath, archiveName, binPaths) { - const sourceArchive = path.join(archivePath, archiveName) - const workingDir = getTmpDir() - try { - const isJar = archiveName.endsWith(`jar`) - - if (isJar) { - run(`jar xf ${sourceArchive}`, workingDir) - } else { - run(`unzip -d${workingDir} ${sourceArchive}`) - } - - for (let binary of binPaths) { - sign(binary, workingDir) - } - - if (isJar) { - if (archiveName.includes(`runner`)) { - run(`jar -cfm ${tmpArchive} META-INF/MANIFEST.MF . `, workingDir) - } else { - run(`jar -cf ${tmpArchive} . `, workingDir) - } - } else { - run(`zip -rm ${tmpArchive} . `, workingDir) - } - - console.log(run(`/bin/mv ${workingDir}/${tmpArchive} ${sourceArchive}`)) - run(`rm -R ${workingDir}`) - console.log( - `Successfully repacked ${sourceArchive} to handle signing inner native dependency.` - ) - } catch (error) { - run(`rm -R ${workingDir}`) - console.error( - `Could not repackage ${archiveName}. Please check the "signArchives.js" task in ` + - `client/tasks to ensure that it's working. This jar has to be treated specially` + - ` because it has a native library and apple's codesign does not sign inner ` + - `native libraries correctly for jar files` - ) - throw error - } -} - -// Archives, and their content that need to be signed in an extra step. If a new archive is added -// to the engine dependencies this also needs to be added here. If an archive is not added here, it -// will show up as a failure to notarise the IDE. The offending archive will be named in the error -// message provided by Apple and can then be added here. -const toSign = [ - { - jarDir: `enso/dist/${engineVersion}/lib/Standard/Database/${engineVersion}/polyglot/java`, - jarName: 'sqlite-jdbc-3.34.0.jar', - jarContent: [ - 'org/sqlite/native/Mac/aarch64/libsqlitejdbc.jnilib', - 'org/sqlite/native/Mac/x86_64/libsqlitejdbc.jnilib', - ], - }, - { - jarDir: `enso/dist/${engineVersion}/component`, - jarName: 'runner.jar', - jarContent: [ - 'org/sqlite/native/Mac/x86_64/libsqlitejdbc.jnilib', - 'com/sun/jna/darwin/libjnidispatch.jnilib', - ], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jartool.jmod', - jarContent: ['bin/jarsigner', 'bin/jar'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jdeps.jmod', - jarContent: ['bin/javap', 'bin/jdeprscan', 'bin/jdeps'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jstatd.jmod', - jarContent: ['bin/jstatd'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.pack.jmod', - jarContent: ['bin/unpack200', 'bin/pack200'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.hotspot.agent.jmod', - jarContent: ['bin/jhsdb'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jfr.jmod', - jarContent: ['bin/jfr'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.rmic.jmod', - jarContent: ['bin/rmic'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'java.rmi.jmod', - jarContent: ['bin/rmid', 'bin/rmiregistry'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'java.base.jmod', - jarContent: ['bin/java', 'bin/keytool', 'lib/jspawnhelper'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jlink.jmod', - jarContent: ['bin/jmod', 'bin/jlink', 'bin/jimage'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.scripting.nashorn.shell.jmod', - jarContent: ['bin/jjs'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jcmd.jmod', - jarContent: ['bin/jstack', 'bin/jcmd', 'bin/jps', 'bin/jmap', 'bin/jstat', 'bin/jinfo'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jshell.jmod', - jarContent: ['bin/jshell'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.compiler.jmod', - jarContent: ['bin/javac', 'bin/serialver'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'java.scripting.jmod', - jarContent: ['bin/jrunscript'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jdi.jmod', - jarContent: ['bin/jdb'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.javadoc.jmod', - jarContent: ['bin/javadoc'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.jconsole.jmod', - jarContent: ['bin/jconsole'], - }, - { - jarDir: `enso/runtime/${GRAALVM}/Contents/Home/jmods`, - jarName: 'jdk.javadoc.jmod', - jarContent: ['bin/javadoc'], - }, -] - -// Extra files that need to be signed. -const extra = [ - `enso/runtime/${GRAALVM}/Contents/MacOS/libjli.dylib`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/llvm/native/bin/ld.lld`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/MASS/libs/MASS.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/cluster/libs/cluster.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/nnet/libs/nnet.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/rpart/libs/rpart.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/lattice/libs/lattice.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/nlme/libs/nlme.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/class/libs/class.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/spatial/libs/spatial.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/foreign/libs/foreign.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/Matrix/libs/Matrix.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/KernSmooth/libs/KernSmooth.so`, - `enso/runtime/${GRAALVM}/Contents/Home/languages/R/library/survival/libs/survival.so`, -] - -// The list of readonly files in the GraalVM distribution. -const readonly = [`enso/runtime/${GRAALVM}/Contents/Home/lib/server/classes.jsa`] - -function beforeSign() { - for (let file of readonly) { - const target = path.join(resRoot, file) - fs.chmodSync(target, 0o644) - } -} - -exports.default = async function () { - // Sign archives. - for (let toSignData of toSign) { - const jarDir = path.join(resRoot, toSignData.jarDir) - const jarName = toSignData.jarName - const jarContent = toSignData.jarContent - console.log({ jarDir, jarName, jarContent }) - signArchive(jarDir, jarName, jarContent) - } - // Sign single binaries. - for (let toSign of extra) { - const target = path.join(resRoot, toSign) - sign(target) - } - // Finally re-sign the top-level enso. - sign(path.join(contentRoot, 'MacOs/Enso')) -} - -module.exports = { beforeSign } diff --git a/app/ide-desktop/lib/client/tasks/signArchivesMacOs.ts b/app/ide-desktop/lib/client/tasks/signArchivesMacOs.ts new file mode 100644 index 0000000000..974366f26c --- /dev/null +++ b/app/ide-desktop/lib/client/tasks/signArchivesMacOs.ts @@ -0,0 +1,337 @@ +/** + This script signs the content of all archives that we have for macOS. For this to work this needs + to run on macOS with `codesign`, and a JDK installed. `codesign` is needed to sign the files, + while the JDK is needed for correct packing and unpacking of java archives. + + We require this extra step as our dependencies contain files that require us to re-sign jar + contents that cannot be opened as pure zip archives, but require a java toolchain to extract + and re-assemble to preserve manifest information. This functionality is not provided by + `electron-osx-sign` out of the box. + + This code is based on https://github.com/electron/electron-osx-sign/pull/231 but our use-case + is unlikely to be supported by electron-osx-sign as it adds a java toolchain as additional + dependency. + This script should be removed once the engine is signed. +**/ + +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import child_process from 'node:child_process' +import glob from 'fast-glob' + +// =============================================== +// === Patterns of entities that need signing. === +// =============================================== + +/** Parts of the GraalVM distribution that need to be signed by us in an extra step. */ +async function graalSignables(resourcesDir: string): Promise { + const archivePatterns: ArchivePattern[] = [ + [`Contents/Home/jmods/jdk.jartool.jmod`, ['bin/jarsigner', 'bin/jar']], + [`Contents/Home/jmods/jdk.jdeps.jmod`, ['bin/javap', 'bin/jdeprscan', 'bin/jdeps']], + [`Contents/Home/jmods/jdk.jstatd.jmod`, ['bin/jstatd']], + [`Contents/Home/jmods/jdk.pack.jmod`, ['bin/unpack200', 'bin/pack200']], + [`Contents/Home/jmods/jdk.hotspot.agent.jmod`, ['bin/jhsdb']], + [`Contents/Home/jmods/jdk.jfr.jmod`, ['bin/jfr']], + [`Contents/Home/jmods/jdk.rmic.jmod`, ['bin/rmic']], + [`Contents/Home/jmods/java.rmi.jmod`, ['bin/rmid', 'bin/rmiregistry']], + [`Contents/Home/jmods/java.base.jmod`, ['bin/java', 'bin/keytool', 'lib/jspawnhelper']], + [`Contents/Home/jmods/jdk.jlink.jmod`, ['bin/jmod', 'bin/jlink', 'bin/jimage']], + [`Contents/Home/jmods/jdk.scripting.nashorn.shell.jmod`, ['bin/jjs']], + [ + `Contents/Home/jmods/jdk.jcmd.jmod`, + ['bin/jstack', 'bin/jcmd', 'bin/jps', 'bin/jmap', 'bin/jstat', 'bin/jinfo'], + ], + [`Contents/Home/jmods/jdk.jshell.jmod`, ['bin/jshell']], + [`Contents/Home/jmods/jdk.compiler.jmod`, ['bin/javac', 'bin/serialver']], + [`Contents/Home/jmods/java.scripting.jmod`, ['bin/jrunscript']], + [`Contents/Home/jmods/jdk.jdi.jmod`, ['bin/jdb']], + [`Contents/Home/jmods/jdk.javadoc.jmod`, ['bin/javadoc']], + [`Contents/Home/jmods/jdk.jconsole.jmod`, ['bin/jconsole']], + [`Contents/Home/jmods/jdk.javadoc.jmod`, ['bin/javadoc']], + ] + + const binariesPatterns = [ + `Contents/Home/languages/llvm/native/bin/graalvm-native-ld`, + `Contents/Home/languages/llvm/native/bin/ld.lld`, + `Contents/Home/languages/R/library/class/libs/class.so`, + `Contents/Home/languages/R/library/cluster/libs/cluster.so`, + `Contents/Home/languages/R/library/foreign/libs/foreign.so`, + `Contents/Home/languages/R/library/KernSmooth/libs/KernSmooth.so`, + `Contents/Home/languages/R/library/lattice/libs/lattice.so`, + `Contents/Home/languages/R/library/MASS/libs/MASS.so`, + `Contents/Home/languages/R/library/Matrix/libs/Matrix.so`, + `Contents/Home/languages/R/library/nlme/libs/nlme.so`, + `Contents/Home/languages/R/library/nnet/libs/nnet.so`, + `Contents/Home/languages/R/library/rpart/libs/rpart.so`, + `Contents/Home/languages/R/library/spatial/libs/spatial.so`, + `Contents/Home/languages/R/library/survival/libs/survival.so`, + `Contents/MacOS/libjli.dylib`, + ] + + // We use `*` for Graal versioned directory to not have to update this script on every GraalVM update. + // Updates might still be needed when the list of binaries to sign changes. + const graalDir = path.join(resourcesDir, 'enso', 'runtime', '*') + const archives = await ArchiveToSign.lookupMany(graalDir, archivePatterns) + const binaries = await BinaryToSign.lookupMany(graalDir, binariesPatterns) + return [...archives, ...binaries] +} + +/** Parts of the Enso Engine distribution that need to be signed by us in an extra step. */ +async function ensoPackageSignables(resourcesDir: string): Promise { + /// Archives, and their content that need to be signed in an extra step. If a new archive is added + /// to the engine dependencies this also needs to be added here. If an archive is not added here, it + /// will show up as a failure to notarise the IDE. The offending archive will be named in the error + /// message provided by Apple and can then be added here. + const engineDir = `${resourcesDir}/enso/dist/*` + const archivePatterns: ArchivePattern[] = [ + [ + `lib/Standard/Database/*/polyglot/java/sqlite-jdbc-*.jar`, + [ + 'org/sqlite/native/Mac/aarch64/libsqlitejdbc.jnilib', + 'org/sqlite/native/Mac/x86_64/libsqlitejdbc.jnilib', + ], + ], + [ + `/component/runner.jar`, + [ + 'org/sqlite/native/Mac/x86_64/libsqlitejdbc.jnilib', + 'com/sun/jna/darwin-aarch64/libjnidispatch.jnilib', + 'com/sun/jna/darwin-x86-64/libjnidispatch.jnilib', + ], + ], + ] + return ArchiveToSign.lookupMany(engineDir, archivePatterns) +} + +// ================ +// === Signing. === +// ================ + +/** Information we need to sign a given binary. */ +interface SigningContext { + /** A digital identity that is stored in a keychain that is on the calling user's keychain + * search list. We rely on this already being set up by the Electron Builder. + */ + identity: string + /** Path to the entitlements file. */ + entitlements: string +} + +/** An entity that we want to sign. */ +interface Signable { + /** Sign this entity. */ + sign(context: SigningContext): Promise +} + +/** Placeholder name for temporary archives. */ +const tmpArchive = 'temporary_archive.zip' + +/** Helper to execute a program in a given directory and return the output. */ +const run = (cmd: string, args: string[], cwd?: string) => { + console.log('Running', cmd, args, cwd) + return child_process.execFileSync(cmd, args, { cwd }).toString() +} + +/** Archive with some binaries that we want to sign. + * + * Can be either a zip or a jar file. + */ +class ArchiveToSign implements Signable { + /** An absolute path to the archive. */ + path: string + + /** A list of patterns for files to sign inside the archive. + * + * Relative to the root of the archive. + */ + binaries: glob.Pattern[] + + /** Create a new instance. */ + constructor(path: string, binaries: glob.Pattern[]) { + this.path = path + this.binaries = binaries + } + + /** + * Sign content of an archive. This function extracts the archive, signs the required files, + * re-packages the archive and replaces the original. + */ + async sign(context: SigningContext) { + console.log(`Signing archive ${this.path}`) + const archiveName = path.basename(this.path) + const workingDir = await getTmpDir() + try { + const isJar = archiveName.endsWith(`jar`) + + if (isJar) { + run(`jar`, ['xf', this.path], workingDir) + } else { + run(`unzip`, ['-d', workingDir, this.path]) + } + + const binariesToSign = await BinaryToSign.lookupMany(workingDir, this.binaries) + for (const binaryToSign of binariesToSign) { + binaryToSign.sign(context) + } + + if (isJar) { + if (archiveName.includes(`runner`)) { + run(`jar`, ['-cfm', tmpArchive, 'META-INF/MANIFEST.MF', '.'], workingDir) + } else { + run(`jar`, ['-cf', tmpArchive, '.'], workingDir) + } + } else { + run(`zip`, ['-rm', tmpArchive, '.'], workingDir) + } + + // We cannot use fs.rename because temp and target might be on different volumes. + console.log(run(`/bin/mv`, [path.join(workingDir, tmpArchive), this.path])) + console.log( + `Successfully repacked ${this.path} to handle signing inner native dependency.` + ) + } catch (error) { + console.error( + `Could not repackage ${archiveName}. Please check the ${import.meta.url} task to ` + + `ensure that it's working. This jar has to be treated specially` + + ` because it has a native library and Apple's codesign does not sign inner ` + + `native libraries correctly for jar files.` + ) + throw error + } finally { + await rmRf(workingDir) + } + } + + /** Looks up for archives to sign using the given path pattern. */ + static async lookup(base: string, [pattern, binaries]: ArchivePattern) { + return lookupHelper(path => new ArchiveToSign(path, binaries))(base, pattern) + } + + /** Looks up for archives to sign using the given path patterns. */ + static lookupMany = lookupManyHelper(ArchiveToSign.lookup) +} + +/** A single code binary file to be signed. */ +class BinaryToSign implements Signable { + /** An absolute path to the binary. */ + path: string + + /** Create a new instance. */ + constructor(path: string) { + this.path = path + } + + /** Sign this binary. */ + async sign({ entitlements, identity }: SigningContext) { + console.log(`Signing ${this.path}`) + run(`codesign`, [ + '-vvv', + '--entitlements', + entitlements, + '--force', + '--options=runtime', + '--sign', + identity, + this.path, + ]) + } + + /** Looks up for binaries to sign using the given path pattern. */ + static lookup = lookupHelper(path => new BinaryToSign(path)) + + /** Looks up for binaries to sign using the given path patterns. */ + static lookupMany = lookupManyHelper(BinaryToSign.lookup) +} + +// ============================== +// === Discovering Signables. === +// ============================== + +/** Helper used to concisely define patterns for an archive to sign. + * + * Consists of pattern of the archive path and set of patterns for files to sign inside the archive. + */ +type ArchivePattern = [glob.Pattern, glob.Pattern[]] + +/** Like `glob` but returns absolute paths by default. */ +async function globAbs(pattern: glob.Pattern, options?: glob.Options): Promise { + const paths = await glob(pattern, { absolute: true, ...options }) + return paths +} + +/** Glob patterns relative to a given base directory. Base directory is allowed to be a pattern as + * well. + **/ +async function globAbsIn( + base: glob.Pattern, + pattern: glob.Pattern, + options?: glob.Options +): Promise { + return globAbs(path.join(base, pattern), options) +} + +/** Generate a lookup function for a given Signable type. */ +function lookupHelper(mapper: (path: string) => R) { + return async (base: string, pattern: glob.Pattern) => { + const paths = await globAbsIn(base, pattern) + return paths.map(mapper) + } +} + +/** Generate a lookup function for a given Signable type. */ +function lookupManyHelper( + lookup: (base: string, pattern: T) => Promise +) { + return async function (base: string, patterns: T[]) { + const results = await Promise.all(patterns.map(pattern => lookup(base, pattern))) + return results.flat() + } +} + +// ================== +// === Utilities. === +// ================== + +/** Remove file recursively. */ +async function rmRf(path: string) { + await fs.rm(path, { recursive: true, force: true }) +} + +/** + * Get a new temporary directory. Caller is responsible for cleaning up the directory. + */ +async function getTmpDir(prefix?: string) { + const ret = await fs.mkdtemp(path.join(os.tmpdir(), prefix ?? 'enso-signing-')) + return ret +} + +// ==================== +// === Entry point. === +// ==================== + +/** Input for this script. */ +interface Input extends SigningContext { + appOutDir: string + productFilename: string +} + +/** Entry point, meant to be used from an afterSign Electron Builder's hook. */ +export default async function (context: Input) { + console.log('Environment: ', process.env) + const { appOutDir, productFilename, entitlements } = context + const appDir = path.join(appOutDir, `${productFilename}.app`) + const contentsDir = path.join(appDir, 'Contents') + const resourcesDir = path.join(contentsDir, 'Resources') + + // Sign archives. + console.log('Signing GraalVM elemenets...') + for (const signable of await graalSignables(resourcesDir)) await signable.sign(context) + + console.log('Signing Engine elements...') + for (const signable of await ensoPackageSignables(resourcesDir)) await signable.sign(context) + + // Finally re-sign the top-level enso. + const topLevelExecutable = new BinaryToSign(path.join(contentsDir, 'MacOS', productFilename)) + await topLevelExecutable.sign(context) +} diff --git a/app/ide-desktop/package.json b/app/ide-desktop/package.json index 1b026cb486..6a0bf5b663 100644 --- a/app/ide-desktop/package.json +++ b/app/ide-desktop/package.json @@ -23,7 +23,11 @@ "lib/icons", "lib/server" ], + "devDependencies": { + "patch-package": "^6.4.7" + }, "scripts": { - "watch": "npm run watch --workspace enso-studio-content" + "watch": "npm run watch --workspace enso-studio-content", + "postinstall": "patch-package" } } diff --git a/app/ide-desktop/patches/README.md b/app/ide-desktop/patches/README.md new file mode 100644 index 0000000000..8c93057b2f --- /dev/null +++ b/app/ide-desktop/patches/README.md @@ -0,0 +1,5 @@ +Patches to be applied on NPM dependencies of the IDE. + +- [app-builder-lib](./app-builder-lib%2B23.3.3.patch) — workaround for + https://github.com/electron-userland/electron-builder/issues/6865, as + discovered by James. diff --git a/app/ide-desktop/patches/app-builder-lib+23.3.3.patch b/app/ide-desktop/patches/app-builder-lib+23.3.3.patch new file mode 100644 index 0000000000..64cbc11082 --- /dev/null +++ b/app/ide-desktop/patches/app-builder-lib+23.3.3.patch @@ -0,0 +1,12 @@ +diff --git a/node_modules/app-builder-lib/out/targets/nsis/NsisTarget.js b/node_modules/app-builder-lib/out/targets/nsis/NsisTarget.js +index e487f90..a09e531 100644 +--- a/node_modules/app-builder-lib/out/targets/nsis/NsisTarget.js ++++ b/node_modules/app-builder-lib/out/targets/nsis/NsisTarget.js +@@ -473,7 +473,6 @@ class NsisTarget extends core_1.Target { + } + async executeMakensis(defines, commands, script) { + const args = this.options.warningsAsErrors === false ? [] : ["-WX"]; +- args.push("-INPUTCHARSET", "UTF8"); + for (const name of Object.keys(defines)) { + const value = defines[name]; + if (value == null) { diff --git a/build-config.yaml b/build-config.yaml index fabfd502e7..b8b8c7f984 100644 --- a/build-config.yaml +++ b/build-config.yaml @@ -1,6 +1,6 @@ # Options intended to be common for all developers. -wasm-size-limit: 14.59 MiB +wasm-size-limit: 14.60 MiB required-versions: cargo-watch: ^8.1.1 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 55ad2fae1d..047bf673e6 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -167,17 +167,6 @@ helper tools for that. We recommend: **For users of M1 Mac**: installing GraalVM on M1 Mac requires manual actions, please refer to a [dedicated documentation](./graalvm-m1-mac.md). -**For users of MacOS Monterey and later**: building desktop IDE currently -requires Python 2 installed in the system. It can be installed using the -following commands: - -```sh -brew install pyenv -pyenv install 2.7.18 -pyenv global 2.7.18 -export PYTHON_PATH=$(pyenv root)/shims/python -``` - The flatbuffers `flatc` compiler can be installed from the following locations: - Using the `conda` package manager (`conda install flatbuffers`). This will