diff --git a/.github/actions/build-rust/action.yml b/.github/actions/build-rust/action.yml index 0eec7aa364..a4de7d6f1f 100644 --- a/.github/actions/build-rust/action.yml +++ b/.github/actions/build-rust/action.yml @@ -29,21 +29,23 @@ runs: if: ${{ inputs.target != 'x86_64-unknown-linux-gnu' && inputs.target != 'aarch64-unknown-linux-gnu' }} shell: bash run: yarn workspace @affine/native build --target ${{ inputs.target }} - env: - CARGO_BUILD_INCREMENTAL: 'false' - name: Build if: ${{ inputs.target == 'x86_64-unknown-linux-gnu' }} uses: addnab/docker-run-action@v3 with: image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian - options: --user 0:0 -e CARGO_BUILD_INCREMENTAL=false -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build - run: yarn workspace @affine/native build --target ${{ inputs.target }} + options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build + run: >- + export CC=x86_64-unknown-linux-gnu-gcc && + export CC_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-gcc && + yarn workspace @affine/native build --target ${{ inputs.target }} - name: Build if: ${{ inputs.target == 'aarch64-unknown-linux-gnu' }} uses: addnab/docker-run-action@v3 with: image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64 - options: --user 0:0 -e CARGO_BUILD_INCREMENTAL=false -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build - run: yarn workspace @affine/native build --target ${{ inputs.target }} + options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build + run: >- + yarn workspace @affine/native build --target ${{ inputs.target }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd3263be8b..ae18d81784 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,8 +118,7 @@ jobs: COVERAGE: true - name: Export static resources - run: yarn export - working-directory: apps/web + run: yarn workspace @affine/web export - name: Upload static resources artifact uses: actions/upload-artifact@v3 @@ -327,10 +326,7 @@ jobs: - name: Run unit tests if: ${{ matrix.spec.test }} shell: bash - run: | - rm -rf apps/electron/node_modules/better-sqlite3/build - yarn --cwd apps/electron/node_modules/better-sqlite3 run install - yarn test:unit + run: yarn test:unit env: NATIVE_TEST: 'true' - name: Build layers @@ -342,12 +338,6 @@ jobs: name: next-js-static path: ./apps/electron/resources/web-static - - name: Rebuild Electron dependences - shell: bash - run: | - rm -rf apps/electron/node_modules/better-sqlite3/build - yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }} - - name: Run desktop tests if: ${{ matrix.spec.test && matrix.spec.os == 'ubuntu-latest' }} run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine/electron test diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index a6e122adc1..8bfab20336 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -128,11 +128,6 @@ jobs: with: name: before-make-web-static path: apps/electron/resources/web-static - - name: Rebuild Electron dependences - shell: bash - run: | - rm -rf apps/electron/node_modules/better-sqlite3/build - yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }} - name: Build layers run: yarn workspace @affine/electron build-layers diff --git a/.github/workflows/release-desktop-app.yml b/.github/workflows/release-desktop-app.yml index e4ed795d2c..a63205fcb0 100644 --- a/.github/workflows/release-desktop-app.yml +++ b/.github/workflows/release-desktop-app.yml @@ -126,12 +126,6 @@ jobs: name: before-make-web-static path: apps/electron/resources/web-static - - name: Rebuild Electron dependences - shell: bash - run: | - rm -rf apps/electron/node_modules/better-sqlite3/build - yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }} - - name: Build layers run: yarn workspace @affine/electron build-layers diff --git a/.gitignore b/.gitignore index 0aab46e78e..97c37bddf8 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ target *.node tsconfig.node.tsbuildinfo lib +affine.db diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 855ac5288e..bf1dadb1bf 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -38,5 +38,7 @@ "tests/unit/**/*.spec.ts", "tests/unit/**/*.spec.tsx" ], - "deepscan.enable": true + "rust-analyzer.check.extraEnv": { + "DATABASE_URL": "sqlite:affine.db" + } } diff --git a/Cargo.lock b/Cargo.lock index 8481fd6cf0..e86a3ee635 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,7 +6,10 @@ version = 3 name = "affine_native" version = "0.0.0" dependencies = [ + "affine_schema", "anyhow", + "chrono", + "dotenv", "napi", "napi-build", "napi-derive", @@ -15,10 +18,37 @@ dependencies = [ "parking_lot", "serde", "serde_json", + "sqlx", "tokio", "uuid", ] +[[package]] +name = "affine_schema" +version = "0.0.0" + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.0.1" @@ -28,18 +58,48 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -48,9 +108,30 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813" +checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -58,12 +139,39 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "time", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "const-oid" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" + [[package]] name = "convert_case" version = "0.6.0" @@ -73,6 +181,36 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -83,6 +221,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -92,6 +240,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctor" version = "0.2.0" @@ -99,7 +257,98 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4056f63fce3b82d852c3da92b08ea59959890813a7f4ce9c0ff85b10cf301b" dependencies = [ "quote", - "syn 2.0.15", + "syn 2.0.18", +] + +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +dependencies = [ + "serde", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", ] [[package]] @@ -110,10 +359,31 @@ checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "windows-sys 0.48.0", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project", + "spin 0.9.8", +] + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -123,6 +393,88 @@ dependencies = [ "libc", ] +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.9" @@ -131,7 +483,40 @@ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.3", +] + +[[package]] +name = "hashlink" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0761a1b9491c4f2e3d66aa0f62d0fba0af9a0e2852e4d48ea506632a4b56e6aa" +dependencies = [ + "hashbrown 0.13.2", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", ] [[package]] @@ -143,6 +528,88 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "inotify" version = "0.9.6" @@ -163,12 +630,50 @@ dependencies = [ "libc", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "js-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kqueue" version = "1.0.7" @@ -189,6 +694,15 @@ dependencies = [ "libc", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] + [[package]] name = "libc" version = "0.2.144" @@ -205,6 +719,29 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "libsqlite3-sys" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "lock_api" version = "0.4.9" @@ -217,11 +754,17 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" + +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "cfg-if", + "digest", ] [[package]] @@ -231,25 +774,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] -name = "mio" -version = "0.8.6" +name = "minimal-lexical" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebffdb73fe72e917997fad08bdbf31ac50b0fa91cec93e69a0662e4264d454c" dependencies = [ "libc", "log", - "wasi", - "windows-sys 0.45.0", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", ] [[package]] name = "napi" -version = "2.12.6" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ac8112fe5998579b22e29903c7b277fc7f91c7860c0236f35792caf8156e18" +checksum = "d7f0a2e93526dd9c8c522d72a4d0c88678be8966fabe9fb8f2947fde6339b682" dependencies = [ "anyhow", - "bitflags 2.2.1", + "bitflags 2.3.1", + "chrono", "ctor", "napi-derive", "napi-sys", @@ -267,9 +817,9 @@ checksum = "882a73d9ef23e8dc2ebbffb6a6ae2ef467c0f18ac10711e4cc59c5485d41df0e" [[package]] name = "napi-derive" -version = "2.12.5" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c47e0f395207c062e680a158f0624ec456c1dfb3c96a8cb888e0401506d50ae9" +checksum = "da1c6a8fa84d549aa8708fcd062372bf8ec6e849de39016ab921067d21bde367" dependencies = [ "cfg-if", "convert_case", @@ -281,9 +831,9 @@ dependencies = [ [[package]] name = "napi-derive-backend" -version = "1.0.51" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a83afae5b4ba6f98ed6e33a52da343fdeb66474f1162a38cde5a3d46eb054e7" +checksum = "20bbc7c69168d06a848f925ec5f0e0997f98e8c8d4f2cc30157f0da51c009e17" dependencies = [ "convert_case", "once_cell", @@ -303,6 +853,16 @@ dependencies = [ "libloading", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "notify" version = "6.0.0" @@ -322,13 +882,61 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2399c9463abc5f909349d8aa9ba080e0b88b3ce2885389b60b993f39b1a56905" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "num_cpus" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -356,17 +964,92 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "windows-sys 0.45.0", ] +[[package]] +name = "paste" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + +[[package]] +name = "pem-rfc7468" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d159833a9105500e0398934e205e0773f0b27529557134ecfc51c27646adac" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + [[package]] name = "pin-project-lite" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff33bdbdfc54cc98a2eca766ebdec3e1b8fb7387523d5c9c9a2891da856f719" +dependencies = [ + "der", + "pkcs8", + "spki", + "zeroize", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -375,18 +1058,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -431,10 +1114,19 @@ dependencies = [ ] [[package]] -name = "regex" -version = "1.8.1" +name = "redox_syscall" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -443,9 +1135,88 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rsa" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55a77d189da1fee555ad95b7e50e7457d91c0e089ec68ca69ad2989413bbdab4" +dependencies = [ + "byteorder", + "digest", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "subtle", + "zeroize", +] + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustls" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +dependencies = [ + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] [[package]] name = "ryu" @@ -468,6 +1239,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "semver" version = "1.0.17" @@ -476,22 +1257,22 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -505,6 +1286,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -514,6 +1317,25 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -530,6 +1352,262 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afd8985c8822235a9ebeedf0bff971462470162759663d3184593c807ab6e898" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c12403de02d88e6808de30eb2153c6997d39cc9511a446b510d5944a3ea6727" +dependencies = [ + "ahash 0.7.6", + "atoi", + "bitflags 1.3.2", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2be74801a0852ace9d86bc8cc8ac36241e7dc712fea26b8f32bd80ce29c98a10" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ce71dd8afc7ad2aeff001bb6affa7128c9087bbdcab07fa97a7952e8ee3d1da" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c430536df19e8b5b048a9ae19b266aba77f9f3e2255b7195f465d678cb2d0a" +dependencies = [ + "atoi", + "base64", + "bitflags 1.3.2", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210e0a1523b6d46ca73db1c5197a233a8e14787596910ce88ff5d47a00da0241" +dependencies = [ + "atoi", + "base64", + "bitflags 1.3.2", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f446c04b2d2d06b49b905e33c877b282e0f70b1b60a22513eacee8bf56d8afbe" +dependencies = [ + "atoi", + "bitflags 1.3.2", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", +] + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -543,9 +1621,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -553,10 +1631,69 @@ dependencies = [ ] [[package]] -name = "tokio" -version = "1.28.0" +name = "tempfile" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.45.0", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -579,14 +1716,79 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] -name = "unicode-ident" -version = "1.0.8" +name = "tokio-stream" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] [[package]] name = "unicode-segmentation" @@ -595,16 +1797,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] -name = "uuid" -version = "1.3.2" +name = "unicode_categories" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "uuid" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ "getrandom", "rand", "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "walkdir" version = "2.3.3" @@ -615,12 +1852,97 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.18", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" + +[[package]] +name = "web-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125" +dependencies = [ + "rustls-webpki", +] + +[[package]] +name = "whoami" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" + [[package]] name = "winapi" version = "0.3.9" @@ -652,6 +1974,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -783,3 +2114,9 @@ name = "windows_x86_64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/Cargo.toml b/Cargo.toml index 8b5f358fa1..02853b04c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,8 @@ [workspace] -members = ["./packages/native"] +members = ["./packages/native", "./packages/native/schema"] + +[profile.dev.package.sqlx-macros] +opt-level = 3 [profile.release] lto = true diff --git a/apps/electron/README.md b/apps/electron/README.md index ec03f0f355..544253b391 100644 --- a/apps/electron/README.md +++ b/apps/electron/README.md @@ -17,22 +17,6 @@ yarn dev # or yarn prod for production build ## Troubleshooting -### better-sqlite3 error - -When running tests or starting electron, you may encounter the following error: - -> Error: The module 'apps/electron/node_modules/better-sqlite3/build/Release/better_sqlite3.node' - -This is due to the fact that the `better-sqlite3` package is built for the Node.js version in Electron & in your machine. To fix this, run the following command based on different cases: - -```sh -# for running unit tests, we are not using Electron's node: -yarn rebuild better-sqlite3 - -# for running Electron, we are using Electron's node: -yarn postinstall -``` - ## Credits Most of the boilerplate code is generously borrowed from the following diff --git a/apps/electron/layers/main/src/__tests__/integration.spec.ts b/apps/electron/layers/main/src/__tests__/integration.spec.ts index 1305c1826c..0c65464b5d 100644 --- a/apps/electron/layers/main/src/__tests__/integration.spec.ts +++ b/apps/electron/layers/main/src/__tests__/integration.spec.ts @@ -1,5 +1,6 @@ import assert from 'node:assert'; import path from 'node:path'; +import { setTimeout } from 'node:timers/promises'; import fs from 'fs-extra'; import { v4 } from 'uuid'; @@ -13,8 +14,6 @@ const registeredHandlers = new Map< ((...args: any[]) => Promise)[] >(); -const delay = (ms: number) => new Promise(r => setTimeout(r, ms)); - type WithoutFirstParameter = T extends (_: any, ...args: infer P) => infer R ? (...args: P) => R : T; @@ -72,11 +71,14 @@ const nativeTheme = { themeSource: 'light', }; -function compareBuffer(a: Uint8Array | null, b: Uint8Array | null) { +function compareBuffer( + a: Uint8Array | null | undefined, + b: Uint8Array | null | undefined +) { if ( - (a === null && b === null) || - a === null || - b === null || + (a == null && b == null) || + a == null || + b == null || a.length !== b.length ) { return false; @@ -105,11 +107,11 @@ const electronModule = { handlers.push(callback); registeredHandlers.set(name, handlers); }, - addEventListener: (...args: any[]) => { + addListener: (...args: any[]) => { // @ts-ignore electronModule.app.on(...args); }, - removeEventListener: () => {}, + removeListener: () => {}, }, BrowserWindow: { getAllWindows: () => { @@ -135,7 +137,6 @@ beforeEach(async () => { const { registerEvents } = await import('../events'); registerEvents(); await fs.mkdirp(SESSION_DATA_PATH); - await import('../db/ensure-db'); registeredHandlers.get('ready')?.forEach(fn => fn()); }); @@ -143,7 +144,10 @@ beforeEach(async () => { afterEach(async () => { // reset registered handlers registeredHandlers.get('before-quit')?.forEach(fn => fn()); - + // wait for the db to be closed on Windows + if (process.platform === 'win32') { + await setTimeout(200); + } await fs.remove(SESSION_DATA_PATH); }); @@ -175,7 +179,7 @@ describe('ensureSQLiteDB', () => { const fileExists = await fs.pathExists(file); expect(fileExists).toBe(true); registeredHandlers.get('before-quit')?.forEach(fn => fn()); - await delay(100); + await setTimeout(100); expect(workspaceDB.db).toBe(null); }); }); @@ -254,7 +258,7 @@ describe('db handlers', () => { test('get non existent blob', async () => { const workspaceId = v4(); const bin = await dispatch('db', 'getBlob', workspaceId, 'non-existent-id'); - expect(bin).toBeNull(); + expect(bin).toBeUndefined(); }); test('list blobs (empty)', async () => { diff --git a/apps/electron/layers/main/src/db/__tests__/ensure-db.spec.ts b/apps/electron/layers/main/src/db/__tests__/ensure-db.spec.ts index 6d82730265..e65e2da7df 100644 --- a/apps/electron/layers/main/src/db/__tests__/ensure-db.spec.ts +++ b/apps/electron/layers/main/src/db/__tests__/ensure-db.spec.ts @@ -1,4 +1,5 @@ import path from 'node:path'; +import { setTimeout } from 'node:timers/promises'; import fs from 'fs-extra'; import { v4 } from 'uuid'; @@ -30,18 +31,20 @@ const electronModule = { handlers.push(callback); registeredHandlers.set(name, handlers); }, - addEventListener: (...args: any[]) => { + addListener: (...args: any[]) => { // @ts-ignore electronModule.app.on(...args); }, - removeEventListener: () => {}, + removeListener: () => {}, }, shell: {} as Partial, dialog: {} as Partial, }; -const runHandler = (key: string) => { - registeredHandlers.get(key)?.forEach(handler => handler()); +const runHandler = async (key: string) => { + await Promise.all( + (registeredHandlers.get(key) ?? []).map(handler => handler()) + ); }; // dynamically import handlers so that we can inject local variables to mocks @@ -51,6 +54,7 @@ vi.doMock('electron', () => { const constructorStub = vi.fn(); const destroyStub = vi.fn(); +destroyStub.mockReturnValue(Promise.resolve()); vi.doMock('../secondary-db', () => { return { @@ -59,6 +63,10 @@ vi.doMock('../secondary-db', () => { constructorStub(...args); } + connectIfNeeded = () => Promise.resolve(); + + pull = () => Promise.resolve(); + destroy = destroyStub; }, }; @@ -69,7 +77,7 @@ beforeEach(() => { }); afterEach(async () => { - runHandler('before-quit'); + await runHandler('before-quit'); await fs.remove(tmpDir); vi.useRealTimers(); }); @@ -98,12 +106,24 @@ test('db should be destroyed when app quits', async () => { expect(db0.db).not.toBeNull(); expect(db1.db).not.toBeNull(); - runHandler('before-quit'); + await runHandler('before-quit'); + + // wait the async `db.destroy()` to be called + await setTimeout(100); expect(db0.db).toBeNull(); expect(db1.db).toBeNull(); }); +test('db should be removed in db$Map after destroyed', async () => { + const { ensureSQLiteDB, db$Map } = await import('../ensure-db'); + const workspaceId = v4(); + const db = await ensureSQLiteDB(workspaceId); + await db.destroy(); + await setTimeout(100); + expect(db$Map.has(workspaceId)).toBe(false); +}); + test('if db has a secondary db path, we should also poll that', async () => { const { ensureSQLiteDB } = await import('../ensure-db'); const { appContext } = await import('../../context'); @@ -115,10 +135,7 @@ test('if db has a secondary db path, we should also poll that', async () => { const db = await ensureSQLiteDB(workspaceId); - await vi.advanceTimersByTimeAsync(1500); - - // not sure why but we still need to wait with real timer - await new Promise(resolve => setTimeout(resolve, 100)); + await setTimeout(10); expect(constructorStub).toBeCalledTimes(1); expect(constructorStub).toBeCalledWith(path.join(tmpDir, 'secondary.db'), db); @@ -128,7 +145,8 @@ test('if db has a secondary db path, we should also poll that', async () => { secondaryDBPath: path.join(tmpDir, 'secondary2.db'), }); - await vi.advanceTimersByTimeAsync(1500); + // wait the async `db.destroy()` to be called + await setTimeout(100); expect(constructorStub).toBeCalledTimes(2); expect(destroyStub).toBeCalledTimes(1); @@ -141,7 +159,7 @@ test('if db has a secondary db path, we should also poll that', async () => { expect(destroyStub).toBeCalledTimes(1); // if primary is destroyed, secondary should also be destroyed - db.destroy(); - await new Promise(resolve => setTimeout(resolve, 100)); + await db.destroy(); + await setTimeout(100); expect(destroyStub).toBeCalledTimes(2); }); diff --git a/apps/electron/layers/main/src/db/__tests__/workspace-db-adapter.spec.ts b/apps/electron/layers/main/src/db/__tests__/workspace-db-adapter.spec.ts index 35deaac532..86b0a49a04 100644 --- a/apps/electron/layers/main/src/db/__tests__/workspace-db-adapter.spec.ts +++ b/apps/electron/layers/main/src/db/__tests__/workspace-db-adapter.spec.ts @@ -16,10 +16,7 @@ const testAppContext: AppContext = { }; afterEach(async () => { - if (process.platform !== 'win32') { - // hmmm .... - await fs.remove(tmpDir); - } + await fs.remove(tmpDir); }); function getTestUpdates() { @@ -40,7 +37,7 @@ test('can create new db file if not exists', async () => { `storage.db` ); expect(await fs.exists(dbPath)).toBe(true); - db.destroy(); + await db.destroy(); }); test('on applyUpdate (from self), will not trigger update', async () => { @@ -52,7 +49,7 @@ test('on applyUpdate (from self), will not trigger update', async () => { db.update$.subscribe(onUpdate); db.applyUpdate(getTestUpdates(), 'self'); expect(onUpdate).not.toHaveBeenCalled(); - db.destroy(); + await db.destroy(); }); test('on applyUpdate (from renderer), will trigger update', async () => { @@ -67,7 +64,7 @@ test('on applyUpdate (from renderer), will trigger update', async () => { db.applyUpdate(getTestUpdates(), 'renderer'); expect(onUpdate).toHaveBeenCalled(); // not yet updated sub.unsubscribe(); - db.destroy(); + await db.destroy(); }); test('on applyUpdate (from external), will trigger update & send external update event', async () => { @@ -83,7 +80,7 @@ test('on applyUpdate (from external), will trigger update & send external update expect(onUpdate).toHaveBeenCalled(); expect(onExternalUpdate).toHaveBeenCalled(); sub.unsubscribe(); - db.destroy(); + await db.destroy(); }); test('on destroy, check if resources have been released', async () => { @@ -95,7 +92,7 @@ test('on destroy, check if resources have been released', async () => { next: vi.fn(), }; db.update$ = updateSub as any; - db.destroy(); + await db.destroy(); expect(db.db).toBe(null); expect(updateSub.complete).toHaveBeenCalled(); }); diff --git a/apps/electron/layers/main/src/db/base-db-adapter.ts b/apps/electron/layers/main/src/db/base-db-adapter.ts index 077b7483d3..a7fa79a35d 100644 --- a/apps/electron/layers/main/src/db/base-db-adapter.ts +++ b/apps/electron/layers/main/src/db/base-db-adapter.ts @@ -1,120 +1,76 @@ +import { SqliteConnection } from '@affine/native'; import assert from 'assert'; -import type { Database } from 'better-sqlite3'; -import sqlite from 'better-sqlite3'; import { logger } from '../logger'; -const schemas = [ - `CREATE TABLE IF NOT EXISTS "updates" ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - data BLOB NOT NULL, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL -)`, - `CREATE TABLE IF NOT EXISTS "blobs" ( - key TEXT PRIMARY KEY NOT NULL, - data BLOB NOT NULL, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL -)`, -]; - -interface UpdateRow { - id: number; - data: Buffer; - timestamp: string; -} - -interface BlobRow { - key: string; - data: Buffer; - timestamp: string; -} - /** * A base class for SQLite DB adapter that provides basic methods around updates & blobs */ export abstract class BaseSQLiteAdapter { - db: Database | null = null; + db: SqliteConnection | null = null; abstract role: string; - constructor(public path: string) {} - - ensureTables() { - assert(this.db, 'db is not connected'); - this.db.exec(schemas.join(';')); + constructor(public readonly path: string) { + logger.info(`[SQLiteAdapter]`, 'path:', path); } - // todo: what if SQLite DB wrapper later is not sync? - connect(): Database | undefined { - if (this.db) { - return this.db; + async connectIfNeeded() { + if (!this.db) { + this.db = new SqliteConnection(this.path); + await this.db.connect(); } - logger.log(`[SQLiteAdapter][${this.role}] open db`, this.path); - const db = (this.db = sqlite(this.path)); - this.ensureTables(); - return db; + return this.db; } - destroy() { - this.db?.close(); + async destroy() { + const { db } = this; this.db = null; + await db?.close(); } - addBlob(key: string, data: Uint8Array) { + async addBlob(key: string, data: Uint8Array) { try { - assert(this.db, 'db is not connected'); - const statement = this.db.prepare( - 'INSERT INTO blobs (key, data) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET data = ?' - ); - statement.run(key, data, data); - return key; + assert(this.db, `${this.path} is not connected`); + await this.db.addBlob(key, data); } catch (error) { logger.error('addBlob', error); } } - getBlob(key: string) { + async getBlob(key: string) { try { - assert(this.db, 'db is not connected'); - const statement = this.db.prepare('SELECT data FROM blobs WHERE key = ?'); - const row = statement.get(key) as BlobRow; - if (!row) { - return null; - } - return row.data; + assert(this.db, `${this.path} is not connected`); + const blob = await this.db.getBlob(key); + return blob?.data; } catch (error) { logger.error('getBlob', error); return null; } } - deleteBlob(key: string) { + async deleteBlob(key: string) { try { - assert(this.db, 'db is not connected'); - const statement = this.db.prepare('DELETE FROM blobs WHERE key = ?'); - statement.run(key); + assert(this.db, `${this.path} is not connected`); + await this.db.deleteBlob(key); } catch (error) { - logger.error('deleteBlob', error); + logger.error(`${this.path} delete blob failed`, error); } } - getBlobKeys() { + async getBlobKeys() { try { - assert(this.db, 'db is not connected'); - const statement = this.db.prepare('SELECT key FROM blobs'); - const rows = statement.all() as BlobRow[]; - return rows.map(row => row.key); + assert(this.db, `${this.path} is not connected`); + return await this.db.getBlobKeys(); } catch (error) { - logger.error('getBlobKeys', error); + logger.error(`getBlobKeys failed`, error); return []; } } - getUpdates() { + async getUpdates() { try { - assert(this.db, 'db is not connected'); - const statement = this.db.prepare('SELECT * FROM updates'); - const rows = statement.all() as UpdateRow[]; - return rows; + assert(this.db, `${this.path} is not connected`); + return await this.db.getUpdates(); } catch (error) { logger.error('getUpdates', error); return []; @@ -122,22 +78,12 @@ export abstract class BaseSQLiteAdapter { } // add a single update to SQLite - addUpdateToSQLite(updates: Uint8Array[]) { + async addUpdateToSQLite(db: SqliteConnection, updates: Uint8Array[]) { // batch write instead write per key stroke? try { - assert(this.db, 'db is not connected'); const start = performance.now(); - const statement = this.db.prepare( - 'INSERT INTO updates (data) VALUES (?)' - ); - const insertMany = this.db.transaction(updates => { - for (const d of updates) { - statement.run(d); - } - }); - - insertMany(updates); - + await db.connect(); + await db.insertUpdates(updates); logger.debug( `[SQLiteAdapter][${this.role}] addUpdateToSQLite`, 'length:', diff --git a/apps/electron/layers/main/src/db/ensure-db.ts b/apps/electron/layers/main/src/db/ensure-db.ts index 1fe8b9b2fd..eaa5f17af3 100644 --- a/apps/electron/layers/main/src/db/ensure-db.ts +++ b/apps/electron/layers/main/src/db/ensure-db.ts @@ -1,12 +1,14 @@ import { app } from 'electron'; +import type { Subject } from 'rxjs'; +import { Observable } from 'rxjs'; import { + concat, defer, - firstValueFrom, from, fromEvent, interval, + lastValueFrom, merge, - Observable, } from 'rxjs'; import { distinctUntilChanged, @@ -17,41 +19,83 @@ import { shareReplay, startWith, switchMap, + take, takeUntil, tap, } from 'rxjs/operators'; import { appContext } from '../context'; import { logger } from '../logger'; -import { getWorkspaceMeta$ } from '../workspace'; +import { getWorkspaceMeta, workspaceSubjects } from '../workspace'; import { SecondaryWorkspaceSQLiteDB } from './secondary-db'; import type { WorkspaceSQLiteDB } from './workspace-db-adapter'; import { openWorkspaceDatabase } from './workspace-db-adapter'; -const db$Map = new Map>(); +// export for testing +export const db$Map = new Map>(); +// use defer to prevent `app` is undefined while running tests const beforeQuit$ = defer(() => fromEvent(app, 'before-quit')); +// return a stream that emit a single event when the subject completes +function completed(subject: Subject) { + return new Observable(subscriber => { + const sub = subject.subscribe({ + complete: () => { + subscriber.next(); + subscriber.complete(); + }, + }); + return () => sub.unsubscribe(); + }); +} + function getWorkspaceDB$(id: string) { if (!db$Map.has(id)) { db$Map.set( id, from(openWorkspaceDatabase(appContext, id)).pipe( - shareReplay(1), - switchMap(db => { - return startPollingSecondaryDB(db).pipe( - ignoreElements(), - startWith(db), - takeUntil(beforeQuit$), - tap({ - complete: () => { - logger.info('[ensureSQLiteDB] close db connection'); - db.destroy(); - db$Map.delete(id); - }, - }) - ); + tap({ + next: db => { + logger.info( + '[ensureSQLiteDB] db connection established', + db.workspaceId + ); + }, }), + switchMap(db => + // takeUntil the polling stream, and then destroy the db + concat( + startPollingSecondaryDB(db).pipe( + ignoreElements(), + startWith(db), + takeUntil(merge(beforeQuit$, completed(db.update$))), + last(), + tap({ + next() { + logger.info( + '[ensureSQLiteDB] polling secondary db complete', + db.workspaceId + ); + }, + }) + ), + defer(async () => { + try { + await db.destroy(); + db$Map.delete(id); + logger.info( + '[ensureSQLiteDB] db connection destroyed', + db.workspaceId + ); + return db; + } catch (err) { + logger.error('[ensureSQLiteDB] destroy db failed', err); + throw err; + } + }) + ).pipe(startWith(db)) + ), shareReplay(1) ) ); @@ -60,51 +104,43 @@ function getWorkspaceDB$(id: string) { } function startPollingSecondaryDB(db: WorkspaceSQLiteDB) { - const meta$ = getWorkspaceMeta$(db.workspaceId); - const secondaryDB$ = meta$.pipe( + return merge( + getWorkspaceMeta(appContext, db.workspaceId), + workspaceSubjects.meta.pipe( + map(({ meta }) => meta), + filter(meta => meta.id === db.workspaceId) + ) + ).pipe( map(meta => meta?.secondaryDBPath), - distinctUntilChanged(), filter((p): p is string => !!p), + distinctUntilChanged(), switchMap(path => { - return new Observable(observer => { - const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db); - observer.next(secondaryDB); - return () => { - logger.info( - '[ensureSQLiteDB] close secondary db connection', - secondaryDB.path - ); - secondaryDB.destroy(); - }; + // on secondary db path change, destroy the old db and create a new one + const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db); + return new Observable(subscriber => { + subscriber.next(secondaryDB); + return () => secondaryDB.destroy(); }); }), - takeUntil(db.update$.pipe(last())), - shareReplay(1) + switchMap(secondaryDB => { + return interval(300000).pipe( + startWith(0), + tap({ + next: () => { + secondaryDB.pull(); + }, + error: err => { + logger.error(`[ensureSQLiteDB] polling secondary db error`, err); + }, + complete: () => { + logger.info('[ensureSQLiteDB] polling secondary db complete'); + }, + }) + ); + }) ); - - const firstDelayedTick$ = defer(() => { - return new Promise(resolve => - setTimeout(() => { - resolve(0); - }, 1000) - ); - }); - - // pull every 30 seconds - const poll$ = merge(firstDelayedTick$, interval(30000)).pipe( - switchMap(() => secondaryDB$), - tap({ - next: secondaryDB => { - secondaryDB.pull(); - }, - }), - takeUntil(db.update$.pipe(last())), - shareReplay(1) - ); - - return poll$; } export function ensureSQLiteDB(id: string) { - return firstValueFrom(getWorkspaceDB$(id)); + return lastValueFrom(getWorkspaceDB$(id).pipe(take(1))); } diff --git a/apps/electron/layers/main/src/db/helper.ts b/apps/electron/layers/main/src/db/helper.ts deleted file mode 100644 index 702fbef192..0000000000 --- a/apps/electron/layers/main/src/db/helper.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { Database } from 'better-sqlite3'; -import sqlite from 'better-sqlite3'; - -import { logger } from '../logger'; - -export function isValidateDB(db: Database) { - // check if db has two tables, one for updates and one for blobs - const statement = db.prepare( - `SELECT name FROM sqlite_schema WHERE type='table'` - ); - const rows = statement.all() as { name: string }[]; - const tableNames = rows.map(row => row.name); - if (!tableNames.includes('updates') || !tableNames.includes('blobs')) { - return false; - } -} - -export function isValidDBFile(path: string) { - let db: Database | null = null; - try { - db = sqlite(path); - // check if db has two tables, one for updates and one for blobs - const statement = db.prepare( - `SELECT name FROM sqlite_schema WHERE type='table'` - ); - const rows = statement.all() as { name: string }[]; - const tableNames = rows.map(row => row.name); - if (!tableNames.includes('updates') || !tableNames.includes('blobs')) { - return false; - } - return true; - } catch (error) { - logger.error('isValidDBFile', error); - return false; - } finally { - db?.close(); - } -} diff --git a/apps/electron/layers/main/src/db/secondary-db.ts b/apps/electron/layers/main/src/db/secondary-db.ts index 181b085168..41f3f1296c 100644 --- a/apps/electron/layers/main/src/db/secondary-db.ts +++ b/apps/electron/layers/main/src/db/secondary-db.ts @@ -1,3 +1,6 @@ +import assert from 'node:assert'; + +import type { SqliteConnection } from '@affine/native'; import { debounce } from 'lodash-es'; import * as Y from 'yjs'; @@ -30,16 +33,12 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId); } - close() { - this.db?.close(); - this.db = null; - } - - override destroy() { - this.flushUpdateQueue(); + override async destroy() { + const { db } = this; + await this.flushUpdateQueue(db); this.unsubscribers.forEach(unsub => unsub()); - this.db?.close(); this.yDoc.destroy(); + await super.destroy(); } get workspaceId() { @@ -48,12 +47,15 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { // do not update db immediately, instead, push to a queue // and flush the queue in a future time - addUpdateToUpdateQueue(update: Uint8Array) { + async addUpdateToUpdateQueue(db: SqliteConnection, update: Uint8Array) { this.updateQueue.push(update); - this.debouncedFlush(); + await this.debouncedFlush(db); } - flushUpdateQueue() { + async flushUpdateQueue(db = this.db) { + if (!db) { + return; // skip if db is not connected + } logger.debug( 'flushUpdateQueue', this.workspaceId, @@ -62,9 +64,8 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { ); const updates = [...this.updateQueue]; this.updateQueue = []; - this.connect(); - this.addUpdateToSQLite(updates); - this.close(); + await db.connect(); + await this.addUpdateToSQLite(db, updates); } // flush after 5s, but will not wait for more than 10s @@ -75,29 +76,31 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { runCounter = 0; // wrap the fn with connect and close - // it only works for sync functions - run = any>(fn: T) => { + async run any>( + fn: T + ): Promise< + (T extends (...args: any[]) => infer U ? Awaited : unknown) | undefined + > { try { - if (this.runCounter === 0) { - this.connect(); - } + await this.connectIfNeeded(); this.runCounter++; - return fn(); + return await fn(); } catch (err) { logger.error(err); } finally { this.runCounter--; if (this.runCounter === 0) { - this.close(); + await super.destroy(); } } - }; + } setupAndListen() { if (this.firstConnected) { return; } this.firstConnected = true; + const { db } = this; const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => { if (origin === 'renderer') { @@ -109,7 +112,7 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { const onSelfUpdate = (update: Uint8Array, origin: YOrigin) => { // for self update from upstream, we need to push it to external DB if (origin === 'upstream') { - this.addUpdateToUpdateQueue(update); + this.addUpdateToUpdateQueue(db!, update); } if (origin === 'self') { @@ -126,13 +129,11 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { this.yDoc.off('update', onSelfUpdate); }); - this.run(() => { + this.run(async () => { // apply all updates from upstream const upstreamUpdate = this.upstream.getDocAsUpdates(); // to initialize the yDoc, we need to apply all updates from the db this.applyUpdate(upstreamUpdate, 'upstream'); - - this.pull(); }); } @@ -141,17 +142,17 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { }; // TODO: have a better solution to handle blobs - syncBlobs() { - this.run(() => { + async syncBlobs() { + await this.run(async () => { // pull blobs - const blobsKeys = this.getBlobKeys(); - const upstreamBlobsKeys = this.upstream.getBlobKeys(); + const blobsKeys = await this.getBlobKeys(); + const upstreamBlobsKeys = await this.upstream.getBlobKeys(); // put every missing blob to upstream for (const key of blobsKeys) { if (!upstreamBlobsKeys.includes(key)) { - const blob = this.getBlob(key); + const blob = await this.getBlob(key); if (blob) { - this.upstream.addBlob(key, blob); + await this.upstream.addBlob(key, blob); logger.debug('syncBlobs', this.workspaceId, key); } } @@ -170,12 +171,17 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter { */ async pull() { const start = performance.now(); - const updates = this.run(() => { + assert(this.upstream.db, 'upstream db should be connected'); + const updates = await this.run(async () => { // TODO: no need to get all updates, just get the latest ones (using a cursor, etc)? - this.syncBlobs(); - return this.getUpdates().map(update => update.data); + await this.syncBlobs(); + return (await this.getUpdates()).map(update => update.data); }); + if (!updates) { + return; + } + const merged = await mergeUpdateWorker(updates); this.applyUpdate(merged, 'self'); diff --git a/apps/electron/layers/main/src/db/workspace-db-adapter.ts b/apps/electron/layers/main/src/db/workspace-db-adapter.ts index 8116ffba9c..f343f19c6a 100644 --- a/apps/electron/layers/main/src/db/workspace-db-adapter.ts +++ b/apps/electron/layers/main/src/db/workspace-db-adapter.ts @@ -1,4 +1,4 @@ -import type { Database } from 'better-sqlite3'; +import type { SqliteConnection } from '@affine/native'; import { Subject } from 'rxjs'; import * as Y from 'yjs'; @@ -21,38 +21,38 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter { super(path); } - override destroy() { - this.db?.close(); - this.db = null; + override async destroy() { + await super.destroy(); this.yDoc.destroy(); // when db is closed, we can safely remove it from ensure-db list this.update$.complete(); + this.firstConnected = false; } getWorkspaceName = () => { return this.yDoc.getMap('space:meta').get('name') as string; }; - async init(): Promise { - const db = super.connect(); + async init() { + const db = await super.connectIfNeeded(); if (!this.firstConnected) { - this.yDoc.on('update', (update: Uint8Array, origin: YOrigin) => { + this.yDoc.on('update', async (update: Uint8Array, origin: YOrigin) => { if (origin === 'renderer') { - this.addUpdateToSQLite([update]); + await this.addUpdateToSQLite(db, [update]); } else if (origin === 'external') { - this.addUpdateToSQLite([update]); - logger.debug('external update', this.workspaceId); dbSubjects.externalUpdate.next({ workspaceId: this.workspaceId, update, }); + await this.addUpdateToSQLite(db, [update]); + logger.debug('external update', this.workspaceId); } }); } - const updates = this.getUpdates(); + const updates = await this.getUpdates(); const merged = await mergeUpdateWorker(updates.map(update => update.data)); // to initialize the yDoc, we need to apply all updates from the db @@ -78,19 +78,19 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter { Y.applyUpdate(this.yDoc, data, origin); }; - override addBlob(key: string, value: Uint8Array) { - const res = super.addBlob(key, value); + override async addBlob(key: string, value: Uint8Array) { + const res = await super.addBlob(key, value); this.update$.next(); return res; } - override deleteBlob(key: string) { + override async deleteBlob(key: string) { super.deleteBlob(key); this.update$.next(); } - override addUpdateToSQLite(data: Uint8Array[]) { - super.addUpdateToSQLite(data); + override async addUpdateToSQLite(db: SqliteConnection, data: Uint8Array[]) { + super.addUpdateToSQLite(db, data); this.update$.next(); } } @@ -102,5 +102,6 @@ export async function openWorkspaceDatabase( const meta = await getWorkspaceMeta(context, workspaceId); const db = new WorkspaceSQLiteDB(meta.mainDBPath, workspaceId); await db.init(); + logger.info(`openWorkspaceDatabase [${workspaceId}]`); return db; } diff --git a/apps/electron/layers/main/src/dialog/dialog.ts b/apps/electron/layers/main/src/dialog/dialog.ts index 10e390a96f..fc83183ba9 100644 --- a/apps/electron/layers/main/src/dialog/dialog.ts +++ b/apps/electron/layers/main/src/dialog/dialog.ts @@ -7,7 +7,6 @@ import { nanoid } from 'nanoid'; import { appContext } from '../context'; import { ensureSQLiteDB } from '../db/ensure-db'; -import { isValidDBFile } from '../db/helper'; import type { WorkspaceSQLiteDB } from '../db/workspace-db-adapter'; import { logger } from '../logger'; import { @@ -208,7 +207,9 @@ export async function loadDBFile(): Promise { return { error: 'DB_FILE_ALREADY_LOADED' }; } - if (!isValidDBFile(filePath)) { + const { SqliteConnection } = await import('@affine/native'); + + if (!(await SqliteConnection.validate(filePath))) { // TODO: report invalid db file error? return { error: 'DB_FILE_INVALID' }; // invalid db file } @@ -305,7 +306,9 @@ export async function moveDBFile( // remove the old db file, but we don't care if it fails if (meta.secondaryDBPath) { - fs.remove(meta.secondaryDBPath); + fs.remove(meta.secondaryDBPath).catch(err => { + logger.error(`[moveDBFile] remove ${meta.secondaryDBPath} failed`, err); + }); } // update meta diff --git a/apps/electron/layers/main/src/workspace/__tests__/handlers.spec.ts b/apps/electron/layers/main/src/workspace/__tests__/handlers.spec.ts index 52003aec39..00be97c510 100644 --- a/apps/electron/layers/main/src/workspace/__tests__/handlers.spec.ts +++ b/apps/electron/layers/main/src/workspace/__tests__/handlers.spec.ts @@ -172,37 +172,3 @@ test('storeWorkspaceMeta', async () => { secondaryDBPath: path.join(tmpDir, 'test.db'), }); }); - -test('getWorkspaceMeta observable', async () => { - const { storeWorkspaceMeta } = await import('../handlers'); - const { getWorkspaceMeta$ } = await import('../index'); - - const workspaceId = v4(); - const workspacePath = path.join( - testAppContext.appDataPath, - 'workspaces', - workspaceId - ); - - const metaChange = vi.fn(); - - const meta$ = getWorkspaceMeta$(workspaceId); - - meta$.subscribe(metaChange); - await new Promise(resolve => setTimeout(resolve, 100)); - - expect(metaChange).toHaveBeenCalledWith({ - id: workspaceId, - mainDBPath: path.join(workspacePath, 'storage.db'), - }); - - await storeWorkspaceMeta(testAppContext, workspaceId, { - secondaryDBPath: path.join(tmpDir, 'test.db'), - }); - - expect(metaChange).toHaveBeenCalledWith({ - id: workspaceId, - mainDBPath: path.join(workspacePath, 'storage.db'), - secondaryDBPath: path.join(tmpDir, 'test.db'), - }); -}); diff --git a/apps/electron/layers/main/src/workspace/handlers.ts b/apps/electron/layers/main/src/workspace/handlers.ts index 03f553aadb..31711df61f 100644 --- a/apps/electron/layers/main/src/workspace/handlers.ts +++ b/apps/electron/layers/main/src/workspace/handlers.ts @@ -41,7 +41,7 @@ export async function deleteWorkspace(context: AppContext, id: string) { ); try { const db = await ensureSQLiteDB(id); - db.destroy(); + await db.destroy(); return await fs.move(basePath, movedPath, { overwrite: true, }); diff --git a/apps/electron/layers/main/src/workspace/index.ts b/apps/electron/layers/main/src/workspace/index.ts index 940653267d..aefa4ffed6 100644 --- a/apps/electron/layers/main/src/workspace/index.ts +++ b/apps/electron/layers/main/src/workspace/index.ts @@ -1,6 +1,3 @@ -import { merge } from 'rxjs'; -import { filter, map } from 'rxjs/operators'; - import { appContext } from '../context'; import type { MainEventListener, @@ -31,14 +28,3 @@ export const workspaceHandlers = { return getWorkspaceMeta(appContext, id); }, } satisfies NamespaceHandlers; - -// used internally. Get a stream of workspace id -> meta -export const getWorkspaceMeta$ = (workspaceId: string) => { - return merge( - getWorkspaceMeta(appContext, workspaceId), - workspaceSubjects.meta.pipe( - map(meta => meta.meta), - filter(meta => meta.id === workspaceId) - ) - ); -}; diff --git a/apps/electron/package.json b/apps/electron/package.json index 0dc99669d5..4435805b88 100644 --- a/apps/electron/package.json +++ b/apps/electron/package.json @@ -10,15 +10,13 @@ "description": "AFFiNE App", "homepage": "https://github.com/toeverything/AFFiNE", "scripts": { - "dev": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs", - "watch": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs --watch", - "prod": "yarn electron-rebuild && yarn node scripts/dev.mjs", + "dev": "yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs", + "watch": "yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs --watch", + "prod": "yarn node scripts/dev.mjs", "build-layers": "zx scripts/build-layers.mjs", "generate-assets": "zx scripts/generate-assets.mjs", "package": "electron-forge package", "make": "electron-forge make", - "rebuild:for-unit-test": "yarn rebuild better-sqlite3", - "rebuild:for-electron": "yarn electron-rebuild", "test": "playwright test" }, "config": { @@ -36,9 +34,7 @@ "@electron-forge/maker-squirrel": "^6.1.1", "@electron-forge/maker-zip": "^6.1.1", "@electron-forge/shared-types": "^6.1.1", - "@electron/rebuild": "^3.2.13", "@electron/remote": "2.0.9", - "@types/better-sqlite3": "^7.6.4", "@types/fs-extra": "^11.0.1", "@types/uuid": "^9.0.1", "cross-env": "7.0.3", @@ -55,8 +51,7 @@ "zx": "^7.2.2" }, "dependencies": { - "better-sqlite3": "^8.4.0", - "chokidar": "^3.5.3", + "cheerio": "^1.0.0-rc.12", "electron-updater": "^5.3.0", "lodash-es": "^4.17.21", "nanoid": "^4.0.2", diff --git a/apps/electron/scripts/common.mjs b/apps/electron/scripts/common.mjs index 4715d3a051..18e0dde292 100644 --- a/apps/electron/scripts/common.mjs +++ b/apps/electron/scripts/common.mjs @@ -46,7 +46,7 @@ export const config = () => { bundle: true, target: `node${NODE_MAJOR_VERSION}`, platform: 'node', - external: ['electron', 'yjs', 'better-sqlite3', 'electron-updater'], + external: ['electron', 'yjs', 'electron-updater'], define: define, format: 'cjs', loader: { diff --git a/packages/native/.env b/packages/native/.env new file mode 100644 index 0000000000..0170a2f443 --- /dev/null +++ b/packages/native/.env @@ -0,0 +1 @@ +DATABASE_URL="sqlite:affine.db" diff --git a/packages/native/.gitignore b/packages/native/.gitignore index e454ccdc89..7f927850c0 100644 --- a/packages/native/.gitignore +++ b/packages/native/.gitignore @@ -1 +1,2 @@ *.fixture +lib diff --git a/packages/native/Cargo.toml b/packages/native/Cargo.toml index 5b6e351aad..c63510021c 100644 --- a/packages/native/Cargo.toml +++ b/packages/native/Cargo.toml @@ -7,13 +7,15 @@ version = "0.0.0" crate-type = ["cdylib"] [dependencies] +affine_schema = { path = "./schema" } anyhow = "1" -# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix +chrono = "0.4" napi = { version = "2", default-features = false, features = [ - "napi4", + "napi5", "tokio_rt", "serde-json", "error_anyhow", + "chrono_date", ] } napi-derive = "2" notify = { version = "6", features = ["serde"] } @@ -21,6 +23,13 @@ once_cell = "1" parking_lot = "0.12" serde = "1" serde_json = "1" +sqlx = { version = "0.7.0-alpha.3", default-features = false, features = [ + "sqlite", + "runtime-tokio", + "tls-rustls", + "chrono", + "macros", +] } tokio = { version = "1", features = ["full"] } uuid = { version = "1", default-features = false, features = [ "serde", @@ -29,4 +38,16 @@ uuid = { version = "1", default-features = false, features = [ ] } [build-dependencies] +affine_schema = { path = "./schema" } +dotenv = "0.15" napi-build = "2" +sqlx = { version = "0.7.0-alpha.3", default-features = false, features = [ + "sqlite", + "runtime-tokio", + "tls-rustls", + "chrono", + "macros", + "migrate", + "json", +] } +tokio = { version = "1", features = ["full"] } diff --git a/packages/native/affine.db b/packages/native/affine.db new file mode 120000 index 0000000000..68060fb15b --- /dev/null +++ b/packages/native/affine.db @@ -0,0 +1 @@ +../../affine.db \ No newline at end of file diff --git a/packages/native/build.rs b/packages/native/build.rs index 9626ce7fdc..bfe845c847 100644 --- a/packages/native/build.rs +++ b/packages/native/build.rs @@ -1,6 +1,23 @@ -extern crate napi_build; +use sqlx::sqlite::SqliteConnectOptions; + +#[tokio::main] +async fn main() -> Result<(), std::io::Error> { + dotenv::dotenv().ok(); -fn main() -> Result<(), std::io::Error> { napi_build::setup(); + let options = SqliteConnectOptions::new() + .filename("../../affine.db") + .journal_mode(sqlx::sqlite::SqliteJournalMode::Off) + .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) + .create_if_missing(true); + let pool = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .unwrap(); + sqlx::query(affine_schema::SCHEMA) + .execute(&pool) + .await + .unwrap(); Ok(()) } diff --git a/packages/native/index.d.ts b/packages/native/index.d.ts index ce480d989e..7e865f9975 100644 --- a/packages/native/index.d.ts +++ b/packages/native/index.d.ts @@ -7,7 +7,7 @@ export interface WatchOptions { recursive?: boolean; } /** Watcher kind enumeration */ -export enum WatcherKind { +export const enum WatcherKind { /** inotify backend (linux) */ Inotify = 'Inotify', /** FS-Event backend (mac) */ @@ -23,6 +23,16 @@ export enum WatcherKind { Unknown = 'Unknown', } export function moveFile(src: string, dst: string): Promise; +export interface BlobRow { + key: string; + data: Buffer; + timestamp: Date; +} +export interface UpdateRow { + id: number; + timestamp: Date; + data: Buffer; +} export class Subscription { toString(): string; unsubscribe(): void; @@ -39,3 +49,16 @@ export class FsWatcher { static unwatch(p: string): void; static close(): void; } +export class SqliteConnection { + constructor(path: string); + connect(): Promise; + addBlob(key: string, blob: Uint8Array): Promise; + getBlob(key: string): Promise; + deleteBlob(key: string): Promise; + getBlobKeys(): Promise>; + getUpdates(): Promise>; + insertUpdates(updates: Array): Promise; + close(): Promise; + get isClose(): boolean; + static validate(path: string): Promise; +} diff --git a/packages/native/index.js b/packages/native/index.js index 45b760b372..98d9a450fe 100644 --- a/packages/native/index.js +++ b/packages/native/index.js @@ -263,9 +263,11 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`); } -const { WatcherKind, Subscription, FsWatcher, moveFile } = nativeBinding; +const { WatcherKind, Subscription, FsWatcher, moveFile, SqliteConnection } = + nativeBinding; module.exports.WatcherKind = WatcherKind; module.exports.Subscription = Subscription; module.exports.FsWatcher = FsWatcher; module.exports.moveFile = moveFile; +module.exports.SqliteConnection = SqliteConnection; diff --git a/packages/native/schema/Cargo.toml b/packages/native/schema/Cargo.toml new file mode 100644 index 0000000000..303ee3ba17 --- /dev/null +++ b/packages/native/schema/Cargo.toml @@ -0,0 +1,4 @@ +[package] +edition = "2021" +name = "affine_schema" +version = "0.0.0" diff --git a/packages/native/schema/README.md b/packages/native/schema/README.md new file mode 100644 index 0000000000..e40e886269 --- /dev/null +++ b/packages/native/schema/README.md @@ -0,0 +1 @@ +A temporary crate to share the schema between AFFiNE native and `build.rs` in the AFFiNE native. diff --git a/packages/native/schema/src/lib.rs b/packages/native/schema/src/lib.rs new file mode 100644 index 0000000000..46726a3530 --- /dev/null +++ b/packages/native/schema/src/lib.rs @@ -0,0 +1,13 @@ +// TODO +// dynamic create it from JavaScript side +// and remove this crate then. +pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + data BLOB NOT NULL, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL +); +CREATE TABLE IF NOT EXISTS "blobs" ( + key TEXT PRIMARY KEY NOT NULL, + data BLOB NOT NULL, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL +);"#; diff --git a/packages/native/src/lib.rs b/packages/native/src/lib.rs index d521fbd77e..2541f255aa 100644 --- a/packages/native/src/lib.rs +++ b/packages/native/src/lib.rs @@ -1 +1,2 @@ pub mod fs; +pub mod sqlite; diff --git a/packages/native/src/sqlite/mod.rs b/packages/native/src/sqlite/mod.rs new file mode 100644 index 0000000000..50c7c493f0 --- /dev/null +++ b/packages/native/src/sqlite/mod.rs @@ -0,0 +1,161 @@ +use chrono::NaiveDateTime; +use napi::bindgen_prelude::{Buffer, Uint8Array}; +use napi_derive::napi; +use sqlx::{ + migrate::MigrateDatabase, + sqlite::{Sqlite, SqliteConnectOptions, SqlitePoolOptions}, + Pool, Row, +}; + +#[napi(object)] +pub struct BlobRow { + pub key: String, + pub data: Buffer, + pub timestamp: NaiveDateTime, +} + +#[napi(object)] +pub struct UpdateRow { + pub id: i64, + pub timestamp: NaiveDateTime, + pub data: Buffer, +} + +#[napi] +pub struct SqliteConnection { + pool: Pool, + path: String, +} + +#[napi] +impl SqliteConnection { + #[napi(constructor)] + pub fn new(path: String) -> napi::Result { + let sqlite_options = SqliteConnectOptions::new() + .filename(&path) + .foreign_keys(false) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Off); + let pool = SqlitePoolOptions::new() + .max_connections(4) + .connect_lazy_with(sqlite_options); + Ok(Self { pool, path }) + } + + #[napi] + pub async fn connect(&self) -> napi::Result<()> { + if !Sqlite::database_exists(&self.path).await.unwrap_or(false) { + Sqlite::create_database(&self.path) + .await + .map_err(anyhow::Error::from)?; + }; + let mut connection = self.pool.acquire().await.map_err(anyhow::Error::from)?; + sqlx::query(affine_schema::SCHEMA) + .execute(connection.as_mut()) + .await + .map_err(anyhow::Error::from)?; + connection.detach(); + Ok(()) + } + + #[napi] + pub async fn add_blob(&self, key: String, blob: Uint8Array) -> napi::Result<()> { + let blob = blob.as_ref(); + sqlx::query_as!( + BlobRow, + "INSERT INTO blobs (key, data) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET data = excluded.data", + key, + blob, + ) + .execute(&self.pool) + .await + .map_err(anyhow::Error::from)?; + Ok(()) + } + + #[napi] + pub async fn get_blob(&self, key: String) -> Option { + sqlx::query_as!(BlobRow, "SELECT * FROM blobs WHERE key = ?", key) + .fetch_one(&self.pool) + .await + .ok() + } + + #[napi] + pub async fn delete_blob(&self, key: String) -> napi::Result<()> { + sqlx::query!("DELETE FROM blobs WHERE key = ?", key) + .execute(&self.pool) + .await + .map_err(anyhow::Error::from)?; + Ok(()) + } + + #[napi] + pub async fn get_blob_keys(&self) -> napi::Result> { + let keys = sqlx::query!("SELECT key FROM blobs") + .fetch_all(&self.pool) + .await + .map(|rows| rows.into_iter().map(|row| row.key).collect()) + .map_err(anyhow::Error::from)?; + Ok(keys) + } + + #[napi] + pub async fn get_updates(&self) -> napi::Result> { + let updates = sqlx::query_as!(UpdateRow, "SELECT * FROM updates") + .fetch_all(&self.pool) + .await + .map_err(anyhow::Error::from)?; + Ok(updates) + } + + #[napi] + pub async fn insert_updates(&self, updates: Vec) -> napi::Result<()> { + let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?; + for update in updates.into_iter() { + let update = update.as_ref(); + sqlx::query_as!(UpdateRow, "INSERT INTO updates (data) VALUES ($1)", update) + .execute(&mut *transaction) + .await + .map_err(anyhow::Error::from)?; + } + transaction.commit().await.map_err(anyhow::Error::from)?; + Ok(()) + } + + #[napi] + pub async fn close(&self) { + self.pool.close().await; + } + + #[napi(getter)] + pub fn is_close(&self) -> bool { + self.pool.is_closed() + } + + #[napi] + pub async fn validate(path: String) -> bool { + if let Ok(pool) = SqlitePoolOptions::new() + .max_connections(1) + .connect(&path) + .await + { + if let Ok(res) = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'") + .fetch_all(&pool) + .await + { + let names = res.iter().map(|row| row.get(0)); + names.fold(0, |acc, cur: String| { + if cur == "updates" || cur == "blobs" { + acc + 1 + } else { + acc + } + }) == 2 + } else { + false + } + } else { + false + } + } +} diff --git a/yarn.lock b/yarn.lock index cbfbcbef3a..2bafe91885 100644 --- a/yarn.lock +++ b/yarn.lock @@ -178,13 +178,10 @@ __metadata: "@electron-forge/maker-squirrel": ^6.1.1 "@electron-forge/maker-zip": ^6.1.1 "@electron-forge/shared-types": ^6.1.1 - "@electron/rebuild": ^3.2.13 "@electron/remote": 2.0.9 - "@types/better-sqlite3": ^7.6.4 "@types/fs-extra": ^11.0.1 "@types/uuid": ^9.0.1 - better-sqlite3: ^8.4.0 - chokidar: ^3.5.3 + cheerio: ^1.0.0-rc.12 cross-env: 7.0.3 electron: 25.0.1 electron-log: ^5.0.0-beta.24 @@ -3317,7 +3314,7 @@ __metadata: languageName: node linkType: hard -"@electron/rebuild@npm:^3.2.10, @electron/rebuild@npm:^3.2.13": +"@electron/rebuild@npm:^3.2.10": version: 3.2.13 resolution: "@electron/rebuild@npm:3.2.13" dependencies: @@ -9233,15 +9230,6 @@ __metadata: languageName: node linkType: hard -"@types/better-sqlite3@npm:^7.6.4": - version: 7.6.4 - resolution: "@types/better-sqlite3@npm:7.6.4" - dependencies: - "@types/node": "*" - checksum: 75ab00d31b56437cc65fe15ff673cf8d1609edca52628083921bcbab1cbd828d135a2859fb4e68af8ef5a4801705ba99d54b96499f997bce65dd306ade3dbe58 - languageName: node - linkType: hard - "@types/body-parser@npm:*": version: 1.19.2 resolution: "@types/body-parser@npm:1.19.2" @@ -11689,17 +11677,6 @@ __metadata: languageName: node linkType: hard -"better-sqlite3@npm:^8.4.0": - version: 8.4.0 - resolution: "better-sqlite3@npm:8.4.0" - dependencies: - bindings: ^1.5.0 - node-gyp: latest - prebuild-install: ^7.1.0 - checksum: f8b180c26428a2d381482e83b4519d0d81a918e00f92cafd255f42eb9583f49b2fed1015121ad49ebe1f3f790cc8c6f4697d1e805193d65e70dacf2e8abbd6af - languageName: node - linkType: hard - "big-integer@npm:^1.6.44": version: 1.6.51 resolution: "big-integer@npm:1.6.51" @@ -11728,15 +11705,6 @@ __metadata: languageName: node linkType: hard -"bindings@npm:^1.5.0": - version: 1.5.0 - resolution: "bindings@npm:1.5.0" - dependencies: - file-uri-to-path: 1.0.0 - checksum: 65b6b48095717c2e6105a021a7da4ea435aa8d3d3cd085cb9e85bcb6e5773cf318c4745c3f7c504412855940b585bdf9b918236612a1c7a7942491de176f1ae7 - languageName: node - linkType: hard - "bl@npm:^4.0.3, bl@npm:^4.1.0": version: 4.1.0 resolution: "bl@npm:4.1.0" @@ -13708,7 +13676,7 @@ __metadata: languageName: node linkType: hard -"detect-libc@npm:^2.0.0, detect-libc@npm:^2.0.1": +"detect-libc@npm:^2.0.1": version: 2.0.1 resolution: "detect-libc@npm:2.0.1" checksum: ccb05fcabbb555beb544d48080179c18523a343face9ee4e1a86605a8715b4169f94d663c21a03c310ac824592f2ba9a5270218819bb411ad7be578a527593d7 @@ -15412,13 +15380,6 @@ __metadata: languageName: node linkType: hard -"expand-template@npm:^2.0.3": - version: 2.0.3 - resolution: "expand-template@npm:2.0.3" - checksum: 588c19847216421ed92befb521767b7018dc88f88b0576df98cb242f20961425e96a92cbece525ef28cc5becceae5d544ae0f5b9b5e2aa05acb13716ca5b3099 - languageName: node - linkType: hard - "expand-tilde@npm:^1.2.2": version: 1.2.2 resolution: "expand-tilde@npm:1.2.2" @@ -15773,13 +15734,6 @@ __metadata: languageName: node linkType: hard -"file-uri-to-path@npm:1.0.0": - version: 1.0.0 - resolution: "file-uri-to-path@npm:1.0.0" - checksum: b648580bdd893a008c92c7ecc96c3ee57a5e7b6c4c18a9a09b44fb5d36d79146f8e442578bc0e173dc027adf3987e254ba1dfd6e3ec998b7c282873010502144 - languageName: node - linkType: hard - "filelist@npm:^1.0.4": version: 1.0.4 resolution: "filelist@npm:1.0.4" @@ -16633,13 +16587,6 @@ __metadata: languageName: node linkType: hard -"github-from-package@npm:0.0.0": - version: 0.0.0 - resolution: "github-from-package@npm:0.0.0" - checksum: 14e448192a35c1e42efee94c9d01a10f42fe790375891a24b25261246ce9336ab9df5d274585aedd4568f7922246c2a78b8a8cd2571bfe99c693a9718e7dd0e3 - languageName: node - linkType: hard - "github-slugger@npm:^1.0.0": version: 1.5.0 resolution: "github-slugger@npm:1.5.0" @@ -20892,7 +20839,7 @@ __metadata: languageName: node linkType: hard -"minimist@npm:^1.1.1, minimist@npm:^1.1.3, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8, minimist@npm:~1.2.5": +"minimist@npm:^1.1.1, minimist@npm:^1.1.3, minimist@npm:^1.2.0, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8, minimist@npm:~1.2.5": version: 1.2.8 resolution: "minimist@npm:1.2.8" checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0 @@ -20976,7 +20923,7 @@ __metadata: languageName: node linkType: hard -"mkdirp-classic@npm:^0.5.2, mkdirp-classic@npm:^0.5.3": +"mkdirp-classic@npm:^0.5.2": version: 0.5.3 resolution: "mkdirp-classic@npm:0.5.3" checksum: 3f4e088208270bbcc148d53b73e9a5bd9eef05ad2cbf3b3d0ff8795278d50dd1d11a8ef1875ff5aea3fa888931f95bfcb2ad5b7c1061cfefd6284d199e6776ac @@ -21255,13 +21202,6 @@ __metadata: languageName: node linkType: hard -"napi-build-utils@npm:^1.0.1": - version: 1.0.2 - resolution: "napi-build-utils@npm:1.0.2" - checksum: 06c14271ee966e108d55ae109f340976a9556c8603e888037145d6522726aebe89dd0c861b4b83947feaf6d39e79e08817559e8693deedc2c94e82c5cbd090c7 - languageName: node - linkType: hard - "natural-compare-lite@npm:^1.4.0": version: 1.4.0 resolution: "natural-compare-lite@npm:1.4.0" @@ -21396,7 +21336,7 @@ __metadata: languageName: node linkType: hard -"node-abi@npm:^3.0.0, node-abi@npm:^3.3.0": +"node-abi@npm:^3.0.0": version: 3.40.0 resolution: "node-abi@npm:3.40.0" dependencies: @@ -22767,28 +22707,6 @@ __metadata: languageName: node linkType: hard -"prebuild-install@npm:^7.1.0": - version: 7.1.1 - resolution: "prebuild-install@npm:7.1.1" - dependencies: - detect-libc: ^2.0.0 - expand-template: ^2.0.3 - github-from-package: 0.0.0 - minimist: ^1.2.3 - mkdirp-classic: ^0.5.3 - napi-build-utils: ^1.0.1 - node-abi: ^3.3.0 - pump: ^3.0.0 - rc: ^1.2.7 - simple-get: ^4.0.0 - tar-fs: ^2.0.0 - tunnel-agent: ^0.6.0 - bin: - prebuild-install: bin.js - checksum: dbf96d0146b6b5827fc8f67f72074d2e19c69628b9a7a0a17d0fad1bf37e9f06922896972e074197fc00a52eae912993e6ef5a0d471652f561df5cb516f3f467 - languageName: node - linkType: hard - "precinct@npm:^8.1.0": version: 8.3.1 resolution: "precinct@npm:8.3.1" @@ -24884,24 +24802,6 @@ __metadata: languageName: node linkType: hard -"simple-concat@npm:^1.0.0": - version: 1.0.1 - resolution: "simple-concat@npm:1.0.1" - checksum: 4d211042cc3d73a718c21ac6c4e7d7a0363e184be6a5ad25c8a1502e49df6d0a0253979e3d50dbdd3f60ef6c6c58d756b5d66ac1e05cda9cacd2e9fc59e3876a - languageName: node - linkType: hard - -"simple-get@npm:^4.0.0": - version: 4.0.1 - resolution: "simple-get@npm:4.0.1" - dependencies: - decompress-response: ^6.0.0 - once: ^1.3.1 - simple-concat: ^1.0.0 - checksum: e4132fd27cf7af230d853fa45c1b8ce900cb430dd0a3c6d3829649fe4f2b26574c803698076c4006450efb0fad2ba8c5455fbb5755d4b0a5ec42d4f12b31d27e - languageName: node - linkType: hard - "simple-git@npm:^3.15.0": version: 3.19.0 resolution: "simple-git@npm:3.19.0" @@ -25841,7 +25741,7 @@ __metadata: languageName: node linkType: hard -"tar-fs@npm:^2.0.0, tar-fs@npm:^2.1.1": +"tar-fs@npm:^2.1.1": version: 2.1.1 resolution: "tar-fs@npm:2.1.1" dependencies: @@ -26354,15 +26254,6 @@ __metadata: languageName: node linkType: hard -"tunnel-agent@npm:^0.6.0": - version: 0.6.0 - resolution: "tunnel-agent@npm:0.6.0" - dependencies: - safe-buffer: ^5.0.1 - checksum: 05f6510358f8afc62a057b8b692f05d70c1782b70db86d6a1e0d5e28a32389e52fa6e7707b6c5ecccacc031462e4bc35af85ecfe4bbc341767917b7cf6965711 - languageName: node - linkType: hard - "turndown@npm:^7.1.1": version: 7.1.2 resolution: "turndown@npm:7.1.2"