From e1ae0d46da89e0786b3a4e22ecbb55f7b5aa1274 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 15 Feb 2024 12:53:57 -0800 Subject: [PATCH] Add an extensions API to the collaboration server (#7807) This PR adds a REST API to the collab server for searching and downloading extensions. Previously, we had implemented this API in zed.dev directly, but this implementation is better, because we use the collab database to store the download counts for extensions. Release Notes: - N/A --------- Co-authored-by: Marshall Bowers Co-authored-by: Marshall Co-authored-by: Conrad --- .github/workflows/ci.yml | 3 + .github/workflows/deploy_collab.yml | 34 +- .gitignore | 5 +- Cargo.lock | 745 ++++++++++++++++-- Cargo.toml | 187 ++--- Procfile | 3 +- crates/collab/.env.toml | 5 + crates/collab/Cargo.toml | 23 +- crates/collab/k8s/collab.template.yml | 25 + .../20221109000000_test_schema.sql | 22 + .../20240214102900_add_extensions.sql | 22 + crates/collab/src/api.rs | 5 + crates/collab/src/api/extensions.rs | 237 ++++++ crates/collab/src/db.rs | 55 +- crates/collab/src/db/ids.rs | 1 + crates/collab/src/db/queries.rs | 1 + crates/collab/src/db/queries/extensions.rs | 205 +++++ crates/collab/src/db/tables.rs | 2 + crates/collab/src/db/tables/extension.rs | 27 + .../collab/src/db/tables/extension_version.rs | 36 + crates/collab/src/db/tests.rs | 1 + crates/collab/src/db/tests/extension_tests.rs | 219 +++++ crates/collab/src/env.rs | 3 +- crates/collab/src/lib.rs | 45 ++ crates/collab/src/main.rs | 7 +- crates/collab/src/tests/test_server.rs | 6 + script/bootstrap | 4 + script/seed-db | 1 - 28 files changed, 1755 insertions(+), 174 deletions(-) create mode 100644 crates/collab/migrations/20240214102900_add_extensions.sql create mode 100644 crates/collab/src/api/extensions.rs create mode 100644 crates/collab/src/db/queries/extensions.rs create mode 100644 crates/collab/src/db/tables/extension.rs create mode 100644 crates/collab/src/db/tables/extension_version.rs create mode 100644 crates/collab/src/db/tests/extension_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 138fa5808e..a3073029c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: submodules: "recursive" fetch-depth: 0 + - name: Remove untracked files + run: git clean -df + - name: Set up default .cargo/config.toml run: cp ./.cargo/ci-config.toml ~/.cargo/config.toml diff --git a/.github/workflows/deploy_collab.yml b/.github/workflows/deploy_collab.yml index eca43501d4..4d3ad09cbc 100644 --- a/.github/workflows/deploy_collab.yml +++ b/.github/workflows/deploy_collab.yml @@ -45,8 +45,18 @@ jobs: submodules: "recursive" fetch-depth: 0 + - name: Install cargo nextest + shell: bash -euxo pipefail {0} + run: | + cargo install cargo-nextest + + - name: Limit target directory size + shell: bash -euxo pipefail {0} + run: script/clear-target-dir-if-larger-than 100 + - name: Run tests - uses: ./.github/actions/run_tests + shell: bash -euxo pipefail {0} + run: cargo nextest run --package collab --no-fail-fast publish: name: Publish collab server image @@ -90,22 +100,26 @@ jobs: - name: Sign into Kubernetes run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} - - name: Determine namespace + - name: Start rollout run: | set -eu if [[ $GITHUB_REF_NAME = "collab-production" ]]; then - echo "Deploying collab:$GITHUB_SHA to production" - echo "KUBE_NAMESPACE=production" >> $GITHUB_ENV + export ZED_KUBE_NAMESPACE=production elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then - echo "Deploying collab:$GITHUB_SHA to staging" - echo "KUBE_NAMESPACE=staging" >> $GITHUB_ENV + export ZED_KUBE_NAMESPACE=staging else echo "cowardly refusing to deploy from an unknown branch" exit 1 fi - - name: Start rollout - run: kubectl -n "$KUBE_NAMESPACE" set image deployment/collab collab=registry.digitalocean.com/zed/collab:${GITHUB_SHA} + echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE" - - name: Wait for rollout to finish - run: kubectl -n "$KUBE_NAMESPACE" rollout status deployment/collab + source script/lib/deploy-helpers.sh + export_vars_for_environment $ZED_KUBE_NAMESPACE + + export ZED_DO_CERTIFICATE_ID=$(doctl compute certificate list --format ID --no-header) + export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}" + + envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - + kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/collab --watch + echo "deployed collab.template.yml to ${ZED_KUBE_NAMESPACE}" diff --git a/.gitignore b/.gitignore index 6f040dd0c5..9b6df52dd1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,12 +5,8 @@ .DS_Store /plugins/bin /script/node_modules -/styles/node_modules -/styles/src/types/zed.ts /crates/theme/schemas/theme.json -/crates/collab/static/styles.css /crates/collab/.admins.json -/vendor/bin /assets/*licenses.md **/venv .build @@ -25,3 +21,4 @@ DerivedData/ **/*.db .pytest_cache .venv +.blob_store diff --git a/Cargo.lock b/Cargo.lock index cdab9f60a1..064192a69f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -714,6 +714,368 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af266887e24cd5f6d2ea7433cacd25dcd4773b7f70e488701968a7cdf51df57" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "hex", + "http 0.2.9", + "hyper", + "ring 0.17.7", + "time", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d56f287a9e65e4914bfedb5b22c056b65e4c232fca512d5509a9df36386759f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6a29eca8ea8982028a4df81883e7001e250a21d323b86418884b5345950a4b" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "http 0.2.9", + "http-body", + "percent-encoding", + "pin-project-lite 0.2.13", + "tracing", + "uuid 1.4.1", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c977e92277652aefb9a76a0fca652b26757d6845dce0d7bf4426da80f13d85b0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "http-body", + "once_cell", + "percent-encoding", + "regex-lite", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d7f527c7b28af1a641f7d89f9e6a4863e8ec00f39d2b731b056fc5ec5ce829" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0be3224cd574ee8ab5fd7c32087876f25c134c27ac603fcb38669ed8d346b0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3167c60d82a13bbaef569da06041644ff41e85c6377e5dad53fa2526ccfe9d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b1cbe0eee57a213039088dbdeca7be9352f24e0d72332d961e8a1cb388f82d" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac 0.12.1", + "http 0.2.9", + "http 1.0.0", + "once_cell", + "p256", + "percent-encoding", + "ring 0.17.7", + "sha2 0.10.7", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "426a5bc369ca7c8d3686439e46edc727f397a47ab3696b13f3ae8c81b3b36132" +dependencies = [ + "futures-util", + "pin-project-lite 0.2.13", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee554133eca2611b66d23548e48f9b44713befdb025ab76bc00185b878397a1" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes 1.5.0", + "crc32c", + "crc32fast", + "hex", + "http 0.2.9", + "http-body", + "md-5", + "pin-project-lite 0.2.13", + "sha1", + "sha2 0.10.7", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" +dependencies = [ + "aws-smithy-types", + "bytes 1.5.0", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85d6a0619f7b67183067fa3b558f94f90753da2df8c04aeb7336d673f804b0b8" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "bytes-utils", + "futures-core", + "http 0.2.9", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite 0.2.13", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1c1b5186b6f5c579bf0de1bcca9dd3d946d6d51361ea1d18131f6a0b64e13ae" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c0a2ce65882e788d2cf83ff28b9b16918de0460c47bf66c5da4f6c17b4c9694" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4cb6b3afa5fc9825a75675975dcc3e21764b5476bc91dbc63df4ea3d30a576e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "h2", + "http 0.2.9", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite 0.2.13", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23165433e80c04e8c09cee66d171292ae7234bae05fa9d5636e33095eae416b2" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes 1.5.0", + "http 0.2.9", + "pin-project-lite 0.2.13", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c94a5bec34850b92c9a054dad57b95c1d47f25125f55973e19f6ad788f0381ff" +dependencies = [ + "base64-simd", + "bytes 1.5.0", + "bytes-utils", + "futures-core", + "http 0.2.9", + "http-body", + "itoa", + "num-integer", + "pin-project-lite 0.2.13", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util 0.7.9", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16f94c9673412b7a72e3c3efec8de89081c320bf59ea12eed34c417a62ad600" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff7e122ee50ca962e9de91f5850cc37e2184b1219611eef6d44aa85929b54f6" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 0.2.9", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.5.17" @@ -727,7 +1089,7 @@ dependencies = [ "bytes 1.5.0", "futures-util", "headers", - "http", + "http 0.2.9", "http-body", "hyper", "itoa", @@ -758,7 +1120,7 @@ dependencies = [ "async-trait", "bytes 1.5.0", "futures-util", - "http", + "http 0.2.9", "http-body", "mime", "tower-layer", @@ -774,7 +1136,7 @@ dependencies = [ "axum", "bytes 1.5.0", "futures-util", - "http", + "http 0.2.9", "mime", "pin-project-lite 0.2.13", "serde", @@ -812,6 +1174,12 @@ dependencies = [ "nix 0.23.2", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -824,6 +1192,16 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -1169,6 +1547,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes 1.5.0", + "either", +] + [[package]] name = "call" version = "0.1.0" @@ -1537,6 +1925,8 @@ dependencies = [ "async-trait", "async-tungstenite", "audio", + "aws-config", + "aws-sdk-s3", "axum", "axum-extra", "base64 0.13.1", @@ -1582,6 +1972,7 @@ dependencies = [ "rpc", "scrypt", "sea-orm", + "semver", "serde", "serde_derive", "serde_json", @@ -2087,6 +2478,15 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +[[package]] +name = "crc32c" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -2149,6 +2549,28 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -2161,9 +2583,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" dependencies = [ "generic-array", "subtle", @@ -2286,6 +2708,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.8" @@ -2513,6 +2945,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "editor" version = "0.1.0" @@ -2579,6 +3023,26 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -2821,6 +3285,16 @@ dependencies = [ "workspace", ] +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "file_finder" version = "0.1.0" @@ -3160,9 +3634,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -3170,9 +3644,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -3198,9 +3672,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -3219,9 +3693,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -3230,21 +3704,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures 0.1.31", "futures-channel", @@ -3546,6 +4020,17 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df00eed8d1f0db937f6be10e46e8072b0671accb504cf0f959c5c52c679f5b9" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.21" @@ -3557,7 +4042,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.9", "indexmap 1.9.3", "slab", "tokio", @@ -3611,7 +4096,7 @@ dependencies = [ "base64 0.21.4", "bytes 1.5.0", "headers-core", - "http", + "http 0.2.9", "httpdate", "mime", "sha1", @@ -3623,7 +4108,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.9", ] [[package]] @@ -3736,6 +4221,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -3743,7 +4239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http", + "http 0.2.9", "pin-project-lite 0.2.13", ] @@ -3788,7 +4284,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "httparse", "httpdate", @@ -3801,6 +4297,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.9", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -4033,7 +4545,7 @@ dependencies = [ "encoding_rs", "event-listener", "futures-lite", - "http", + "http 0.2.9", "log", "mime", "once_cell", @@ -5578,12 +6090,29 @@ dependencies = [ "workspace", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.7", +] + [[package]] name = "palette" version = "0.7.3" @@ -5688,9 +6217,9 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.2.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e0b28ace46c5a396546bcf443bf422b57049617433d8854227352a4a9b24e7" +checksum = "c1a5d4e9c205d2c1ae73b84aab6240e98218c0e72e63b50422cfb2d1ca952282" dependencies = [ "base64ct", "rand_core 0.6.4", @@ -5902,9 +6431,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.8", + "pkcs8 0.10.2", + "spki 0.7.2", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -5913,8 +6452,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.8", + "spki 0.7.2", ] [[package]] @@ -6705,6 +7244,12 @@ dependencies = [ "regex-syntax 0.8.2", ] +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -6752,7 +7297,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-tls", @@ -6793,6 +7338,17 @@ dependencies = [ "usvg", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac 0.12.1", + "zeroize", +] + [[package]] name = "rgb" version = "0.8.36" @@ -6831,11 +7387,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi 0.3.9", ] +[[package]] +name = "ring" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +dependencies = [ + "cc", + "getrandom 0.2.10", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rkyv" version = "0.7.42" @@ -6989,10 +7559,10 @@ dependencies = [ "num-iter", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.1.0", + "spki 0.7.2", "subtle", "zeroize", ] @@ -7124,15 +7694,28 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ - "ring", + "log", + "ring 0.17.7", "rustls-webpki", "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.3" @@ -7144,12 +7727,12 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.7", + "untrusted 0.9.0", ] [[package]] @@ -7302,8 +7885,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -7430,6 +8013,20 @@ dependencies = [ "workspace", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -7747,6 +8344,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.1.0" @@ -7934,6 +8541,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.2" @@ -7941,7 +8558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der", + "der 0.7.8", ] [[package]] @@ -8301,9 +8918,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "sum_tree" @@ -8913,6 +9530,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -9004,7 +9631,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-timeout", @@ -9052,7 +9679,7 @@ dependencies = [ "bytes 1.5.0", "futures-core", "futures-util", - "http", + "http 0.2.9", "http-body", "http-range-header", "pin-project-lite 0.2.13", @@ -9624,7 +10251,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.9", "httparse", "log", "native-tls", @@ -9644,7 +10271,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.9", "httparse", "log", "rand 0.8.5", @@ -9786,6 +10413,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.4.1" @@ -10000,6 +10633,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vte" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index 170a58ea7e..b155bf2232 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,91 +1,91 @@ [workspace] members = [ - "crates/activity_indicator", - "crates/ai", - "crates/assets", - "crates/assistant", - "crates/audio", - "crates/auto_update", - "crates/breadcrumbs", - "crates/call", - "crates/channel", - "crates/cli", - "crates/client", - "crates/clock", - "crates/collab", - "crates/collab_ui", - "crates/collections", - "crates/command_palette", - "crates/copilot", - "crates/copilot_ui", - "crates/db", - "crates/diagnostics", - "crates/editor", - "crates/extension", - "crates/extensions_ui", - "crates/feature_flags", - "crates/feedback", - "crates/file_finder", - "crates/fs", - "crates/fsevent", - "crates/fuzzy", - "crates/git", - "crates/go_to_line", - "crates/gpui", - "crates/gpui_macros", - "crates/install_cli", - "crates/journal", - "crates/language", - "crates/language_selector", - "crates/language_tools", - "crates/live_kit_client", - "crates/live_kit_server", - "crates/lsp", - "crates/markdown_preview", - "crates/media", - "crates/menu", - "crates/multi_buffer", - "crates/node_runtime", - "crates/notifications", - "crates/outline", - "crates/picker", - "crates/plugin", - "crates/plugin_macros", - "crates/prettier", - "crates/project", - "crates/project_panel", - "crates/project_symbols", - "crates/quick_action_bar", - "crates/recent_projects", - "crates/refineable", - "crates/refineable/derive_refineable", - "crates/release_channel", - "crates/rich_text", - "crates/rope", - "crates/rpc", - "crates/search", - "crates/semantic_index", - "crates/settings", - "crates/snippet", - "crates/sqlez", - "crates/sqlez_macros", - "crates/story", - "crates/storybook", - "crates/sum_tree", - "crates/terminal", - "crates/terminal_view", - "crates/text", - "crates/theme", - "crates/theme_importer", - "crates/theme_selector", - "crates/ui", - "crates/util", - "crates/vcs_menu", - "crates/vim", - "crates/welcome", - "crates/workspace", - "crates/zed", - "crates/zed_actions", + "crates/activity_indicator", + "crates/ai", + "crates/assets", + "crates/assistant", + "crates/audio", + "crates/auto_update", + "crates/breadcrumbs", + "crates/call", + "crates/channel", + "crates/cli", + "crates/client", + "crates/clock", + "crates/collab", + "crates/collab_ui", + "crates/collections", + "crates/command_palette", + "crates/copilot", + "crates/copilot_ui", + "crates/db", + "crates/diagnostics", + "crates/editor", + "crates/extension", + "crates/extensions_ui", + "crates/feature_flags", + "crates/feedback", + "crates/file_finder", + "crates/fs", + "crates/fsevent", + "crates/fuzzy", + "crates/git", + "crates/go_to_line", + "crates/gpui", + "crates/gpui_macros", + "crates/install_cli", + "crates/journal", + "crates/language", + "crates/language_selector", + "crates/language_tools", + "crates/live_kit_client", + "crates/live_kit_server", + "crates/lsp", + "crates/markdown_preview", + "crates/media", + "crates/menu", + "crates/multi_buffer", + "crates/node_runtime", + "crates/notifications", + "crates/outline", + "crates/picker", + "crates/plugin", + "crates/plugin_macros", + "crates/prettier", + "crates/project", + "crates/project_panel", + "crates/project_symbols", + "crates/quick_action_bar", + "crates/recent_projects", + "crates/refineable", + "crates/refineable/derive_refineable", + "crates/release_channel", + "crates/rich_text", + "crates/rope", + "crates/rpc", + "crates/search", + "crates/semantic_index", + "crates/settings", + "crates/snippet", + "crates/sqlez", + "crates/sqlez_macros", + "crates/story", + "crates/storybook", + "crates/sum_tree", + "crates/terminal", + "crates/terminal_view", + "crates/text", + "crates/theme", + "crates/theme_importer", + "crates/theme_selector", + "crates/ui", + "crates/util", + "crates/vcs_menu", + "crates/vim", + "crates/welcome", + "crates/workspace", + "crates/zed", + "crates/zed_actions", ] default-members = ["crates/zed"] resolver = "2" @@ -191,8 +191,8 @@ globset = "0.4" indoc = "1" # We explicitly disable a http2 support in isahc. isahc = { version = "1.7.2", default-features = false, features = [ - "static-curl", - "text-decoding", + "static-curl", + "text-decoding", ] } lazy_static = "1.4.0" log = { version = "0.4.16", features = ["kv_unstable_serde"] } @@ -208,12 +208,13 @@ regex = "1.5" rusqlite = { version = "0.29.0", features = ["blob", "array", "modern_sqlite"] } rust-embed = { version = "8.0", features = ["include-exclude"] } schemars = "0.8" +semver = { version = "1.0" } serde = { version = "1.0", features = ["derive", "rc"] } serde_derive = { version = "1.0", features = ["deserialize_in_place"] } serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] } serde_json_lenient = { version = "0.1", features = [ - "preserve_order", - "raw_value", + "preserve_order", + "raw_value", ] } serde_repr = "0.1" smallvec = { version = "1.6", features = ["union"] } @@ -223,7 +224,11 @@ sysinfo = "0.29.10" tempfile = "3.9.0" thiserror = "1.0.29" tiktoken-rs = "0.5.7" -time = { version = "0.3", features = ["serde", "serde-well-known"] } +time = { version = "0.3", features = [ + "serde", + "serde-well-known", + "formatting", +] } toml = "0.5" tree-sitter = { version = "0.20", features = ["wasm"] } tree-sitter-astro = { git = "https://github.com/virchau13/tree-sitter-astro.git", rev = "e924787e12e8a03194f36a113290ac11d6dc10f3" } diff --git a/Procfile b/Procfile index 7bd9114dad..288842ebd3 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ -collab: cd crates/collab && RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run serve +collab: RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run --package=collab serve livekit: livekit-server --dev +blob_store: MINIO_ROOT_USER=the-blob-store-access-key MINIO_ROOT_PASSWORD=the-blob-store-secret-key minio server .blob_store diff --git a/crates/collab/.env.toml b/crates/collab/.env.toml index 01866012ea..7340a71cd9 100644 --- a/crates/collab/.env.toml +++ b/crates/collab/.env.toml @@ -7,6 +7,11 @@ ZED_ENVIRONMENT = "development" LIVE_KIT_SERVER = "http://localhost:7880" LIVE_KIT_KEY = "devkey" LIVE_KIT_SECRET = "secret" +BLOB_STORE_ACCESS_KEY = "the-blob-store-access-key" +BLOB_STORE_SECRET_KEY = "the-blob-store-secret-key" +BLOB_STORE_BUCKET = "the-extensions-bucket" +BLOB_STORE_URL = "http://127.0.0.1:9000" +BLOB_STORE_REGION = "the-region" # RUST_LOG=info # LOG_JSON=true diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index 02b2e5eb48..6c9e07f3be 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -15,9 +15,11 @@ name = "seed" required-features = ["seed-support"] [dependencies] -anyhow.workspace = true -async-tungstenite = "0.16" axum = { version = "0.5", features = ["json", "headers", "ws"] } +anyhow.workspace = true +aws-config = { version = "1.1.5" } +aws-sdk-s3 = { version = "1.15.0" } +async-tungstenite = "0.16" axum-extra = { version = "0.3", features = ["erased-json"] } base64 = "0.13" chrono.workspace = true @@ -40,13 +42,26 @@ rand.workspace = true reqwest = { version = "0.11", features = ["json"], optional = true } rpc.workspace = true scrypt = "0.7" -sea-orm = { version = "0.12.x", features = ["sqlx-postgres", "postgres-array", "runtime-tokio-rustls", "with-uuid"] } +sea-orm = { version = "0.12.x", features = [ + "sqlx-postgres", + "postgres-array", + "runtime-tokio-rustls", + "with-uuid", +] } +semver.workspace = true serde.workspace = true serde_derive.workspace = true serde_json.workspace = true sha-1 = "0.9" smallvec.workspace = true -sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "any"] } +sqlx = { version = "0.7", features = [ + "runtime-tokio-rustls", + "postgres", + "json", + "time", + "uuid", + "any", +] } text.workspace = true time.workspace = true tokio = { version = "1", features = ["full"] } diff --git a/crates/collab/k8s/collab.template.yml b/crates/collab/k8s/collab.template.yml index 120e5f592f..9ff7cee9e1 100644 --- a/crates/collab/k8s/collab.template.yml +++ b/crates/collab/k8s/collab.template.yml @@ -105,6 +105,31 @@ spec: secretKeyRef: name: livekit key: secret + - name: BLOB_STORE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: blob-store + key: access_key + - name: BLOB_STORE_SECRET_KEY + valueFrom: + secretKeyRef: + name: blob-store + key: secret_key + - name: BLOB_STORE_URL + valueFrom: + secretKeyRef: + name: blob-store + key: url + - name: BLOB_STORE_REGION + valueFrom: + secretKeyRef: + name: blob-store + key: region + - name: BLOB_STORE_BUCKET + valueFrom: + secretKeyRef: + name: blob-store + key: bucket - name: INVITE_LINK_PREFIX value: ${INVITE_LINK_PREFIX} - name: RUST_BACKTRACE diff --git a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql index 1026fdea0d..fef10f987e 100644 --- a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql +++ b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql @@ -353,3 +353,25 @@ CREATE TABLE contributors ( signed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (user_id) ); + +CREATE TABLE extensions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + external_id TEXT NOT NULL, + name TEXT NOT NULL, + latest_version TEXT NOT NULL, + total_download_count INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE extension_versions ( + extension_id INTEGER REFERENCES extensions(id), + version TEXT NOT NULL, + published_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + authors TEXT NOT NULL, + repository TEXT NOT NULL, + description TEXT NOT NULL, + download_count INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (extension_id, version) +); + +CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id"); +CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count"); diff --git a/crates/collab/migrations/20240214102900_add_extensions.sql b/crates/collab/migrations/20240214102900_add_extensions.sql new file mode 100644 index 0000000000..b32094036d --- /dev/null +++ b/crates/collab/migrations/20240214102900_add_extensions.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS extensions ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + external_id TEXT NOT NULL, + latest_version TEXT NOT NULL, + total_download_count BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS extension_versions ( + extension_id INTEGER REFERENCES extensions(id), + version TEXT NOT NULL, + published_at TIMESTAMP NOT NULL DEFAULT now(), + authors TEXT NOT NULL, + repository TEXT NOT NULL, + description TEXT NOT NULL, + download_count BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY(extension_id, version) +); + +CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id"); +CREATE INDEX "trigram_index_extensions_name" ON "extensions" USING GIN(name gin_trgm_ops); +CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count"); diff --git a/crates/collab/src/api.rs b/crates/collab/src/api.rs index 59d176b047..44d6fc3eb5 100644 --- a/crates/collab/src/api.rs +++ b/crates/collab/src/api.rs @@ -1,3 +1,5 @@ +mod extensions; + use crate::{ auth, db::{ContributorSelector, User, UserId}, @@ -20,6 +22,8 @@ use std::sync::Arc; use tower::ServiceBuilder; use tracing::instrument; +pub use extensions::fetch_extensions_from_blob_store_periodically; + pub fn routes(rpc_server: Arc, state: Arc) -> Router { Router::new() .route("/user", get(get_authenticated_user)) @@ -28,6 +32,7 @@ pub fn routes(rpc_server: Arc, state: Arc) -> Router Router { + Router::new() + .route("/extensions", get(get_extensions)) + .route( + "/extensions/:extension_id/:version/download", + get(download_extension), + ) +} + +#[derive(Debug, Deserialize)] +struct GetExtensionsParams { + filter: Option, +} + +#[derive(Debug, Deserialize)] +struct DownloadExtensionParams { + extension_id: String, + version: String, +} + +#[derive(Debug, Serialize)] +struct GetExtensionsResponse { + pub data: Vec, +} + +#[derive(Deserialize)] +struct ExtensionManifest { + name: String, + version: String, + description: Option, + authors: Vec, + repository: String, +} + +async fn get_extensions( + Extension(app): Extension>, + Query(params): Query, +) -> Result> { + let extensions = app.db.get_extensions(params.filter.as_deref(), 30).await?; + Ok(Json(GetExtensionsResponse { data: extensions })) +} + +async fn download_extension( + Extension(app): Extension>, + Path(params): Path, +) -> Result { + let Some((blob_store_client, bucket)) = app + .blob_store_client + .clone() + .zip(app.config.blob_store_bucket.clone()) + else { + Err(Error::Http( + StatusCode::NOT_IMPLEMENTED, + "not supported".into(), + ))? + }; + + let DownloadExtensionParams { + extension_id, + version, + } = params; + + let version_exists = app + .db + .record_extension_download(&extension_id, &version) + .await?; + + if !version_exists { + Err(Error::Http( + StatusCode::NOT_FOUND, + "unknown extension version".into(), + ))?; + } + + let url = blob_store_client + .get_object() + .bucket(bucket) + .key(format!( + "extensions/{extension_id}/{version}/archive.tar.gz" + )) + .presigned(PresigningConfig::expires_in(EXTENSION_DOWNLOAD_URL_LIFETIME).unwrap()) + .await + .map_err(|e| anyhow!("failed to create presigned extension download url {e}"))?; + + Ok(Redirect::temporary(url.uri())) +} + +const EXTENSION_FETCH_INTERVAL: Duration = Duration::from_secs(5 * 60); +const EXTENSION_DOWNLOAD_URL_LIFETIME: Duration = Duration::from_secs(3 * 60); + +pub fn fetch_extensions_from_blob_store_periodically(app_state: Arc, executor: Executor) { + let Some(blob_store_client) = app_state.blob_store_client.clone() else { + log::info!("no blob store client"); + return; + }; + let Some(blob_store_bucket) = app_state.config.blob_store_bucket.clone() else { + log::info!("no blob store bucket"); + return; + }; + + executor.spawn_detached({ + let executor = executor.clone(); + async move { + loop { + fetch_extensions_from_blob_store( + &blob_store_client, + &blob_store_bucket, + &app_state, + ) + .await + .log_err(); + executor.sleep(EXTENSION_FETCH_INTERVAL).await; + } + } + }); +} + +async fn fetch_extensions_from_blob_store( + blob_store_client: &aws_sdk_s3::Client, + blob_store_bucket: &String, + app_state: &Arc, +) -> anyhow::Result<()> { + let list = blob_store_client + .list_objects() + .bucket(blob_store_bucket) + .prefix("extensions/") + .send() + .await?; + + let objects = list + .contents + .ok_or_else(|| anyhow!("missing bucket contents"))?; + + let mut published_versions = HashMap::<&str, Vec<&str>>::default(); + for object in &objects { + let Some(key) = object.key.as_ref() else { + continue; + }; + let mut parts = key.split('/'); + let Some(_) = parts.next().filter(|part| *part == "extensions") else { + continue; + }; + let Some(extension_id) = parts.next() else { + continue; + }; + let Some(version) = parts.next() else { + continue; + }; + published_versions + .entry(extension_id) + .or_default() + .push(version); + } + + let known_versions = app_state.db.get_known_extension_versions().await?; + + let mut new_versions = HashMap::<&str, Vec>::default(); + let empty = Vec::new(); + for (extension_id, published_versions) in published_versions { + let known_versions = known_versions.get(extension_id).unwrap_or(&empty); + + for published_version in published_versions { + if known_versions + .binary_search_by_key(&published_version, String::as_str) + .is_err() + { + let object = blob_store_client + .get_object() + .bucket(blob_store_bucket) + .key(format!( + "extensions/{extension_id}/{published_version}/manifest.json" + )) + .send() + .await?; + let manifest_bytes = object + .body + .collect() + .await + .map(|data| data.into_bytes()) + .with_context(|| format!("failed to download manifest for extension {extension_id} version {published_version}"))? + .to_vec(); + let manifest = serde_json::from_slice::(&manifest_bytes) + .with_context(|| format!("invalid manifest for extension {extension_id} version {published_version}: {}", String::from_utf8_lossy(&manifest_bytes)))?; + + let published_at = object.last_modified.ok_or_else(|| anyhow!("missing last modified timestamp for extension {extension_id} version {published_version}"))?; + let published_at = + time::OffsetDateTime::from_unix_timestamp_nanos(published_at.as_nanos())?; + let published_at = PrimitiveDateTime::new(published_at.date(), published_at.time()); + + let version = semver::Version::parse(&manifest.version).with_context(|| { + format!( + "invalid version for extension {extension_id} version {published_version}" + ) + })?; + + new_versions + .entry(extension_id) + .or_default() + .push(NewExtensionVersion { + name: manifest.name, + version, + description: manifest.description.unwrap_or_default(), + authors: manifest.authors, + repository: manifest.repository, + published_at, + }); + } + } + } + + app_state + .db + .insert_extension_versions(&new_versions) + .await?; + + Ok(()) +} diff --git a/crates/collab/src/db.rs b/crates/collab/src/db.rs index 6ccea25a60..08e502a42f 100644 --- a/crates/collab/src/db.rs +++ b/crates/collab/src/db.rs @@ -1,12 +1,8 @@ -#[cfg(test)] -pub mod tests; - -#[cfg(test)] -pub use tests::TestDb; - mod ids; mod queries; mod tables; +#[cfg(test)] +pub mod tests; use crate::{executor::Executor, Error, Result}; use anyhow::anyhow; @@ -25,7 +21,7 @@ use sea_orm::{ FromQueryResult, IntoActiveModel, IsolationLevel, JoinType, QueryOrder, QuerySelect, Statement, TransactionTrait, }; -use serde::{Deserialize, Serialize}; +use serde::{ser::Error as _, Deserialize, Serialize, Serializer}; use sqlx::{ migrate::{Migrate, Migration, MigrationSource}, Connection, @@ -40,13 +36,17 @@ use std::{ sync::Arc, time::Duration, }; -pub use tables::*; +use time::{format_description::well_known::iso8601, PrimitiveDateTime}; use tokio::sync::{Mutex, OwnedMutexGuard}; +#[cfg(test)] +pub use tests::TestDb; + pub use ids::*; pub use queries::contributors::ContributorSelector; pub use sea_orm::ConnectOptions; pub use tables::user::Model as User; +pub use tables::*; /// Database gives you a handle that lets you access the database. /// It handles pooling internally. @@ -717,3 +717,42 @@ pub struct WorktreeSettingsFile { pub path: String, pub content: String, } + +pub struct NewExtensionVersion { + pub name: String, + pub version: semver::Version, + pub description: String, + pub authors: Vec, + pub repository: String, + pub published_at: PrimitiveDateTime, +} + +#[derive(Debug, Serialize, PartialEq)] +pub struct ExtensionMetadata { + pub id: String, + pub name: String, + pub version: String, + pub authors: Vec, + pub repository: String, + #[serde(serialize_with = "serialize_iso8601")] + pub published_at: PrimitiveDateTime, + pub download_count: u64, +} + +pub fn serialize_iso8601( + datetime: &PrimitiveDateTime, + serializer: S, +) -> Result { + const SERDE_CONFIG: iso8601::EncodedConfig = iso8601::Config::DEFAULT + .set_year_is_six_digits(false) + .set_time_precision(iso8601::TimePrecision::Second { + decimal_digits: None, + }) + .encode(); + + datetime + .assume_utc() + .format(&time::format_description::well_known::Iso8601::) + .map_err(S::Error::custom)? + .serialize(serializer) +} diff --git a/crates/collab/src/db/ids.rs b/crates/collab/src/db/ids.rs index d69e19643a..44a5db6a75 100644 --- a/crates/collab/src/db/ids.rs +++ b/crates/collab/src/db/ids.rs @@ -85,6 +85,7 @@ id_type!(SignupId); id_type!(UserId); id_type!(ChannelBufferCollaboratorId); id_type!(FlagId); +id_type!(ExtensionId); id_type!(NotificationId); id_type!(NotificationKindId); diff --git a/crates/collab/src/db/queries.rs b/crates/collab/src/db/queries.rs index f6bba13ede..7d9043f595 100644 --- a/crates/collab/src/db/queries.rs +++ b/crates/collab/src/db/queries.rs @@ -5,6 +5,7 @@ pub mod buffers; pub mod channels; pub mod contacts; pub mod contributors; +pub mod extensions; pub mod messages; pub mod notifications; pub mod projects; diff --git a/crates/collab/src/db/queries/extensions.rs b/crates/collab/src/db/queries/extensions.rs new file mode 100644 index 0000000000..78e80ea190 --- /dev/null +++ b/crates/collab/src/db/queries/extensions.rs @@ -0,0 +1,205 @@ +use super::*; + +impl Database { + pub async fn get_extensions( + &self, + filter: Option<&str>, + limit: usize, + ) -> Result> { + self.transaction(|tx| async move { + let mut condition = Condition::all(); + if let Some(filter) = filter { + let fuzzy_name_filter = Self::fuzzy_like_string(filter); + condition = condition.add(Expr::cust_with_expr("name ILIKE $1", fuzzy_name_filter)); + } + + let extensions = extension::Entity::find() + .filter(condition) + .order_by_desc(extension::Column::TotalDownloadCount) + .order_by_asc(extension::Column::Id) + .limit(Some(limit as u64)) + .filter( + extension::Column::LatestVersion + .into_expr() + .eq(extension_version::Column::Version.into_expr()), + ) + .inner_join(extension_version::Entity) + .select_also(extension_version::Entity) + .all(&*tx) + .await?; + + Ok(extensions + .into_iter() + .filter_map(|(extension, latest_version)| { + let version = latest_version?; + Some(ExtensionMetadata { + id: extension.external_id, + name: extension.name, + version: version.version, + authors: version + .authors + .split(',') + .map(|author| author.trim().to_string()) + .collect::>(), + repository: version.repository, + published_at: version.published_at, + download_count: extension.total_download_count as u64, + }) + }) + .collect()) + }) + .await + } + + pub async fn get_known_extension_versions<'a>(&self) -> Result>> { + self.transaction(|tx| async move { + let mut extension_external_ids_by_id = HashMap::default(); + + let mut rows = extension::Entity::find().stream(&*tx).await?; + while let Some(row) = rows.next().await { + let row = row?; + extension_external_ids_by_id.insert(row.id, row.external_id); + } + drop(rows); + + let mut known_versions_by_extension_id: HashMap> = + HashMap::default(); + let mut rows = extension_version::Entity::find().stream(&*tx).await?; + while let Some(row) = rows.next().await { + let row = row?; + + let Some(extension_id) = extension_external_ids_by_id.get(&row.extension_id) else { + continue; + }; + + let versions = known_versions_by_extension_id + .entry(extension_id.clone()) + .or_default(); + if let Err(ix) = versions.binary_search(&row.version) { + versions.insert(ix, row.version); + } + } + drop(rows); + + Ok(known_versions_by_extension_id) + }) + .await + } + + pub async fn insert_extension_versions( + &self, + versions_by_extension_id: &HashMap<&str, Vec>, + ) -> Result<()> { + self.transaction(|tx| async move { + for (external_id, versions) in versions_by_extension_id { + if versions.is_empty() { + continue; + } + + let latest_version = versions + .iter() + .max_by_key(|version| &version.version) + .unwrap(); + + let insert = extension::Entity::insert(extension::ActiveModel { + name: ActiveValue::Set(latest_version.name.clone()), + external_id: ActiveValue::Set(external_id.to_string()), + id: ActiveValue::NotSet, + latest_version: ActiveValue::Set(latest_version.version.to_string()), + total_download_count: ActiveValue::NotSet, + }) + .on_conflict( + OnConflict::columns([extension::Column::ExternalId]) + .update_column(extension::Column::ExternalId) + .to_owned(), + ); + + let extension = if tx.support_returning() { + insert.exec_with_returning(&*tx).await? + } else { + // Sqlite + insert.exec_without_returning(&*tx).await?; + extension::Entity::find() + .filter(extension::Column::ExternalId.eq(*external_id)) + .one(&*tx) + .await? + .ok_or_else(|| anyhow!("failed to insert extension"))? + }; + + extension_version::Entity::insert_many(versions.iter().map(|version| { + extension_version::ActiveModel { + extension_id: ActiveValue::Set(extension.id), + published_at: ActiveValue::Set(version.published_at), + version: ActiveValue::Set(version.version.to_string()), + authors: ActiveValue::Set(version.authors.join(", ")), + repository: ActiveValue::Set(version.repository.clone()), + description: ActiveValue::Set(version.description.clone()), + download_count: ActiveValue::NotSet, + } + })) + .on_conflict(OnConflict::new().do_nothing().to_owned()) + .exec_without_returning(&*tx) + .await?; + + if let Ok(db_version) = semver::Version::parse(&extension.latest_version) { + if db_version >= latest_version.version { + continue; + } + } + + let mut extension = extension.into_active_model(); + extension.latest_version = ActiveValue::Set(latest_version.version.to_string()); + extension.name = ActiveValue::set(latest_version.name.clone()); + extension::Entity::update(extension).exec(&*tx).await?; + } + + Ok(()) + }) + .await + } + + pub async fn record_extension_download(&self, extension: &str, version: &str) -> Result { + self.transaction(|tx| async move { + #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] + enum QueryId { + Id, + } + + let extension_id: Option = extension::Entity::find() + .filter(extension::Column::ExternalId.eq(extension)) + .select_only() + .column(extension::Column::Id) + .into_values::<_, QueryId>() + .one(&*tx) + .await?; + let Some(extension_id) = extension_id else { + return Ok(false); + }; + + extension_version::Entity::update_many() + .col_expr( + extension_version::Column::DownloadCount, + extension_version::Column::DownloadCount.into_expr().add(1), + ) + .filter( + extension_version::Column::ExtensionId + .eq(extension_id) + .and(extension_version::Column::Version.eq(version)), + ) + .exec(&*tx) + .await?; + + extension::Entity::update_many() + .col_expr( + extension::Column::TotalDownloadCount, + extension::Column::TotalDownloadCount.into_expr().add(1), + ) + .filter(extension::Column::Id.eq(extension_id)) + .exec(&*tx) + .await?; + + Ok(true) + }) + .await + } +} diff --git a/crates/collab/src/db/tables.rs b/crates/collab/src/db/tables.rs index 646447c91f..72d9835032 100644 --- a/crates/collab/src/db/tables.rs +++ b/crates/collab/src/db/tables.rs @@ -10,6 +10,8 @@ pub mod channel_message; pub mod channel_message_mention; pub mod contact; pub mod contributor; +pub mod extension; +pub mod extension_version; pub mod feature_flag; pub mod follower; pub mod language_server; diff --git a/crates/collab/src/db/tables/extension.rs b/crates/collab/src/db/tables/extension.rs new file mode 100644 index 0000000000..5a1462c701 --- /dev/null +++ b/crates/collab/src/db/tables/extension.rs @@ -0,0 +1,27 @@ +use crate::db::ExtensionId; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "extensions")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: ExtensionId, + pub external_id: String, + pub name: String, + pub latest_version: String, + pub total_download_count: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_one = "super::extension_version::Entity")] + LatestVersion, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::LatestVersion.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/crates/collab/src/db/tables/extension_version.rs b/crates/collab/src/db/tables/extension_version.rs new file mode 100644 index 0000000000..459f2296b1 --- /dev/null +++ b/crates/collab/src/db/tables/extension_version.rs @@ -0,0 +1,36 @@ +use crate::db::ExtensionId; +use sea_orm::entity::prelude::*; +use time::PrimitiveDateTime; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "extension_versions")] +pub struct Model { + #[sea_orm(primary_key)] + pub extension_id: ExtensionId, + #[sea_orm(primary_key)] + pub version: String, + pub published_at: PrimitiveDateTime, + pub authors: String, + pub repository: String, + pub description: String, + pub download_count: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::extension::Entity", + from = "Column::ExtensionId", + to = "super::extension::Column::Id" + on_condition = r#"super::extension::Column::LatestVersion.into_expr().eq(Column::Version.into_expr())"# + )] + Extension, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Extension.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/crates/collab/src/db/tests.rs b/crates/collab/src/db/tests.rs index a85aae4fa3..7790b951b2 100644 --- a/crates/collab/src/db/tests.rs +++ b/crates/collab/src/db/tests.rs @@ -2,6 +2,7 @@ mod buffer_tests; mod channel_tests; mod contributor_tests; mod db_tests; +mod extension_tests; mod feature_flag_tests; mod message_tests; diff --git a/crates/collab/src/db/tests/extension_tests.rs b/crates/collab/src/db/tests/extension_tests.rs new file mode 100644 index 0000000000..8963ffcbe9 --- /dev/null +++ b/crates/collab/src/db/tests/extension_tests.rs @@ -0,0 +1,219 @@ +use super::Database; +use crate::{ + db::{ExtensionMetadata, NewExtensionVersion}, + test_both_dbs, +}; +use std::sync::Arc; +use time::{OffsetDateTime, PrimitiveDateTime}; + +test_both_dbs!( + test_extensions, + test_extensions_postgres, + test_extensions_sqlite +); + +async fn test_extensions(db: &Arc) { + let versions = db.get_known_extension_versions().await.unwrap(); + assert!(versions.is_empty()); + + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert!(extensions.is_empty()); + + let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap(); + let t0 = PrimitiveDateTime::new(t0.date(), t0.time()); + + db.insert_extension_versions( + &[ + ( + "ext1", + vec![ + NewExtensionVersion { + name: "Extension 1".into(), + version: semver::Version::parse("0.0.1").unwrap(), + description: "an extension".into(), + authors: vec!["max".into()], + repository: "ext1/repo".into(), + published_at: t0, + }, + NewExtensionVersion { + name: "Extension One".into(), + version: semver::Version::parse("0.0.2").unwrap(), + description: "a good extension".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + }, + ], + ), + ( + "ext2", + vec![NewExtensionVersion { + name: "Extension Two".into(), + version: semver::Version::parse("0.2.0").unwrap(), + description: "a great extension".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + }], + ), + ] + .into_iter() + .collect(), + ) + .await + .unwrap(); + + let versions = db.get_known_extension_versions().await.unwrap(); + assert_eq!( + versions, + [ + ("ext1".into(), vec!["0.0.1".into(), "0.0.2".into()]), + ("ext2".into(), vec!["0.2.0".into()]) + ] + .into_iter() + .collect() + ); + + // The latest version of each extension is returned. + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 0, + }, + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 0 + }, + ] + ); + + // Record extensions being downloaded. + for _ in 0..7 { + assert!(db.record_extension_download("ext2", "0.0.2").await.unwrap()); + } + + for _ in 0..3 { + assert!(db.record_extension_download("ext1", "0.0.1").await.unwrap()); + } + + for _ in 0..2 { + assert!(db.record_extension_download("ext1", "0.0.2").await.unwrap()); + } + + // Record download returns false if the extension does not exist. + assert!(!db + .record_extension_download("no-such-extension", "0.0.2") + .await + .unwrap()); + + // Extensions are returned in descending order of total downloads. + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 7 + }, + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 5, + }, + ] + ); + + // Add more extensions, including a new version of `ext1`, and backfilling + // an older version of `ext2`. + db.insert_extension_versions( + &[ + ( + "ext1", + vec![NewExtensionVersion { + name: "Extension One".into(), + version: semver::Version::parse("0.0.3").unwrap(), + description: "a real good extension".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + }], + ), + ( + "ext2", + vec![NewExtensionVersion { + name: "Extension Two".into(), + version: semver::Version::parse("0.1.0").unwrap(), + description: "an old extension".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + }], + ), + ] + .into_iter() + .collect(), + ) + .await + .unwrap(); + + let versions = db.get_known_extension_versions().await.unwrap(); + assert_eq!( + versions, + [ + ( + "ext1".into(), + vec!["0.0.1".into(), "0.0.2".into(), "0.0.3".into()] + ), + ("ext2".into(), vec!["0.1.0".into(), "0.2.0".into()]) + ] + .into_iter() + .collect() + ); + + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 7 + }, + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.3".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 5, + }, + ] + ); +} diff --git a/crates/collab/src/env.rs b/crates/collab/src/env.rs index 58c29b0205..4e6fe3b3a3 100644 --- a/crates/collab/src/env.rs +++ b/crates/collab/src/env.rs @@ -3,7 +3,8 @@ use std::fs; pub fn load_dotenv() -> anyhow::Result<()> { let env: toml::map::Map = toml::de::from_str( - &fs::read_to_string("./.env.toml").map_err(|_| anyhow!("no .env.toml file found"))?, + &fs::read_to_string("./crates/collab/.env.toml") + .map_err(|_| anyhow!("no .env.toml file found"))?, )?; for (key, value) in env { diff --git a/crates/collab/src/lib.rs b/crates/collab/src/lib.rs index aba9bd75d1..195ed7b11d 100644 --- a/crates/collab/src/lib.rs +++ b/crates/collab/src/lib.rs @@ -8,11 +8,14 @@ pub mod rpc; #[cfg(test)] mod tests; +use anyhow::anyhow; +use aws_config::{BehaviorVersion, Region}; use axum::{http::StatusCode, response::IntoResponse}; use db::Database; use executor::Executor; use serde::Deserialize; use std::{path::PathBuf, sync::Arc}; +use util::ResultExt; pub type Result = std::result::Result; @@ -100,6 +103,11 @@ pub struct Config { pub live_kit_secret: Option, pub rust_log: Option, pub log_json: Option, + pub blob_store_url: Option, + pub blob_store_region: Option, + pub blob_store_access_key: Option, + pub blob_store_secret_key: Option, + pub blob_store_bucket: Option, pub zed_environment: Arc, } @@ -118,6 +126,7 @@ pub struct MigrateConfig { pub struct AppState { pub db: Arc, pub live_kit_client: Option>, + pub blob_store_client: Option, pub config: Config, } @@ -146,8 +155,44 @@ impl AppState { let this = Self { db: Arc::new(db), live_kit_client, + blob_store_client: build_blob_store_client(&config).await.log_err(), config, }; Ok(Arc::new(this)) } } + +async fn build_blob_store_client(config: &Config) -> anyhow::Result { + let keys = aws_sdk_s3::config::Credentials::new( + config + .blob_store_access_key + .clone() + .ok_or_else(|| anyhow!("missing blob_store_access_key"))?, + config + .blob_store_secret_key + .clone() + .ok_or_else(|| anyhow!("missing blob_store_secret_key"))?, + None, + None, + "env", + ); + + let s3_config = aws_config::defaults(BehaviorVersion::latest()) + .endpoint_url( + config + .blob_store_url + .as_ref() + .ok_or_else(|| anyhow!("missing blob_store_url"))?, + ) + .region(Region::new( + config + .blob_store_region + .clone() + .ok_or_else(|| anyhow!("missing blob_store_region"))?, + )) + .credentials_provider(keys) + .load() + .await; + + Ok(aws_sdk_s3::Client::new(&s3_config)) +} diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index a2fda0dd33..b80e8961df 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -1,6 +1,9 @@ use anyhow::anyhow; use axum::{routing::get, Extension, Router}; -use collab::{db, env, executor::Executor, AppState, Config, MigrateConfig, Result}; +use collab::{ + api::fetch_extensions_from_blob_store_periodically, db, env, executor::Executor, AppState, + Config, MigrateConfig, Result, +}; use db::Database; use std::{ env::args, @@ -50,6 +53,8 @@ async fn main() -> Result<()> { let rpc_server = collab::rpc::Server::new(epoch, state.clone(), Executor::Production); rpc_server.start().await?; + fetch_extensions_from_blob_store_periodically(state.clone(), Executor::Production); + let app = collab::api::routes(rpc_server.clone(), state.clone()) .merge(collab::rpc::routes(rpc_server.clone())) .merge( diff --git a/crates/collab/src/tests/test_server.rs b/crates/collab/src/tests/test_server.rs index 62870b860c..39292ead44 100644 --- a/crates/collab/src/tests/test_server.rs +++ b/crates/collab/src/tests/test_server.rs @@ -479,6 +479,7 @@ impl TestServer { Arc::new(AppState { db: test_db.db().clone(), live_kit_client: Some(Arc::new(fake_server.create_api_client())), + blob_store_client: None, config: Config { http_port: 0, database_url: "".into(), @@ -491,6 +492,11 @@ impl TestServer { rust_log: None, log_json: None, zed_environment: "test".into(), + blob_store_url: None, + blob_store_region: None, + blob_store_access_key: None, + blob_store_secret_key: None, + blob_store_bucket: None, }, }) } diff --git a/script/bootstrap b/script/bootstrap index 16ae872dbd..054daccf42 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,6 +3,10 @@ echo "installing foreman..." which foreman > /dev/null || brew install foreman +echo "installing minio..." +which minio > /dev/null || brew install minio/stable/minio +mkdir -p .blob_store/the-extensions-bucket + echo "creating database..." script/sqlx database create diff --git a/script/seed-db b/script/seed-db index 277ea89ba3..5079e01955 100755 --- a/script/seed-db +++ b/script/seed-db @@ -1,5 +1,4 @@ #!/bin/bash set -e -cd crates/collab cargo run --quiet --package=collab --features seed-support --bin seed -- $@