diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 138fa5808e..a3073029c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: submodules: "recursive" fetch-depth: 0 + - name: Remove untracked files + run: git clean -df + - name: Set up default .cargo/config.toml run: cp ./.cargo/ci-config.toml ~/.cargo/config.toml diff --git a/.github/workflows/deploy_collab.yml b/.github/workflows/deploy_collab.yml index eca43501d4..4d3ad09cbc 100644 --- a/.github/workflows/deploy_collab.yml +++ b/.github/workflows/deploy_collab.yml @@ -45,8 +45,18 @@ jobs: submodules: "recursive" fetch-depth: 0 + - name: Install cargo nextest + shell: bash -euxo pipefail {0} + run: | + cargo install cargo-nextest + + - name: Limit target directory size + shell: bash -euxo pipefail {0} + run: script/clear-target-dir-if-larger-than 100 + - name: Run tests - uses: ./.github/actions/run_tests + shell: bash -euxo pipefail {0} + run: cargo nextest run --package collab --no-fail-fast publish: name: Publish collab server image @@ -90,22 +100,26 @@ jobs: - name: Sign into Kubernetes run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} - - name: Determine namespace + - name: Start rollout run: | set -eu if [[ $GITHUB_REF_NAME = "collab-production" ]]; then - echo "Deploying collab:$GITHUB_SHA to production" - echo "KUBE_NAMESPACE=production" >> $GITHUB_ENV + export ZED_KUBE_NAMESPACE=production elif [[ $GITHUB_REF_NAME = "collab-staging" ]]; then - echo "Deploying collab:$GITHUB_SHA to staging" - echo "KUBE_NAMESPACE=staging" >> $GITHUB_ENV + export ZED_KUBE_NAMESPACE=staging else echo "cowardly refusing to deploy from an unknown branch" exit 1 fi - - name: Start rollout - run: kubectl -n "$KUBE_NAMESPACE" set image deployment/collab collab=registry.digitalocean.com/zed/collab:${GITHUB_SHA} + echo "Deploying collab:$GITHUB_SHA to $ZED_KUBE_NAMESPACE" - - name: Wait for rollout to finish - run: kubectl -n "$KUBE_NAMESPACE" rollout status deployment/collab + source script/lib/deploy-helpers.sh + export_vars_for_environment $ZED_KUBE_NAMESPACE + + export ZED_DO_CERTIFICATE_ID=$(doctl compute certificate list --format ID --no-header) + export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${GITHUB_SHA}" + + envsubst < crates/collab/k8s/collab.template.yml | kubectl apply -f - + kubectl -n "$ZED_KUBE_NAMESPACE" rollout status deployment/collab --watch + echo "deployed collab.template.yml to ${ZED_KUBE_NAMESPACE}" diff --git a/.gitignore b/.gitignore index 6f040dd0c5..9b6df52dd1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,12 +5,8 @@ .DS_Store /plugins/bin /script/node_modules -/styles/node_modules -/styles/src/types/zed.ts /crates/theme/schemas/theme.json -/crates/collab/static/styles.css /crates/collab/.admins.json -/vendor/bin /assets/*licenses.md **/venv .build @@ -25,3 +21,4 @@ DerivedData/ **/*.db .pytest_cache .venv +.blob_store diff --git a/Cargo.lock b/Cargo.lock index cdab9f60a1..064192a69f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -714,6 +714,368 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af266887e24cd5f6d2ea7433cacd25dcd4773b7f70e488701968a7cdf51df57" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "hex", + "http 0.2.9", + "hyper", + "ring 0.17.7", + "time", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d56f287a9e65e4914bfedb5b22c056b65e4c232fca512d5509a9df36386759f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6a29eca8ea8982028a4df81883e7001e250a21d323b86418884b5345950a4b" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "http 0.2.9", + "http-body", + "percent-encoding", + "pin-project-lite 0.2.13", + "tracing", + "uuid 1.4.1", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c977e92277652aefb9a76a0fca652b26757d6845dce0d7bf4426da80f13d85b0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "http-body", + "once_cell", + "percent-encoding", + "regex-lite", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d7f527c7b28af1a641f7d89f9e6a4863e8ec00f39d2b731b056fc5ec5ce829" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0be3224cd574ee8ab5fd7c32087876f25c134c27ac603fcb38669ed8d346b0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.5.0", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3167c60d82a13bbaef569da06041644ff41e85c6377e5dad53fa2526ccfe9d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.9", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b1cbe0eee57a213039088dbdeca7be9352f24e0d72332d961e8a1cb388f82d" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac 0.12.1", + "http 0.2.9", + "http 1.0.0", + "once_cell", + "p256", + "percent-encoding", + "ring 0.17.7", + "sha2 0.10.7", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "426a5bc369ca7c8d3686439e46edc727f397a47ab3696b13f3ae8c81b3b36132" +dependencies = [ + "futures-util", + "pin-project-lite 0.2.13", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee554133eca2611b66d23548e48f9b44713befdb025ab76bc00185b878397a1" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes 1.5.0", + "crc32c", + "crc32fast", + "hex", + "http 0.2.9", + "http-body", + "md-5", + "pin-project-lite 0.2.13", + "sha1", + "sha2 0.10.7", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" +dependencies = [ + "aws-smithy-types", + "bytes 1.5.0", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85d6a0619f7b67183067fa3b558f94f90753da2df8c04aeb7336d673f804b0b8" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "bytes-utils", + "futures-core", + "http 0.2.9", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite 0.2.13", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1c1b5186b6f5c579bf0de1bcca9dd3d946d6d51361ea1d18131f6a0b64e13ae" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c0a2ce65882e788d2cf83ff28b9b16918de0460c47bf66c5da4f6c17b4c9694" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4cb6b3afa5fc9825a75675975dcc3e21764b5476bc91dbc63df4ea3d30a576e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.5.0", + "fastrand 2.0.0", + "h2", + "http 0.2.9", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite 0.2.13", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23165433e80c04e8c09cee66d171292ae7234bae05fa9d5636e33095eae416b2" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes 1.5.0", + "http 0.2.9", + "pin-project-lite 0.2.13", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c94a5bec34850b92c9a054dad57b95c1d47f25125f55973e19f6ad788f0381ff" +dependencies = [ + "base64-simd", + "bytes 1.5.0", + "bytes-utils", + "futures-core", + "http 0.2.9", + "http-body", + "itoa", + "num-integer", + "pin-project-lite 0.2.13", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util 0.7.9", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16f94c9673412b7a72e3c3efec8de89081c320bf59ea12eed34c417a62ad600" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff7e122ee50ca962e9de91f5850cc37e2184b1219611eef6d44aa85929b54f6" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 0.2.9", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.5.17" @@ -727,7 +1089,7 @@ dependencies = [ "bytes 1.5.0", "futures-util", "headers", - "http", + "http 0.2.9", "http-body", "hyper", "itoa", @@ -758,7 +1120,7 @@ dependencies = [ "async-trait", "bytes 1.5.0", "futures-util", - "http", + "http 0.2.9", "http-body", "mime", "tower-layer", @@ -774,7 +1136,7 @@ dependencies = [ "axum", "bytes 1.5.0", "futures-util", - "http", + "http 0.2.9", "mime", "pin-project-lite 0.2.13", "serde", @@ -812,6 +1174,12 @@ dependencies = [ "nix 0.23.2", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -824,6 +1192,16 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -1169,6 +1547,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes 1.5.0", + "either", +] + [[package]] name = "call" version = "0.1.0" @@ -1537,6 +1925,8 @@ dependencies = [ "async-trait", "async-tungstenite", "audio", + "aws-config", + "aws-sdk-s3", "axum", "axum-extra", "base64 0.13.1", @@ -1582,6 +1972,7 @@ dependencies = [ "rpc", "scrypt", "sea-orm", + "semver", "serde", "serde_derive", "serde_json", @@ -2087,6 +2478,15 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +[[package]] +name = "crc32c" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -2149,6 +2549,28 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -2161,9 +2583,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" dependencies = [ "generic-array", "subtle", @@ -2286,6 +2708,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.8" @@ -2513,6 +2945,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "editor" version = "0.1.0" @@ -2579,6 +3023,26 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -2821,6 +3285,16 @@ dependencies = [ "workspace", ] +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "file_finder" version = "0.1.0" @@ -3160,9 +3634,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -3170,9 +3644,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -3198,9 +3672,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -3219,9 +3693,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -3230,21 +3704,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures 0.1.31", "futures-channel", @@ -3546,6 +4020,17 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df00eed8d1f0db937f6be10e46e8072b0671accb504cf0f959c5c52c679f5b9" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.21" @@ -3557,7 +4042,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.9", "indexmap 1.9.3", "slab", "tokio", @@ -3611,7 +4096,7 @@ dependencies = [ "base64 0.21.4", "bytes 1.5.0", "headers-core", - "http", + "http 0.2.9", "httpdate", "mime", "sha1", @@ -3623,7 +4108,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.9", ] [[package]] @@ -3736,6 +4221,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -3743,7 +4239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes 1.5.0", - "http", + "http 0.2.9", "pin-project-lite 0.2.13", ] @@ -3788,7 +4284,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "httparse", "httpdate", @@ -3801,6 +4297,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.9", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -4033,7 +4545,7 @@ dependencies = [ "encoding_rs", "event-listener", "futures-lite", - "http", + "http 0.2.9", "log", "mime", "once_cell", @@ -5578,12 +6090,29 @@ dependencies = [ "workspace", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.7", +] + [[package]] name = "palette" version = "0.7.3" @@ -5688,9 +6217,9 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.2.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e0b28ace46c5a396546bcf443bf422b57049617433d8854227352a4a9b24e7" +checksum = "c1a5d4e9c205d2c1ae73b84aab6240e98218c0e72e63b50422cfb2d1ca952282" dependencies = [ "base64ct", "rand_core 0.6.4", @@ -5902,9 +6431,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.8", + "pkcs8 0.10.2", + "spki 0.7.2", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -5913,8 +6452,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.8", + "spki 0.7.2", ] [[package]] @@ -6705,6 +7244,12 @@ dependencies = [ "regex-syntax 0.8.2", ] +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -6752,7 +7297,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-tls", @@ -6793,6 +7338,17 @@ dependencies = [ "usvg", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac 0.12.1", + "zeroize", +] + [[package]] name = "rgb" version = "0.8.36" @@ -6831,11 +7387,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi 0.3.9", ] +[[package]] +name = "ring" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +dependencies = [ + "cc", + "getrandom 0.2.10", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rkyv" version = "0.7.42" @@ -6989,10 +7559,10 @@ dependencies = [ "num-iter", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.1.0", + "spki 0.7.2", "subtle", "zeroize", ] @@ -7124,15 +7694,28 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ - "ring", + "log", + "ring 0.17.7", "rustls-webpki", "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.3" @@ -7144,12 +7727,12 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.7", + "untrusted 0.9.0", ] [[package]] @@ -7302,8 +7885,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -7430,6 +8013,20 @@ dependencies = [ "workspace", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -7747,6 +8344,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.1.0" @@ -7934,6 +8541,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.2" @@ -7941,7 +8558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der", + "der 0.7.8", ] [[package]] @@ -8301,9 +8918,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "sum_tree" @@ -8913,6 +9530,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -9004,7 +9631,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.9", "http-body", "hyper", "hyper-timeout", @@ -9052,7 +9679,7 @@ dependencies = [ "bytes 1.5.0", "futures-core", "futures-util", - "http", + "http 0.2.9", "http-body", "http-range-header", "pin-project-lite 0.2.13", @@ -9624,7 +10251,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.9", "httparse", "log", "native-tls", @@ -9644,7 +10271,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes 1.5.0", - "http", + "http 0.2.9", "httparse", "log", "rand 0.8.5", @@ -9786,6 +10413,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.4.1" @@ -10000,6 +10633,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vte" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index 170a58ea7e..b155bf2232 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,91 +1,91 @@ [workspace] members = [ - "crates/activity_indicator", - "crates/ai", - "crates/assets", - "crates/assistant", - "crates/audio", - "crates/auto_update", - "crates/breadcrumbs", - "crates/call", - "crates/channel", - "crates/cli", - "crates/client", - "crates/clock", - "crates/collab", - "crates/collab_ui", - "crates/collections", - "crates/command_palette", - "crates/copilot", - "crates/copilot_ui", - "crates/db", - "crates/diagnostics", - "crates/editor", - "crates/extension", - "crates/extensions_ui", - "crates/feature_flags", - "crates/feedback", - "crates/file_finder", - "crates/fs", - "crates/fsevent", - "crates/fuzzy", - "crates/git", - "crates/go_to_line", - "crates/gpui", - "crates/gpui_macros", - "crates/install_cli", - "crates/journal", - "crates/language", - "crates/language_selector", - "crates/language_tools", - "crates/live_kit_client", - "crates/live_kit_server", - "crates/lsp", - "crates/markdown_preview", - "crates/media", - "crates/menu", - "crates/multi_buffer", - "crates/node_runtime", - "crates/notifications", - "crates/outline", - "crates/picker", - "crates/plugin", - "crates/plugin_macros", - "crates/prettier", - "crates/project", - "crates/project_panel", - "crates/project_symbols", - "crates/quick_action_bar", - "crates/recent_projects", - "crates/refineable", - "crates/refineable/derive_refineable", - "crates/release_channel", - "crates/rich_text", - "crates/rope", - "crates/rpc", - "crates/search", - "crates/semantic_index", - "crates/settings", - "crates/snippet", - "crates/sqlez", - "crates/sqlez_macros", - "crates/story", - "crates/storybook", - "crates/sum_tree", - "crates/terminal", - "crates/terminal_view", - "crates/text", - "crates/theme", - "crates/theme_importer", - "crates/theme_selector", - "crates/ui", - "crates/util", - "crates/vcs_menu", - "crates/vim", - "crates/welcome", - "crates/workspace", - "crates/zed", - "crates/zed_actions", + "crates/activity_indicator", + "crates/ai", + "crates/assets", + "crates/assistant", + "crates/audio", + "crates/auto_update", + "crates/breadcrumbs", + "crates/call", + "crates/channel", + "crates/cli", + "crates/client", + "crates/clock", + "crates/collab", + "crates/collab_ui", + "crates/collections", + "crates/command_palette", + "crates/copilot", + "crates/copilot_ui", + "crates/db", + "crates/diagnostics", + "crates/editor", + "crates/extension", + "crates/extensions_ui", + "crates/feature_flags", + "crates/feedback", + "crates/file_finder", + "crates/fs", + "crates/fsevent", + "crates/fuzzy", + "crates/git", + "crates/go_to_line", + "crates/gpui", + "crates/gpui_macros", + "crates/install_cli", + "crates/journal", + "crates/language", + "crates/language_selector", + "crates/language_tools", + "crates/live_kit_client", + "crates/live_kit_server", + "crates/lsp", + "crates/markdown_preview", + "crates/media", + "crates/menu", + "crates/multi_buffer", + "crates/node_runtime", + "crates/notifications", + "crates/outline", + "crates/picker", + "crates/plugin", + "crates/plugin_macros", + "crates/prettier", + "crates/project", + "crates/project_panel", + "crates/project_symbols", + "crates/quick_action_bar", + "crates/recent_projects", + "crates/refineable", + "crates/refineable/derive_refineable", + "crates/release_channel", + "crates/rich_text", + "crates/rope", + "crates/rpc", + "crates/search", + "crates/semantic_index", + "crates/settings", + "crates/snippet", + "crates/sqlez", + "crates/sqlez_macros", + "crates/story", + "crates/storybook", + "crates/sum_tree", + "crates/terminal", + "crates/terminal_view", + "crates/text", + "crates/theme", + "crates/theme_importer", + "crates/theme_selector", + "crates/ui", + "crates/util", + "crates/vcs_menu", + "crates/vim", + "crates/welcome", + "crates/workspace", + "crates/zed", + "crates/zed_actions", ] default-members = ["crates/zed"] resolver = "2" @@ -191,8 +191,8 @@ globset = "0.4" indoc = "1" # We explicitly disable a http2 support in isahc. isahc = { version = "1.7.2", default-features = false, features = [ - "static-curl", - "text-decoding", + "static-curl", + "text-decoding", ] } lazy_static = "1.4.0" log = { version = "0.4.16", features = ["kv_unstable_serde"] } @@ -208,12 +208,13 @@ regex = "1.5" rusqlite = { version = "0.29.0", features = ["blob", "array", "modern_sqlite"] } rust-embed = { version = "8.0", features = ["include-exclude"] } schemars = "0.8" +semver = { version = "1.0" } serde = { version = "1.0", features = ["derive", "rc"] } serde_derive = { version = "1.0", features = ["deserialize_in_place"] } serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] } serde_json_lenient = { version = "0.1", features = [ - "preserve_order", - "raw_value", + "preserve_order", + "raw_value", ] } serde_repr = "0.1" smallvec = { version = "1.6", features = ["union"] } @@ -223,7 +224,11 @@ sysinfo = "0.29.10" tempfile = "3.9.0" thiserror = "1.0.29" tiktoken-rs = "0.5.7" -time = { version = "0.3", features = ["serde", "serde-well-known"] } +time = { version = "0.3", features = [ + "serde", + "serde-well-known", + "formatting", +] } toml = "0.5" tree-sitter = { version = "0.20", features = ["wasm"] } tree-sitter-astro = { git = "https://github.com/virchau13/tree-sitter-astro.git", rev = "e924787e12e8a03194f36a113290ac11d6dc10f3" } diff --git a/Procfile b/Procfile index 7bd9114dad..288842ebd3 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ -collab: cd crates/collab && RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run serve +collab: RUST_LOG=${RUST_LOG:-warn,collab=info} cargo run --package=collab serve livekit: livekit-server --dev +blob_store: MINIO_ROOT_USER=the-blob-store-access-key MINIO_ROOT_PASSWORD=the-blob-store-secret-key minio server .blob_store diff --git a/crates/collab/.env.toml b/crates/collab/.env.toml index 01866012ea..7340a71cd9 100644 --- a/crates/collab/.env.toml +++ b/crates/collab/.env.toml @@ -7,6 +7,11 @@ ZED_ENVIRONMENT = "development" LIVE_KIT_SERVER = "http://localhost:7880" LIVE_KIT_KEY = "devkey" LIVE_KIT_SECRET = "secret" +BLOB_STORE_ACCESS_KEY = "the-blob-store-access-key" +BLOB_STORE_SECRET_KEY = "the-blob-store-secret-key" +BLOB_STORE_BUCKET = "the-extensions-bucket" +BLOB_STORE_URL = "http://127.0.0.1:9000" +BLOB_STORE_REGION = "the-region" # RUST_LOG=info # LOG_JSON=true diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index 02b2e5eb48..6c9e07f3be 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -15,9 +15,11 @@ name = "seed" required-features = ["seed-support"] [dependencies] -anyhow.workspace = true -async-tungstenite = "0.16" axum = { version = "0.5", features = ["json", "headers", "ws"] } +anyhow.workspace = true +aws-config = { version = "1.1.5" } +aws-sdk-s3 = { version = "1.15.0" } +async-tungstenite = "0.16" axum-extra = { version = "0.3", features = ["erased-json"] } base64 = "0.13" chrono.workspace = true @@ -40,13 +42,26 @@ rand.workspace = true reqwest = { version = "0.11", features = ["json"], optional = true } rpc.workspace = true scrypt = "0.7" -sea-orm = { version = "0.12.x", features = ["sqlx-postgres", "postgres-array", "runtime-tokio-rustls", "with-uuid"] } +sea-orm = { version = "0.12.x", features = [ + "sqlx-postgres", + "postgres-array", + "runtime-tokio-rustls", + "with-uuid", +] } +semver.workspace = true serde.workspace = true serde_derive.workspace = true serde_json.workspace = true sha-1 = "0.9" smallvec.workspace = true -sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "any"] } +sqlx = { version = "0.7", features = [ + "runtime-tokio-rustls", + "postgres", + "json", + "time", + "uuid", + "any", +] } text.workspace = true time.workspace = true tokio = { version = "1", features = ["full"] } diff --git a/crates/collab/k8s/collab.template.yml b/crates/collab/k8s/collab.template.yml index 120e5f592f..9ff7cee9e1 100644 --- a/crates/collab/k8s/collab.template.yml +++ b/crates/collab/k8s/collab.template.yml @@ -105,6 +105,31 @@ spec: secretKeyRef: name: livekit key: secret + - name: BLOB_STORE_ACCESS_KEY + valueFrom: + secretKeyRef: + name: blob-store + key: access_key + - name: BLOB_STORE_SECRET_KEY + valueFrom: + secretKeyRef: + name: blob-store + key: secret_key + - name: BLOB_STORE_URL + valueFrom: + secretKeyRef: + name: blob-store + key: url + - name: BLOB_STORE_REGION + valueFrom: + secretKeyRef: + name: blob-store + key: region + - name: BLOB_STORE_BUCKET + valueFrom: + secretKeyRef: + name: blob-store + key: bucket - name: INVITE_LINK_PREFIX value: ${INVITE_LINK_PREFIX} - name: RUST_BACKTRACE diff --git a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql index 1026fdea0d..fef10f987e 100644 --- a/crates/collab/migrations.sqlite/20221109000000_test_schema.sql +++ b/crates/collab/migrations.sqlite/20221109000000_test_schema.sql @@ -353,3 +353,25 @@ CREATE TABLE contributors ( signed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (user_id) ); + +CREATE TABLE extensions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + external_id TEXT NOT NULL, + name TEXT NOT NULL, + latest_version TEXT NOT NULL, + total_download_count INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE extension_versions ( + extension_id INTEGER REFERENCES extensions(id), + version TEXT NOT NULL, + published_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + authors TEXT NOT NULL, + repository TEXT NOT NULL, + description TEXT NOT NULL, + download_count INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (extension_id, version) +); + +CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id"); +CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count"); diff --git a/crates/collab/migrations/20240214102900_add_extensions.sql b/crates/collab/migrations/20240214102900_add_extensions.sql new file mode 100644 index 0000000000..b32094036d --- /dev/null +++ b/crates/collab/migrations/20240214102900_add_extensions.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS extensions ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + external_id TEXT NOT NULL, + latest_version TEXT NOT NULL, + total_download_count BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS extension_versions ( + extension_id INTEGER REFERENCES extensions(id), + version TEXT NOT NULL, + published_at TIMESTAMP NOT NULL DEFAULT now(), + authors TEXT NOT NULL, + repository TEXT NOT NULL, + description TEXT NOT NULL, + download_count BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY(extension_id, version) +); + +CREATE UNIQUE INDEX "index_extensions_external_id" ON "extensions" ("external_id"); +CREATE INDEX "trigram_index_extensions_name" ON "extensions" USING GIN(name gin_trgm_ops); +CREATE INDEX "index_extensions_total_download_count" ON "extensions" ("total_download_count"); diff --git a/crates/collab/src/api.rs b/crates/collab/src/api.rs index 59d176b047..44d6fc3eb5 100644 --- a/crates/collab/src/api.rs +++ b/crates/collab/src/api.rs @@ -1,3 +1,5 @@ +mod extensions; + use crate::{ auth, db::{ContributorSelector, User, UserId}, @@ -20,6 +22,8 @@ use std::sync::Arc; use tower::ServiceBuilder; use tracing::instrument; +pub use extensions::fetch_extensions_from_blob_store_periodically; + pub fn routes(rpc_server: Arc, state: Arc) -> Router { Router::new() .route("/user", get(get_authenticated_user)) @@ -28,6 +32,7 @@ pub fn routes(rpc_server: Arc, state: Arc) -> Router Router { + Router::new() + .route("/extensions", get(get_extensions)) + .route( + "/extensions/:extension_id/:version/download", + get(download_extension), + ) +} + +#[derive(Debug, Deserialize)] +struct GetExtensionsParams { + filter: Option, +} + +#[derive(Debug, Deserialize)] +struct DownloadExtensionParams { + extension_id: String, + version: String, +} + +#[derive(Debug, Serialize)] +struct GetExtensionsResponse { + pub data: Vec, +} + +#[derive(Deserialize)] +struct ExtensionManifest { + name: String, + version: String, + description: Option, + authors: Vec, + repository: String, +} + +async fn get_extensions( + Extension(app): Extension>, + Query(params): Query, +) -> Result> { + let extensions = app.db.get_extensions(params.filter.as_deref(), 30).await?; + Ok(Json(GetExtensionsResponse { data: extensions })) +} + +async fn download_extension( + Extension(app): Extension>, + Path(params): Path, +) -> Result { + let Some((blob_store_client, bucket)) = app + .blob_store_client + .clone() + .zip(app.config.blob_store_bucket.clone()) + else { + Err(Error::Http( + StatusCode::NOT_IMPLEMENTED, + "not supported".into(), + ))? + }; + + let DownloadExtensionParams { + extension_id, + version, + } = params; + + let version_exists = app + .db + .record_extension_download(&extension_id, &version) + .await?; + + if !version_exists { + Err(Error::Http( + StatusCode::NOT_FOUND, + "unknown extension version".into(), + ))?; + } + + let url = blob_store_client + .get_object() + .bucket(bucket) + .key(format!( + "extensions/{extension_id}/{version}/archive.tar.gz" + )) + .presigned(PresigningConfig::expires_in(EXTENSION_DOWNLOAD_URL_LIFETIME).unwrap()) + .await + .map_err(|e| anyhow!("failed to create presigned extension download url {e}"))?; + + Ok(Redirect::temporary(url.uri())) +} + +const EXTENSION_FETCH_INTERVAL: Duration = Duration::from_secs(5 * 60); +const EXTENSION_DOWNLOAD_URL_LIFETIME: Duration = Duration::from_secs(3 * 60); + +pub fn fetch_extensions_from_blob_store_periodically(app_state: Arc, executor: Executor) { + let Some(blob_store_client) = app_state.blob_store_client.clone() else { + log::info!("no blob store client"); + return; + }; + let Some(blob_store_bucket) = app_state.config.blob_store_bucket.clone() else { + log::info!("no blob store bucket"); + return; + }; + + executor.spawn_detached({ + let executor = executor.clone(); + async move { + loop { + fetch_extensions_from_blob_store( + &blob_store_client, + &blob_store_bucket, + &app_state, + ) + .await + .log_err(); + executor.sleep(EXTENSION_FETCH_INTERVAL).await; + } + } + }); +} + +async fn fetch_extensions_from_blob_store( + blob_store_client: &aws_sdk_s3::Client, + blob_store_bucket: &String, + app_state: &Arc, +) -> anyhow::Result<()> { + let list = blob_store_client + .list_objects() + .bucket(blob_store_bucket) + .prefix("extensions/") + .send() + .await?; + + let objects = list + .contents + .ok_or_else(|| anyhow!("missing bucket contents"))?; + + let mut published_versions = HashMap::<&str, Vec<&str>>::default(); + for object in &objects { + let Some(key) = object.key.as_ref() else { + continue; + }; + let mut parts = key.split('/'); + let Some(_) = parts.next().filter(|part| *part == "extensions") else { + continue; + }; + let Some(extension_id) = parts.next() else { + continue; + }; + let Some(version) = parts.next() else { + continue; + }; + published_versions + .entry(extension_id) + .or_default() + .push(version); + } + + let known_versions = app_state.db.get_known_extension_versions().await?; + + let mut new_versions = HashMap::<&str, Vec>::default(); + let empty = Vec::new(); + for (extension_id, published_versions) in published_versions { + let known_versions = known_versions.get(extension_id).unwrap_or(&empty); + + for published_version in published_versions { + if known_versions + .binary_search_by_key(&published_version, String::as_str) + .is_err() + { + let object = blob_store_client + .get_object() + .bucket(blob_store_bucket) + .key(format!( + "extensions/{extension_id}/{published_version}/manifest.json" + )) + .send() + .await?; + let manifest_bytes = object + .body + .collect() + .await + .map(|data| data.into_bytes()) + .with_context(|| format!("failed to download manifest for extension {extension_id} version {published_version}"))? + .to_vec(); + let manifest = serde_json::from_slice::(&manifest_bytes) + .with_context(|| format!("invalid manifest for extension {extension_id} version {published_version}: {}", String::from_utf8_lossy(&manifest_bytes)))?; + + let published_at = object.last_modified.ok_or_else(|| anyhow!("missing last modified timestamp for extension {extension_id} version {published_version}"))?; + let published_at = + time::OffsetDateTime::from_unix_timestamp_nanos(published_at.as_nanos())?; + let published_at = PrimitiveDateTime::new(published_at.date(), published_at.time()); + + let version = semver::Version::parse(&manifest.version).with_context(|| { + format!( + "invalid version for extension {extension_id} version {published_version}" + ) + })?; + + new_versions + .entry(extension_id) + .or_default() + .push(NewExtensionVersion { + name: manifest.name, + version, + description: manifest.description.unwrap_or_default(), + authors: manifest.authors, + repository: manifest.repository, + published_at, + }); + } + } + } + + app_state + .db + .insert_extension_versions(&new_versions) + .await?; + + Ok(()) +} diff --git a/crates/collab/src/db.rs b/crates/collab/src/db.rs index 6ccea25a60..08e502a42f 100644 --- a/crates/collab/src/db.rs +++ b/crates/collab/src/db.rs @@ -1,12 +1,8 @@ -#[cfg(test)] -pub mod tests; - -#[cfg(test)] -pub use tests::TestDb; - mod ids; mod queries; mod tables; +#[cfg(test)] +pub mod tests; use crate::{executor::Executor, Error, Result}; use anyhow::anyhow; @@ -25,7 +21,7 @@ use sea_orm::{ FromQueryResult, IntoActiveModel, IsolationLevel, JoinType, QueryOrder, QuerySelect, Statement, TransactionTrait, }; -use serde::{Deserialize, Serialize}; +use serde::{ser::Error as _, Deserialize, Serialize, Serializer}; use sqlx::{ migrate::{Migrate, Migration, MigrationSource}, Connection, @@ -40,13 +36,17 @@ use std::{ sync::Arc, time::Duration, }; -pub use tables::*; +use time::{format_description::well_known::iso8601, PrimitiveDateTime}; use tokio::sync::{Mutex, OwnedMutexGuard}; +#[cfg(test)] +pub use tests::TestDb; + pub use ids::*; pub use queries::contributors::ContributorSelector; pub use sea_orm::ConnectOptions; pub use tables::user::Model as User; +pub use tables::*; /// Database gives you a handle that lets you access the database. /// It handles pooling internally. @@ -717,3 +717,42 @@ pub struct WorktreeSettingsFile { pub path: String, pub content: String, } + +pub struct NewExtensionVersion { + pub name: String, + pub version: semver::Version, + pub description: String, + pub authors: Vec, + pub repository: String, + pub published_at: PrimitiveDateTime, +} + +#[derive(Debug, Serialize, PartialEq)] +pub struct ExtensionMetadata { + pub id: String, + pub name: String, + pub version: String, + pub authors: Vec, + pub repository: String, + #[serde(serialize_with = "serialize_iso8601")] + pub published_at: PrimitiveDateTime, + pub download_count: u64, +} + +pub fn serialize_iso8601( + datetime: &PrimitiveDateTime, + serializer: S, +) -> Result { + const SERDE_CONFIG: iso8601::EncodedConfig = iso8601::Config::DEFAULT + .set_year_is_six_digits(false) + .set_time_precision(iso8601::TimePrecision::Second { + decimal_digits: None, + }) + .encode(); + + datetime + .assume_utc() + .format(&time::format_description::well_known::Iso8601::) + .map_err(S::Error::custom)? + .serialize(serializer) +} diff --git a/crates/collab/src/db/ids.rs b/crates/collab/src/db/ids.rs index d69e19643a..44a5db6a75 100644 --- a/crates/collab/src/db/ids.rs +++ b/crates/collab/src/db/ids.rs @@ -85,6 +85,7 @@ id_type!(SignupId); id_type!(UserId); id_type!(ChannelBufferCollaboratorId); id_type!(FlagId); +id_type!(ExtensionId); id_type!(NotificationId); id_type!(NotificationKindId); diff --git a/crates/collab/src/db/queries.rs b/crates/collab/src/db/queries.rs index f6bba13ede..7d9043f595 100644 --- a/crates/collab/src/db/queries.rs +++ b/crates/collab/src/db/queries.rs @@ -5,6 +5,7 @@ pub mod buffers; pub mod channels; pub mod contacts; pub mod contributors; +pub mod extensions; pub mod messages; pub mod notifications; pub mod projects; diff --git a/crates/collab/src/db/queries/extensions.rs b/crates/collab/src/db/queries/extensions.rs new file mode 100644 index 0000000000..78e80ea190 --- /dev/null +++ b/crates/collab/src/db/queries/extensions.rs @@ -0,0 +1,205 @@ +use super::*; + +impl Database { + pub async fn get_extensions( + &self, + filter: Option<&str>, + limit: usize, + ) -> Result> { + self.transaction(|tx| async move { + let mut condition = Condition::all(); + if let Some(filter) = filter { + let fuzzy_name_filter = Self::fuzzy_like_string(filter); + condition = condition.add(Expr::cust_with_expr("name ILIKE $1", fuzzy_name_filter)); + } + + let extensions = extension::Entity::find() + .filter(condition) + .order_by_desc(extension::Column::TotalDownloadCount) + .order_by_asc(extension::Column::Id) + .limit(Some(limit as u64)) + .filter( + extension::Column::LatestVersion + .into_expr() + .eq(extension_version::Column::Version.into_expr()), + ) + .inner_join(extension_version::Entity) + .select_also(extension_version::Entity) + .all(&*tx) + .await?; + + Ok(extensions + .into_iter() + .filter_map(|(extension, latest_version)| { + let version = latest_version?; + Some(ExtensionMetadata { + id: extension.external_id, + name: extension.name, + version: version.version, + authors: version + .authors + .split(',') + .map(|author| author.trim().to_string()) + .collect::>(), + repository: version.repository, + published_at: version.published_at, + download_count: extension.total_download_count as u64, + }) + }) + .collect()) + }) + .await + } + + pub async fn get_known_extension_versions<'a>(&self) -> Result>> { + self.transaction(|tx| async move { + let mut extension_external_ids_by_id = HashMap::default(); + + let mut rows = extension::Entity::find().stream(&*tx).await?; + while let Some(row) = rows.next().await { + let row = row?; + extension_external_ids_by_id.insert(row.id, row.external_id); + } + drop(rows); + + let mut known_versions_by_extension_id: HashMap> = + HashMap::default(); + let mut rows = extension_version::Entity::find().stream(&*tx).await?; + while let Some(row) = rows.next().await { + let row = row?; + + let Some(extension_id) = extension_external_ids_by_id.get(&row.extension_id) else { + continue; + }; + + let versions = known_versions_by_extension_id + .entry(extension_id.clone()) + .or_default(); + if let Err(ix) = versions.binary_search(&row.version) { + versions.insert(ix, row.version); + } + } + drop(rows); + + Ok(known_versions_by_extension_id) + }) + .await + } + + pub async fn insert_extension_versions( + &self, + versions_by_extension_id: &HashMap<&str, Vec>, + ) -> Result<()> { + self.transaction(|tx| async move { + for (external_id, versions) in versions_by_extension_id { + if versions.is_empty() { + continue; + } + + let latest_version = versions + .iter() + .max_by_key(|version| &version.version) + .unwrap(); + + let insert = extension::Entity::insert(extension::ActiveModel { + name: ActiveValue::Set(latest_version.name.clone()), + external_id: ActiveValue::Set(external_id.to_string()), + id: ActiveValue::NotSet, + latest_version: ActiveValue::Set(latest_version.version.to_string()), + total_download_count: ActiveValue::NotSet, + }) + .on_conflict( + OnConflict::columns([extension::Column::ExternalId]) + .update_column(extension::Column::ExternalId) + .to_owned(), + ); + + let extension = if tx.support_returning() { + insert.exec_with_returning(&*tx).await? + } else { + // Sqlite + insert.exec_without_returning(&*tx).await?; + extension::Entity::find() + .filter(extension::Column::ExternalId.eq(*external_id)) + .one(&*tx) + .await? + .ok_or_else(|| anyhow!("failed to insert extension"))? + }; + + extension_version::Entity::insert_many(versions.iter().map(|version| { + extension_version::ActiveModel { + extension_id: ActiveValue::Set(extension.id), + published_at: ActiveValue::Set(version.published_at), + version: ActiveValue::Set(version.version.to_string()), + authors: ActiveValue::Set(version.authors.join(", ")), + repository: ActiveValue::Set(version.repository.clone()), + description: ActiveValue::Set(version.description.clone()), + download_count: ActiveValue::NotSet, + } + })) + .on_conflict(OnConflict::new().do_nothing().to_owned()) + .exec_without_returning(&*tx) + .await?; + + if let Ok(db_version) = semver::Version::parse(&extension.latest_version) { + if db_version >= latest_version.version { + continue; + } + } + + let mut extension = extension.into_active_model(); + extension.latest_version = ActiveValue::Set(latest_version.version.to_string()); + extension.name = ActiveValue::set(latest_version.name.clone()); + extension::Entity::update(extension).exec(&*tx).await?; + } + + Ok(()) + }) + .await + } + + pub async fn record_extension_download(&self, extension: &str, version: &str) -> Result { + self.transaction(|tx| async move { + #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] + enum QueryId { + Id, + } + + let extension_id: Option = extension::Entity::find() + .filter(extension::Column::ExternalId.eq(extension)) + .select_only() + .column(extension::Column::Id) + .into_values::<_, QueryId>() + .one(&*tx) + .await?; + let Some(extension_id) = extension_id else { + return Ok(false); + }; + + extension_version::Entity::update_many() + .col_expr( + extension_version::Column::DownloadCount, + extension_version::Column::DownloadCount.into_expr().add(1), + ) + .filter( + extension_version::Column::ExtensionId + .eq(extension_id) + .and(extension_version::Column::Version.eq(version)), + ) + .exec(&*tx) + .await?; + + extension::Entity::update_many() + .col_expr( + extension::Column::TotalDownloadCount, + extension::Column::TotalDownloadCount.into_expr().add(1), + ) + .filter(extension::Column::Id.eq(extension_id)) + .exec(&*tx) + .await?; + + Ok(true) + }) + .await + } +} diff --git a/crates/collab/src/db/tables.rs b/crates/collab/src/db/tables.rs index 646447c91f..72d9835032 100644 --- a/crates/collab/src/db/tables.rs +++ b/crates/collab/src/db/tables.rs @@ -10,6 +10,8 @@ pub mod channel_message; pub mod channel_message_mention; pub mod contact; pub mod contributor; +pub mod extension; +pub mod extension_version; pub mod feature_flag; pub mod follower; pub mod language_server; diff --git a/crates/collab/src/db/tables/extension.rs b/crates/collab/src/db/tables/extension.rs new file mode 100644 index 0000000000..5a1462c701 --- /dev/null +++ b/crates/collab/src/db/tables/extension.rs @@ -0,0 +1,27 @@ +use crate::db::ExtensionId; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "extensions")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: ExtensionId, + pub external_id: String, + pub name: String, + pub latest_version: String, + pub total_download_count: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_one = "super::extension_version::Entity")] + LatestVersion, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::LatestVersion.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/crates/collab/src/db/tables/extension_version.rs b/crates/collab/src/db/tables/extension_version.rs new file mode 100644 index 0000000000..459f2296b1 --- /dev/null +++ b/crates/collab/src/db/tables/extension_version.rs @@ -0,0 +1,36 @@ +use crate::db::ExtensionId; +use sea_orm::entity::prelude::*; +use time::PrimitiveDateTime; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "extension_versions")] +pub struct Model { + #[sea_orm(primary_key)] + pub extension_id: ExtensionId, + #[sea_orm(primary_key)] + pub version: String, + pub published_at: PrimitiveDateTime, + pub authors: String, + pub repository: String, + pub description: String, + pub download_count: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::extension::Entity", + from = "Column::ExtensionId", + to = "super::extension::Column::Id" + on_condition = r#"super::extension::Column::LatestVersion.into_expr().eq(Column::Version.into_expr())"# + )] + Extension, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Extension.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/crates/collab/src/db/tests.rs b/crates/collab/src/db/tests.rs index a85aae4fa3..7790b951b2 100644 --- a/crates/collab/src/db/tests.rs +++ b/crates/collab/src/db/tests.rs @@ -2,6 +2,7 @@ mod buffer_tests; mod channel_tests; mod contributor_tests; mod db_tests; +mod extension_tests; mod feature_flag_tests; mod message_tests; diff --git a/crates/collab/src/db/tests/extension_tests.rs b/crates/collab/src/db/tests/extension_tests.rs new file mode 100644 index 0000000000..8963ffcbe9 --- /dev/null +++ b/crates/collab/src/db/tests/extension_tests.rs @@ -0,0 +1,219 @@ +use super::Database; +use crate::{ + db::{ExtensionMetadata, NewExtensionVersion}, + test_both_dbs, +}; +use std::sync::Arc; +use time::{OffsetDateTime, PrimitiveDateTime}; + +test_both_dbs!( + test_extensions, + test_extensions_postgres, + test_extensions_sqlite +); + +async fn test_extensions(db: &Arc) { + let versions = db.get_known_extension_versions().await.unwrap(); + assert!(versions.is_empty()); + + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert!(extensions.is_empty()); + + let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap(); + let t0 = PrimitiveDateTime::new(t0.date(), t0.time()); + + db.insert_extension_versions( + &[ + ( + "ext1", + vec![ + NewExtensionVersion { + name: "Extension 1".into(), + version: semver::Version::parse("0.0.1").unwrap(), + description: "an extension".into(), + authors: vec!["max".into()], + repository: "ext1/repo".into(), + published_at: t0, + }, + NewExtensionVersion { + name: "Extension One".into(), + version: semver::Version::parse("0.0.2").unwrap(), + description: "a good extension".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + }, + ], + ), + ( + "ext2", + vec![NewExtensionVersion { + name: "Extension Two".into(), + version: semver::Version::parse("0.2.0").unwrap(), + description: "a great extension".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + }], + ), + ] + .into_iter() + .collect(), + ) + .await + .unwrap(); + + let versions = db.get_known_extension_versions().await.unwrap(); + assert_eq!( + versions, + [ + ("ext1".into(), vec!["0.0.1".into(), "0.0.2".into()]), + ("ext2".into(), vec!["0.2.0".into()]) + ] + .into_iter() + .collect() + ); + + // The latest version of each extension is returned. + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 0, + }, + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 0 + }, + ] + ); + + // Record extensions being downloaded. + for _ in 0..7 { + assert!(db.record_extension_download("ext2", "0.0.2").await.unwrap()); + } + + for _ in 0..3 { + assert!(db.record_extension_download("ext1", "0.0.1").await.unwrap()); + } + + for _ in 0..2 { + assert!(db.record_extension_download("ext1", "0.0.2").await.unwrap()); + } + + // Record download returns false if the extension does not exist. + assert!(!db + .record_extension_download("no-such-extension", "0.0.2") + .await + .unwrap()); + + // Extensions are returned in descending order of total downloads. + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 7 + }, + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.2".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 5, + }, + ] + ); + + // Add more extensions, including a new version of `ext1`, and backfilling + // an older version of `ext2`. + db.insert_extension_versions( + &[ + ( + "ext1", + vec![NewExtensionVersion { + name: "Extension One".into(), + version: semver::Version::parse("0.0.3").unwrap(), + description: "a real good extension".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + }], + ), + ( + "ext2", + vec![NewExtensionVersion { + name: "Extension Two".into(), + version: semver::Version::parse("0.1.0").unwrap(), + description: "an old extension".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + }], + ), + ] + .into_iter() + .collect(), + ) + .await + .unwrap(); + + let versions = db.get_known_extension_versions().await.unwrap(); + assert_eq!( + versions, + [ + ( + "ext1".into(), + vec!["0.0.1".into(), "0.0.2".into(), "0.0.3".into()] + ), + ("ext2".into(), vec!["0.1.0".into(), "0.2.0".into()]) + ] + .into_iter() + .collect() + ); + + let extensions = db.get_extensions(None, 5).await.unwrap(); + assert_eq!( + extensions, + &[ + ExtensionMetadata { + id: "ext2".into(), + name: "Extension Two".into(), + version: "0.2.0".into(), + authors: vec!["marshall".into()], + repository: "ext2/repo".into(), + published_at: t0, + download_count: 7 + }, + ExtensionMetadata { + id: "ext1".into(), + name: "Extension One".into(), + version: "0.0.3".into(), + authors: vec!["max".into(), "marshall".into()], + repository: "ext1/repo".into(), + published_at: t0, + download_count: 5, + }, + ] + ); +} diff --git a/crates/collab/src/env.rs b/crates/collab/src/env.rs index 58c29b0205..4e6fe3b3a3 100644 --- a/crates/collab/src/env.rs +++ b/crates/collab/src/env.rs @@ -3,7 +3,8 @@ use std::fs; pub fn load_dotenv() -> anyhow::Result<()> { let env: toml::map::Map = toml::de::from_str( - &fs::read_to_string("./.env.toml").map_err(|_| anyhow!("no .env.toml file found"))?, + &fs::read_to_string("./crates/collab/.env.toml") + .map_err(|_| anyhow!("no .env.toml file found"))?, )?; for (key, value) in env { diff --git a/crates/collab/src/lib.rs b/crates/collab/src/lib.rs index aba9bd75d1..195ed7b11d 100644 --- a/crates/collab/src/lib.rs +++ b/crates/collab/src/lib.rs @@ -8,11 +8,14 @@ pub mod rpc; #[cfg(test)] mod tests; +use anyhow::anyhow; +use aws_config::{BehaviorVersion, Region}; use axum::{http::StatusCode, response::IntoResponse}; use db::Database; use executor::Executor; use serde::Deserialize; use std::{path::PathBuf, sync::Arc}; +use util::ResultExt; pub type Result = std::result::Result; @@ -100,6 +103,11 @@ pub struct Config { pub live_kit_secret: Option, pub rust_log: Option, pub log_json: Option, + pub blob_store_url: Option, + pub blob_store_region: Option, + pub blob_store_access_key: Option, + pub blob_store_secret_key: Option, + pub blob_store_bucket: Option, pub zed_environment: Arc, } @@ -118,6 +126,7 @@ pub struct MigrateConfig { pub struct AppState { pub db: Arc, pub live_kit_client: Option>, + pub blob_store_client: Option, pub config: Config, } @@ -146,8 +155,44 @@ impl AppState { let this = Self { db: Arc::new(db), live_kit_client, + blob_store_client: build_blob_store_client(&config).await.log_err(), config, }; Ok(Arc::new(this)) } } + +async fn build_blob_store_client(config: &Config) -> anyhow::Result { + let keys = aws_sdk_s3::config::Credentials::new( + config + .blob_store_access_key + .clone() + .ok_or_else(|| anyhow!("missing blob_store_access_key"))?, + config + .blob_store_secret_key + .clone() + .ok_or_else(|| anyhow!("missing blob_store_secret_key"))?, + None, + None, + "env", + ); + + let s3_config = aws_config::defaults(BehaviorVersion::latest()) + .endpoint_url( + config + .blob_store_url + .as_ref() + .ok_or_else(|| anyhow!("missing blob_store_url"))?, + ) + .region(Region::new( + config + .blob_store_region + .clone() + .ok_or_else(|| anyhow!("missing blob_store_region"))?, + )) + .credentials_provider(keys) + .load() + .await; + + Ok(aws_sdk_s3::Client::new(&s3_config)) +} diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index a2fda0dd33..b80e8961df 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -1,6 +1,9 @@ use anyhow::anyhow; use axum::{routing::get, Extension, Router}; -use collab::{db, env, executor::Executor, AppState, Config, MigrateConfig, Result}; +use collab::{ + api::fetch_extensions_from_blob_store_periodically, db, env, executor::Executor, AppState, + Config, MigrateConfig, Result, +}; use db::Database; use std::{ env::args, @@ -50,6 +53,8 @@ async fn main() -> Result<()> { let rpc_server = collab::rpc::Server::new(epoch, state.clone(), Executor::Production); rpc_server.start().await?; + fetch_extensions_from_blob_store_periodically(state.clone(), Executor::Production); + let app = collab::api::routes(rpc_server.clone(), state.clone()) .merge(collab::rpc::routes(rpc_server.clone())) .merge( diff --git a/crates/collab/src/tests/test_server.rs b/crates/collab/src/tests/test_server.rs index 62870b860c..39292ead44 100644 --- a/crates/collab/src/tests/test_server.rs +++ b/crates/collab/src/tests/test_server.rs @@ -479,6 +479,7 @@ impl TestServer { Arc::new(AppState { db: test_db.db().clone(), live_kit_client: Some(Arc::new(fake_server.create_api_client())), + blob_store_client: None, config: Config { http_port: 0, database_url: "".into(), @@ -491,6 +492,11 @@ impl TestServer { rust_log: None, log_json: None, zed_environment: "test".into(), + blob_store_url: None, + blob_store_region: None, + blob_store_access_key: None, + blob_store_secret_key: None, + blob_store_bucket: None, }, }) } diff --git a/script/bootstrap b/script/bootstrap index 16ae872dbd..054daccf42 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,6 +3,10 @@ echo "installing foreman..." which foreman > /dev/null || brew install foreman +echo "installing minio..." +which minio > /dev/null || brew install minio/stable/minio +mkdir -p .blob_store/the-extensions-bucket + echo "creating database..." script/sqlx database create diff --git a/script/seed-db b/script/seed-db index 277ea89ba3..5079e01955 100755 --- a/script/seed-db +++ b/script/seed-db @@ -1,5 +1,4 @@ #!/bin/bash set -e -cd crates/collab cargo run --quiet --package=collab --features seed-support --bin seed -- $@