From ac4c6c60f185d0fc0ae49b0c328676731122a06c Mon Sep 17 00:00:00 2001 From: Conrad Irwin Date: Wed, 20 Mar 2024 21:00:02 -0600 Subject: [PATCH] Make it (a tiny bit) easier to run your own collab (#9557) * Allow creating channels when seeding * Allow configuring a custom `SEED_PATH` * Seed the database when creating/migrating it so you don't need a separate step for this. Release Notes: - N/A --- crates/collab/.admins.default.json | 8 - crates/collab/.env.toml | 2 + crates/collab/Cargo.toml | 5 +- crates/collab/README.md | 10 +- crates/collab/seed.default.json | 12 ++ crates/collab/src/bin/seed.rs | 97 ------------- crates/collab/src/db.rs | 6 - crates/collab/src/lib.rs | 12 +- crates/collab/src/main.rs | 42 ++++-- crates/collab/src/seed.rs | 137 ++++++++++++++++++ crates/collab/src/tests/test_server.rs | 2 + .../src/developing_zed__building_zed_linux.md | 6 - .../developing_zed__local_collaboration.md | 42 ++++-- script/bootstrap | 18 +-- script/linux | 9 ++ script/seed-db | 2 +- script/zed-local | 13 +- 17 files changed, 246 insertions(+), 177 deletions(-) delete mode 100644 crates/collab/.admins.default.json create mode 100644 crates/collab/seed.default.json delete mode 100644 crates/collab/src/bin/seed.rs create mode 100644 crates/collab/src/seed.rs diff --git a/crates/collab/.admins.default.json b/crates/collab/.admins.default.json deleted file mode 100644 index 75153ae06b..0000000000 --- a/crates/collab/.admins.default.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - "nathansobo", - "as-cii", - "maxbrunsfeld", - "iamnbutler", - "mikayla-maki", - "JosephTLyons" -] diff --git a/crates/collab/.env.toml b/crates/collab/.env.toml index b244e83eb2..ee01d75782 100644 --- a/crates/collab/.env.toml +++ b/crates/collab/.env.toml @@ -1,4 +1,5 @@ DATABASE_URL = "postgres://postgres@localhost/zed" +# DATABASE_URL = "sqlite:////home/zed/.config/zed/db.sqlite3?mode=rwc" DATABASE_MAX_CONNECTIONS = 5 HTTP_PORT = 8080 API_TOKEN = "secret" @@ -13,6 +14,7 @@ BLOB_STORE_BUCKET = "the-extensions-bucket" BLOB_STORE_URL = "http://127.0.0.1:9000" BLOB_STORE_REGION = "the-region" ZED_CLIENT_CHECKSUM_SEED = "development-checksum-seed" +SEED_PATH = "crates/collab/seed.default.json" # CLICKHOUSE_URL = "" # CLICKHOUSE_USER = "default" diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index 6fcea2ae04..ad61aa8379 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -13,8 +13,9 @@ workspace = true [[bin]] name = "collab" -[[bin]] -name = "seed" +[features] +sqlite = ["sea-orm/sqlx-sqlite", "sqlx/sqlite"] +test-support = ["sqlite"] [dependencies] anyhow.workspace = true diff --git a/crates/collab/README.md b/crates/collab/README.md index 4c43407474..bb3c76b15b 100644 --- a/crates/collab/README.md +++ b/crates/collab/README.md @@ -6,21 +6,21 @@ It contains our back-end logic for collaboration, to which we connect from the Z # Local Development - Detailed instructions on getting started are [here](https://zed.dev/docs/local-collaboration). +Detailed instructions on getting started are [here](https://zed.dev/docs/local-collaboration). # Deployment We run two instances of collab: -* Staging (https://staging-collab.zed.dev) -* Production (https://collab.zed.dev) +- Staging (https://staging-collab.zed.dev) +- Production (https://collab.zed.dev) Both of these run on the Kubernetes cluster hosted in Digital Ocean. Deployment is triggered by pushing to the `collab-staging` (or `collab-production`) tag in Github. The best way to do this is: -* `./script/deploy-collab staging` -* `./script/deploy-collab production` +- `./script/deploy-collab staging` +- `./script/deploy-collab production` You can tell what is currently deployed with `./script/what-is-deployed`. diff --git a/crates/collab/seed.default.json b/crates/collab/seed.default.json new file mode 100644 index 0000000000..ded1dc862b --- /dev/null +++ b/crates/collab/seed.default.json @@ -0,0 +1,12 @@ +{ + "admins": [ + "nathansobo", + "as-cii", + "maxbrunsfeld", + "iamnbutler", + "mikayla-maki", + "JosephTLyons" + ], + "channels": ["zed"], + "number_of_users": 100 +} diff --git a/crates/collab/src/bin/seed.rs b/crates/collab/src/bin/seed.rs deleted file mode 100644 index 66828c75bc..0000000000 --- a/crates/collab/src/bin/seed.rs +++ /dev/null @@ -1,97 +0,0 @@ -use collab::{ - db::{self, NewUserParams}, - env::load_dotenv, - executor::Executor, -}; -use db::{ConnectOptions, Database}; -use serde::{de::DeserializeOwned, Deserialize}; -use std::{fmt::Write, fs}; - -#[derive(Debug, Deserialize)] -struct GitHubUser { - id: i32, - login: String, - email: Option, -} - -#[tokio::main] -async fn main() { - load_dotenv().expect("failed to load .env.toml file"); - - let mut admin_logins = load_admins("crates/collab/.admins.default.json") - .expect("failed to load default admins file"); - if let Ok(other_admins) = load_admins("./.admins.json") { - admin_logins.extend(other_admins); - } - - let database_url = std::env::var("DATABASE_URL").expect("missing DATABASE_URL env var"); - let db = Database::new(ConnectOptions::new(database_url), Executor::Production) - .await - .expect("failed to connect to postgres database"); - let client = reqwest::Client::new(); - - // Create admin users for all of the users in `.admins.toml` or `.admins.default.toml`. - for admin_login in admin_logins { - let user = fetch_github::( - &client, - &format!("https://api.github.com/users/{admin_login}"), - ) - .await; - db.create_user( - &user.email.unwrap_or(format!("{admin_login}@example.com")), - true, - NewUserParams { - github_login: user.login, - github_user_id: user.id, - }, - ) - .await - .expect("failed to create admin user"); - } - - // Fetch 100 other random users from GitHub and insert them into the database. - let mut user_count = db - .get_all_users(0, 200) - .await - .expect("failed to load users from db") - .len(); - let mut last_user_id = None; - while user_count < 100 { - let mut uri = "https://api.github.com/users?per_page=100".to_string(); - if let Some(last_user_id) = last_user_id { - write!(&mut uri, "&since={}", last_user_id).unwrap(); - } - let users = fetch_github::>(&client, &uri).await; - - for github_user in users { - last_user_id = Some(github_user.id); - user_count += 1; - db.get_or_create_user_by_github_account( - &github_user.login, - Some(github_user.id), - github_user.email.as_deref(), - None, - ) - .await - .expect("failed to insert user"); - } - } -} - -fn load_admins(path: &str) -> anyhow::Result> { - let file_content = fs::read_to_string(path)?; - Ok(serde_json::from_str(&file_content)?) -} - -async fn fetch_github(client: &reqwest::Client, url: &str) -> T { - let response = client - .get(url) - .header("user-agent", "zed") - .send() - .await - .unwrap_or_else(|_| panic!("failed to fetch '{}'", url)); - response - .json() - .await - .unwrap_or_else(|_| panic!("failed to deserialize github user from '{}'", url)) -} diff --git a/crates/collab/src/db.rs b/crates/collab/src/db.rs index dea287eb6a..71565e8444 100644 --- a/crates/collab/src/db.rs +++ b/crates/collab/src/db.rs @@ -128,12 +128,6 @@ impl Database { Ok(new_migrations) } - /// Initializes static data that resides in the database by upserting it. - pub async fn initialize_static_data(&mut self) -> Result<()> { - self.initialize_notification_kinds().await?; - Ok(()) - } - /// Transaction runs things in a transaction. If you want to call other methods /// and pass the transaction around you need to reborrow the transaction at each /// call site with: `&*tx`. diff --git a/crates/collab/src/lib.rs b/crates/collab/src/lib.rs index 9cc4271e2d..7826978b45 100644 --- a/crates/collab/src/lib.rs +++ b/crates/collab/src/lib.rs @@ -6,6 +6,7 @@ pub mod env; pub mod executor; mod rate_limiter; pub mod rpc; +pub mod seed; #[cfg(test)] mod tests; @@ -111,6 +112,8 @@ impl std::error::Error for Error {} pub struct Config { pub http_port: u16, pub database_url: String, + pub migrations_path: Option, + pub seed_path: Option, pub database_max_connections: u32, pub api_token: String, pub clickhouse_url: Option, @@ -142,12 +145,6 @@ impl Config { } } -#[derive(Default, Deserialize)] -pub struct MigrateConfig { - pub database_url: String, - pub migrations_path: Option, -} - pub struct AppState { pub db: Arc, pub live_kit_client: Option>, @@ -162,8 +159,7 @@ impl AppState { pub async fn new(config: Config, executor: Executor) -> Result> { let mut db_options = db::ConnectOptions::new(config.database_url.clone()); db_options.max_connections(config.database_max_connections); - let mut db = Database::new(db_options, Executor::Production).await?; - db.initialize_notification_kinds().await?; + let db = Database::new(db_options, Executor::Production).await?; let live_kit_client = if let Some(((server, key), secret)) = config .live_kit_server diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index 6d92d2e8bd..7f25e74560 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -7,7 +7,7 @@ use axum::{ }; use collab::{ api::fetch_extensions_from_blob_store_periodically, db, env, executor::Executor, AppState, - Config, MigrateConfig, RateLimiter, Result, + Config, RateLimiter, Result, }; use db::Database; use std::{ @@ -43,7 +43,16 @@ async fn main() -> Result<()> { println!("collab v{} ({})", VERSION, REVISION.unwrap_or("unknown")); } Some("migrate") => { - run_migrations().await?; + let config = envy::from_env::().expect("error loading config"); + run_migrations(&config).await?; + } + Some("seed") => { + let config = envy::from_env::().expect("error loading config"); + let db_options = db::ConnectOptions::new(config.database_url.clone()); + let mut db = Database::new(db_options, Executor::Production).await?; + db.initialize_notification_kinds().await?; + + collab::seed::seed(&config, &db, true).await?; } Some("serve") => { let (is_api, is_collab) = if let Some(next) = args.next() { @@ -53,14 +62,14 @@ async fn main() -> Result<()> { }; if !is_api && !is_collab { Err(anyhow!( - "usage: collab " + "usage: collab " ))?; } let config = envy::from_env::().expect("error loading config"); init_tracing(&config); - run_migrations().await?; + run_migrations(&config).await?; let state = AppState::new(config, Executor::Production).await?; @@ -155,22 +164,25 @@ async fn main() -> Result<()> { } _ => { Err(anyhow!( - "usage: collab " + "usage: collab " ))?; } } Ok(()) } -async fn run_migrations() -> Result<()> { - let config = envy::from_env::().expect("error loading config"); +async fn run_migrations(config: &Config) -> Result<()> { let db_options = db::ConnectOptions::new(config.database_url.clone()); - let db = Database::new(db_options, Executor::Production).await?; + let mut db = Database::new(db_options, Executor::Production).await?; - let migrations_path = config - .migrations_path - .as_deref() - .unwrap_or_else(|| Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/migrations"))); + let migrations_path = config.migrations_path.as_deref().unwrap_or_else(|| { + #[cfg(feature = "sqlite")] + let default_migrations = concat!(env!("CARGO_MANIFEST_DIR"), "/migrations.sqlite"); + #[cfg(not(feature = "sqlite"))] + let default_migrations = concat!(env!("CARGO_MANIFEST_DIR"), "/migrations"); + + Path::new(default_migrations) + }); let migrations = db.migrate(&migrations_path, false).await?; for (migration, duration) in migrations { @@ -182,6 +194,12 @@ async fn run_migrations() -> Result<()> { ); } + db.initialize_notification_kinds().await?; + + if config.seed_path.is_some() { + collab::seed::seed(&config, &db, false).await?; + } + return Ok(()); } diff --git a/crates/collab/src/seed.rs b/crates/collab/src/seed.rs new file mode 100644 index 0000000000..b851aa239d --- /dev/null +++ b/crates/collab/src/seed.rs @@ -0,0 +1,137 @@ +use crate::db::{self, ChannelRole, NewUserParams}; + +use anyhow::Context; +use db::Database; +use serde::{de::DeserializeOwned, Deserialize}; +use std::{fmt::Write, fs, path::Path}; + +use crate::Config; + +#[derive(Debug, Deserialize)] +struct GitHubUser { + id: i32, + login: String, + email: Option, +} + +#[derive(Deserialize)] +struct SeedConfig { + // Which users to create as admins. + admins: Vec, + // Which channels to create (all admins are invited to all channels) + channels: Vec, + // Number of random users to create from the Github API + number_of_users: Option, +} + +pub async fn seed(config: &Config, db: &Database, force: bool) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + + if !db.get_all_users(0, 1).await?.is_empty() && !force { + return Ok(()); + } + + let seed_path = config + .seed_path + .as_ref() + .context("called seed with no SEED_PATH")?; + + let seed_config = load_admins(seed_path) + .context(format!("failed to load {}", seed_path.to_string_lossy()))?; + + let mut first_user = None; + let mut others = vec![]; + + for admin_login in seed_config.admins { + let user = fetch_github::( + &client, + &format!("https://api.github.com/users/{admin_login}"), + ) + .await; + let user = db + .create_user( + &user.email.unwrap_or(format!("{admin_login}@example.com")), + true, + NewUserParams { + github_login: user.login, + github_user_id: user.id, + }, + ) + .await + .context("failed to create admin user")?; + if first_user.is_none() { + first_user = Some(user.user_id); + } else { + others.push(user.user_id) + } + } + + for channel in seed_config.channels { + let (channel, _) = db + .create_channel(&channel, None, first_user.unwrap()) + .await + .context("failed to create channel")?; + + for user_id in &others { + db.invite_channel_member( + channel.id, + *user_id, + first_user.unwrap(), + ChannelRole::Admin, + ) + .await + .context("failed to add user to channel")?; + } + } + + if let Some(number_of_users) = seed_config.number_of_users { + // Fetch 100 other random users from GitHub and insert them into the database + // (for testing autocompleters, etc.) + let mut user_count = db + .get_all_users(0, 200) + .await + .expect("failed to load users from db") + .len(); + let mut last_user_id = None; + while user_count < number_of_users { + let mut uri = "https://api.github.com/users?per_page=100".to_string(); + if let Some(last_user_id) = last_user_id { + write!(&mut uri, "&since={}", last_user_id).unwrap(); + } + let users = fetch_github::>(&client, &uri).await; + + for github_user in users { + last_user_id = Some(github_user.id); + user_count += 1; + db.get_or_create_user_by_github_account( + &github_user.login, + Some(github_user.id), + github_user.email.as_deref(), + None, + ) + .await + .expect("failed to insert user"); + } + } + } + + Ok(()) +} + +fn load_admins(path: impl AsRef) -> anyhow::Result { + let file_content = fs::read_to_string(path)?; + Ok(serde_json::from_str(&file_content)?) +} + +async fn fetch_github(client: &reqwest::Client, url: &str) -> T { + let response = client + .get(url) + .header("user-agent", "zed") + .send() + .await + .unwrap_or_else(|_| panic!("failed to fetch '{}'", url)); + response + .json() + .await + .unwrap_or_else(|_| panic!("failed to deserialize github user from '{}'", url)) +} diff --git a/crates/collab/src/tests/test_server.rs b/crates/collab/src/tests/test_server.rs index 1aa13a2e6c..e5ca052a2f 100644 --- a/crates/collab/src/tests/test_server.rs +++ b/crates/collab/src/tests/test_server.rs @@ -515,6 +515,8 @@ impl TestServer { zed_client_checksum_seed: None, slack_panics_webhook: None, auto_join_channel_id: None, + migrations_path: None, + seed_path: None, }, }) } diff --git a/docs/src/developing_zed__building_zed_linux.md b/docs/src/developing_zed__building_zed_linux.md index 394fe4bd83..398ff361cc 100644 --- a/docs/src/developing_zed__building_zed_linux.md +++ b/docs/src/developing_zed__building_zed_linux.md @@ -16,12 +16,6 @@ git submodule update --init --recursive rustup update ``` -- Install the Rust wasm toolchain: - - ```bash - rustup target add wasm32-wasi - ``` - - Install the necessary system libraries: ```bash diff --git a/docs/src/developing_zed__local_collaboration.md b/docs/src/developing_zed__local_collaboration.md index 7477c64be5..fa47ee9a27 100644 --- a/docs/src/developing_zed__local_collaboration.md +++ b/docs/src/developing_zed__local_collaboration.md @@ -12,21 +12,19 @@ script/bootstrap This script will set up the `zed` Postgres database, and populate it with some users. It requires internet access, because it fetches some users from the GitHub API. -The script will create several _admin_ users, who you'll sign in as by default when developing locally. The GitHub logins for these default admin users are specified in this file: +The script will seed the database with various content defined by: ``` -cat crates/collab/.admins.default.json +cat crates/collab/seed.default.json ``` -To use a different set of admin users, you can create a file called `.admins.json` in the same directory: +To use a different set of admin users, you can create your own version of that json file and export the `SEED_PATH` environment variable. Note that the usernames listed in the admins list currently must correspond to valid Github users. -``` -cat > crates/collab/.admins.json < /dev/null || brew install foreman - -echo "creating database..." -script/sqlx database create - -echo "migrating database..." -cargo run -p collab -- migrate - -echo "seeding database..." -script/seed-db - if [[ "$OSTYPE" == "linux-gnu"* ]]; then echo "Linux dependencies..." script/linux +else + echo "installing foreman..." + which foreman > /dev/null || brew install foreman fi + +echo "creating database..." +script/sqlx database create diff --git a/script/linux b/script/linux index 1d6c40f839..bb12006c6f 100755 --- a/script/linux +++ b/script/linux @@ -2,6 +2,9 @@ set -e +# install the wasm toolchain +rustup target add wasm32-wasi + # if sudo is not installed, define an empty alias maysudo=$(command -v sudo || command -v doas || true) @@ -27,6 +30,8 @@ fi dnf=$(command -v dnf || true) if [[ -n $dnf ]]; then deps=( + gcc + g++ alsa-lib-devel fontconfig-devel wayland-devel @@ -35,6 +40,10 @@ if [[ -n $dnf ]]; then libzstd-devel vulkan-loader ) + # libxkbcommon-x11-devel is in the crb repo + $maysudo "$dnf" config-manager --set-enabled crb + $maysudo "$dnf" install epel-release epel-next-release + $maysudo "$dnf" install -y "${deps[@]}" exit 0 fi diff --git a/script/seed-db b/script/seed-db index d3ee89e4f0..3b329c054c 100755 --- a/script/seed-db +++ b/script/seed-db @@ -1,4 +1,4 @@ #!/bin/bash set -e -cargo run --quiet --package=collab --bin seed -- $@ +cargo run -p collab migrate diff --git a/script/zed-local b/script/zed-local index 1b1852043b..62b6685355 100755 --- a/script/zed-local +++ b/script/zed-local @@ -19,16 +19,9 @@ OPTIONS const { spawn, execFileSync } = require("child_process"); const assert = require("assert"); -const defaultUsers = require("../crates/collab/.admins.default.json"); -let users = defaultUsers; -try { - const customUsers = require("../crates/collab/.admins.json"); - assert(customUsers.length > 0); - assert(customUsers.every((user) => typeof user === "string")); - users = customUsers.concat( - defaultUsers.filter((user) => !customUsers.includes(user)), - ); -} catch (_) {} +const users = require( + process.env.SEED_PATH || "../crates/collab/seed.default.json", +).admins; const RESOLUTION_REGEX = /(\d+) x (\d+)/; const DIGIT_FLAG_REGEX = /^--?(\d+)$/;