zed-industries/zed
1
1
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2024-10-06 10:57:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

Make it (a tiny bit) easier to run your own collab (#9557)

Browse Source 
* Allow creating channels when seeding
* Allow configuring a custom `SEED_PATH`
* Seed the database when creating/migrating it so you don't need a
  separate step for this.

Release Notes:

- N/A
This commit is contained in:
Conrad Irwin 2024-03-20 21:00:02 -06:00 committed by GitHub
parent 1062c5bd26
commit ac4c6c60f1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 246 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
crates/collab/.admins.default.json
View File

@ -1,8 +0,0 @@
[
"nathansobo",
"as-cii",
"maxbrunsfeld",
"iamnbutler",
"mikayla-maki",
"JosephTLyons"
]

2
crates/collab/.env.toml
View File

@ -1,4 +1,5 @@
DATABASE_URL = "postgres://postgres@localhost/zed"
# DATABASE_URL = "sqlite:////home/zed/.config/zed/db.sqlite3?mode=rwc"
DATABASE_MAX_CONNECTIONS = 5
HTTP_PORT = 8080
API_TOKEN = "secret"
@ -13,6 +14,7 @@ BLOB_STORE_BUCKET = "the-extensions-bucket"
BLOB_STORE_URL = "http://127.0.0.1:9000"
BLOB_STORE_REGION = "the-region"
ZED_CLIENT_CHECKSUM_SEED = "development-checksum-seed"
SEED_PATH = "crates/collab/seed.default.json"
# CLICKHOUSE_URL = ""
# CLICKHOUSE_USER = "default"

5
crates/collab/Cargo.toml
View File

@ -13,8 +13,9 @@ workspace = true
[[bin]]
name = "collab"
[[bin]]
name = "seed"
[features]
sqlite = ["sea-orm/sqlx-sqlite", "sqlx/sqlite"]
test-support = ["sqlite"]
[dependencies]
anyhow.workspace = true

10
crates/collab/README.md
View File

@ -6,21 +6,21 @@ It contains our back-end logic for collaboration, to which we connect from the Z
# Local Development
Detailed instructions on getting started are [here](https://zed.dev/docs/local-collaboration).
Detailed instructions on getting started are [here](https://zed.dev/docs/local-collaboration).
# Deployment
We run two instances of collab:
* Staging (https://staging-collab.zed.dev)
* Production (https://collab.zed.dev)
- Staging (https://staging-collab.zed.dev)
- Production (https://collab.zed.dev)
Both of these run on the Kubernetes cluster hosted in Digital Ocean.
Deployment is triggered by pushing to the `collab-staging` (or `collab-production`) tag in Github. The best way to do this is:
* `./script/deploy-collab staging`
* `./script/deploy-collab production`
- `./script/deploy-collab staging`
- `./script/deploy-collab production`
You can tell what is currently deployed with `./script/what-is-deployed`.

12
crates/collab/seed.default.json Normal file
View File

@ -0,0 +1,12 @@
{
"admins": [
"nathansobo",
"as-cii",
"maxbrunsfeld",
"iamnbutler",
"mikayla-maki",
"JosephTLyons"
],
"channels": ["zed"],
"number_of_users": 100
}

97
crates/collab/src/bin/seed.rs
View File

@ -1,97 +0,0 @@
use collab::{
db::{self, NewUserParams},
env::load_dotenv,
executor::Executor,
};
use db::{ConnectOptions, Database};
use serde::{de::DeserializeOwned, Deserialize};
use std::{fmt::Write, fs};
#[derive(Debug, Deserialize)]
struct GitHubUser {
id: i32,
login: String,
email: Option<String>,
}
#[tokio::main]
async fn main() {
load_dotenv().expect("failed to load .env.toml file");
let mut admin_logins = load_admins("crates/collab/.admins.default.json")
.expect("failed to load default admins file");
if let Ok(other_admins) = load_admins("./.admins.json") {
admin_logins.extend(other_admins);
}
let database_url = std::env::var("DATABASE_URL").expect("missing DATABASE_URL env var");
let db = Database::new(ConnectOptions::new(database_url), Executor::Production)
.await
.expect("failed to connect to postgres database");
let client = reqwest::Client::new();
// Create admin users for all of the users in `.admins.toml` or `.admins.default.toml`.
for admin_login in admin_logins {
let user = fetch_github::<GitHubUser>(
&client,
&format!("https://api.github.com/users/{admin_login}"),
)
.await;
db.create_user(
&user.email.unwrap_or(format!("{admin_login}@example.com")),
true,
NewUserParams {
github_login: user.login,
github_user_id: user.id,
},
)
.await
.expect("failed to create admin user");
}
// Fetch 100 other random users from GitHub and insert them into the database.
let mut user_count = db
.get_all_users(0, 200)
.await
.expect("failed to load users from db")
.len();
let mut last_user_id = None;
while user_count < 100 {
let mut uri = "https://api.github.com/users?per_page=100".to_string();
if let Some(last_user_id) = last_user_id {
write!(&mut uri, "&since={}", last_user_id).unwrap();
}
let users = fetch_github::<Vec<GitHubUser>>(&client, &uri).await;
for github_user in users {
last_user_id = Some(github_user.id);
user_count += 1;
db.get_or_create_user_by_github_account(
&github_user.login,
Some(github_user.id),
github_user.email.as_deref(),
None,
)
.await
.expect("failed to insert user");
}
}
}
fn load_admins(path: &str) -> anyhow::Result<Vec<String>> {
let file_content = fs::read_to_string(path)?;
Ok(serde_json::from_str(&file_content)?)
}
async fn fetch_github<T: DeserializeOwned>(client: &reqwest::Client, url: &str) -> T {
let response = client
.get(url)
.header("user-agent", "zed")
.send()
.await
.unwrap_or_else(|_| panic!("failed to fetch '{}'", url));
response
.json()
.await
.unwrap_or_else(|_| panic!("failed to deserialize github user from '{}'", url))
}

6
crates/collab/src/db.rs
View File

@ -128,12 +128,6 @@ impl Database {
Ok(new_migrations)
}
/// Initializes static data that resides in the database by upserting it.
pub async fn initialize_static_data(&mut self) -> Result<()> {
self.initialize_notification_kinds().await?;
Ok(())
}
/// Transaction runs things in a transaction. If you want to call other methods
/// and pass the transaction around you need to reborrow the transaction at each
/// call site with: `&*tx`.

12
crates/collab/src/lib.rs
View File

@ -6,6 +6,7 @@ pub mod env;
pub mod executor;
mod rate_limiter;
pub mod rpc;
pub mod seed;
#[cfg(test)]
mod tests;
@ -111,6 +112,8 @@ impl std::error::Error for Error {}
pub struct Config {
pub http_port: u16,
pub database_url: String,
pub migrations_path: Option<PathBuf>,
pub seed_path: Option<PathBuf>,
pub database_max_connections: u32,
pub api_token: String,
pub clickhouse_url: Option<String>,
@ -142,12 +145,6 @@ impl Config {
}
}
#[derive(Default, Deserialize)]
pub struct MigrateConfig {
pub database_url: String,
pub migrations_path: Option<PathBuf>,
}
pub struct AppState {
pub db: Arc<Database>,
pub live_kit_client: Option<Arc<dyn live_kit_server::api::Client>>,
@ -162,8 +159,7 @@ impl AppState {
pub async fn new(config: Config, executor: Executor) -> Result<Arc<Self>> {
let mut db_options = db::ConnectOptions::new(config.database_url.clone());
db_options.max_connections(config.database_max_connections);
let mut db = Database::new(db_options, Executor::Production).await?;
db.initialize_notification_kinds().await?;
let db = Database::new(db_options, Executor::Production).await?;
let live_kit_client = if let Some(((server, key), secret)) = config
.live_kit_server

42
crates/collab/src/main.rs
View File

@ -7,7 +7,7 @@ use axum::{
};
use collab::{
api::fetch_extensions_from_blob_store_periodically, db, env, executor::Executor, AppState,
Config, MigrateConfig, RateLimiter, Result,
Config, RateLimiter, Result,
};
use db::Database;
use std::{
@ -43,7 +43,16 @@ async fn main() -> Result<()> {
println!("collab v{} ({})", VERSION, REVISION.unwrap_or("unknown"));
}
Some("migrate") => {
run_migrations().await?;
let config = envy::from_env::<Config>().expect("error loading config");
run_migrations(&config).await?;
}
Some("seed") => {
let config = envy::from_env::<Config>().expect("error loading config");
let db_options = db::ConnectOptions::new(config.database_url.clone());
let mut db = Database::new(db_options, Executor::Production).await?;
db.initialize_notification_kinds().await?;
collab::seed::seed(&config, &db, true).await?;
}
Some("serve") => {
let (is_api, is_collab) = if let Some(next) = args.next() {
@ -53,14 +62,14 @@ async fn main() -> Result<()> {
};
if !is_api && !is_collab {
Err(anyhow!(
"usage: collab <version | migrate | serve [api|collab]>"
"usage: collab <version | migrate | seed | serve [api|collab]>"
))?;
}
let config = envy::from_env::<Config>().expect("error loading config");
init_tracing(&config);
run_migrations().await?;
run_migrations(&config).await?;
let state = AppState::new(config, Executor::Production).await?;
@ -155,22 +164,25 @@ async fn main() -> Result<()> {
}
_ => {
Err(anyhow!(
"usage: collab <version | migrate | serve [api|collab]>"
"usage: collab <version | migrate | seed | serve [api|collab]>"
))?;
}
}
Ok(())
}
async fn run_migrations() -> Result<()> {
let config = envy::from_env::<MigrateConfig>().expect("error loading config");
async fn run_migrations(config: &Config) -> Result<()> {
let db_options = db::ConnectOptions::new(config.database_url.clone());
let db = Database::new(db_options, Executor::Production).await?;
let mut db = Database::new(db_options, Executor::Production).await?;
let migrations_path = config
.migrations_path
.as_deref()
.unwrap_or_else(|| Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/migrations")));
let migrations_path = config.migrations_path.as_deref().unwrap_or_else(|| {
#[cfg(feature = "sqlite")]
let default_migrations = concat!(env!("CARGO_MANIFEST_DIR"), "/migrations.sqlite");
#[cfg(not(feature = "sqlite"))]
let default_migrations = concat!(env!("CARGO_MANIFEST_DIR"), "/migrations");
Path::new(default_migrations)
});
let migrations = db.migrate(&migrations_path, false).await?;
for (migration, duration) in migrations {
@ -182,6 +194,12 @@ async fn run_migrations() -> Result<()> {
);
}
db.initialize_notification_kinds().await?;
if config.seed_path.is_some() {
collab::seed::seed(&config, &db, false).await?;
}
return Ok(());
}

137
crates/collab/src/seed.rs Normal file
View File

@ -0,0 +1,137 @@
use crate::db::{self, ChannelRole, NewUserParams};
use anyhow::Context;
use db::Database;
use serde::{de::DeserializeOwned, Deserialize};
use std::{fmt::Write, fs, path::Path};
use crate::Config;
#[derive(Debug, Deserialize)]
struct GitHubUser {
id: i32,
login: String,
email: Option<String>,
}
#[derive(Deserialize)]
struct SeedConfig {
// Which users to create as admins.
admins: Vec<String>,
// Which channels to create (all admins are invited to all channels)
channels: Vec<String>,
// Number of random users to create from the Github API
number_of_users: Option<usize>,
}
pub async fn seed(config: &Config, db: &Database, force: bool) -> anyhow::Result<()> {
let client = reqwest::Client::new();
if !db.get_all_users(0, 1).await?.is_empty() && !force {
return Ok(());
}
let seed_path = config
.seed_path
.as_ref()
.context("called seed with no SEED_PATH")?;
let seed_config = load_admins(seed_path)
.context(format!("failed to load {}", seed_path.to_string_lossy()))?;
let mut first_user = None;
let mut others = vec![];
for admin_login in seed_config.admins {
let user = fetch_github::<GitHubUser>(
&client,
&format!("https://api.github.com/users/{admin_login}"),
)
.await;
let user = db
.create_user(
&user.email.unwrap_or(format!("{admin_login}@example.com")),
true,
NewUserParams {
github_login: user.login,
github_user_id: user.id,
},
)
.await
.context("failed to create admin user")?;
if first_user.is_none() {
first_user = Some(user.user_id);
} else {
others.push(user.user_id)
}
}
for channel in seed_config.channels {
let (channel, _) = db
.create_channel(&channel, None, first_user.unwrap())
.await
.context("failed to create channel")?;
for user_id in &others {
db.invite_channel_member(
channel.id,
*user_id,
first_user.unwrap(),
ChannelRole::Admin,
)
.await
.context("failed to add user to channel")?;
}
}
if let Some(number_of_users) = seed_config.number_of_users {
// Fetch 100 other random users from GitHub and insert them into the database
// (for testing autocompleters, etc.)
let mut user_count = db
.get_all_users(0, 200)
.await
.expect("failed to load users from db")
.len();
let mut last_user_id = None;
while user_count < number_of_users {
let mut uri = "https://api.github.com/users?per_page=100".to_string();
if let Some(last_user_id) = last_user_id {
write!(&mut uri, "&since={}", last_user_id).unwrap();
}
let users = fetch_github::<Vec<GitHubUser>>(&client, &uri).await;
for github_user in users {
last_user_id = Some(github_user.id);
user_count += 1;
db.get_or_create_user_by_github_account(
&github_user.login,
Some(github_user.id),
github_user.email.as_deref(),
None,
)
.await
.expect("failed to insert user");
}
}
}
Ok(())
}
fn load_admins(path: impl AsRef<Path>) -> anyhow::Result<SeedConfig> {
let file_content = fs::read_to_string(path)?;
Ok(serde_json::from_str(&file_content)?)
}
async fn fetch_github<T: DeserializeOwned>(client: &reqwest::Client, url: &str) -> T {
let response = client
.get(url)
.header("user-agent", "zed")
.send()
.await
.unwrap_or_else(|_| panic!("failed to fetch '{}'", url));
response
.json()
.await
.unwrap_or_else(|_| panic!("failed to deserialize github user from '{}'", url))
}

2
crates/collab/src/tests/test_server.rs
View File

@ -515,6 +515,8 @@ impl TestServer {
zed_client_checksum_seed: None,
slack_panics_webhook: None,
auto_join_channel_id: None,
migrations_path: None,
seed_path: None,
},
})
}

6
docs/src/developing_zed__building_zed_linux.md
View File

@ -16,12 +16,6 @@ git submodule update --init --recursive
rustup update
```
- Install the Rust wasm toolchain:
```bash
rustup target add wasm32-wasi
```
- Install the necessary system libraries:
```bash

42
docs/src/developing_zed__local_collaboration.md
View File

@ -12,21 +12,19 @@ script/bootstrap
This script will set up the `zed` Postgres database, and populate it with some users. It requires internet access, because it fetches some users from the GitHub API.
The script will create several _admin_ users, who you'll sign in as by default when developing locally. The GitHub logins for these default admin users are specified in this file:
The script will seed the database with various content defined by:
```
cat crates/collab/.admins.default.json
cat crates/collab/seed.default.json
```
To use a different set of admin users, you can create a file called `.admins.json` in the same directory:
To use a different set of admin users, you can create your own version of that json file and export the `SEED_PATH` environment variable. Note that the usernames listed in the admins list currently must correspond to valid Github users.
```
cat > crates/collab/.admins.json <<JSON
[
"your-github-login",
"another-github-login"
]
JSON
```json
{
"admins": ["admin1", "admin2"],
"channels": ["zed"]
}
```
## Testing collaborative features locally
@ -34,13 +32,37 @@ JSON
In one terminal, run Zed's collaboration server and the `livekit` dev server:
```
foreman start
```
In a second terminal, run two or more instances of Zed.
```
script/zed-local -2
```
This script starts one to four instances of Zed, depending on the `-2`, `-3` or `-4` flags. Each instance will be connected to the local `collab` server, signed in as a different user from `.admins.json` or `.admins.default.json`.
## Running a local collab server
If you want to run your own version of the zed collaboration service, you can, but note that this is still under development, and there is no good support for authentication nor extensions.
Configuration is done through environment variables. By default it will read the configuration from [`.env.toml`](../../crates/collab/.env.toml) and you should use that as a guide for setting this up.
By default Zed assumes that the DATABASE_URL is a Postgres database, but you can make it use Sqlite by compiling with `--features sqlite` and using a sqlite DATABASE_URL with `?mode=rwc`.
To authenticate you must first configure the server by creating a seed.json file that contains at a minimum your github handle. This will be used to create the user on demand.
```json
{
"admins": ["nathansobo"]
}
```
By default the collab server will seed the database when first creating it, but if you want to add more users you can explicitly reseed them with `SEED_PATH=./seed.json cargo run -p collab seed`
Then when running the zed client you must specify two environment variables, `ZED_ADMIN_API_TOKEN` (which should match the value of `API_TOKEN` in .env.toml) and `ZED_IMPERSONATE` (which should match one of the users in your seed.json)

18
script/bootstrap
View File

@ -1,18 +1,12 @@
#!/usr/bin/env bash
echo "installing foreman..."
which foreman > /dev/null || brew install foreman
echo "creating database..."
script/sqlx database create
echo "migrating database..."
cargo run -p collab -- migrate
echo "seeding database..."
script/seed-db
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
echo "Linux dependencies..."
script/linux
else
echo "installing foreman..."
which foreman > /dev/null || brew install foreman
fi
echo "creating database..."
script/sqlx database create

9
script/linux
View File

@ -2,6 +2,9 @@
set -e
# install the wasm toolchain
rustup target add wasm32-wasi
# if sudo is not installed, define an empty alias
maysudo=$(command -v sudo || command -v doas || true)
@ -27,6 +30,8 @@ fi
dnf=$(command -v dnf || true)
if [[ -n $dnf ]]; then
deps=(
gcc
g++
alsa-lib-devel
fontconfig-devel
wayland-devel
@ -35,6 +40,10 @@ if [[ -n $dnf ]]; then
libzstd-devel
vulkan-loader
)
# libxkbcommon-x11-devel is in the crb repo
$maysudo "$dnf" config-manager --set-enabled crb
$maysudo "$dnf" install epel-release epel-next-release
$maysudo "$dnf" install -y "${deps[@]}"
exit 0
fi

2
script/seed-db
View File

@ -1,4 +1,4 @@
#!/bin/bash
set -e
cargo run --quiet --package=collab --bin seed -- $@
cargo run -p collab migrate

13
script/zed-local
View File

@ -19,16 +19,9 @@ OPTIONS
const { spawn, execFileSync } = require("child_process");
const assert = require("assert");
const defaultUsers = require("../crates/collab/.admins.default.json");
let users = defaultUsers;
try {
const customUsers = require("../crates/collab/.admins.json");
assert(customUsers.length > 0);
assert(customUsers.every((user) => typeof user === "string"));
users = customUsers.concat(
defaultUsers.filter((user) => !customUsers.includes(user)),
);
} catch (_) {}
const users = require(
process.env.SEED_PATH || "../crates/collab/seed.default.json",
).admins;
const RESOLUTION_REGEX = /(\d+) x (\d+)/;
const DIGIT_FLAG_REGEX = /^--?(\d+)$/;