mirror of
https://github.com/toeverything/AFFiNE.git
synced 2024-12-24 19:22:49 +03:00
feat: try use a yrs doc to merge state before store to sqlite, but not working
This commit is contained in:
linonetwo
parent
c4c6844fbe
commit
765cd983fc
253
client-app/src-tauri/Cargo.lock
generated
253
client-app/src-tauri/Cargo.lock
generated
@ -15,6 +15,7 @@ dependencies = [
|
||||
"jwst",
|
||||
"jwst-storage",
|
||||
"lib0",
|
||||
"moka",
|
||||
"project-root",
|
||||
"schemars",
|
||||
"serde",
|
||||
@ -22,6 +23,7 @@ dependencies = [
|
||||
"tauri",
|
||||
"tauri-build",
|
||||
"tokio",
|
||||
"y-sync",
|
||||
"yrs",
|
||||
]
|
||||
|
||||
@ -117,6 +119,36 @@ version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
|
||||
|
||||
[[package]]
|
||||
name = "async-io"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794"
|
||||
dependencies = [
|
||||
"async-lock",
|
||||
"autocfg",
|
||||
"concurrent-queue",
|
||||
"futures-lite",
|
||||
"libc",
|
||||
"log",
|
||||
"parking",
|
||||
"polling",
|
||||
"slab",
|
||||
"socket2",
|
||||
"waker-fn",
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-lock"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685"
|
||||
dependencies = [
|
||||
"event-listener",
|
||||
"futures-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
version = "0.3.3"
|
||||
@ -372,6 +404,12 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytecount"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.12.3"
|
||||
@ -414,6 +452,15 @@ dependencies = [
|
||||
"system-deps 6.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "camino"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cang-jie"
|
||||
version = "0.14.0"
|
||||
@ -423,6 +470,28 @@ dependencies = [
|
||||
"tantivy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cargo-platform"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cargo_metadata"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
|
||||
dependencies = [
|
||||
"camino",
|
||||
"cargo-platform",
|
||||
"semver 1.0.14",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cargo_toml"
|
||||
version = "0.13.0"
|
||||
@ -634,6 +703,15 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "concurrent-queue"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.4.0"
|
||||
@ -1026,6 +1104,15 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "error-chain"
|
||||
version = "0.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "2.5.3"
|
||||
@ -1225,6 +1312,21 @@ version = "0.3.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
|
||||
|
||||
[[package]]
|
||||
name = "futures-lite"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"memchr",
|
||||
"parking",
|
||||
"pin-project-lite",
|
||||
"waker-fn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.25"
|
||||
@ -2095,6 +2197,15 @@ dependencies = [
|
||||
"time 0.3.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mach"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "malloc_buf"
|
||||
version = "0.0.6"
|
||||
@ -2203,6 +2314,32 @@ dependencies = [
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "moka"
|
||||
version = "0.9.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b49a05f67020456541f4f29cbaa812016a266a86ec76f96d3873d459c68fe5e"
|
||||
dependencies = [
|
||||
"async-io",
|
||||
"async-lock",
|
||||
"crossbeam-channel",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"futures-util",
|
||||
"num_cpus",
|
||||
"once_cell",
|
||||
"parking_lot 0.12.1",
|
||||
"quanta",
|
||||
"rustc_version 0.4.0",
|
||||
"scheduled-thread-pool",
|
||||
"skeptic",
|
||||
"smallvec",
|
||||
"tagptr",
|
||||
"thiserror",
|
||||
"triomphe",
|
||||
"uuid 1.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "murmurhash32"
|
||||
version = "0.2.0"
|
||||
@ -2572,6 +2709,12 @@ dependencies = [
|
||||
"system-deps 6.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
@ -2857,6 +3000,20 @@ dependencies = [
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polling"
|
||||
version = "2.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"log",
|
||||
"wepoll-ffi",
|
||||
"windows-sys 0.42.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
@ -2954,6 +3111,33 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d9cc634bc78768157b5cbfe988ffcd1dcba95cd2b2f03a88316c08c6d00ed63"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quanta"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
"libc",
|
||||
"mach",
|
||||
"once_cell",
|
||||
"raw-cpuid",
|
||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||
"web-sys",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.23.1"
|
||||
@ -3053,6 +3237,15 @@ dependencies = [
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "raw-cpuid"
|
||||
version = "10.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "raw-window-handle"
|
||||
version = "0.5.0"
|
||||
@ -3322,6 +3515,15 @@ dependencies = [
|
||||
"windows-sys 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scheduled-thread-pool"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf"
|
||||
dependencies = [
|
||||
"parking_lot 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "0.8.11"
|
||||
@ -3765,6 +3967,21 @@ version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
|
||||
|
||||
[[package]]
|
||||
name = "skeptic"
|
||||
version = "0.13.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
|
||||
dependencies = [
|
||||
"bytecount",
|
||||
"cargo_metadata",
|
||||
"error-chain",
|
||||
"glob",
|
||||
"pulldown-cmark",
|
||||
"tempfile",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.7"
|
||||
@ -4062,6 +4279,12 @@ dependencies = [
|
||||
"version-compare 0.1.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tagptr"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "tantivy"
|
||||
version = "0.19.0"
|
||||
@ -4698,6 +4921,12 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "triomphe"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1ee9bd9239c339d714d657fac840c6d2a4f9c45f4f9ec7b0975113458be78db"
|
||||
|
||||
[[package]]
|
||||
name = "type-map"
|
||||
version = "0.5.0"
|
||||
@ -4719,6 +4948,15 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.8"
|
||||
@ -4859,6 +5097,12 @@ version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||
|
||||
[[package]]
|
||||
name = "waker-fn"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.3.2"
|
||||
@ -5068,6 +5312,15 @@ dependencies = [
|
||||
"windows-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wepoll-ffi"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
||||
@ -33,7 +33,9 @@ dotenvy = "0.15.6"
|
||||
tauri = { version = "1.2", features = ["api-all", "devtools"] }
|
||||
tokio = { version = "1.23.0", features = ["rt", "macros"] }
|
||||
lib0 = "0.12.0"
|
||||
moka = { version = "0.9.6", features = ["future"] }
|
||||
yrs = { path = "../src-OctoBase/libs/vendors/y-crdt/yrs" }
|
||||
y-sync = { path = "../src-OctoBase/libs/vendors/y-sync" }
|
||||
|
||||
[features]
|
||||
# by default Tauri runs in production mode
|
||||
|
||||
@ -1,13 +1,15 @@
|
||||
use std::io::ErrorKind;
|
||||
|
||||
use ipc_types::document::{
|
||||
CreateDocumentParameter, GetDocumentParameter, GetDocumentResponse, YDocumentUpdate,
|
||||
};
|
||||
use jwst::encode_update;
|
||||
use jwst::DocStorage;
|
||||
use jwst::Workspace as OctoBaseWorkspace;
|
||||
use lib0::any::Any;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use y_sync::sync::Message;
|
||||
use y_sync::sync::MessageReader;
|
||||
use y_sync::sync::SyncMessage;
|
||||
use yrs::updates::decoder::DecoderV1;
|
||||
use yrs::{updates::decoder::Decode, Doc, StateVector, Update};
|
||||
|
||||
use crate::state::AppState;
|
||||
@ -30,7 +32,7 @@ pub async fn create_doc<'s>(
|
||||
.0
|
||||
.lock()
|
||||
.await
|
||||
.doc_storage
|
||||
.doc_db
|
||||
.write_doc(parameters.workspace_id.clone(), workspace_doc.doc())
|
||||
.await
|
||||
{
|
||||
@ -51,9 +53,12 @@ pub async fn get_doc<'s>(
|
||||
) -> Result<GetDocumentResponse, String> {
|
||||
// TODO: check user permission
|
||||
|
||||
if let Ok(all_updates_of_workspace) = &state.0.lock().await.doc_storage.all(¶meters.id).await {
|
||||
if let Ok(all_updates_of_workspace) = &state.0.lock().await.doc_db.all(¶meters.id).await {
|
||||
Ok(GetDocumentResponse {
|
||||
updates: all_updates_of_workspace.iter().map(|model| model.blob.clone()).collect::<Vec<Vec<u8>>>(),
|
||||
updates: all_updates_of_workspace
|
||||
.iter()
|
||||
.map(|model| model.blob.clone())
|
||||
.collect::<Vec<Vec<u8>>>(),
|
||||
})
|
||||
} else {
|
||||
Err(format!(
|
||||
@ -68,17 +73,47 @@ pub async fn update_y_document<'s>(
|
||||
state: tauri::State<'s, AppState>,
|
||||
parameters: YDocumentUpdate,
|
||||
) -> Result<bool, String> {
|
||||
if let Some(success) = &state
|
||||
.0
|
||||
.lock()
|
||||
.await
|
||||
.doc_storage
|
||||
.write_update(parameters.id.clone(), ¶meters.update)
|
||||
.await
|
||||
.ok()
|
||||
{
|
||||
Ok(*success)
|
||||
} else {
|
||||
Err(format!("Failed to update yDoc to {}", parameters.id))
|
||||
let state = &state.0.lock().await;
|
||||
let doc_store = &state.doc_store;
|
||||
let doc_db = &state.doc_db;
|
||||
let update = Update::decode_v1(¶meters.update).unwrap();
|
||||
let mut decoder = DecoderV1::from(¶meters.update[..]);
|
||||
for msg in MessageReader::new(&mut decoder) {
|
||||
msg.ok().and_then(|msg| match msg {
|
||||
Message::Sync(msg) => match msg {
|
||||
SyncMessage::SyncStep1(sv) => {
|
||||
Some(())
|
||||
}
|
||||
SyncMessage::SyncStep2(update) => {
|
||||
Some(())
|
||||
}
|
||||
SyncMessage::Update(update) => {
|
||||
let mut tx = doc_store.doc().transact();
|
||||
tx.apply_update(Update::decode_v1(&update).unwrap());
|
||||
tx.commit();
|
||||
let merged_update = tx.encode_update_v1();
|
||||
Some(())
|
||||
}
|
||||
},
|
||||
Message::Auth(reason) => {
|
||||
Some(())
|
||||
}
|
||||
Message::AwarenessQuery => {
|
||||
Some(())
|
||||
}
|
||||
Message::Awareness(update) => {
|
||||
Some(())
|
||||
}
|
||||
Message::Custom(tag, data) => {
|
||||
Some(())
|
||||
}
|
||||
});
|
||||
}
|
||||
let merged_update = doc_store.doc().transact().encode_update_v1();
|
||||
doc_db
|
||||
.insert(¶meters.id.clone(), &merged_update)
|
||||
.await
|
||||
.ok();
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@ -1,11 +1,14 @@
|
||||
use cloud_database::SqliteDBContext;
|
||||
use jwst::Workspace;
|
||||
use jwst_storage::{BlobAutoStorage, DocAutoStorage};
|
||||
use std::{fs, path::Path};
|
||||
use tauri::api::path::document_dir;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
pub struct AppStateRaw {
|
||||
pub doc_storage: DocAutoStorage,
|
||||
pub doc_db: DocAutoStorage,
|
||||
/// yDoc for receiving yjs update and merge them, before serialize update into sqlite
|
||||
pub doc_store: Workspace,
|
||||
pub blob_storage: BlobAutoStorage,
|
||||
pub metadata_db: SqliteDBContext,
|
||||
}
|
||||
@ -37,7 +40,10 @@ impl AppStateRaw {
|
||||
fs::create_dir_all(affine_document_path.clone()).unwrap();
|
||||
|
||||
Some(Self {
|
||||
doc_storage: DocAutoStorage::init_pool(&doc_db_env).await.unwrap(),
|
||||
doc_db: DocAutoStorage::init_pool(&doc_db_env).await.unwrap(),
|
||||
// with fake id, we only use yDoc inside of it
|
||||
// TODO: use workspace pool, to handle multiple workspace
|
||||
doc_store: Workspace::new(""),
|
||||
blob_storage: BlobAutoStorage::init_pool(&blob_db_env).await.unwrap(),
|
||||
metadata_db: SqliteDBContext::new(metadata_db_env).await,
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user