data migration preparations

This commit is contained in:
Nikita Galaiko 2023-04-12 14:14:39 +02:00
parent 490e696da5
commit 4fc2f424e8
12 changed files with 687 additions and 102 deletions

View File

@ -0,0 +1,38 @@
use crate::{projects, sessions};
use anyhow::{anyhow, Context, Ok, Result};
pub struct Repository {
pub(crate) project_id: String,
pub(crate) git_repository: git2::Repository,
}
impl Repository {
pub fn open(project: &projects::Project) -> Result<Self> {
let git_repository = git2::Repository::open(&project.path)
.with_context(|| format!("{}: failed to open git repository", project.path))?;
Ok(Self {
project_id: project.id.clone(),
git_repository,
})
}
pub fn sessions(&self) -> Result<Vec<sessions::Session>> {
Err(anyhow!("TODO"))
}
pub(crate) fn session_path(&self) -> std::path::PathBuf {
self.git_repository.path().parent().unwrap().join("session")
}
pub(crate) fn deltas_path(&self) -> std::path::PathBuf {
self.session_path().join("deltas")
}
pub(crate) fn wd_path(&self) -> std::path::PathBuf {
self.session_path().join("wd")
}
pub(crate) fn logs_path(&self) -> std::path::PathBuf {
self.git_repository.path().parent().unwrap().join("logs")
}
}

7
src-tauri/src/app/mod.rs Normal file
View File

@ -0,0 +1,7 @@
mod gb_repository;
pub mod reader;
pub mod session;
mod writer;
#[cfg(test)]
mod reader_tests;

117
src-tauri/src/app/reader.rs Normal file
View File

@ -0,0 +1,117 @@
use anyhow::{Context, Result};
use crate::fs;
pub trait Reader {
fn read_to_string(&self, file_path: &str) -> Result<String>;
fn list_files(&self, dir_path: &str) -> Result<Vec<String>>;
}
pub struct WdReader<'reader> {
git_repository: &'reader git2::Repository,
}
impl WdReader<'_> {
pub fn read_to_string(&self, path: &str) -> Result<String> {
let contents =
std::fs::read_to_string(self.git_repository.path().parent().unwrap().join(path))
.with_context(|| format!("{}: not found", path))?;
Ok(contents)
}
}
impl Reader for WdReader<'_> {
fn read_to_string(&self, path: &str) -> Result<String> {
self.read_to_string(path)
}
fn list_files(&self, dir_path: &str) -> Result<Vec<String>> {
let files: Vec<String> =
fs::list_files(self.git_repository.path().parent().unwrap().join(dir_path))?
.iter()
.map(|f| f.to_str().unwrap().to_string())
.filter(|f| !f.starts_with(".git"))
.collect();
Ok(files)
}
}
pub fn get_working_directory_reader(git_repository: &git2::Repository) -> WdReader {
WdReader { git_repository }
}
pub struct CommitReader<'reader> {
repository: &'reader git2::Repository,
commit_oid: git2::Oid,
tree: git2::Tree<'reader>,
}
impl CommitReader<'_> {
pub fn get_commit_oid(&self) -> git2::Oid {
self.commit_oid
}
}
impl Reader for CommitReader<'_> {
fn read_to_string(&self, path: &str) -> Result<String> {
let entry = self
.tree
.get_path(std::path::Path::new(path))
.with_context(|| format!("{}: tree entry not found", path))?;
let blob = self
.repository
.find_blob(entry.id())
.with_context(|| format!("{}: blob not found", entry.id()))?;
let contents = String::from_utf8_lossy(blob.content()).to_string();
Ok(contents)
}
fn list_files(&self, dir_path: &str) -> Result<Vec<String>> {
let mut files: Vec<String> = Vec::new();
let repo_root = self.repository.path().parent().unwrap();
self.tree
.walk(git2::TreeWalkMode::PreOrder, |root, entry| {
if entry.name().is_none() {
return git2::TreeWalkResult::Ok;
}
let abs_dir_path = repo_root.join(dir_path);
let abs_entry_path = repo_root.join(root).join(entry.name().unwrap());
if !abs_entry_path.starts_with(&abs_dir_path) {
return git2::TreeWalkResult::Ok;
}
if abs_dir_path.eq(&abs_entry_path) {
return git2::TreeWalkResult::Ok;
}
if entry.kind() == Some(git2::ObjectType::Tree) {
return git2::TreeWalkResult::Ok;
}
let relpath = abs_entry_path.strip_prefix(abs_dir_path).unwrap();
files.push(relpath.to_str().unwrap().to_string());
git2::TreeWalkResult::Ok
})
.with_context(|| format!("{}: tree walk failed", dir_path))?;
Ok(files)
}
}
pub fn get_commit_reader<'reader>(
repository: &'reader git2::Repository,
commit_oid: git2::Oid,
) -> Result<CommitReader<'reader>> {
let commit = repository
.find_commit(commit_oid)
.with_context(|| format!("{}: commit not found", commit_oid))?;
let tree = commit
.tree()
.with_context(|| format!("{}: tree not found", commit_oid))?;
Ok(CommitReader {
repository,
tree,
commit_oid,
})
}

View File

@ -0,0 +1,130 @@
use super::reader::Reader;
use anyhow::Result;
use tempfile::tempdir;
fn commit(repository: &git2::Repository) -> Result<git2::Oid> {
let mut index = repository.index()?;
index.add_all(&["."], git2::IndexAddOption::DEFAULT, None)?;
index.write()?;
let oid = index.write_tree()?;
let signature = git2::Signature::now("test", "test@email.com").unwrap();
let commit_oid = repository.commit(
Some("HEAD"),
&signature,
&signature,
"some commit",
&repository.find_tree(oid)?,
&[&repository.find_commit(repository.refname_to_id("HEAD")?)?],
)?;
Ok(commit_oid)
}
fn test_repository() -> Result<git2::Repository> {
let path = tempdir()?.path().to_str().unwrap().to_string();
let repository = git2::Repository::init(&path)?;
let mut index = repository.index()?;
let oid = index.write_tree()?;
let signature = git2::Signature::now("test", "test@email.com").unwrap();
repository.commit(
Some("HEAD"),
&signature,
&signature,
"Initial commit",
&repository.find_tree(oid)?,
&[],
)?;
Ok(repository)
}
#[test]
fn test_working_directory_reader_read_file() -> Result<()> {
let repository = test_repository()?;
let file_path = "test.txt";
std::fs::write(&repository.path().parent().unwrap().join(file_path), "test")?;
let reader = super::reader::get_working_directory_reader(&repository);
assert_eq!(reader.read_to_string(&file_path)?, "test");
Ok(())
}
#[test]
fn test_commit_reader_read_file() -> Result<()> {
let repository = test_repository()?;
let file_path = "test.txt";
std::fs::write(&repository.path().parent().unwrap().join(file_path), "test")?;
let oid = commit(&repository)?;
std::fs::write(
&repository.path().parent().unwrap().join(file_path),
"test2",
)?;
let reader = super::reader::get_commit_reader(&repository, oid)?;
assert_eq!(reader.read_to_string(&file_path)?, "test");
Ok(())
}
#[test]
fn test_working_directory_reader_list_files() -> Result<()> {
let repository = test_repository()?;
std::fs::write(
&repository.path().parent().unwrap().join("test.txt"),
"test",
)?;
std::fs::create_dir(&repository.path().parent().unwrap().join("dir"))?;
std::fs::write(
&repository
.path()
.parent()
.unwrap()
.join("dir")
.join("test.txt"),
"test",
)?;
let reader = super::reader::get_working_directory_reader(&repository);
let files = reader.list_files(".")?;
assert_eq!(files.len(), 2);
assert!(files.contains(&"test.txt".to_string()));
assert!(files.contains(&"dir/test.txt".to_string()));
Ok(())
}
#[test]
fn test_commit_reader_list_files() -> Result<()> {
let repository = test_repository()?;
std::fs::write(
&repository.path().parent().unwrap().join("test.txt"),
"test",
)?;
std::fs::create_dir(&repository.path().parent().unwrap().join("dir"))?;
std::fs::write(
&repository
.path()
.parent()
.unwrap()
.join("dir")
.join("test.txt"),
"test",
)?;
let oid = commit(&repository)?;
std::fs::remove_dir_all(&repository.path().parent().unwrap().join("dir"))?;
let reader = super::reader::get_commit_reader(&repository, oid)?;
let files = reader.list_files(".")?;
assert_eq!(files.len(), 2);
assert!(files.contains(&"test.txt".to_string()));
assert!(files.contains(&"dir/test.txt".to_string()));
Ok(())
}

View File

@ -0,0 +1,287 @@
use std::collections::HashMap;
use super::{
gb_repository as repository, reader,
writer::{self, Writer},
};
use crate::{deltas, pty, sessions};
use anyhow::{anyhow, Context, Result};
pub struct SessionWriter<'writer> {
repository: &'writer repository::Repository,
writer: Box<dyn writer::Writer + 'writer>,
}
impl<'writer> SessionWriter<'writer> {
pub fn open(
repository: &'writer repository::Repository,
session: &'writer sessions::Session,
) -> Result<Self> {
let reader = reader::get_working_directory_reader(&repository.git_repository);
let current_session_id = reader.read_to_string(
repository
.session_path()
.join("meta")
.join("id")
.to_str()
.unwrap(),
);
if current_session_id.is_ok() && !current_session_id.as_ref().unwrap().eq(&session.id) {
return Err(anyhow!(
"{}: can not open writer for {} because a writer for {} is still open",
repository.project_id,
session.id,
current_session_id.unwrap()
));
}
let writer = writer::get_working_directory_writer(&repository.git_repository);
writer
.write_string(
repository
.session_path()
.join("meta")
.join("last")
.to_str()
.unwrap(),
&session.meta.last_timestamp_ms.to_string(),
)
.with_context(|| "failed to write last timestamp")?;
if current_session_id.is_ok() && current_session_id.as_ref().unwrap().eq(&session.id) {
let writer = SessionWriter {
repository: &repository,
writer: Box::new(writer),
};
return Ok(writer);
}
writer
.write_string(
repository
.session_path()
.join("meta")
.join("id")
.to_str()
.unwrap(),
session.id.as_str(),
)
.with_context(|| "failed to write id")?;
writer
.write_string(
repository
.session_path()
.join("meta")
.join("start")
.to_str()
.unwrap(),
session.meta.start_timestamp_ms.to_string().as_str(),
)
.with_context(|| "failed to write start timestamp")?;
if let Some(branch) = session.meta.branch.as_ref() {
writer
.write_string(
repository
.session_path()
.join("meta")
.join("branch")
.to_str()
.unwrap(),
branch,
)
.with_context(|| "failed to write branch")?;
}
if let Some(commit) = session.meta.commit.as_ref() {
writer
.write_string(
repository
.session_path()
.join("meta")
.join("commit")
.to_str()
.unwrap(),
commit,
)
.with_context(|| "failed to write commit")?;
}
let writer = SessionWriter {
repository: &repository,
writer: Box::new(writer),
};
Ok(writer)
}
pub fn append_pty(&self, record: &pty::Record) -> Result<()> {
log::info!(
"{}: writing pty record to pty.jsonl",
self.repository.project_id
);
serde_json::to_string(record)?;
serde_jsonlines::append_json_lines(
&self.repository.session_path().join("pty.jsonl"),
[record],
)?;
Ok(())
}
pub fn write_logs<P: AsRef<std::path::Path>>(&self, path: P, contents: &str) -> Result<()> {
let path = path.as_ref();
log::info!(
"{}: writing logs to {}",
self.repository.project_id,
path.display()
);
self.writer.write_string(
&self.repository.logs_path().join(path).to_str().unwrap(),
contents,
)?;
Ok(())
}
pub fn write_file<P: AsRef<std::path::Path>>(&self, path: P, contents: &str) -> Result<()> {
let path = path.as_ref();
log::info!(
"{}: writing file to {}",
self.repository.project_id,
path.display()
);
self.writer.write_string(
&self.repository.wd_path().join(path).to_str().unwrap(),
contents,
)?;
Ok(())
}
pub fn write_deltas<P: AsRef<std::path::Path>>(
&self,
path: P,
deltas: Vec<deltas::Delta>,
) -> Result<()> {
let path = path.as_ref();
log::info!(
"{}: writing deltas to {}",
self.repository.project_id,
path.display()
);
let raw_deltas = serde_json::to_string(&deltas)?;
self.writer.write_string(
&self.repository.deltas_path().join(path).to_str().unwrap(),
&raw_deltas,
)?;
Ok(())
}
}
pub struct SessionReader<'reader> {
repository: &'reader repository::Repository,
reader: Box<dyn reader::Reader + 'reader>,
}
impl<'reader> SessionReader<'reader> {
pub fn open(
repository: &'reader repository::Repository,
session: sessions::Session,
) -> Result<Self> {
let wd_reader = reader::get_working_directory_reader(&repository.git_repository);
let current_session_id = wd_reader.read_to_string(
&repository
.session_path()
.join("meta")
.join("id")
.to_str()
.unwrap(),
);
if current_session_id.is_ok() && current_session_id.as_ref().unwrap() == &session.id {
return Ok(SessionReader {
reader: Box::new(wd_reader),
repository,
});
}
let session_hash = if let Some(hash) = session.hash {
hash
} else {
return Err(anyhow!(
"can not open reader for {} because it has no commit hash nor it is a current session",
session.id
));
};
let oid = git2::Oid::from_str(&session_hash)
.with_context(|| format!("failed to parse commit hash {}", session_hash))?;
let commit_reader = reader::get_commit_reader(&repository.git_repository, oid)?;
Ok(SessionReader {
reader: Box::new(commit_reader),
repository,
})
}
pub fn files(&self, paths: Option<Vec<&str>>) -> Result<HashMap<String, String>> {
let files = self
.reader
.list_files(&self.repository.wd_path().to_str().unwrap())?;
let files_with_content = files
.iter()
.filter(|file| {
if let Some(paths) = paths.as_ref() {
paths.iter().any(|path| file.starts_with(path))
} else {
true
}
})
.map(|file| {
let content = self
.reader
.read_to_string(&self.repository.wd_path().join(file).to_str().unwrap())
.unwrap();
(file.to_string(), content)
})
.collect();
Ok(files_with_content)
}
pub fn deltas(&self, paths: Option<Vec<&str>>) -> Result<HashMap<String, Vec<deltas::Delta>>> {
let files = self
.reader
.list_files(&self.repository.deltas_path().to_str().unwrap())?;
let files_with_content = files
.iter()
.filter(|file| {
if let Some(paths) = paths.as_ref() {
paths.iter().any(|path| file.starts_with(path))
} else {
true
}
})
.map(|file| {
let content = self
.reader
.read_to_string(&self.repository.deltas_path().join(file).to_str().unwrap())
.unwrap();
let deltas: Vec<deltas::Delta> = serde_json::from_str(&content).unwrap();
(file.to_string(), deltas)
})
.collect();
Ok(files_with_content)
}
}

View File

@ -0,0 +1,38 @@
use anyhow::{Context, Result};
use std::io::Write;
pub trait Writer {
fn write_string(&self, path: &str, contents: &str) -> Result<()>;
fn append_string(&self, path: &str, contents: &str) -> Result<()>;
}
pub struct WdWriter<'writer> {
git_repository: &'writer git2::Repository,
}
pub fn get_working_directory_writer<'writer>(
git_repository: &'writer git2::Repository,
) -> WdWriter {
WdWriter { git_repository }
}
impl Writer for WdWriter<'_> {
fn write_string(&self, path: &str, contents: &str) -> Result<()> {
let file_path = self.git_repository.path().parent().unwrap().join(path);
let dir_path = file_path.parent().unwrap();
std::fs::create_dir_all(dir_path)?;
std::fs::write(path, contents)?;
Ok(())
}
fn append_string(&self, path: &str, contents: &str) -> Result<()> {
let file_path = self.git_repository.path().parent().unwrap().join(path);
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(file_path)
.with_context(|| format!("failed to open file: {}", path))?;
file.write_all(contents.as_bytes())?;
Ok(())
}
}

View File

@ -1,3 +1,4 @@
pub mod activity;
#[cfg(test)]
mod activity_tests;

View File

@ -1,3 +1,4 @@
mod app;
mod deltas;
mod events;
mod fs;

View File

@ -3,3 +3,4 @@ mod recorder;
mod connection;
pub use server::start_server;
pub use recorder::Record;

View File

@ -1,8 +1,7 @@
use std::path::Path;
use crate::git::activity;
use crate::{app::reader, git::activity};
use anyhow::{anyhow, Context, Result};
use serde::Serialize;
use std::path::Path;
#[derive(Debug, Clone, Serialize, PartialEq)]
#[serde(rename_all = "camelCase")]
@ -24,88 +23,61 @@ pub struct Session {
// if hash is not set, the session is not saved aka current
pub hash: Option<String>,
pub meta: Meta,
// TODO: make this a method instead
pub activity: Vec<activity::Activity>,
}
impl Session {
pub fn from_commit(repo: &git2::Repository, commit: &git2::Commit) -> Result<Self> {
let tree = commit.tree().with_context(|| {
format!("failed to get tree from commit {}", commit.id().to_string())
})?;
impl<'reader> TryFrom<Box<dyn reader::Reader + 'reader>> for Session {
type Error = anyhow::Error;
let start_timestamp_ms = read_as_string(repo, &tree, Path::new("session/meta/start"))?
fn try_from(reader: Box<dyn reader::Reader + 'reader>) -> Result<Self, Self::Error> {
let id = reader
.read_to_string("session/meta/id")
.with_context(|| "failed to read session id")?;
let start_timestamp_ms = reader
.read_to_string("session/meta/start")
.with_context(|| "failed to read session start timestamp")?
.parse::<u128>()
.with_context(|| {
format!(
"failed to parse start timestamp from commit {}",
commit.id().to_string()
)
})?;
.with_context(|| "failed to parse session start timestamp")?;
let last_timestamp_ms = reader
.read_to_string("session/meta/last")
.with_context(|| "failed to read session last timestamp")?
.parse::<u128>()
.with_context(|| "failed to parse session last timestamp")?;
let branch = reader.read_to_string("session/meta/branch");
let commit = reader.read_to_string("session/meta/commit");
let logs_path = Path::new("logs/HEAD");
let activity = match tree.get_path(logs_path).is_ok() {
true => read_as_string(repo, &tree, logs_path)
.with_context(|| {
format!(
"failed to read reflog from commit {}",
commit.id().to_string()
)
})?
.lines()
.filter_map(|line| activity::parse_reflog_line(line).ok())
.filter(|activity| activity.timestamp_ms >= start_timestamp_ms)
.collect::<Vec<activity::Activity>>(),
false => Vec::new(),
};
let branch_path = Path::new("session/meta/branch");
let session_branch = match tree.get_path(branch_path).is_ok() {
true => read_as_string(repo, &tree, branch_path)
.with_context(|| {
format!(
"failed to read branch name from commit {}",
commit.id().to_string()
)
})?
.into(),
false => None,
};
let commit_path = Path::new("session/meta/commit");
let session_commit = match tree.get_path(commit_path).is_ok() {
true => read_as_string(repo, &tree, commit_path)
.with_context(|| {
format!(
"failed to read branch name from commit {}",
commit.id().to_string()
)
})?
.into(),
false => None,
};
Ok(Session {
id: read_as_string(repo, &tree, Path::new("session/meta/id")).with_context(|| {
format!(
"failed to read session id from commit {}",
commit.id().to_string()
)
})?,
hash: Some(commit.id().to_string()),
Ok(Self {
id,
hash: None,
meta: Meta {
start_timestamp_ms,
last_timestamp_ms: read_as_string(repo, &tree, Path::new("session/meta/last"))?
.parse::<u128>()
.with_context(|| {
format!(
"failed to parse last timestamp from commit {}",
commit.id().to_string()
)
})?,
branch: session_branch,
commit: session_commit,
last_timestamp_ms,
branch: if branch.is_err() {
None
} else {
Some(branch.unwrap())
},
commit: if commit.is_err() {
None
} else {
Some(commit.unwrap())
},
},
activity,
activity: vec![],
})
}
}
impl<'reader> TryFrom<reader::CommitReader<'reader>> for Session {
type Error = anyhow::Error;
fn try_from(reader: reader::CommitReader<'reader>) -> Result<Self, Self::Error> {
let commit_oid = reader.get_commit_oid().to_string();
let session = Session::try_from(Box::new(reader) as Box<dyn reader::Reader + 'reader>)?;
Ok(Session {
hash: Some(commit_oid),
..session
})
}
}

View File

@ -1,11 +1,10 @@
use crate::{app::reader, projects, users};
use anyhow::Result;
use std::{
collections::HashMap,
path::Path,
sync::{Arc, Mutex},
};
use crate::{projects, users};
use anyhow::Result;
use tempfile::tempdir;
fn test_user() -> users::User {
@ -219,23 +218,23 @@ fn test_flush_with_user() {
}
#[test]
fn test_get_persistent() {
fn test_get_persistent() -> Result<()> {
let (repo, project) = test_project().unwrap();
let store = super::Store::new(Arc::new(Mutex::new(clone_repo(&repo))), project.clone());
let created_session = store.create_current();
assert!(created_session.is_ok());
let mut created_session = created_session.unwrap();
let mut created_session = store.create_current()?;
created_session = store.flush(&created_session, None).unwrap();
created_session = store.flush(&created_session, None)?;
let commid_oid = git2::Oid::from_str(&created_session.hash.as_ref().unwrap()).unwrap();
let commit = repo.find_commit(commid_oid).unwrap();
let commit_oid = git2::Oid::from_str(&created_session.hash.as_ref().unwrap())?;
let reconstructed = super::sessions::Session::from_commit(&repo, &commit);
let reader = reader::get_commit_reader(&repo, commit_oid)?;
let reconstructed = super::sessions::Session::try_from(reader);
assert!(reconstructed.is_ok());
let reconstructed = reconstructed.unwrap();
assert_eq!(reconstructed, created_session);
Ok(())
}
fn clone_repo(repo: &git2::Repository) -> git2::Repository {

View File

@ -1,4 +1,7 @@
use crate::{fs, projects, sessions, users};
use crate::{
app::reader::{self, Reader},
fs, projects, sessions, users,
};
use anyhow::{Context, Result};
use filetime::FileTime;
use sha2::{Digest, Sha256};
@ -54,12 +57,9 @@ impl Store {
walker.set_sorting(git2::Sort::TIME)?;
for commit_id in walker {
let commit = git_repository.find_commit(commit_id?)?;
if sessions::id_from_commit(&git_repository, &commit)? == session_id {
return Ok(Some(sessions::Session::from_commit(
&git_repository,
&commit,
)?));
let reader = reader::get_commit_reader(&git_repository, commit_id?)?;
if reader.read_to_string("session/meta/id")? == session_id {
return Ok(Some(sessions::Session::try_from(reader)?));
}
}
@ -206,14 +206,8 @@ impl Store {
let mut sessions: Vec<sessions::Session> = vec![];
for id in walker {
let id = id?;
let commit = git_repository.find_commit(id).with_context(|| {
format!(
"failed to find commit {} in repository {}",
id.to_string(),
git_repository.path().display()
)
})?;
let session = sessions::Session::from_commit(&git_repository, &commit)?;
let reader = reader::get_commit_reader(&git_repository, id)?;
let session = sessions::Session::try_from(reader)?;
sessions.push(session);
}