zed-industries/zed
1
1
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2024-09-19 02:17:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

working on initialization + index breakup

Browse Source
This commit is contained in:
KCaverly 2023-08-22 14:45:27 +02:00
parent ced2b2aec3
commit aabdfa210f
2 changed files with 197 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/search/src/project_search.rs
View File

@ -849,7 +849,7 @@ impl ProjectSearchView {
let model = model.read(cx);
project = model.project.clone();
SemanticIndex::global(cx).map(|semantic| {
semantic.update(cx, |this, cx| this.initialize_project(project, cx))
semantic.update(cx, |this, cx| this.initialize_project(project.clone(), cx));
});
}

308
crates/semantic_index/src/semantic_index.rs
View File

@ -96,6 +96,7 @@ struct ProjectState {
subscription: gpui::Subscription,
outstanding_job_count_rx: watch::Receiver<usize>,
_outstanding_job_count_tx: Arc<Mutex<watch::Sender<usize>>>,
queue: HashMap<WorktreeId, Vec<IndexOperation>>,
}
#[derive(Clone)]
@ -130,9 +131,25 @@ impl ProjectState {
outstanding_job_count_rx,
_outstanding_job_count_tx,
subscription,
queue: HashMap::new(),
}
}
fn add_to_queue(&mut self, worktree_id: WorktreeId, operation: IndexOperation) {
if let Some(worktree_queue) = self.queue.get_mut(&worktree_id) {
worktree_queue.push(operation);
} else {
self.queue.insert(worktree_id, vec![operation]);
}
}
fn pop(&mut self) -> Option<IndexOperation> {
self.queue
.iter_mut()
.next()
.and_then(|(_, mut entry)| entry.pop())
}
fn db_id_for_worktree_id(&self, id: WorktreeId) -> Option<i64> {
self.worktree_db_ids
.iter()
@ -158,6 +175,7 @@ impl ProjectState {
}
}
#[derive(Clone)]
pub struct PendingFile {
worktree_db_id: i64,
relative_path: PathBuf,
@ -167,6 +185,12 @@ pub struct PendingFile {
job_handle: JobHandle,
}
#[derive(Clone)]
enum IndexOperation {
IndexFile { file: PendingFile },
DeleteFile { file: PendingFile },
}
pub struct SearchResult {
pub buffer: ModelHandle<Buffer>,
pub range: Range<Anchor>,
@ -628,102 +652,12 @@ impl SemanticIndex {
}
});
cx.spawn(|this, mut cx| async move {
futures::future::join_all(worktree_scans_complete).await;
let worktree_db_ids = futures::future::join_all(worktree_db_ids).await;
let worktrees = project.read_with(&cx, |project, cx| {
project
.worktrees(cx)
.map(|worktree| worktree.read(cx).snapshot())
.collect::<Vec<_>>()
});
let mut worktree_file_mtimes = HashMap::new();
let mut db_ids_by_worktree_id = HashMap::new();
for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) {
let db_id = db_id?;
db_ids_by_worktree_id.insert(worktree.id(), db_id);
worktree_file_mtimes.insert(
worktree.id(),
this.read_with(&cx, |this, _| this.get_file_mtimes(db_id))
.await?,
);
}
let worktree_db_ids = db_ids_by_worktree_id
.iter()
.map(|(a, b)| (*a, *b))
.collect();
let (job_count_tx, job_count_rx) = watch::channel_with(0);
let job_count_tx = Arc::new(Mutex::new(job_count_tx));
this.update(&mut cx, |this, _| {
let project_state = ProjectState::new(
_subscription,
worktree_db_ids,
worktree_file_mtimes.clone(),
job_count_rx,
job_count_tx,
);
this.projects.insert(project.downgrade(), project_state);
});
anyhow::Ok(())
})
.detach_and_log_err(cx)
}
pub fn index_project(
&mut self,
project: ModelHandle<Project>,
cx: &mut ModelContext<Self>,
) -> Task<Result<(usize, watch::Receiver<usize>)>> {
let t0 = Instant::now();
let worktree_scans_complete = project
.read(cx)
.worktrees(cx)
.map(|worktree| {
let scan_complete = worktree.read(cx).as_local().unwrap().scan_complete();
async move {
scan_complete.await;
}
})
.collect::<Vec<_>>();
let worktree_db_ids = project
.read(cx)
.worktrees(cx)
.map(|worktree| {
self.find_or_create_worktree(worktree.read(cx).abs_path().to_path_buf())
})
.collect::<Vec<_>>();
let language_registry = self.language_registry.clone();
let db_update_tx = self.db_update_tx.clone();
let parsing_files_tx = self.parsing_files_tx.clone();
let state = self.projects.get(&project.downgrade());
let state = if state.is_none() {
return Task::Ready(Some(Err(anyhow!("Project not yet initialized"))));
} else {
state.unwrap()
};
let state = state.clone().to_owned();
let _subscription = cx.subscribe(&project, |this, project, event, _cx| {
if let project::Event::WorktreeUpdatedEntries(worktree_id, changes) = event {
todo!();
// this.project_entries_changed(project, changes, cx, worktree_id);
}
});
cx.spawn(|this, mut cx| async move {
futures::future::join_all(worktree_scans_complete).await;
let worktree_db_ids = futures::future::join_all(worktree_db_ids).await;
let worktrees = project.read_with(&cx, |project, cx| {
project
.worktrees(cx)
@ -733,6 +667,7 @@ impl SemanticIndex {
let mut worktree_file_mtimes = HashMap::new();
let mut db_ids_by_worktree_id = HashMap::new();
for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) {
let db_id = db_id?;
db_ids_by_worktree_id.insert(worktree.id(), db_id);
@ -761,10 +696,12 @@ impl SemanticIndex {
this.projects.insert(project.downgrade(), project_state);
});
cx.background()
let worktree_files = cx
.background()
.spawn(async move {
let mut count = 0;
let mut worktree_files = HashMap::new();
for worktree in worktrees.into_iter() {
let mut candidate_files = Vec::new();
let mut file_mtimes = worktree_file_mtimes.remove(&worktree.id()).unwrap();
for file in worktree.files(false, 0) {
let absolute_path = worktree.absolutize(&file.path);
@ -773,6 +710,7 @@ impl SemanticIndex {
.language_for_file(&absolute_path, None)
.await
{
// Test if file is valid parseable file
if !PARSEABLE_ENTIRE_FILE_TYPES.contains(&language.name().as_ref())
&& &language.name().as_ref() != &"Markdown"
&& language
@ -789,40 +727,186 @@ impl SemanticIndex {
.map_or(false, |existing_mtime| existing_mtime == file.mtime);
if !already_stored {
count += 1;
let job_handle = JobHandle::new(&job_count_tx);
parsing_files_tx
.try_send(PendingFile {
candidate_files.push(IndexOperation::IndexFile {
file: PendingFile {
worktree_db_id: db_ids_by_worktree_id[&worktree.id()],
relative_path: path_buf,
absolute_path,
language,
job_handle,
modified_time: file.mtime,
})
.unwrap();
},
});
}
}
}
for file in file_mtimes.keys() {
db_update_tx
.try_send(DbOperation::Delete {
worktree_id: db_ids_by_worktree_id[&worktree.id()],
path: file.to_owned(),
})
.unwrap();
}
worktree_files.insert(worktree.id(), candidate_files);
}
log::trace!(
"walking worktree took {:?} milliseconds",
t0.elapsed().as_millis()
);
anyhow::Ok((count, job_count_rx))
anyhow::Ok(worktree_files)
})
.await
.await?;
this.update(&mut cx, |this, cx| {
if let Some(project_state) = this.projects.get_mut(&project.downgrade()) {
for (worktree_id, index_operations) in &worktree_files {
for op in index_operations {
project_state.add_to_queue(*worktree_id, op.clone());
}
}
}
});
cx.background().spawn(async move { anyhow::Ok(()) }).await
})
.detach_and_log_err(cx)
}
pub fn index_project(
&mut self,
project: ModelHandle<Project>,
cx: &mut ModelContext<Self>,
) -> Task<Result<(usize, watch::Receiver<usize>)>> {
let state = self.projects.get_mut(&project.downgrade());
let state = if state.is_none() {
return Task::Ready(Some(Err(anyhow!("Project not yet initialized"))));
} else {
state.unwrap()
};
let parsing_files_tx = self.parsing_files_tx.clone();
let db_update_tx = self.db_update_tx.clone();
let job_count_rx = state.outstanding_job_count_rx.clone();
let count = state.queue.values().map(Vec::len).sum();
cx.spawn(|this, mut cx| async move {
this.update(&mut cx, |this, cx| {
let Some(mut state) = this.projects.get_mut(&project.downgrade()) else {
return;
};
let Some(mut index_operation) = state.pop() else { return;};
let _ = match index_operation {
IndexOperation::IndexFile { file } => {
parsing_files_tx.try_send(file);
}
IndexOperation::DeleteFile { file } => {
db_update_tx.try_send(DbOperation::Delete {
worktree_id: file.worktree_db_id,
path: file.relative_path,
});
}
};
});
})
.detach();
Task::Ready(Some(Ok((count, job_count_rx))))
// cx.spawn(|this, mut cx| async move {
// futures::future::join_all(worktree_scans_complete).await;
// let worktree_db_ids = futures::future::join_all(worktree_db_ids).await;
// let worktrees = project.read_with(&cx, |project, cx| {
// project
// .worktrees(cx)
// .map(|worktree| worktree.read(cx).snapshot())
// .collect::<Vec<_>>()
// });
// let mut worktree_file_mtimes = HashMap::new();
// let mut db_ids_by_worktree_id = HashMap::new();
// for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) {
// let db_id = db_id?;
// db_ids_by_worktree_id.insert(worktree.id(), db_id);
// worktree_file_mtimes.insert(
// worktree.id(),
// this.read_with(&cx, |this, _| this.get_file_mtimes(db_id))
// .await?,
// );
// }
// let worktree_db_ids = db_ids_by_worktree_id
// .iter()
// .map(|(a, b)| (*a, *b))
// .collect();
// let (job_count_tx, job_count_rx) = watch::channel_with(0);
// let job_count_tx = Arc::new(Mutex::new(job_count_tx));
// this.update(&mut cx, |this, _| {
// let project_state = ProjectState::new(
// _subscription,
// worktree_db_ids,
// worktree_file_mtimes.clone(),
// job_count_rx.clone(),
// job_count_tx.clone(),
// );
// this.projects.insert(project.downgrade(), project_state);
// });
// cx.background()
// .spawn(async move {
// let mut count = 0;
// for worktree in worktrees.into_iter() {
// let mut file_mtimes = worktree_file_mtimes.remove(&worktree.id()).unwrap();
// for file in worktree.files(false, 0) {
// let absolute_path = worktree.absolutize(&file.path);
// if let Ok(language) = language_registry
// .language_for_file(&absolute_path, None)
// .await
// {
// if !PARSEABLE_ENTIRE_FILE_TYPES.contains(&language.name().as_ref())
// && &language.name().as_ref() != &"Markdown"
// && language
// .grammar()
// .and_then(|grammar| grammar.embedding_config.as_ref())
// .is_none()
// {
// continue;
// }
// let path_buf = file.path.to_path_buf();
// let stored_mtime = file_mtimes.remove(&file.path.to_path_buf());
// let already_stored = stored_mtime
// .map_or(false, |existing_mtime| existing_mtime == file.mtime);
// if !already_stored {
// count += 1;
// let job_handle = JobHandle::new(&job_count_tx);
// parsing_files_tx
// .try_send(PendingFile {
// worktree_db_id: db_ids_by_worktree_id[&worktree.id()],
// relative_path: path_buf,
// absolute_path,
// language,
// job_handle,
// modified_time: file.mtime,
// })
// .unwrap();
// }
// }
// }
// for file in file_mtimes.keys() {
// db_update_tx
// .try_send(DbOperation::Delete {
// worktree_id: db_ids_by_worktree_id[&worktree.id()],
// path: file.to_owned(),
// })
// .unwrap();
// }
// }
// log::trace!(
// "walking worktree took {:?} milliseconds",
// t0.elapsed().as_millis()
// );
// anyhow::Ok((count, job_count_rx))
// })
// .await
// })
}
pub fn outstanding_job_count_rx(