tweag/nickel
1
1
Fork 0
mirror of https://github.com/tweag/nickel.git synced 2024-09-11 11:47:03 +03:00
Code Issues Projects Releases Wiki Activity

Nuke the linearizer (#1658)

Browse Source 
* Convert hover

* WIP

* Slight cleanups

* Rename LinRegistry
This commit is contained in:
jneem 2023-10-02 09:29:14 -05:00 committed by GitHub
parent c1798cd52e
commit 432983cd10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 352 additions and 2806 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
core/src/term/mod.rs
View File

@ -1747,6 +1747,22 @@ pub trait Traverse<T>: Sized {
f: &mut dyn FnMut(&T, &S) -> TraverseControl<S, U>,
scope: &S,
) -> Option<U>;
fn find_map<S>(&self, mut pred: impl FnMut(&T) -> Option<S>) -> Option<S>
where
T: Clone,
{
self.traverse_ref(
&mut |t, _state: &()| {
if let Some(s) = pred(t) {
TraverseControl::Return(s)
} else {
TraverseControl::Continue
}
},
&(),
)
}
}
impl Traverse<RichTerm> for RichTerm {

11
core/src/typ.rs
View File

@ -1059,13 +1059,10 @@ impl Type {
/// Searches for a `TypeF::Flat`. If one is found, returns the term it contains.
pub fn find_flat(&self) -> Option<RichTerm> {
self.traverse_ref(
&mut |ty: &Type, _: &()| match &ty.typ {
TypeF::Flat(f) => TraverseControl::Return(f.clone()),
_ => TraverseControl::Continue,
},
&(),
)
self.find_map(|ty: &Type| match &ty.typ {
TypeF::Flat(f) => Some(f.clone()),
_ => None,
})
}
}

1
lsp/nls/Cargo.toml
View File

@ -17,7 +17,6 @@ path = "src/main.rs"
[features]
default = ["format"]
format = ["nickel-lang-core/format"]
old-completer = []
[build-dependencies]
lalrpop.workspace = true

262
lsp/nls/src/analysis.rs Normal file
View File

@ -0,0 +1,262 @@
use std::collections::HashMap;
use codespan::FileId;
use nickel_lang_core::{
term::{RichTerm, Traverse, TraverseControl},
typ::{Type, TypeF},
typecheck::{linearization::Linearizer, reporting::NameReg, Extra, UnifType},
};
use crate::{
field_walker::DefWithPath,
identifier::LocIdent,
position::PositionLookup,
term::RichTermPtr,
usage::{Environment, UsageLookup},
};
#[derive(Default, Debug)]
pub struct ParentLookup {
table: HashMap<RichTermPtr, RichTerm>,
}
impl ParentLookup {
pub fn new(rt: &RichTerm) -> Self {
let mut table = HashMap::new();
let mut traverse_merge =
|rt: &RichTerm, parent: &Option<RichTerm>| -> TraverseControl<Option<RichTerm>, ()> {
if let Some(parent) = parent {
table.insert(RichTermPtr(rt.clone()), parent.clone());
}
TraverseControl::ContinueWithScope(Some(rt.clone()))
};
rt.traverse_ref(&mut traverse_merge, &None);
ParentLookup { table }
}
pub fn parent(&self, rt: &RichTerm) -> Option<&RichTerm> {
self.table.get(&RichTermPtr(rt.clone()))
}
pub fn parent_chain<'a>(&'a self, rt: &'a RichTerm) -> ParentChainIter<'_> {
ParentChainIter {
table: self,
next: Some(rt),
}
}
}
pub struct ParentChainIter<'a> {
table: &'a ParentLookup,
next: Option<&'a RichTerm>,
}
impl<'a> Iterator for ParentChainIter<'a> {
type Item = &'a RichTerm;
fn next(&mut self) -> Option<Self::Item> {
if let Some(next) = self.next {
self.next = self.table.parent(next);
Some(next)
} else {
None
}
}
}
/// The initial analysis that we collect for a file.
///
/// This analysis is re-collected from scratch each time the file is updated.
#[derive(Default, Debug)]
pub struct Analysis {
pub position_lookup: PositionLookup,
pub usage_lookup: UsageLookup,
pub parent_lookup: ParentLookup,
pub type_lookup: CollectedTypes<Type>,
}
impl Analysis {
pub fn new(
term: &RichTerm,
type_lookup: CollectedTypes<Type>,
initial_env: &Environment,
) -> Self {
Self {
position_lookup: PositionLookup::new(term),
usage_lookup: UsageLookup::new(term, initial_env),
parent_lookup: ParentLookup::new(term),
type_lookup,
}
}
}
/// The collection of analyses for every file that we know about.
#[derive(Default, Debug)]
pub struct AnalysisRegistry {
// Most of the fields of `Analysis` are themselves hash tables. Having
// a table of tables requires more lookups than necessary, but it makes
// it easy to invalidate a whole file.
pub analysis: HashMap<FileId, Analysis>,
}
impl AnalysisRegistry {
pub fn insert(
&mut self,
file_id: FileId,
type_lookups: CollectedTypes<Type>,
term: &RichTerm,
initial_env: &crate::usage::Environment,
) {
self.analysis
.insert(file_id, Analysis::new(term, type_lookups, initial_env));
}
/// Inserts a new file into the analysis, but only generates usage analysis for it.
///
/// This is useful for temporary little pieces of input (like parts extracted from incomplete input)
/// that need variable resolution but not the full analysis.
pub fn insert_usage(&mut self, file_id: FileId, term: &RichTerm, initial_env: &Environment) {
self.analysis.insert(
file_id,
Analysis {
usage_lookup: UsageLookup::new(term, initial_env),
..Default::default()
},
);
}
pub fn remove(&mut self, file_id: FileId) {
self.analysis.remove(&file_id);
}
pub fn get_def(&self, ident: &LocIdent) -> Option<&DefWithPath> {
let file = ident.pos.as_opt_ref()?.src_id;
self.analysis.get(&file)?.usage_lookup.def(ident)
}
pub fn get_usages(&self, ident: &LocIdent) -> impl Iterator<Item = &LocIdent> {
fn inner<'a>(
slf: &'a AnalysisRegistry,
ident: &LocIdent,
) -> Option<impl Iterator<Item = &'a LocIdent>> {
let file = ident.pos.as_opt_ref()?.src_id;
Some(slf.analysis.get(&file)?.usage_lookup.usages(ident))
}
inner(self, ident).into_iter().flatten()
}
pub fn get_env(&self, rt: &RichTerm) -> Option<&crate::usage::Environment> {
let file = rt.pos.as_opt_ref()?.src_id;
self.analysis.get(&file)?.usage_lookup.env(rt)
}
pub fn get_type(&self, rt: &RichTerm) -> Option<&Type> {
let file = rt.pos.as_opt_ref()?.src_id;
self.analysis
.get(&file)?
.type_lookup
.terms
.get(&RichTermPtr(rt.clone()))
}
pub fn get_type_for_ident(&self, id: &LocIdent) -> Option<&Type> {
let file = id.pos.as_opt_ref()?.src_id;
self.analysis.get(&file)?.type_lookup.idents.get(id)
}
pub fn get_parent_chain<'a>(&'a self, rt: &'a RichTerm) -> Option<ParentChainIter<'a>> {
let file = rt.pos.as_opt_ref()?.src_id;
Some(self.analysis.get(&file)?.parent_lookup.parent_chain(rt))
}
}
#[derive(Default)]
pub struct TypeCollector {
// Store a copy of the terms we've added so far. The index in this array is their ItemId.
term_ids: Vec<RichTermPtr>,
}
#[derive(Clone, Debug)]
pub struct CollectedTypes<Ty> {
pub terms: HashMap<RichTermPtr, Ty>,
pub idents: HashMap<LocIdent, Ty>,
}
impl<Ty> Default for CollectedTypes<Ty> {
fn default() -> Self {
Self {
terms: HashMap::new(),
idents: HashMap::new(),
}
}
}
impl Linearizer for TypeCollector {
type Building = CollectedTypes<UnifType>;
type Completed = CollectedTypes<Type>;
type CompletionExtra = Extra;
type ItemId = usize;
fn scope(&mut self) -> Self {
TypeCollector::default()
}
fn scope_meta(&mut self) -> Self {
TypeCollector::default()
}
fn add_term(&mut self, lin: &mut Self::Building, rt: &RichTerm, ty: UnifType) -> Option<usize> {
self.term_ids.push(RichTermPtr(rt.clone()));
lin.terms.insert(RichTermPtr(rt.clone()), ty);
Some(self.term_ids.len() - 1)
}
fn complete(
self,
lin: Self::Building,
Extra {
table,
names,
wildcards,
}: &Extra,
) -> Self::Completed {
let mut name_reg = NameReg::new(names.clone());
let mut transform_type = |uty: UnifType| -> Type {
let ty = name_reg.to_type(table, uty);
match ty.typ {
TypeF::Wildcard(i) => wildcards.get(i).unwrap_or(&ty).clone(),
_ => ty,
}
};
let terms = lin
.terms
.into_iter()
.map(|(rt, uty)| (rt, transform_type(uty)))
.collect();
let idents = lin
.idents
.into_iter()
.map(|(id, uty)| (id, transform_type(uty)))
.collect();
CollectedTypes { terms, idents }
}
fn retype(&mut self, lin: &mut Self::Building, item_id: Option<usize>, new_type: UnifType) {
if let Some(id) = item_id {
lin.terms.insert(self.term_ids[id].clone(), new_type);
}
}
fn retype_ident(
&mut self,
lin: &mut Self::Building,
ident: &nickel_lang_core::identifier::LocIdent,
new_type: UnifType,
) {
lin.idents.insert((*ident).into(), new_type);
}
}

66
lsp/nls/src/cache.rs
View File

@ -1,8 +1,6 @@
use std::collections::HashMap;
use codespan::{ByteIndex, FileId};
use lsp_types::TextDocumentPositionParams;
use nickel_lang_core::position::TermPos;
use nickel_lang_core::term::{RichTerm, Term, Traverse};
use nickel_lang_core::{
cache::{Cache, CacheError, CacheOp, EntryState, SourcePath, TermEntry},
error::{Error, ImportError},
@ -10,17 +8,15 @@ use nickel_lang_core::{
typecheck::{self},
};
use crate::linearization::{building::Building, AnalysisHost, Environment, LinRegistry};
use crate::linearization::{CollectedTypes, CombinedLinearizer, TypeCollector};
use crate::analysis::{AnalysisRegistry, CollectedTypes, TypeCollector};
pub trait CacheExt {
fn typecheck_with_analysis(
&mut self,
file_id: FileId,
initial_ctxt: &typecheck::Context,
initial_env: &Environment,
initial_term_env: &crate::usage::Environment,
lin_registry: &mut LinRegistry,
registry: &mut AnalysisRegistry,
) -> Result<CacheOp<()>, CacheError<Vec<Error>>>;
fn position(&self, lsp_pos: &TextDocumentPositionParams)
@ -32,9 +28,8 @@ impl CacheExt for Cache {
&mut self,
file_id: FileId,
initial_ctxt: &typecheck::Context,
initial_env: &Environment,
initial_term_env: &crate::usage::Environment,
lin_registry: &mut LinRegistry,
registry: &mut AnalysisRegistry,
) -> Result<CacheOp<()>, CacheError<Vec<Error>>> {
if !self.terms().contains_key(&file_id) {
return Err(CacheError::NotParsed);
@ -46,51 +41,37 @@ impl CacheExt for Cache {
import_errors = errors;
// Reverse the imports, so we try to typecheck the leaf dependencies first.
for &id in ids.iter().rev() {
let _ = self.typecheck_with_analysis(
id,
initial_ctxt,
initial_env,
initial_term_env,
lin_registry,
);
let _ = self.typecheck_with_analysis(id, initial_ctxt, initial_term_env, registry);
}
}
for id in self.get_imports(file_id) {
// If we have typechecked a file correctly, its imports should be
// in the `lin_registry`. The imports that are not in `lin_registry`
// in the `registry`. The imports that are not in `registry`
// were not typechecked correctly.
if !lin_registry.map.contains_key(&id) {
if !registry.analysis.contains_key(&id) {
typecheck_import_diagnostics.push(id);
}
}
// After self.parse(), the cache must be populated
let TermEntry { term, state, .. } = self.terms().get(&file_id).unwrap();
let TermEntry { term, state, .. } = self.terms().get(&file_id).unwrap().clone();
let result = if *state > EntryState::Typechecked && lin_registry.map.contains_key(&file_id)
let result = if state > EntryState::Typechecked && registry.analysis.contains_key(&file_id)
{
Ok(CacheOp::Cached(()))
} else if *state >= EntryState::Parsed {
let host = AnalysisHost::new(file_id, initial_env.clone());
} else if state >= EntryState::Parsed {
let types = TypeCollector::default();
let lin = CombinedLinearizer(host, types);
let building = Building {
lin_registry,
linearization: Vec::new(),
import_locations: HashMap::new(),
cache: self,
};
let (_, (linearized, type_lookups)) = typecheck::type_check_linearize(
term,
let (_, type_lookups) = typecheck::type_check_linearize(
&term,
initial_ctxt.clone(),
self,
lin,
(building, CollectedTypes::default()),
types,
CollectedTypes::default(),
)
.map_err(|err| vec![Error::TypecheckError(err)])?;
lin_registry.insert(file_id, linearized, type_lookups, term, initial_term_env);
registry.insert(file_id, type_lookups, &term, initial_term_env);
self.update_state(file_id, EntryState::Typechecked);
Ok(CacheOp::Done(()))
} else {
@ -107,16 +88,15 @@ impl CacheExt for Cache {
let typecheck_import_diagnostics = typecheck_import_diagnostics.into_iter().map(|id| {
let message = "This import could not be resolved \
because its content has failed to typecheck correctly.";
// The unwrap is safe here because (1) we have linearized `file_id` and it must be
// in the `lin_registry` and (2) every resolved import has a corresponding position
// in the linearization of the file that imports it.
let pos = lin_registry
.map
.get(&file_id)
.and_then(|lin| lin.import_locations.get(&id))
.unwrap_or(&TermPos::None);
// Find a position (one is enough) that the import came from.
let pos = term
.find_map(|rt: &RichTerm| match rt.as_ref() {
Term::ResolvedImport(_) => Some(rt.pos),
_ => None,
})
.unwrap_or_default();
let name: String = self.name(id).to_str().unwrap().into();
ImportError::IOError(name, String::from(message), *pos)
ImportError::IOError(name, String::from(message), pos)
});
import_errors.extend(typecheck_import_diagnostics);

4
lsp/nls/src/field_walker.rs
View File

@ -229,7 +229,7 @@ impl<'a> FieldResolver<'a> {
}
Term::Var(id) => self
.server
.lin_registry
.analysis
.get_def(&(*id).into())
.map(|def| {
log::info!("got def {def:?}");
@ -262,7 +262,7 @@ impl<'a> FieldResolver<'a> {
_ => Default::default(),
};
let typ_fields = if let Some(typ) = self.server.lin_registry.get_type(rt) {
let typ_fields = if let Some(typ) = self.server.analysis.get_type(rt) {
log::info!("got inferred type {typ:?}");
self.resolve_type(typ)
} else {

9
lsp/nls/src/files.rs
View File

@ -81,11 +81,7 @@ pub fn handle_save(server: &mut Server, params: DidChangeTextDocumentParams) ->
// re-parsed).
let invalid = server.cache.get_rev_imports_transitive(file_id);
for f in &invalid {
server.lin_registry.map.remove(f);
server.lin_registry.position_lookups.remove(f);
server.lin_registry.usage_lookups.remove(f);
server.lin_registry.parent_lookups.remove(f);
server.lin_registry.type_lookups.remove(f);
server.analysis.remove(*f);
}
// TODO: make this part more abstracted
@ -110,9 +106,8 @@ pub(crate) fn typecheck(
.typecheck_with_analysis(
file_id,
&server.initial_ctxt,
&server.initial_env,
&server.initial_term_env,
&mut server.lin_registry,
&mut server.analysis,
)
.map_err(|error| match error {
CacheError::Error(tc_error) => tc_error

11
lsp/nls/src/incomplete.rs
View File

@ -15,11 +15,7 @@ use nickel_lang_core::{
transform::import_resolution,
};
use crate::{
files::typecheck,
server::Server,
usage::{Environment, UsageLookup},
};
use crate::{files::typecheck, server::Server, usage::Environment};
// Take a bunch of tokens and the end of a possibly-delimited sequence, and return the
// index of the beginning of the possibly-delimited sequence. The sequence might not
@ -161,10 +157,7 @@ pub fn parse_path_from_incomplete_input(
match server.cache.parse_nocache(file_id) {
Ok((rt, _errors)) if !matches!(rt.as_ref(), Term::ParseError(_)) => {
server
.lin_registry
.usage_lookups
.insert(file_id, UsageLookup::new(&rt, env));
server.analysis.insert_usage(file_id, &rt, env);
Some(resolve_imports(rt, server))
}
_ => None,

316
lsp/nls/src/linearization/building.rs
View File

@ -1,316 +0,0 @@
use std::{collections::HashMap, mem};
use codespan::FileId;
use log::debug;
use nickel_lang_core::{
cache::Cache,
identifier::Ident,
position::TermPos,
term::{record::Field, IndexMap, RichTerm},
typ::TypeF,
typecheck::UnifType,
};
use crate::linearization::interface::{TermKind, UsageState};
use super::{
interface::{Unresolved, ValueState},
Environment, ItemId, LinRegistry, LinearizationItem,
};
/// Holds any inner datatype that can be used as stable resource
/// while recording terms.
pub struct Building<'a> {
pub linearization: Vec<LinearizationItem<Unresolved>>,
pub import_locations: HashMap<FileId, TermPos>,
pub lin_registry: &'a mut LinRegistry,
pub cache: &'a Cache,
}
impl<'b> Building<'b> {
pub(super) fn push(&mut self, item: LinearizationItem<Unresolved>) {
self.linearization.push(item)
}
fn get_item(&self, id: ItemId) -> Option<&LinearizationItem<Unresolved>> {
self.linearization.get(id.index)
}
fn get_item_mut(&mut self, id: ItemId) -> Option<&mut LinearizationItem<Unresolved>> {
self.linearization.get_mut(id.index)
}
fn get_item_kind_with_id(
&self,
current_file: FileId,
id: ItemId,
) -> Option<(&ItemId, &TermKind)> {
if current_file == id.file_id {
// This usage references an item in the file we're currently linearizing
let item = self.get_item(id)?;
Some((&item.id, &item.kind))
} else {
// This usage references an item in another file (that has already been linearized)
let item = self.lin_registry.get_item(id)?;
Some((&item.id, &item.kind))
}
}
fn get_item_kind(&self, current_file: FileId, id: ItemId) -> Option<&TermKind> {
let (_, kind) = self.get_item_kind_with_id(current_file, id)?;
Some(kind)
}
fn get_item_kind_mut(&mut self, current_file: FileId, id: ItemId) -> Option<&mut TermKind> {
if current_file == id.file_id {
// This usage references an item in the file we're currently linearizing
let item = self.get_item_mut(id)?;
Some(&mut item.kind)
} else {
// This usage references an item in another file (that has already been linearized)
let item = self
.lin_registry
.map
.get_mut(&id.file_id)?
.get_item_mut(id)?;
Some(&mut item.kind)
}
}
pub(super) fn add_usage(&mut self, current_file: FileId, decl: ItemId, usage: ItemId) {
match self
.get_item_kind_mut(current_file, decl)
.expect("Could not find parent")
{
// In principle, we shouldn't add an usage to a bare record. However, the way the
// stdlib is currently loaded makes usage of `std` referring to a record. In this
// specific case, we simply ignore it (meaning that one can't do goto references for
// `std` currently).
TermKind::Record(_) => (),
// unreachable()!: add_usage can only be called on let bindings, functions and record
// fields, only referring to items which support usages.
TermKind::Type(_) | TermKind::Structure | TermKind::Usage(_) => unreachable!(),
TermKind::Declaration { ref mut usages, .. }
| TermKind::RecordField { ref mut usages, .. } => usages.push(usage),
};
}
pub(super) fn inform_declaration(
&mut self,
current_file: FileId,
declaration: ItemId,
value: ItemId,
) {
let kind = self.get_item_kind_mut(current_file, declaration);
if let Some(TermKind::Declaration {
value: value_state, ..
}) = kind
{
*value_state = ValueState::Known(value)
}
}
pub(super) fn register_fields(
&mut self,
current_file: FileId,
record_term: &RichTerm,
record_fields: &IndexMap<nickel_lang_core::identifier::LocIdent, Field>,
record: ItemId,
env: &mut Environment,
) {
for (ident, field) in record_fields.iter() {
let id = ItemId {
file_id: current_file,
index: self.next_id(),
};
self.push(LinearizationItem {
env: env.clone(),
term: record_term.clone(),
id,
pos: ident.pos,
// temporary, the actual type is resolved later and the item retyped
ty: UnifType::concrete(TypeF::Dyn),
kind: TermKind::RecordField {
record,
ident: *ident,
usages: Vec::new(),
value: ValueState::Unknown,
},
metadata: Some(field.metadata.clone()),
});
env.insert(ident.ident(), id);
self.add_record_field(current_file, record, (ident.ident(), id))
}
}
pub(super) fn add_record_field(
&mut self,
current_file: FileId,
record: ItemId,
(field_ident, reference_id): (Ident, ItemId),
) {
match self
.get_item_kind_mut(current_file, record)
.expect("Could not find record")
{
TermKind::Record(ref mut fields) => {
fields.insert(field_ident, reference_id);
}
_ => panic!(),
}
}
pub(super) fn resolve_reference<'a>(
&'a self,
current_file: FileId,
item: &'a TermKind,
) -> Option<&'a TermKind> {
match item {
// if declaration is a record field, resolve its value
TermKind::RecordField { value, .. } => {
debug!("parent referenced a record field {:?}", value);
value
// retrieve record
.as_option()
.and_then(|value_index| self.get_item_kind(current_file, value_index))
}
TermKind::Declaration {
value: ValueState::Known(value),
path: Some(idents),
..
} => {
let item = self.get_item_kind(current_file, *value)?;
let item = self.resolve_reference(current_file, item)?;
let mut ids = idents.clone();
let (mut prev_item, mut curr_item) = (item, item);
while let Some(id) = ids.pop() {
match curr_item {
TermKind::Record(ref fields) => {
let item = fields.get(&id.ident())?;
let item_kind = self.get_item_kind(current_file, *item)?;
match item_kind {
TermKind::RecordField {
value: ValueState::Known(next_item),
..
} => {
prev_item = item_kind;
let item_kind = self.get_item_kind(current_file, *next_item)?;
curr_item = item_kind;
continue;
}
TermKind::RecordField {
value: ValueState::Unknown,
..
} => break,
// the value from a record is always a record field
_ => unreachable!(),
}
}
_ => break,
}
}
// return the `prev_item` because that was the record field we wanted to
// resolve, and `curr_item` points to the record field's value.
Some(prev_item)
}
// if declaration is a let binding, resolve its value
TermKind::Declaration {
value: ValueState::Known(value),
..
} => self.get_item_kind(current_file, *value),
TermKind::Usage(UsageState::Resolved(pointed)) => {
let kind = self.get_item_kind(current_file, *pointed)?;
self.resolve_reference(current_file, kind)
}
// if something else was referenced, stop.
_ => Some(item),
}
}
/// `resolve_record_references` tries to resolve the references passed to it, and returns the
/// ids of the items it couldn't resolve.
pub(super) fn resolve_record_references(
&mut self,
current_file: FileId,
mut defers: Vec<(ItemId, ItemId, Ident)>,
) -> Vec<(ItemId, ItemId, Ident)> {
let mut unresolved: Vec<(ItemId, ItemId, Ident)> = Vec::new();
while let Some(deferred) = defers.pop() {
// child_item: current deferred usage item
// i.e.: root.<child>
// parent_accessor_id: id of the parent usage
// i.e.: <parent>.child
// child_ident: identifier the child item references
let (child_item, parent_accessor_id, child_ident) = &deferred;
// resolve the value referenced by the parent accessor element
// get the parent accessor, and read its resolved reference
let parent_referenced = self.get_item_kind(current_file, *parent_accessor_id);
if let Some(TermKind::Usage(UsageState::Deferred { .. })) = parent_referenced {
debug!("parent references deferred usage");
unresolved.push(deferred);
continue;
}
// load the parent referenced declaration (i.e.: a declaration or record field term)
let parent_declaration = parent_referenced.and_then(|parent_usage_value| {
self.resolve_reference(current_file, parent_usage_value)
});
if let Some(TermKind::Usage(UsageState::Deferred { .. })) = parent_declaration {
debug!("parent references deferred usage");
unresolved.push(deferred);
continue;
}
let referenced_declaration = parent_declaration
// resolve indirection by following the usage
.and_then(|parent_declaration| {
self.resolve_reference(current_file, parent_declaration)
})
// get record field
.and_then(|parent_declaration| match &parent_declaration {
TermKind::Record(fields) => {
fields.get(child_ident).and_then(|child_declaration_id| {
self.get_item_kind_with_id(current_file, *child_declaration_id)
})
}
_ => None,
});
let referenced_id = referenced_declaration.map(|(id, _)| *id);
debug!(
"Associating child {} to value {:?}",
child_ident, referenced_declaration
);
{
let child = self.get_item_kind_mut(current_file, *child_item).unwrap();
*child = TermKind::Usage(UsageState::from(referenced_id));
}
if let Some(referenced_id) = referenced_id {
self.add_usage(current_file, referenced_id, *child_item);
}
}
if defers.is_empty() && !unresolved.is_empty() {
defers = mem::take(&mut unresolved);
defers
} else {
Vec::new()
}
}
/// Return the id of the next item to be inserted. Note that `next_id` doesn't mutate anything:
/// as long as no new item is inserted in the linearization, calling `next_id` multiple time
/// will return the same result, namely the size of the current linearization.
pub(super) fn next_id(&self) -> usize {
self.linearization.len()
}
}

207
lsp/nls/src/linearization/completed.rs
View File

@ -1,207 +0,0 @@
#![allow(unused_imports)] // until we get rid of feature(old-completer)
use std::{collections::HashMap, hash::Hash};
use codespan::FileId;
use nickel_lang_core::{
position::{RawPos, TermPos},
term::{record::FieldMetadata, SharedTerm, Term},
};
use super::{
interface::{Resolved, TermKind, UsageState, ValueState},
ItemId, LinRegistry, LinearizationItem,
};
#[derive(Clone, Debug)]
struct SharedTermPtr(SharedTerm);
impl PartialEq for SharedTermPtr {
fn eq(&self, other: &Self) -> bool {
SharedTerm::ptr_eq(&self.0, &other.0)
}
}
impl Eq for SharedTermPtr {}
impl Hash for SharedTermPtr {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
(self.0.as_ref() as *const Term).hash(state);
}
}
#[derive(Debug, Default, Clone)]
pub struct Completed {
pub linearization: Vec<LinearizationItem<Resolved>>,
pub import_locations: HashMap<FileId, TermPos>,
id_to_index: HashMap<ItemId, usize>,
}
impl Completed {
pub fn new(
linearization: Vec<LinearizationItem<Resolved>>,
id_to_index: HashMap<ItemId, usize>,
import_locations: HashMap<FileId, TermPos>,
) -> Self {
Self {
linearization,
import_locations,
id_to_index,
}
}
/// Returns the closest item to the left (if any) and to the right (if any) of
/// a specified item. The "closeness" metric in this context is just the source
/// position.
#[cfg(feature = "old-completer")]
pub fn get_items_adjacent(
&self,
id: ItemId,
) -> (
Option<&LinearizationItem<Resolved>>,
Option<&LinearizationItem<Resolved>>,
) {
let Some(index) = self.id_to_index.get(&id).copied() else {
return (None, None);
};
let (left_index, right_index) = (index - 1, index + 1);
let left = self.linearization.get(left_index);
let right = self.linearization.get(right_index);
(left, right)
}
pub fn get_item(&self, id: ItemId) -> Option<&LinearizationItem<Resolved>> {
self.id_to_index
.get(&id)
.and_then(|index| self.linearization.get(*index))
}
/// Try to retrieve the item from the current linearization first, and if that fails, look into
/// the registry if there is a linearization corresponding to the item's file id.
#[cfg(feature = "old-completer")]
pub fn get_item_with_reg<'a>(
&'a self,
id: ItemId,
lin_registry: &'a LinRegistry,
) -> Option<&'a LinearizationItem<Resolved>> {
self.get_item(id).or_else(|| lin_registry.get_item(id))
}
pub fn get_item_mut(&mut self, id: ItemId) -> Option<&mut LinearizationItem<Resolved>> {
let index = self.id_to_index.get(&id)?;
self.linearization.get_mut(*index)
}
/// Finds the index of a linearization item for a given location
/// The linearization is a list of items that are sorted by their physical occurrence.
/// - Each element has a corresponding span in the source
/// - Spans are either equal (same starting point, same length)
/// or shorter but never intersecting
///
/// (start_element_2 >= start_element_1 AND end_element_2 <= end_element_1)
///
/// For any location a binary search is used to efficiently find the index
/// of the *last* element that starts at this position.
/// This corresponds to the most concrete Element as the linearization is
/// 1. produced by a stable sort and
/// 2. lower elements are more concrete
///
/// If a perfect match cannot be found, the binary search still provides an
/// anchor point from which we reversely find the first element that *contains*
/// the location looked up
///
/// If neither is possible `None` is returned as no corresponding linearization
/// item could be found.
///
pub fn item_at(&self, pos: RawPos) -> Option<&LinearizationItem<Resolved>> {
let linearization = &self.linearization;
let item = match linearization.binary_search_by(|item| {
item.pos
.as_opt_ref()
.and_then(|span| span.start_pos().partial_cmp(&pos))
.unwrap_or(std::cmp::Ordering::Less)
}) {
// Found item(s) starting at `locator`
// search for most precise element
Ok(index) => linearization[index..]
.iter()
.take_while(|item| {
// Here because None is smaller than everything, if binary search succeeds,
// we can safely unwrap the position.
item.pos.unwrap().start_pos() == pos
})
.last(),
// No perfect match found
// iterate back finding the first wrapping linearization item
Err(index) => linearization[..index].iter().rfind(|item| {
item.pos
.as_opt_ref()
.map(|span| span.contains(pos))
// if the item found is None, we can not find a better one.
.unwrap_or(true)
}),
};
item
}
/// Return all the items in the scope of the given linearization item.
#[cfg(feature = "old-completer")]
pub fn get_in_scope<'a>(
&'a self,
LinearizationItem { env, .. }: &'a LinearizationItem<Resolved>,
lin_registry: &'a LinRegistry,
) -> Vec<&'a LinearizationItem<Resolved>> {
env.iter()
.filter_map(|(_, id)| self.get_item_with_reg(*id, lin_registry))
.collect()
}
/// Retrive the type and the metadata of a linearization item. Requires a registry as this code
/// tries to jump to the definitions of objects to find the relevant data, which might lie in a
/// different file (and thus in a different linearization within the registry).
#[cfg(feature = "old-completer")]
pub fn get_type_and_metadata(
&self,
item: &LinearizationItem<Resolved>,
lin_registry: &LinRegistry,
) -> (Resolved, Vec<String>) {
let mut extra = Vec::new();
let item = match item.kind {
TermKind::Usage(UsageState::Resolved(declaration)) => self
.get_item_with_reg(declaration, lin_registry)
.and_then(|decl| match decl.kind {
TermKind::Declaration {
value: ValueState::Known(value),
..
}
| TermKind::RecordField {
value: ValueState::Known(value),
..
} => self.get_item_with_reg(value, lin_registry),
_ => None,
})
.unwrap_or(item),
TermKind::Declaration {
value: ValueState::Known(value),
..
} => self.get_item_with_reg(value, lin_registry).unwrap_or(item),
_ => item,
};
if let Some(FieldMetadata {
doc, annotation, ..
}) = item.metadata.as_ref()
{
if let Some(doc) = doc {
extra.push(doc.to_owned());
}
if let Some(contracts) = annotation.contracts_to_string() {
extra.push(contracts);
}
}
(item.ty.to_owned(), extra)
}
}

84
lsp/nls/src/linearization/interface.rs
View File

@ -1,84 +0,0 @@
use std::collections::HashMap;
use nickel_lang_core::{
identifier::{Ident, LocIdent},
typ::Type,
typecheck::UnifType,
};
use super::ItemId;
pub trait ResolutionState {}
/// Types are available as [nickel_lang_core::typecheck::UnifType] only during recording. They are
/// resolved after typechecking as [nickel_lang_core::typ::Type]
pub type Unresolved = UnifType;
impl ResolutionState for Unresolved {}
/// When resolved a concrete [Type] is known
pub type Resolved = Type;
impl ResolutionState for Resolved {}
/// Abstract term kinds.
/// Currently tracks
/// 1. Declarations
/// 2. Usages
/// 3. Records, listing their fields
/// 4. wildcard (Structure) for any other kind of term.
/// Can be extended later to represent Contracts, Records, etc.
#[derive(Debug, Clone, PartialEq)]
pub enum TermKind {
Declaration {
id: LocIdent,
usages: Vec<ItemId>,
value: ValueState,
// This is the path to a bound variable. If we have
// `let { a = {b = {c, ..}, ..}, ..} = ...`, the `path` will
// be `Some([a, b, c])`, and `ident` will be `c`.
// If we have `let { a = {b = {c = somevar, ..}, ..}, ..} = ...`
// instead, the `path` remains the same, but the ident will be `somevar`
// If there is no pattern variable bound, the `path` is `None`
path: Option<Vec<LocIdent>>,
},
Usage(UsageState),
Record(HashMap<Ident, ItemId>),
RecordField {
ident: LocIdent,
record: ItemId,
usages: Vec<ItemId>,
value: ValueState,
},
Type(Type),
Structure,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ValueState {
Unknown,
Known(ItemId),
}
impl ValueState {
pub fn as_option(&self) -> Option<ItemId> {
match self {
ValueState::Unknown => None,
ValueState::Known(value) => Some(*value),
}
}
}
/// Some usages cannot be fully resolved in a first pass (i.e. recursive record fields)
/// In these cases we defer the resolution to a second pass during linearization
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum UsageState {
Unbound,
Resolved(ItemId),
Deferred { parent: ItemId, child: LocIdent },
}
impl From<Option<ItemId>> for UsageState {
fn from(option: Option<ItemId>) -> Self {
match option {
Some(id) => UsageState::Resolved(id),
None => UsageState::Unbound,
}
}
}

950
lsp/nls/src/linearization/mod.rs
View File

@ -1,950 +0,0 @@
use std::{collections::HashMap, marker::PhantomData};
use codespan::FileId;
use log::debug;
use nickel_lang_core::{
identifier::Ident,
position::TermPos,
term::{
record::{Field, FieldMetadata},
RichTerm, Term, Traverse, TraverseControl, UnaryOp,
},
typ::TypeF,
typecheck::{linearization::Linearizer, reporting::NameReg, UnifType},
};
use crate::{
field_walker::DefWithPath, identifier::LocIdent, position::PositionLookup, term::RichTermPtr,
usage::UsageLookup,
};
use self::{
building::Building,
completed::Completed,
interface::{ResolutionState, Resolved, TermKind, UsageState, ValueState},
};
pub mod building;
pub mod completed;
pub mod interface;
pub type Environment = nickel_lang_core::environment::Environment<Ident, ItemId>;
#[derive(Clone, Debug)]
pub struct ParentLookup {
table: HashMap<RichTermPtr, RichTerm>,
}
impl ParentLookup {
pub fn new(rt: &RichTerm) -> Self {
let mut table = HashMap::new();
let mut traverse_merge =
|rt: &RichTerm, parent: &Option<RichTerm>| -> TraverseControl<Option<RichTerm>, ()> {
if let Some(parent) = parent {
table.insert(RichTermPtr(rt.clone()), parent.clone());
}
TraverseControl::ContinueWithScope(Some(rt.clone()))
};
rt.traverse_ref(&mut traverse_merge, &None);
ParentLookup { table }
}
pub fn parent(&self, rt: &RichTerm) -> Option<&RichTerm> {
self.table.get(&RichTermPtr(rt.clone()))
}
pub fn parent_chain<'a>(&'a self, rt: &'a RichTerm) -> ParentChainIter<'_> {
ParentChainIter {
table: self,
next: Some(rt),
}
}
}
pub struct ParentChainIter<'a> {
table: &'a ParentLookup,
next: Option<&'a RichTerm>,
}
impl<'a> Iterator for ParentChainIter<'a> {
type Item = &'a RichTerm;
fn next(&mut self) -> Option<Self::Item> {
if let Some(next) = self.next {
self.next = self.table.parent(next);
Some(next)
} else {
None
}
}
}
/// A registry mapping file ids to their corresponding linearization. The registry stores the
/// linearization of every file that has been imported and analyzed, including the main open
/// document.
///
/// `LinRegistry` wraps some methods of [completed::Completed] by first selecting the linearization
/// with a matching file id and then calling to the corresponding method on it.
#[derive(Clone, Default, Debug)]
pub struct LinRegistry {
pub map: HashMap<FileId, Completed>,
// TODO: these are supposed to eventually *replace* part of the linearization, at
// which point we'll rename `LinRegistry` (and probably just have one
// HashMap<FileId, everything>)
//
// Most of these tables do one more lookup than necessary: they look up a
// file id and then they look up a term in an inner table. This is a little
// inefficient for lookups, but it makes it easy to invalidate a whole file
// in one go.
pub position_lookups: HashMap<FileId, PositionLookup>,
pub usage_lookups: HashMap<FileId, UsageLookup>,
pub parent_lookups: HashMap<FileId, ParentLookup>,
pub type_lookups: HashMap<FileId, CollectedTypes<Type>>,
}
impl LinRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn insert(
&mut self,
file_id: FileId,
linearization: Completed,
type_lookups: CollectedTypes<Type>,
term: &RichTerm,
initial_env: &crate::usage::Environment,
) {
self.map.insert(file_id, linearization);
self.position_lookups
.insert(file_id, PositionLookup::new(term));
self.usage_lookups
.insert(file_id, UsageLookup::new(term, initial_env));
self.parent_lookups.insert(file_id, ParentLookup::new(term));
self.type_lookups.insert(file_id, type_lookups);
}
/// Look for the linearization corresponding to an item's id, and return the corresponding item
/// from the linearization, if any.
pub fn get_item(&self, id: ItemId) -> Option<&LinearizationItem<Resolved>> {
let lin = self.map.get(&id.file_id).unwrap();
lin.get_item(id)
}
pub fn get_def(&self, ident: &LocIdent) -> Option<&DefWithPath> {
let file = ident.pos.as_opt_ref()?.src_id;
self.usage_lookups.get(&file)?.def(ident)
}
pub fn get_usages(&self, ident: &LocIdent) -> impl Iterator<Item = &LocIdent> {
fn inner<'a>(
slf: &'a LinRegistry,
ident: &LocIdent,
) -> Option<impl Iterator<Item = &'a LocIdent>> {
let file = ident.pos.as_opt_ref()?.src_id;
Some(slf.usage_lookups.get(&file)?.usages(ident))
}
inner(self, ident).into_iter().flatten()
}
pub fn get_env(&self, rt: &RichTerm) -> Option<&crate::usage::Environment> {
let file = rt.pos.as_opt_ref()?.src_id;
self.usage_lookups.get(&file)?.env(rt)
}
pub fn get_type(&self, rt: &RichTerm) -> Option<&Type> {
let file = rt.pos.as_opt_ref()?.src_id;
self.type_lookups
.get(&file)?
.terms
.get(&RichTermPtr(rt.clone()))
}
pub fn get_type_for_ident(&self, id: &LocIdent) -> Option<&Type> {
let file = id.pos.as_opt_ref()?.src_id;
self.type_lookups.get(&file)?.idents.get(id)
}
pub fn get_parent_chain<'a>(&'a self, rt: &'a RichTerm) -> Option<ParentChainIter<'a>> {
let file = rt.pos.as_opt_ref()?.src_id;
Some(self.parent_lookups.get(&file)?.parent_chain(rt))
}
}
#[derive(PartialEq, Copy, Debug, Clone, Eq, Hash)]
pub struct ItemId {
pub file_id: FileId,
pub index: usize,
}
/// A recorded item of a given state of resolution state
/// Tracks a unique id used to build a reference table after finalizing
/// the linearization using the LSP [AnalysisHost]
#[derive(Debug, Clone, PartialEq)]
pub struct LinearizationItem<S: ResolutionState> {
/// The term that spawned this linearization item. Note that there is not a one-to-one
/// relationship between terms and linearization items. For example, a single `let` term
/// with a pattern binding can create multiple `declaration` linearization items.
pub term: RichTerm,
pub env: Environment,
pub id: ItemId,
/// This is not necessarily the same as `term.pos`, for example if this linearization
/// item points to a single pattern binding in a `let` term then `pos` will be the position
/// of just the identifier.
pub pos: TermPos,
pub ty: S,
pub kind: TermKind,
pub metadata: Option<FieldMetadata>,
}
/// [Linearizer] used by the LSP
///
/// Tracks a _scope stable_ environment managing variable ident
/// resolution
pub struct AnalysisHost<'a> {
// We need the lifetime on `AnalysisHost` to be able to
// have a lifetime for the associated `Building` data structure.
phantom: PhantomData<&'a usize>,
file: FileId,
env: Environment,
meta: Option<FieldMetadata>,
/// Indexing a record will store a reference to the record as
/// well as its fields.
/// `Self::scope` will produce a host with a single **`pop`ed**
/// Ident. As fields are typechecked in the same order, each
/// in their own scope immediately after the record, which
/// gives the corresponding record field _term_ to the ident
/// useable to construct a vale declaration.
record_fields: Option<(ItemId, Vec<(ItemId, LocIdent)>)>,
bindings: Option<Vec<ItemId>>,
/// Accesses to nested records are recorded recursively.
///
/// ```
/// outer.middle.inner -> inner(middle(outer))
/// ```
///
/// To resolve those inner fields, accessors (`inner`, `middle`)
/// are recorded first until a variable (`outer`) is found.
/// Then, access to all nested records are resolved at once.
access: Option<Vec<LocIdent>>,
}
impl<'a> AnalysisHost<'a> {
pub fn new(file: FileId, env: Environment) -> Self {
Self {
phantom: PhantomData,
file,
env,
meta: Default::default(),
record_fields: Default::default(),
bindings: Default::default(),
access: Default::default(),
}
}
fn next_id(&self, lin: &Building) -> ItemId {
ItemId {
file_id: self.file,
index: lin.next_id(),
}
}
// Helper with logic common to let/fun and let pattern/fun pattern.
//
// If the term is a let-binding, this function calls to the provided `push` callback to add the next
// item id to the relevant list of bindings.
//
// If the term is a function, this function generates a stub item for it.
//
// Returns the value state of the corresponding declaration.
//
// Panic if `rt` is neither a let/let pattern nor a fun/fun pattern.
fn setup_decl(
&mut self,
lin: &mut Building,
rt: &RichTerm,
ty: &UnifType,
pos: TermPos,
push: impl FnOnce(ItemId),
) -> ValueState {
let next_id = self.next_id(lin);
match rt.as_ref() {
Term::LetPattern(..) | Term::Let(..) => {
push(next_id);
ValueState::Unknown
}
Term::FunPattern(..) | Term::Fun(..) => {
// stub object, representing the whole function
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: next_id,
ty: ty.clone(),
pos,
kind: TermKind::Structure,
metadata: self.meta.take(),
});
ValueState::Known(self.next_id(lin))
}
_ => panic!("expected only a let pattern or a fun pattern in this function"),
}
}
}
use nickel_lang_core::typ::Type;
use nickel_lang_core::typecheck::Extra;
impl<'a> Linearizer for AnalysisHost<'a> {
type Building = Building<'a>;
type Completed = Completed;
type CompletionExtra = Extra;
type ItemId = ItemId;
fn add_term(&mut self, lin: &mut Building, rt: &RichTerm, ty: UnifType) -> Option<ItemId> {
let pos = rt.pos;
let term = rt.term.as_ref();
debug!("adding term: {:?} @ {:?}", term, pos);
// The id of the main item added. Some terms give rise to several items (records or
// functions for example), but this is the id of the first item added, which is considered
// to be the item attached to the term - the rest is extra declaration/usage/etc
let main_id = ItemId {
file_id: self.file,
index: lin.next_id(),
};
// Register record field if appropriate
// `record` is the id [LinearizatonItem] of the enclosing record
// `offset` is used to find the [LinearizationItem] representing the field
// Field items are inserted immediately after the record
if !matches!(
term,
Term::Op1(UnaryOp::StaticAccess(_), _) | Term::Annotated(..)
) {
if let Some((record, (offset, _))) = self
.record_fields
// We call take because each record field will be linearized in a different scope.
// In particular, each record field gets its own copy of `record_fields`, so we can
// take it.
.take()
.map(|(record, mut fields)| (record, fields.pop().unwrap()))
{
if let Some(field) = lin.linearization.get_mut(record.index + offset.index) {
let usage_offset = if matches!(term, Term::Var(_)) {
debug!(
"associating nested field {:?} with chain {:?}",
term,
self.access.as_ref()
);
self.access.as_ref().map(|v| v.len()).unwrap_or(0)
} else {
0
};
match field.kind {
TermKind::RecordField { ref mut value, .. } => {
*value = ValueState::Known(ItemId {
file_id: self.file,
index: main_id.index + usage_offset,
});
}
// The linearization item of a record with n fields is expected to be
// followed by n linearization items representing each field
_ => unreachable!(),
}
}
}
if let Some(decls) = self.bindings.take() {
let offset = self.access.as_ref().map(|v| v.len()).unwrap_or(0);
for decl in decls {
lin.inform_declaration(
self.file,
decl,
ItemId {
file_id: self.file,
index: main_id.index + offset,
},
);
}
}
}
if pos == TermPos::None {
return None;
}
match term {
Term::LetPattern(ident, destruct, ..) | Term::FunPattern(ident, destruct, _) => {
let mut pattern_bindings = Vec::new();
if let Some(ident) = ident {
let value_ptr =
self.setup_decl(lin, rt, &ty, pos, |id| pattern_bindings.push(id));
let next_id = self.next_id(lin);
self.env.insert(ident.ident(), next_id);
let kind = TermKind::Declaration {
id: ident.to_owned(),
usages: Vec::new(),
value: value_ptr,
path: None,
};
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: next_id,
ty,
pos: ident.pos,
kind,
metadata: None,
});
}
for (path, bind_ident, field) in destruct
.to_owned()
.inner()
.into_iter()
.flat_map(|matched| matched.to_flattened_bindings())
{
let decl_id = self.next_id(lin);
pattern_bindings.push(decl_id);
self.env.insert(bind_ident.ident(), decl_id);
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: decl_id,
// TODO: get type from pattern
ty: UnifType::concrete(TypeF::Dyn),
pos: bind_ident.pos,
kind: TermKind::Declaration {
id: bind_ident,
usages: Vec::new(),
value: ValueState::Unknown,
path: Some(path),
},
metadata: Some(field.metadata),
});
}
self.bindings = Some(pattern_bindings);
}
Term::Let(ident, ..) | Term::Fun(ident, ..) => {
let mut binding = None;
let value_ptr = self.setup_decl(lin, rt, &ty, pos, |id| binding = Some(id));
self.bindings = binding.map(|id| vec![id]);
let next_id = self.next_id(lin);
self.env.insert(ident.ident(), next_id);
let kind = TermKind::Declaration {
id: ident.to_owned(),
usages: Vec::new(),
value: value_ptr,
path: None,
};
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: next_id,
ty,
pos: ident.pos,
kind,
metadata: self.meta.take(),
});
}
Term::Var(ident) => {
debug!(
"adding usage of variable {} followed by chain {:?}",
ident, self.access
);
let pointed = self.env.get(&ident.ident()).copied();
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: main_id,
pos: ident.pos,
ty: UnifType::concrete(TypeF::Dyn),
kind: TermKind::Usage(UsageState::from(pointed)),
metadata: self.meta.take(),
});
if let Some(referenced) = pointed {
lin.add_usage(self.file, referenced, main_id)
}
if let Some(chain) = self.access.take() {
let chain: Vec<_> = chain.into_iter().rev().collect();
for accessor in chain.iter() {
let id = self.next_id(lin);
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id,
pos: accessor.pos,
ty: UnifType::concrete(TypeF::Dyn),
kind: TermKind::Usage(UsageState::Deferred {
parent: ItemId {
file_id: self.file,
index: id.index - 1,
},
child: accessor.to_owned().into(),
}),
metadata: self.meta.take(),
});
}
}
}
Term::Record(record) | Term::RecRecord(record, ..) => {
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: main_id,
pos,
ty,
kind: TermKind::Record(HashMap::new()),
metadata: self.meta.take(),
});
lin.register_fields(self.file, rt, &record.fields, main_id, &mut self.env);
let mut field_names = record.fields.keys().cloned().collect::<Vec<_>>();
field_names.sort_unstable();
self.record_fields = Some((
ItemId {
file_id: main_id.file_id,
index: main_id.index + 1,
},
field_names
.into_iter()
.enumerate()
.map(|(id, ident)| {
(
ItemId {
file_id: self.file,
index: id,
},
ident.into(),
)
})
.rev()
.collect(),
));
}
Term::Op1(UnaryOp::StaticAccess(ident), _) => {
let x = self.access.get_or_insert(Vec::with_capacity(1));
x.push(ident.to_owned().into())
}
Term::Annotated(annot, _) => {
// Notice 1: No push to lin for the `FieldMetadata` itself
// Notice 2: we discard the encoded value as anything we
// would do with the value will be handled in the following
// call to [Self::add_term]
self.meta = Some(FieldMetadata {
annotation: annot.clone(),
..Default::default()
})
}
Term::ResolvedImport(file) => {
fn final_term_pos(term: &RichTerm) -> &TermPos {
let RichTerm { term, pos } = term;
match term.as_ref() {
Term::Let(_, _, body, _) | Term::LetPattern(_, _, _, body) => {
final_term_pos(body)
}
Term::Op1(UnaryOp::StaticAccess(field), _) => &field.pos,
_ => pos,
}
}
lin.import_locations.insert(*file, pos);
let linearization = lin.lin_registry.map.get(file)?;
// This is safe because the import file is resolved before we linearize the
// containing file, therefore the cache MUST have the term stored.
let term = lin.cache.get_owned(*file).unwrap();
let position = final_term_pos(&term);
// unwrap(): this unwrap fails only when position is a `TermPos::None`, which only
// happens if the `RichTerm` has been transformed or evaluated. None of these
// happen before linearization.
let term_id = linearization.item_at(position.unwrap().start_pos())?.id;
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: main_id,
pos,
ty,
kind: TermKind::Usage(UsageState::Resolved(term_id)),
metadata: self.meta.take(),
})
}
Term::Type(t) => {
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: main_id,
pos,
ty,
kind: TermKind::Type(t.clone()),
metadata: self.meta.take(),
});
}
other => {
debug!("Add wildcard item: {:?}", other);
lin.push(LinearizationItem {
env: self.env.clone(),
term: rt.clone(),
id: main_id,
pos,
ty,
kind: TermKind::Structure,
metadata: self.meta.take(),
})
}
}
Some(main_id)
}
fn add_field_metadata(&mut self, _lin: &mut Building, field: &Field) {
// Notice 1: No push to lin for the `FieldMetadata` itself
// Notice 2: we discard the encoded value as anything we
// would do with the value will be handled in the following
// call to [Self::add_term]
self.meta = Some(field.metadata.clone())
}
/// [Self::add_term] produces a depth first representation or the
/// traversed AST. This function indexes items by _source position_.
/// Elements are reorderd to allow efficient lookup of elemts by
/// their location in the source.
///
/// Additionally, resolves concrete types for all items.
fn complete(
self,
mut lin: Building,
Extra {
table,
names: reported_names,
wildcards,
}: &Extra,
) -> Completed {
debug!("linearizing {:?}", self.file);
let mut name_reg = NameReg::new(reported_names.clone());
// TODO: Storing defers while linearizing?
let mut defers: Vec<_> = lin
.linearization
.iter()
.filter_map(|item| match &item.kind {
TermKind::Usage(UsageState::Deferred { parent, child }) => {
Some((item.id, *parent, child.ident()))
}
_ => None,
})
.collect();
defers.reverse();
let unresolved = lin.resolve_record_references(self.file, defers);
debug!("unresolved references: {:?}", unresolved);
let Building {
mut linearization,
import_locations,
..
} = lin;
linearization.sort_by(
|it1, it2| match (it1.pos.as_opt_ref(), it2.pos.as_opt_ref()) {
(None, None) => std::cmp::Ordering::Equal,
(None, _) => std::cmp::Ordering::Less,
(_, None) => std::cmp::Ordering::Greater,
(Some(pos1), Some(pos2)) => {
(pos1.src_id, pos1.start).cmp(&(pos2.src_id, pos2.start))
}
},
);
// create an index of id -> new position
let mut id_mapping = HashMap::new();
linearization
.iter()
.enumerate()
.for_each(|(index, LinearizationItem { id, .. })| {
id_mapping.insert(*id, index);
});
fn transform_wildcard(wildcards: &[Type], t: Type) -> Type {
match t.typ {
TypeF::Wildcard(i) => wildcards.get(i).unwrap_or(&t).clone(),
_ => t,
}
}
// resolve types
let lin_: Vec<_> = linearization
.into_iter()
.map(
|LinearizationItem {
env,
term,
id,
pos,
ty,
kind,
metadata: meta,
}| LinearizationItem {
ty: name_reg.to_type(table, ty),
term,
env,
id,
pos,
kind,
metadata: meta,
},
)
.map(|item| LinearizationItem {
ty: transform_wildcard(wildcards, item.ty),
..item
})
.collect();
Completed::new(lin_, id_mapping, import_locations)
}
fn scope(&mut self) -> Self {
AnalysisHost {
phantom: PhantomData,
file: self.file,
env: self.env.clone(),
meta: self.meta.clone(),
record_fields: self.record_fields.as_mut().and_then(|(record, fields)| {
Some(*record).zip(fields.pop().map(|field| vec![field]))
}),
bindings: self.bindings.take(),
access: self.access.clone(),
}
}
fn scope_meta(&mut self) -> Self {
AnalysisHost {
phantom: PhantomData,
file: self.file,
env: self.env.clone(),
// Metadata must be attached to the original scope of the value (`self`), while the new
// scope for metadata should be clean.
// In general, the scope for the metadata shouldn't interfere with any of the previous
// state (record fields, let binding, etc.), which is kept intact inside `self`, while
// this new scope gets a cleared stated.
meta: None,
record_fields: None,
bindings: None,
access: None,
}
}
fn retype_ident(
&mut self,
lin: &mut Building,
ident: &nickel_lang_core::identifier::LocIdent,
new_type: UnifType,
) {
if let Some(item) = self
.env
.get(&ident.ident())
.and_then(|item_id| lin.linearization.get_mut(item_id.index))
{
debug!("retyping {:?} to {:?}", ident, new_type);
item.ty = new_type;
} else {
debug!(
"retype_indent failed! Environment miss: {}",
self.env.get(&ident.ident()).is_none()
);
}
}
fn retype(&mut self, lin: &mut Building, item_id: Option<ItemId>, new_type: UnifType) {
let Some(item_id) = item_id else {
return;
};
if let Some(item) = lin.linearization.get_mut(item_id.index) {
debug!("retyping item {:?} to {:?}", item_id, new_type);
item.ty = new_type;
} else {
debug!("retype item failed (item not found)!");
}
}
}
#[derive(Default)]
pub struct TypeCollector {
// Store a copy of the terms we've added so far. The index in this array is their ItemId.
term_ids: Vec<RichTermPtr>,
}
#[derive(Clone, Debug)]
pub struct CollectedTypes<Ty> {
pub terms: HashMap<RichTermPtr, Ty>,
pub idents: HashMap<LocIdent, Ty>,
}
impl<Ty> Default for CollectedTypes<Ty> {
fn default() -> Self {
Self {
terms: HashMap::new(),
idents: HashMap::new(),
}
}
}
impl Linearizer for TypeCollector {
type Building = CollectedTypes<UnifType>;
type Completed = CollectedTypes<Type>;
type CompletionExtra = Extra;
type ItemId = usize;
fn scope(&mut self) -> Self {
TypeCollector::default()
}
fn scope_meta(&mut self) -> Self {
TypeCollector::default()
}
fn add_term(&mut self, lin: &mut Self::Building, rt: &RichTerm, ty: UnifType) -> Option<usize> {
self.term_ids.push(RichTermPtr(rt.clone()));
lin.terms.insert(RichTermPtr(rt.clone()), ty);
Some(self.term_ids.len() - 1)
}
fn complete(
self,
lin: Self::Building,
Extra {
table,
names,
wildcards,
}: &Extra,
) -> Self::Completed {
let mut name_reg = NameReg::new(names.clone());
let mut transform_type = |uty: UnifType| -> Type {
let ty = name_reg.to_type(table, uty);
match ty.typ {
TypeF::Wildcard(i) => wildcards.get(i).unwrap_or(&ty).clone(),
_ => ty,
}
};
let terms = lin
.terms
.into_iter()
.map(|(rt, uty)| (rt, transform_type(uty)))
.collect();
let idents = lin
.idents
.into_iter()
.map(|(id, uty)| (id, transform_type(uty)))
.collect();
CollectedTypes { terms, idents }
}
fn retype(&mut self, lin: &mut Self::Building, item_id: Option<usize>, new_type: UnifType) {
if let Some(id) = item_id {
lin.terms.insert(self.term_ids[id].clone(), new_type);
}
}
fn retype_ident(
&mut self,
lin: &mut Self::Building,
ident: &nickel_lang_core::identifier::LocIdent,
new_type: UnifType,
) {
lin.idents.insert((*ident).into(), new_type);
}
}
pub struct CombinedLinearizer<T, U>(pub T, pub U);
impl<T: Linearizer, U: Linearizer> Linearizer for CombinedLinearizer<T, U>
where
T: Linearizer<CompletionExtra = U::CompletionExtra>,
{
type Building = (T::Building, U::Building);
type Completed = (T::Completed, U::Completed);
type ItemId = (Option<T::ItemId>, Option<U::ItemId>);
// Maybe this should be (T::CompletionExtra, U::CompletionExtra) but in practice
// CompletionExtra is always Extra anyway.
type CompletionExtra = T::CompletionExtra;
fn scope(&mut self) -> Self {
CombinedLinearizer(self.0.scope(), self.1.scope())
}
fn scope_meta(&mut self) -> Self {
CombinedLinearizer(self.0.scope_meta(), self.1.scope_meta())
}
fn add_term(
&mut self,
lin: &mut Self::Building,
term: &RichTerm,
ty: UnifType,
) -> Option<Self::ItemId> {
let id0 = self.0.add_term(&mut lin.0, term, ty.clone());
let id1 = self.1.add_term(&mut lin.1, term, ty);
Some((id0, id1))
}
fn add_field_metadata(&mut self, lin: &mut Self::Building, field: &Field) {
self.0.add_field_metadata(&mut lin.0, field);
self.1.add_field_metadata(&mut lin.1, field);
}
fn retype_ident(
&mut self,
lin: &mut Self::Building,
ident: &nickel_lang_core::identifier::LocIdent,
new_type: UnifType,
) {
self.0.retype_ident(&mut lin.0, ident, new_type.clone());
self.1.retype_ident(&mut lin.1, ident, new_type);
}
fn complete(self, lin: Self::Building, extra: &Self::CompletionExtra) -> Self::Completed
where
Self: Sized,
{
(self.0.complete(lin.0, extra), self.1.complete(lin.1, extra))
}
fn retype(
&mut self,
lin: &mut Self::Building,
item_id: Option<Self::ItemId>,
new_type: UnifType,
) {
if let Some((id0, id1)) = item_id {
self.0.retype(&mut lin.0, id0, new_type.clone());
self.1.retype(&mut lin.1, id1, new_type);
}
}
}

2
lsp/nls/src/main.rs
View File

@ -5,6 +5,7 @@ use anyhow::Result;
use log::debug;
use lsp_server::Connection;
mod analysis;
mod cache;
mod diagnostic;
mod error;
@ -12,7 +13,6 @@ mod field_walker;
mod files;
mod identifier;
mod incomplete;
mod linearization;
mod position;
mod requests;
mod server;

2
lsp/nls/src/position.rs
View File

@ -99,7 +99,7 @@ fn make_disjoint<T: Clone>(mut all_ranges: Vec<(Range<u32>, T)>) -> Vec<(Range<u
///
/// Overlapping positions are resolved in favor of the smaller one; i.e., lookups return the
/// most specific term for a given position.
#[derive(Clone, Debug)]
#[derive(Default, Clone, Debug)]
pub struct PositionLookup {
// The intervals here are sorted and disjoint.
term_ranges: Vec<(Range<u32>, RichTermPtr)>,

1115
lsp/nls/src/requests/completion.rs
View File

File diff suppressed because it is too large Load Diff

8
lsp/nls/src/requests/goto.rs
View File

@ -11,11 +11,7 @@ use crate::{
fn get_defs(term: &RichTerm, server: &Server) -> Vec<LocIdent> {
match term.as_ref() {
Term::Var(id) => {
if let Some(loc) = server
.lin_registry
.get_def(&(*id).into())
.map(|def| def.ident)
{
if let Some(loc) = server.analysis.get_def(&(*id).into()).map(|def| def.ident) {
vec![loc]
} else {
vec![]
@ -92,7 +88,7 @@ pub fn handle_references(
let mut usages: Vec<_> = def_locs
.iter()
.flat_map(|id| server.lin_registry.get_usages(id))
.flat_map(|id| server.analysis.get_usages(id))
.cloned()
.collect();

6
lsp/nls/src/requests/hover.rs
View File

@ -56,7 +56,7 @@ fn values_and_metadata_from_field(
}
fn ident_hover(ident: LocIdent, server: &Server) -> Option<HoverData> {
let ty = server.lin_registry.get_type_for_ident(&ident).cloned();
let ty = server.analysis.get_type_for_ident(&ident).cloned();
let span = ident.pos.into_opt()?;
let mut ret = HoverData {
values: Vec::new(),
@ -65,7 +65,7 @@ fn ident_hover(ident: LocIdent, server: &Server) -> Option<HoverData> {
ty,
};
if let Some(def) = server.lin_registry.get_def(&ident) {
if let Some(def) = server.analysis.get_def(&ident) {
let resolver = FieldResolver::new(server);
if let Some(((last, path), val)) = def.path.split_last().zip(def.value.as_ref()) {
let parents = resolver.resolve_term_path(val, path);
@ -81,7 +81,7 @@ fn ident_hover(ident: LocIdent, server: &Server) -> Option<HoverData> {
}
fn term_hover(rt: &RichTerm, server: &Server) -> Option<HoverData> {
let ty = server.lin_registry.get_type(rt).cloned();
let ty = server.analysis.get_type(rt).cloned();
let span = rt.pos.into_opt()?;
match rt.as_ref() {

9
lsp/nls/src/requests/symbols.rs
View File

@ -17,13 +17,8 @@ pub fn handle_document_symbols(
.id_of(&SourcePath::Path(path))
.ok_or_else(|| crate::error::Error::FileNotFound(params.text_document.uri.clone()))?;
let usage_lookups = server.lin_registry.usage_lookups.get(&file_id).unwrap();
let type_lookups = server
.lin_registry
.type_lookups
.get(&file_id)
.ok_or_else(|| crate::error::Error::FileNotFound(params.text_document.uri.clone()))?;
let usage_lookups = &server.file_analysis(file_id)?.usage_lookup;
let type_lookups = &server.file_analysis(file_id)?.type_lookup;
let mut symbols = usage_lookups
.symbols()

64
lsp/nls/src/server.rs
View File

@ -20,7 +20,6 @@ use lsp_types::{
use nickel_lang_core::{
cache::{Cache, ErrorTolerance},
identifier::LocIdent,
position::{RawPos, TermPos},
stdlib::StdlibModule,
term::RichTerm,
@ -28,17 +27,14 @@ use nickel_lang_core::{
use nickel_lang_core::{stdlib, typecheck::Context};
use crate::{
analysis::{Analysis, AnalysisRegistry},
cache::CacheExt,
diagnostic::DiagnosticCompat,
field_walker::DefWithPath,
linearization::{Environment, ItemId, LinRegistry},
requests::{completion, formatting, goto, hover, symbols},
trace::Trace,
};
#[cfg(feature = "old-completer")]
use crate::linearization::completed::Completed;
pub const COMPLETIONS_TRIGGERS: &[&str] = &[".", "\"", "/"];
pub struct Server {
@ -46,9 +42,8 @@ pub struct Server {
pub cache: Cache,
/// In order to return diagnostics, we store the URL of each file we know about.
pub file_uris: HashMap<FileId, Url>,
pub lin_registry: LinRegistry,
pub analysis: AnalysisRegistry,
pub initial_ctxt: Context,
pub initial_env: Environment,
pub initial_term_env: crate::usage::Environment,
}
@ -90,32 +85,12 @@ impl Server {
connection,
cache,
file_uris: HashMap::new(),
lin_registry: LinRegistry::new(),
analysis: AnalysisRegistry::default(),
initial_ctxt,
initial_env: Environment::new(),
initial_term_env: crate::usage::Environment::new(),
}
}
pub fn initialize_stdlib_environment(&mut self) -> Option<()> {
let modules = stdlib::modules();
for module in modules {
// This module has a different format from the rest of the stdlib items
// Also, users are not supposed to use the internal module directly
if module == StdlibModule::Internals {
continue;
}
// The module is bound to its name in the environment.
let name: LocIdent = LocIdent::from(module.name());
let file_id = self.cache.get_submodule_file_id(module)?;
// We're using the ID 0 to get the top-level value, which is the body of the module.
let content_id = ItemId { file_id, index: 0 };
self.initial_env.insert(name.ident(), content_id);
}
Some(())
}
pub(crate) fn reply(&mut self, response: Response) {
trace!("Sending response: {:#?}", response);
@ -159,9 +134,8 @@ impl Server {
.typecheck_with_analysis(
file_id,
&self.initial_ctxt,
&self.initial_env,
&self.initial_term_env,
&mut self.lin_registry,
&mut self.analysis,
)
.unwrap();
@ -189,7 +163,6 @@ impl Server {
pub fn run(&mut self) -> Result<()> {
trace!("Running...");
self.linearize_stdlib()?;
self.initialize_stdlib_environment().unwrap();
while let Ok(msg) = self.connection.receiver.recv() {
trace!("Message: {:#?}", msg);
match msg {
@ -290,11 +263,10 @@ impl Server {
Ok(())
}
#[cfg(feature = "old-completer")]
pub fn lin_cache_get(&self, file_id: &FileId) -> Result<&Completed, ResponseError> {
self.lin_registry
.map
.get(file_id)
pub fn file_analysis(&self, file: FileId) -> Result<&Analysis, ResponseError> {
self.analysis
.analysis
.get(&file)
.ok_or_else(|| ResponseError {
data: None,
message: "File has not yet been parsed or cached.".to_owned(),
@ -304,14 +276,8 @@ impl Server {
pub fn lookup_term_by_position(&self, pos: RawPos) -> Result<Option<&RichTerm>, ResponseError> {
Ok(self
.lin_registry
.position_lookups
.get(&pos.src_id)
.ok_or_else(|| ResponseError {
data: None,
message: "File has not yet been parsed or cached.".to_owned(),
code: ErrorCode::ParseError as i32,
})?
.file_analysis(pos.src_id)?
.position_lookup
.get(pos.index))
}
@ -320,14 +286,8 @@ impl Server {
pos: RawPos,
) -> Result<Option<crate::identifier::LocIdent>, ResponseError> {
Ok(self
.lin_registry
.position_lookups
.get(&pos.src_id)
.ok_or_else(|| ResponseError {
data: None,
message: "File has not yet been parsed or cached.".to_owned(),
code: ErrorCode::ParseError as i32,
})?
.file_analysis(pos.src_id)?
.position_lookup
.get_ident(pos.index))
}

15
lsp/nls/src/trace.rs
View File

@ -219,24 +219,9 @@ impl<T, E: Display> ResultExt<E> for Result<T, E> {
}
pub mod param {
use crate::linearization::completed::Completed;
use super::{Enrich, ResultExt, Trace};
use lsp_server::RequestId;
impl Enrich<&Completed> for Trace {
fn enrich(id: &RequestId, param: &Completed) {
Self::with_trace(|mut t| {
t.received.entry(id.to_owned()).and_modify(|item| {
item.params.linearization_size = Some(param.linearization.len());
});
Ok(())
})
.report();
}
}
pub struct FileUpdate<'a> {
pub content: &'a str,
}