digital-lsp-server/src/sources.rs
2024-09-30 15:57:03 +08:00

681 lines
24 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use crate::core::fast_hdlparam::FastHdlparam;
use crate::core::sv_parser::make_fast_from_syntaxtree;
use crate::core::vhdl_parser::make_fast_from_design_file;
use crate::core::vhdl_parser::vhdl_parse;
use crate::definition::def_types::*;
use crate::definition::get_scopes_from_syntax_tree;
use crate::diagnostics::{get_diagnostics, is_hidden};
use crate::server::LSPServer;
use crate::utils::to_escape_path;
#[allow(unused)]
use log::info;
use log::{debug, error};
use pathdiff::diff_paths;
use ropey::{Rope, RopeSlice};
use vhdl_lang::ast::DesignFile;
use std::cmp::min;
use std::collections::HashMap;
use std::env::current_dir;
use std::fs;
#[allow(unused)]
use std::ops::Deref;
use std::ops::Range as StdRange;
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::{Arc, Condvar, Mutex, RwLock};
use std::thread;
use sv_parser::*;
use thread::JoinHandle;
use tower_lsp::lsp_types::*;
use walkdir::WalkDir;
macro_rules! unwrap_result {
($expr:expr) => {
match $expr {
Ok(e) => e,
Err(_) => return
}
};
}
impl LSPServer {
pub fn did_open(&self, params: DidOpenTextDocumentParams) -> PublishDiagnosticsParams {
info!("[LSPServer] did open");
let document: TextDocumentItem = params.text_document;
let uri = document.uri.clone();
// check if doc is already added
if self.srcs.names.read().unwrap().contains_key(&document.uri) {
// convert to a did_change that replace the entire text
self.did_change(DidChangeTextDocumentParams {
text_document: VersionedTextDocumentIdentifier::new(document.uri, document.version),
content_changes: vec![TextDocumentContentChangeEvent {
range: None,
range_length: None,
text: document.text,
}],
});
} else {
self.srcs.add(document);
}
// diagnostics
let urls = self.srcs.names.read().unwrap().keys().cloned().collect();
let file_id = self.srcs.get_id(&uri);
let file = self.srcs.get_file(file_id).unwrap();
let file = file.read().unwrap();
get_diagnostics(uri, &file.text, urls, &self.conf.read().unwrap())
}
pub fn did_change(&self, params: DidChangeTextDocumentParams) {
info!("[LSPServer] did change, change content: {:?}", params.content_changes);
let file_id = self.srcs.get_id(&params.text_document.uri);
let file = self.srcs.get_file(file_id).unwrap();
let mut file = file.write().unwrap();
// loop through changes and apply
for change in params.content_changes {
if change.range.is_none() {
file.text = Rope::from_str(&change.text);
} else {
file.text.apply_change(&change);
}
file.last_change_range = change.range;
}
file.version = params.text_document.version;
drop(file);
// invalidate syntaxtree and wake parse thread
let meta_data = self.srcs.get_meta_data(file_id).unwrap();
let (lock, cvar) = &*meta_data.read().unwrap().valid_parse;
let mut valid = lock.lock().unwrap();
*valid = false;
cvar.notify_all();
}
pub fn did_save(&self, params: DidSaveTextDocumentParams) -> PublishDiagnosticsParams {
info!("[LSPServer] did save");
let urls = self.srcs.names.read().unwrap().keys().cloned().collect();
let file_id = self.srcs.get_id(&params.text_document.uri);
let file = self.srcs.get_file(file_id).unwrap();
let file = file.read().unwrap();
get_diagnostics(
params.text_document.uri,
&file.text,
urls,
&self.conf.read().unwrap(),
)
}
}
/// The Source struct holds all file specific information
pub struct Source {
/// id
pub id: usize,
/// uri
pub uri: Url,
/// 代码文本信息
pub text: Rope,
/// 版本号
pub version: i32,
/// 解析的 IR分为 DesignFileVHDL和 SyntaxTreeSV两种
pub parse_ir: Option<ParseIR>,
/// 用于在解析失败时恢复解析的量
pub last_change_range: Option<Range>,
}
pub enum ParseIR {
/// 基于 rust_hdl 的IR存放 VHDL
DesignFile(vhdl_lang::ast::DesignFile),
/// 存放 sv 的 IR
SyntaxTree(sv_parser::SyntaxTree)
}
/// file metadata, including whether or not the syntax tree is up to date
pub struct SourceMeta {
pub id: usize,
pub valid_parse: Arc<(Mutex<bool>, Condvar)>,
#[allow(unused)]
pub parse_handle: JoinHandle<()>,
}
/// find SystemVerilog/Verilog sources recursively from opened files
fn find_src_paths(dirs: &[PathBuf]) -> Vec<PathBuf> {
let mut paths: Vec<PathBuf> = Vec::new();
for dir in dirs {
let walker = WalkDir::new(dir).into_iter();
for entry in walker.filter_entry(|e| !is_hidden(e)) {
let entry = entry.unwrap();
if entry.file_type().is_file() && entry.path().extension().is_some() {
let extension = entry.path().extension().unwrap();
if extension == "sv" || extension == "svh" || extension == "v" || extension == "vh"
{
let entry_path = entry.path().to_path_buf();
if !paths.contains(&entry_path) {
paths.push(entry_path);
}
}
}
}
}
paths
}
/// The Sources struct manages all source files
pub struct Sources {
// all files
pub files: Arc<RwLock<Vec<Arc<RwLock<Source>>>>>,
// map file urls to id
pub names: Arc<RwLock<HashMap<Url, usize>>>,
// file metadata
pub meta: Arc<RwLock<Vec<Arc<RwLock<SourceMeta>>>>>,
/// scope tree 类型的树形结构,用于提供 sv 的 lsp
pub scope_tree: Arc<RwLock<Option<GenericScope>>>,
/// 存储 vhdl design file ir 的对象
pub design_file_map: Arc<RwLock<HashMap<String, DesignFile>>>,
// include directories, passed to parser to resolve `include
pub include_dirs: Arc<RwLock<Vec<PathBuf>>>,
// source directories
pub source_dirs: Arc<RwLock<Vec<PathBuf>>>,
// fast result
pub fast_map: Arc<RwLock<HashMap<String, FastHdlparam>>>
}
impl std::default::Default for Sources {
fn default() -> Self {
Self::new()
}
}
impl Sources {
pub fn new() -> Self {
Self {
files: Arc::new(RwLock::new(Vec::new())),
names: Arc::new(RwLock::new(HashMap::new())),
meta: Arc::new(RwLock::new(Vec::new())),
scope_tree: Arc::new(RwLock::new(None)),
design_file_map: Arc::new(RwLock::new(HashMap::new())),
include_dirs: Arc::new(RwLock::new(Vec::new())),
source_dirs: Arc::new(RwLock::new(Vec::new())),
fast_map: Arc::new(RwLock::new(HashMap::new()))
}
}
pub fn init(&self) {
let mut paths: Vec<PathBuf> = Vec::new();
for path in &*self.include_dirs.read().unwrap() {
paths.push(path.clone());
}
for path in &*self.source_dirs.read().unwrap() {
paths.push(path.clone());
}
// find and add all source/header files recursively from configured include and source directories
let src_paths = find_src_paths(&paths);
for path in src_paths {
let url = unwrap_result!(Url::from_file_path(&path));
let text = unwrap_result!(fs::read_to_string(&path));
let doc = TextDocumentItem::new(
url,
"systemverilog".to_string(),
-1,
text,
);
self.add(doc);
}
}
/// 增加一个 hdl 文件,并为该文件添加单独的解析线程
pub fn add(&self, doc: TextDocumentItem) {
// use a condvar to synchronize the parse thread
// the valid bool decides whether or not the file
// needs to be re-parsed
#[allow(clippy::mutex_atomic)] // https://github.com/rust-lang/rust-clippy/issues/1516
let valid_parse = Arc::new((Mutex::new(false), Condvar::new()));
let valid_parse2 = valid_parse.clone();
let mut files = self.files.write().unwrap();
let source = Arc::new(RwLock::new(Source {
id: files.len(),
uri: doc.uri.clone(),
text: Rope::from_str(&doc.text),
version: doc.version,
parse_ir: None,
last_change_range: None,
}));
let source_handle = source.clone();
let scope_handle = self.scope_tree.clone();
let design_file_handle = self.design_file_map.clone();
let fast_map_handle = self.fast_map.clone();
let inc_dirs = self.include_dirs.clone();
info!("launch worker to parse {:?}", doc.uri.to_string());
let language_id = doc.language_id.to_string();
// spawn parse thread
let parse_handle = thread::spawn(move || {
let (lock, cvar) = &*valid_parse2;
loop {
let file = source_handle.read().unwrap();
let text = file.text.clone();
let uri = &file.uri.clone();
let range = &file.last_change_range.clone();
drop(file);
info!("do parse in {:?}, language_id: {:?}", uri.to_string(), language_id);
match language_id.as_str() {
"vhdl" => {
vhdl_parser_pipeline(
&design_file_handle,
&fast_map_handle,
uri,
);
},
"verilog" | "systemverilog" => {
sv_parser_pipeline(
&source_handle,
&scope_handle,
&fast_map_handle,
&text,
uri,
range,
&inc_dirs.read().unwrap()
);
},
_ => {}
}
// 通过条件锁进行控制
let mut valid = lock.lock().unwrap();
*valid = true;
cvar.notify_all();
while *valid {
valid = cvar.wait(valid).unwrap();
}
}
});
files.push(source);
let fid = files.len() - 1;
self.meta
.write()
.unwrap()
.push(Arc::new(RwLock::new(SourceMeta {
id: fid,
valid_parse,
parse_handle,
})));
self.names.write().unwrap().insert(doc.uri, fid);
}
/// get file by id
pub fn get_file(&self, id: usize) -> Option<Arc<RwLock<Source>>> {
let files = self.files.read().ok()?;
for file in files.iter() {
let source = file.read().ok()?;
if source.id == id {
return Some(file.clone());
}
}
None
}
/// get metadata by file id
pub fn get_meta_data(&self, id: usize) -> Option<Arc<RwLock<SourceMeta>>> {
let meta = self.meta.read().ok()?;
for data in meta.iter() {
let i = data.read().ok()?;
if i.id == id {
return Some(data.clone());
}
}
None
}
/// wait for a valid parse
pub fn wait_parse_ready(&self, id: usize, wait_valid: bool) {
let file = self.get_file(id).unwrap();
let file = file.read().unwrap();
if file.parse_ir.is_none() || wait_valid {
drop(file);
let meta_data = self.get_meta_data(id).unwrap();
let (lock, cvar) = &*meta_data.read().unwrap().valid_parse;
let mut valid = lock.lock().unwrap();
while !*valid {
valid = cvar.wait(valid).unwrap();
}
}
}
/// get file id from url
pub fn get_id(&self, uri: &Url) -> usize {
*self.names.read().unwrap().get(uri).unwrap()
}
/// compute identifier completions
pub fn get_completions(
&self,
token: &str,
byte_idx: usize,
url: &Url,
) -> Option<CompletionList> {
debug!("retrieving identifier completion for token: {}", &token);
Some(CompletionList {
is_incomplete: false,
items: self
.scope_tree
.read()
.ok()?
.as_ref()?
.get_completion(token, byte_idx, url),
})
}
/// compute dot completions
pub fn get_dot_completions(
&self,
token: &str,
byte_idx: usize,
url: &Url,
) -> Option<CompletionList> {
debug!("retrieving dot completion for token: {}", &token);
let tree = self.scope_tree.read().ok()?;
Some(CompletionList {
is_incomplete: false,
items: tree
.as_ref()?
.get_dot_completion(token, byte_idx, url, tree.as_ref()?),
})
}
}
/// 更加稳定地解析 sv 和 v
/// 支持遇到错误进行自动修复,然后再解析
pub fn recovery_sv_parse(
doc: &Rope,
uri: &Url,
last_change_range: &Option<Range>,
inc_paths: &[PathBuf],
) -> Option<SyntaxTree> {
let mut parse_iterations = 1;
let mut i = 0;
let mut includes: Vec<PathBuf> = inc_paths.to_vec();
let mut defines = HashMap::new();
let mut reverted_change = false;
let mut text = doc.clone();
while i < parse_iterations {
i += 1;
match parse_sv_str(
&text.to_string(),
uri.to_file_path().unwrap(),
&defines,
&includes,
true,
true
) {
Ok((syntax_tree, _)) => {
return Some(syntax_tree);
}
Err(err) => {
match err {
// 语法错误
sv_parser::Error::Parse(trace) => match trace {
Some((_, bpos)) => {
let mut line_start = text.byte_to_line(bpos);
let mut line_end = text.byte_to_line(bpos) + 1;
if !reverted_change {
if let Some(range) = last_change_range {
line_start = range.start.line as usize;
line_end = range.end.line as usize + 1;
reverted_change = true;
}
}
// 把 last_change 处的地方替换成空格
for line_idx in line_start .. line_end {
let line = text.line(line_idx);
let start_char = text.line_to_char(line_idx);
let line_length = line.len_chars();
text.remove(start_char..(start_char + line_length - 1));
text.insert(start_char, &" ".to_owned().repeat(line_length));
}
parse_iterations += 1;
}
None => return None,
},
// 遇到 include 错误,那就把提示中的 include 加入解析中再次解析
sv_parser::Error::Include { source: x } => {
if let sv_parser::Error::File { source: _, path: z } = *x {
// Include paths have to be relative to the working directory
// so we have to convert a source file relative path to a working directory
// relative path. This should have been handled by sv-parser
let mut inc_path_given = z.clone();
let mut uri_path = uri.to_file_path().unwrap();
uri_path.pop();
let rel_path = diff_paths(uri_path, current_dir().unwrap()).unwrap();
inc_path_given.pop();
let inc_path = rel_path.join(inc_path_given);
if !includes.contains(&inc_path) {
includes.push(inc_path);
} else {
error!("parser: include error: {:?}", z);
break;
}
parse_iterations += 1;
}
}
// 宏定义不存在的错误
sv_parser::Error::DefineNotFound(not_found_macro_name) => {
let com_define = Define {
identifier: not_found_macro_name.to_string(),
arguments: Vec::new(),
text: Some(DefineText {text: "undefined".to_string(), origin: None})
};
defines.insert(not_found_macro_name, Some(com_define));
parse_iterations += 1;
}
_ => error!("parse error, {:?}", err),
};
}
}
}
None
}
pub fn sv_parser_pipeline(
source_handle: &Arc<RwLock<Source>>,
scope_handle: &Arc<RwLock<Option<GenericScope>>>,
fast_map_handle: &Arc<RwLock<HashMap<String, FastHdlparam>>>,
doc: &Rope,
uri: &Url,
last_change_range: &Option<Range>,
include_dirs: &Vec<PathBuf>
) {
let path = match PathBuf::from_str(uri.path()) {
Ok(path) => path,
Err(error) => {
info!("error happen in <goto_include_definition>: {:?}", error);
return;
}
};
let escape_path = to_escape_path(&path);
let escape_path_string = escape_path.to_str().unwrap_or("");
if escape_path_string.len() == 0 {
info!("error happen in [sv_parser_pipeline], escape_path_string is empty");
return;
}
let syntax_tree = recovery_sv_parse(
doc,
uri,
last_change_range,
include_dirs
);
// 更新 scope tree
let mut scope_tree = match &syntax_tree {
Some(tree) => get_scopes_from_syntax_tree(tree, uri),
None => None,
};
info!("finish parse {:?}", uri.to_string());
let mut file = source_handle.write().unwrap();
// 加入语法树 & 更新 fast
if let Some(syntax_tree) = syntax_tree {
if let Ok(hdlparam) = make_fast_from_syntaxtree(&syntax_tree, &escape_path) {
let mut fast_map = fast_map_handle.write().unwrap();
fast_map.insert(escape_path_string.to_string(), hdlparam);
}
let parse_ir = ParseIR::SyntaxTree(syntax_tree);
file.parse_ir = Some(parse_ir);
} else {
file.parse_ir = None;
}
// file.syntax_tree = syntax_tree;
drop(file);
// 更新 global_scope用于 sv 的解析
// global_scope 为全局最大的那个 scope它的 scopes 和 defs 下的元素和每一个文件一一对应
let mut global_scope = scope_handle.write().unwrap();
match &mut *global_scope {
Some(scope) => match &mut scope_tree {
Some(tree) => {
// 更新所有 uri 为当前 uri 的文件结构
scope.defs.retain(|x| &x.url() != uri);
scope.scopes.retain(|x| &x.url() != uri);
scope.defs.append(&mut tree.defs);
scope.scopes.append(&mut tree.scopes);
}
None => (),
},
// 使用 scope_tree 来更新全局的 scope
None => *global_scope = scope_tree,
}
// eprintln!("{:#?}", *global_scope);
drop(global_scope);
}
pub fn vhdl_parser_pipeline(
design_file_handle: &Arc<RwLock<HashMap<String, DesignFile>>>,
fast_map_handle: &Arc<RwLock<HashMap<String, FastHdlparam>>>,
uri: &Url
) {
let path = match PathBuf::from_str(uri.path()) {
Ok(path) => path,
Err(error) => {
info!("error happen in <goto_include_definition>: {:?}", error);
return;
}
};
let escape_path = to_escape_path(&path);
let escape_path_string = escape_path.to_str().unwrap_or("");
if escape_path_string.len() == 0 {
info!("error happen in [vhdl_parser_pipeline], escape_path_string is empty");
return;
}
if let Some(design_file) = vhdl_parse(&escape_path) {
let mut design_files = design_file_handle.write().unwrap();
if let Some(hdlparam) = make_fast_from_design_file(&design_file) {
let mut fast_map = fast_map_handle.write().unwrap();
fast_map.insert(escape_path_string.to_string(), hdlparam);
}
design_files.insert(escape_path_string.to_string(), design_file);
}
}
//TODO: add bounds checking for utf8<->utf16 conversions
/// This trait defines some helper functions to convert between lsp types
/// and char/byte positions
pub trait LSPSupport {
fn pos_to_byte(&self, pos: &Position) -> usize;
fn pos_to_char(&self, pos: &Position) -> usize;
fn byte_to_pos(&self, byte_idx: usize) -> Position;
fn char_to_pos(&self, char_idx: usize) -> Position;
fn range_to_char_range(&self, range: &Range) -> StdRange<usize>;
fn char_range_to_range(&self, range: StdRange<usize>) -> Range;
fn apply_change(&mut self, change: &TextDocumentContentChangeEvent);
}
/// Extend ropey's Rope type with lsp convenience functions
impl LSPSupport for Rope {
fn pos_to_byte(&self, pos: &Position) -> usize {
return self.char_to_byte(self.pos_to_char(pos));
}
fn pos_to_char(&self, pos: &Position) -> usize {
let line_slice = self.line(pos.line as usize);
return self.line_to_char(pos.line as usize) + line_slice.utf16_cu_to_char(pos.character as usize);
}
fn byte_to_pos(&self, byte_idx: usize) -> Position {
return self.char_to_pos(self.byte_to_char(min(byte_idx, self.len_bytes() - 1)));
}
fn char_to_pos(&self, char_idx: usize) -> Position {
let line = self.char_to_line(char_idx);
let line_slice = self.line(line);
return Position {
line: line as u32,
character: line_slice.char_to_utf16_cu(char_idx - self.line_to_char(line)) as u32,
};
}
fn range_to_char_range(&self, range: &Range) -> StdRange<usize> {
return self.pos_to_char(&range.start)..self.pos_to_char(&range.end);
}
fn char_range_to_range(&self, range: StdRange<usize>) -> Range {
return Range {
start: self.char_to_pos(range.start),
end: self.char_to_pos(range.end),
};
}
fn apply_change(&mut self, change: &TextDocumentContentChangeEvent) {
if let Some(range) = change.range {
let char_range = self.range_to_char_range(&range);
self.remove(char_range.clone());
if !change.text.is_empty() {
self.insert(char_range.start, &change.text);
}
}
}
}
impl<'a> LSPSupport for RopeSlice<'a> {
fn pos_to_byte(&self, pos: &Position) -> usize {
self.char_to_byte(self.pos_to_char(pos))
}
fn pos_to_char(&self, pos: &Position) -> usize {
let line_slice = self.line(pos.line as usize);
self.line_to_char(pos.line as usize) + line_slice.utf16_cu_to_char(pos.character as usize)
}
fn byte_to_pos(&self, byte_idx: usize) -> Position {
self.char_to_pos(self.byte_to_char(min(byte_idx, self.len_bytes() - 1)))
}
fn char_to_pos(&self, char_idx: usize) -> Position {
let line = self.char_to_line(char_idx);
let line_slice = self.line(line);
Position {
line: line as u32,
character: line_slice.char_to_utf16_cu(char_idx - self.line_to_char(line)) as u32,
}
}
fn range_to_char_range(&self, range: &Range) -> StdRange<usize> {
self.pos_to_char(&range.start)..self.pos_to_char(&range.end)
}
fn char_range_to_range(&self, range: StdRange<usize>) -> Range {
Range {
start: self.char_to_pos(range.start),
end: self.char_to_pos(range.end),
}
}
fn apply_change(&mut self, _: &TextDocumentContentChangeEvent) {
panic!("can't edit a rope slice");
}
}