From a6411df61ad08dca81b5468ecd032f1d40b2f551 Mon Sep 17 00:00:00 2001 From: LSTM-Kirigaya <1193466151@qq.com> Date: Wed, 2 Oct 2024 22:40:54 +0800 Subject: [PATCH] save --- .gitignore | 3 +- Cargo.lock | 21 ++++++- Cargo.toml | 3 +- src/core/cache_storage.rs | 123 ++++++++++++++++++++++++++++++++++++-- src/core/sv_parser.rs | 2 +- src/hover/feature.rs | 2 +- src/main.rs | 4 +- src/request/mod.rs | 29 +++++---- src/server.rs | 8 +-- src/sources.rs | 77 +++++++++++++++++------- src/test/mod.rs | 71 +++++++++++++++++++++- src/utils/file.rs | 82 ++++++++++++++++++++++++- 12 files changed, 369 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 074a64a..657e4ff 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ test.txt build.bat -.DS_Store \ No newline at end of file +.DS_Store +.cache \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index cb468f2..d6f3413 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,6 +160,15 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -324,9 +333,9 @@ name = "digital-lsp" version = "0.0.1" dependencies = [ "anyhow", + "bincode", "flexi_logger", "log", - "once_cell", "path-clean", "pathdiff", "percent-encoding", @@ -342,6 +351,7 @@ dependencies = [ "tempdir", "tokio", "tower-lsp", + "uuid", "vhdl_lang", "walkdir", "which", @@ -1719,6 +1729,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", +] + [[package]] name = "vec_map" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 29788d0..e204fc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,8 @@ edition = "2018" [dependencies] sv-parser = { version = "0.13.3", path = "sv-parser/sv-parser"} vhdl_lang = { version = "^0.83.0", path = "rust_hdl/vhdl_lang" } -once_cell = "1.8" +uuid = { version = "1.0", features = ["v4"] } +bincode = "1.3" percent-encoding = "2.1.0" log = "0.4.19" tower-lsp = "0.20.0" diff --git a/src/core/cache_storage.rs b/src/core/cache_storage.rs index 0ca4134..0cf70f8 100644 --- a/src/core/cache_storage.rs +++ b/src/core/cache_storage.rs @@ -1,12 +1,127 @@ +use std::{collections::HashMap, fs::{self, File}, io::Read, path::PathBuf, str::FromStr, sync::{Arc, RwLock}}; + +use log::info; +use serde::{Deserialize, Serialize}; + +use crate::utils::{file_size_in_kb, get_last_modified_time, k_deserialize}; + +use super::hdlparam::FastHdlparam; + +pub enum CacheResult { + Ok(T), + NotNeedCache, + CacheNotFound, + NotHit, + FailDeserialize, + Error(String) +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CacheItem { + /// 文件名字 + pub file_name: String, + /// 文件大小，单位 KB + pub size: u64, + /// 版本，根据特定算法得出 + pub version: String, + /// 缓存文件的名字 + pub cache_name: String +} + + /// 用于进行高效 IR 缓存的模块 +pub struct CacheManager { + /// 缓存文件夹根目录 + pub root_dir: PathBuf, + /// meta 文件内容 + pub meta_name: String, + /// meta 内容 + pub meta: Arc>> +} -#[allow(unused)] -pub fn dump_ir_storage() { + +impl CacheManager { + pub fn new(root_dir: &str) -> Self { + // 读入 meta 文件 + let root_dir = PathBuf::from_str(root_dir).unwrap(); + let meta_name = "index.cache"; + let meta_path = root_dir.join(meta_name); + let meta = get_or_init_meta(&meta_path); + + // 如果不存在 root dir，则创建 + if !root_dir.exists() { + match fs::create_dir_all(&root_dir) { + Ok(_) => {}, + Err(err) => info!("error happen when create {root_dir:?}: {err:?}") + } + } + + CacheManager { + root_dir, + meta_name: meta_name.to_string(), + meta: Arc::new(RwLock::new(meta)) + } + } + + pub fn is_big_file(&self, path: &PathBuf) -> bool { + if let Ok(size) = file_size_in_kb(path.to_str().unwrap()) { + return size >= 1024; + } + + false + } + + fn get_version(&self, path: &PathBuf) -> String { + let path_str = path.to_str().unwrap(); + if let Ok(modify_ts) = get_last_modified_time(path_str) { + return modify_ts.to_string(); + } + "".to_string() + } + + /// ## 说明 + /// 尝试获取当前的路径文件的 fast 的缓存 + /// + /// ## None + /// + /// * 如果本来就没有缓存 + /// * 缓冲的 version 对不上 + pub fn try_get_fast_cache(&self, path: &PathBuf) -> CacheResult { + if !self.is_big_file(path) { + return CacheResult::NotNeedCache + } + let path_string = path.to_str().unwrap(); + let meta_handle = self.meta.read().unwrap(); + if let Some(cache_item) = meta_handle.get(path_string) { + let current_version = self.get_version(path); + let cache_path = self.root_dir.join(&cache_item.cache_name); + if current_version == cache_item.version && cache_path.exists() { + // hit cache + if let Ok(fast) = k_deserialize::(&cache_path) { + return CacheResult::Ok(fast); + } + } + } + CacheResult::CacheNotFound + } + + pub fn update_cache() { + + } } -#[allow(unused)] -pub fn load_ir_storage() { + +fn get_or_init_meta(meta_path: &PathBuf) -> HashMap { + if meta_path.exists() { + // 读取文件 + if let Ok(meta_map) = k_deserialize::>(meta_path) { + return meta_map; + } + } + + // 重新创建新的 + HashMap::::new() } \ No newline at end of file diff --git a/src/core/sv_parser.rs b/src/core/sv_parser.rs index f60247b..c79e931 100644 --- a/src/core/sv_parser.rs +++ b/src/core/sv_parser.rs @@ -28,7 +28,7 @@ pub fn sv_parser(path: &str) -> Option { let doc = Rope::from_str(&text); let uri = Url::from_file_path(&path).unwrap(); let result = recovery_sv_parse(&doc, &uri, &None, &includes); - + if let Some(syntax_tree) = result { if let Ok(hdlparam) = make_fast_from_syntaxtree(&syntax_tree, &path) { return Some(hdlparam); diff --git a/src/hover/feature.rs b/src/hover/feature.rs index 58573b7..1e9ed94 100644 --- a/src/hover/feature.rs +++ b/src/hover/feature.rs @@ -346,7 +346,7 @@ pub fn hover_module_declaration( let port_num = module.ports.len(); let param_num = module.params.len(); let instance_num = module.instances.len(); - let port_desc = format!("`port` {port_num}, `param` {param_num}, `instantiation` {instance_num}"); + let port_desc = format!("`$(instance-param) ` {port_num}, `param` {param_num}, `instantiation` {instance_num}"); // 统计 dir let mut input_count = 0 as u32; diff --git a/src/main.rs b/src/main.rs index 5672be0..e1dad98 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,7 +20,7 @@ mod server; mod sources; mod request; -use server::{Backend, GLOBAL_BACKEND}; +use server::Backend; #[derive(StructOpt, Debug)] #[structopt(name = "Digtal LSP", about = "LSP for Digital IDE")] @@ -40,8 +40,6 @@ async fn main() { let (service, socket) = LspService::build(|client| { let backend = Arc::new(Backend::new(client, log_handle)); - - let _ = GLOBAL_BACKEND.set(backend.clone()); backend }) .custom_method("custom/request", CustomRequest) diff --git a/src/request/mod.rs b/src/request/mod.rs index 05cdc27..9a365ae 100644 --- a/src/request/mod.rs +++ b/src/request/mod.rs @@ -16,7 +16,7 @@ use crate::core::sv_parser::make_fast_from_syntaxtree; use crate::core::vhdl_parser::{make_fast_from_design_file, vhdl_parse}; use crate::utils::*; -use crate::server::{Backend, GLOBAL_BACKEND}; +use crate::server::Backend; use crate::sources::recovery_sv_parse; @@ -92,7 +92,7 @@ pub fn do_fast<'a>(path: String, backend: &Arc) -> Result info!("parse fast \"{}\"", path); let language_id = get_language_id_by_path_str(&path); - match language_id.as_str() { + let res = match language_id.as_str() { "vhdl" => do_vhdl_fast(&path, backend), "verilog" | "systemverilog" => do_sv_fast(&path, backend), _ => Err(tower_lsp::jsonrpc::Error { @@ -100,7 +100,10 @@ pub fn do_fast<'a>(path: String, backend: &Arc) -> Result message: Cow::Owned(format!("invalid file: {path}, expect vhdl, verilog or system verilog!")), data: None }) - } + }; + + info!("finish parse"); + res } fn do_sv_fast(path: &str, backend: &Arc) -> Result { @@ -189,16 +192,16 @@ impl Notification for UpdateFastNotification { type Params = Self; } -#[allow(unused)] -pub async fn update_fast_to_client(fast: FastHdlparam, path: &PathBuf) { - // info!("send fast to foreend {:?}", fast); +// #[allow(unused)] +// pub async fn update_fast_to_client(fast: FastHdlparam, path: &PathBuf) { +// // info!("send fast to foreend {:?}", fast); - info!("send fast to foreend"); - let backend = GLOBAL_BACKEND.get().unwrap(); - let path = path.to_str().unwrap_or("").to_string(); +// info!("send fast to foreend"); +// let backend = GLOBAL_BACKEND.get().unwrap(); +// let path = path.to_str().unwrap_or("").to_string(); - let params = UpdateFastNotification { fast, path }; - info!("backend client: {:?}", backend.client); +// let params = UpdateFastNotification { fast, path }; +// info!("backend client: {:?}", backend.client); - backend.client.send_notification::(params).await; -} \ No newline at end of file +// backend.client.send_notification::(params).await; +// } \ No newline at end of file diff --git a/src/server.rs b/src/server.rs index d66d4a5..3bdf3f6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,18 +1,15 @@ +use crate::core::cache_storage::CacheManager; use crate::sources::*; use crate::completion::keyword::*; use flexi_logger::LoggerHandle; #[allow(unused)] use log::{debug, info, warn}; -use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; use std::string::ToString; use std::sync::{Arc, Mutex, RwLock}; use tower_lsp::jsonrpc::Result; use tower_lsp::lsp_types::*; use tower_lsp::{Client, LanguageServer}; - -pub static GLOBAL_BACKEND: OnceCell> = OnceCell::new(); - pub struct LSPServer { pub srcs: Sources, pub key_comps: Vec, @@ -169,8 +166,9 @@ impl Default for VeribleFormat { #[tower_lsp::async_trait] impl LanguageServer for Backend { - async fn initialize(&self, _: InitializeParams) -> Result { + async fn initialize(&self, params: InitializeParams) -> Result { self.server.srcs.init(); + // params.client_info.unwrap().name // 申明 LSP 的基本信息和提供的能力 let server_info = Some(ServerInfo { diff --git a/src/sources.rs b/src/sources.rs index 77f54f1..afad191 100644 --- a/src/sources.rs +++ b/src/sources.rs @@ -393,6 +393,7 @@ impl Sources { } } + /// 更加稳定地解析 sv 和 v /// 支持遇到错误进行自动修复，然后再解析 pub fn recovery_sv_parse( @@ -408,7 +409,32 @@ pub fn recovery_sv_parse( let mut reverted_change = false; let mut text = doc.clone(); - while i < parse_iterations { + let recover_text_by_byte_idx = |byte_idx: usize, reverted_change: &mut bool, text: &mut Rope| { + let mut line_start = text.byte_to_line(byte_idx); + let mut line_end = text.byte_to_line(byte_idx) + 1; + + // println!("byte: {byte_idx}, line_start: {line_start}, line_end: {line_end}"); + + if !(*reverted_change) { + if let Some(range) = last_change_range { + line_start = range.start.line as usize; + line_end = range.end.line as usize + 1; + *reverted_change = true; + } + } + + // 把 last_change 处的地方替换成空格 + for line_idx in line_start .. line_end { + let line = text.line(line_idx); + let start_char = text.line_to_char(line_idx); + let line_length = line.len_chars(); + text.remove(start_char..(start_char + line_length - 1)); + text.insert(start_char, &" ".to_owned().repeat(line_length)); + } + }; + + // 最多解析 500 次 + while i < parse_iterations && i < 500 { i += 1; match parse_sv_str( &text.to_string(), @@ -422,28 +448,13 @@ pub fn recovery_sv_parse( return Some(syntax_tree); } Err(err) => { + println!("err: {err:?}"); match err { // 语法错误 sv_parser::Error::Parse(trace) => match trace { - Some((_, bpos)) => { - let mut line_start = text.byte_to_line(bpos); - let mut line_end = text.byte_to_line(bpos) + 1; - if !reverted_change { - if let Some(range) = last_change_range { - line_start = range.start.line as usize; - line_end = range.end.line as usize + 1; - reverted_change = true; - } - } - - // 把 last_change 处的地方替换成空格 - for line_idx in line_start .. line_end { - let line = text.line(line_idx); - let start_char = text.line_to_char(line_idx); - let line_length = line.len_chars(); - text.remove(start_char..(start_char + line_length - 1)); - text.insert(start_char, &" ".to_owned().repeat(line_length)); - } + Some((_, byte_idx)) => { + // 把出错的地方替换成空格 + recover_text_by_byte_idx(byte_idx, &mut reverted_change, &mut text); parse_iterations += 1; } None => return None, @@ -476,11 +487,35 @@ pub fn recovery_sv_parse( let com_define = Define { identifier: not_found_macro_name.to_string(), arguments: Vec::new(), - text: Some(DefineText {text: "undefined".to_string(), origin: None}) + text: Some(DefineText {text: "dide-undefined".to_string(), origin: None}) }; defines.insert(not_found_macro_name, Some(com_define)); parse_iterations += 1; } + sv_parser::Error::Preprocess(trace) => match trace { + Some((_, byte_idx)) => { + // 把出错的地方替换成空格 + // println!("text {text:?}"); + recover_text_by_byte_idx(byte_idx, &mut reverted_change, &mut text); + parse_iterations += 1; + } + None => return None + } + sv_parser::Error::DefineArgNotFound(trace) => match trace { + Some((_, start_byte_idx, _)) => { + recover_text_by_byte_idx(start_byte_idx, &mut reverted_change, &mut text); + // recover_text_by_byte_idx(end_byte_idx, &mut reverted_change, &mut text); + parse_iterations += 1; + } + _ => return None + } + sv_parser::Error::DefineNoArgs(trace) => match trace { + Some((_, start_byte_idx, _)) => { + recover_text_by_byte_idx(start_byte_idx, &mut reverted_change, &mut text); + parse_iterations += 1; + } + _ => return None + } _ => error!("parse error, {:?}", err), }; } diff --git a/src/test/mod.rs b/src/test/mod.rs index 79af1e0..914714a 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -6,6 +6,9 @@ const TESTFILES_DIR: &str = "testfiles"; #[allow(unused)] const DIGTIAL_IDE_TEST: &str = "/home/dide/project/Digital-Test/Digital-IDE-test/user"; +#[allow(unused)] +const MIPS_DESIGN_TEST: &str = "/home/dide/project/Digital-Test/MipsDesign"; + #[allow(unused)] const TEST_FILE: &str = "/home/dide/project/Digital-Test/MipsDesign/src/MyCpu.v"; @@ -37,19 +40,21 @@ macro_rules! unwrap_option { #[cfg(test)] mod test_fast { - use std::{fs, path::Path}; + use std::{env, fs, path::Path}; + use crate::core::sv_parser::sv_parser; use super::*; #[test] fn test_mycpu() { - let hdlparam = sv_parser(TEST_FILE); - println!("{hdlparam:?}"); + let hdlparam = sv_parser("/home/dide/project/Digital-Test/Digital-IDE-test/user/src/svlog/tools/ivtest/undef.sv"); + // println!("{hdlparam:?}"); assert!(hdlparam.is_some()); } #[test] fn test_testfiles() { + let ws_path = env::current_dir().unwrap(); let entries = unwrap_result!(fs::read_dir(TESTFILES_DIR)); for entry in entries { let entry = unwrap_result!(entry); @@ -58,13 +63,18 @@ mod test_fast { let extension = unwrap_option!(file_path.extension()); let file_path = unwrap_option!(file_path.to_str()); if extension == "v" || extension == "sv" { + + let file_path = ws_path.join(file_path); + let file_path = file_path.to_str().unwrap(); println!("Test file: {:?}", file_path); + let _ = sv_parser(file_path); } } } } + // cargo test --release --package digital-lsp --lib -- test::test_fast::test_digital_ide_test --nocapture #[test] fn test_digital_ide_test() { // 判断路径是否存在且为文件夹 @@ -79,6 +89,20 @@ mod test_fast { } } + #[test] + fn test_mips_design() { + // 判断路径是否存在且为文件夹 + let path = Path::new(MIPS_DESIGN_TEST); + if path.exists() && path.is_dir() { + // 递归遍历文件夹 + if let Err(e) = traverse_directory(path) { + eprintln!("Error: {}", e); + } + } else { + eprintln!("Path does not exist or is not a directory"); + } + } + #[test] fn test_incomplete_example() { let path = Path::new(INCOMPLETE_EXAMPLE); @@ -119,6 +143,13 @@ mod test_fast { if path.to_str().unwrap() == "/home/dide/project/Digital-Test/Digital-IDE-test/user/src/svlog/tools/hdlconv/pri_encoder_using_assign.sv" { continue; } + if path.to_str().unwrap() == "/home/dide/project/Digital-Test/Digital-IDE-test/user/src/svlog/tools/ivtest/macro_with_args.sv" { + continue; + } + if path.to_str().unwrap() == "/home/dide/project/Digital-Test/Digital-IDE-test/user/src/vlog/element/macro.v" { + continue; + } + let file_path = path.to_str().unwrap(); let result = sv_parser(file_path); assert!(result.is_some()); @@ -358,4 +389,38 @@ mod test_scope_tree { } Ok(()) } +} + +#[cfg(test)] +mod test_file { + use std::fs; + + use crate::utils::{file_size_in_kb, get_language_id_by_pathbuf, RecursiveFileIterator}; + use super::*; + + #[test] + fn test_big_file_detect() { + let file_iter = RecursiveFileIterator::new(DIGTIAL_IDE_TEST); + // 超过 1 MB 的文件，我们都认为是大文件 + let threshold: u64 = 1024; + for pathbuf in file_iter { + let language_id = get_language_id_by_pathbuf(&pathbuf); + if language_id == "verilog" || language_id == "systemverilog" { + let size = file_size_in_kb(pathbuf.to_str().unwrap()); + if let Ok(size) = size { + if size >= threshold { + println!("detect big file : {:?}, size: {}", pathbuf, size); + } + } else { + eprintln!("error to get size of {:?}", pathbuf); + } + } + } + } + + #[test] + fn test_cache() { + let test_path = "/home/dide/project/digital-lsp-server/.cache"; + fs::create_dir_all("/home/dide/project/digital-lsp-server/.cache"); + } } \ No newline at end of file diff --git a/src/utils/file.rs b/src/utils/file.rs index 5315bed..dc856e7 100644 --- a/src/utils/file.rs +++ b/src/utils/file.rs @@ -1,7 +1,8 @@ -use std::{fs, path::PathBuf, str::FromStr}; +use std::{fmt::Debug, fs::{self, File}, io::{Read, Write}, path::PathBuf, str::FromStr, time::UNIX_EPOCH}; use log::info; use ropey::Rope; +use serde::{Deserialize, Serialize}; use tower_lsp::lsp_types::*; use crate::sources::LSPSupport; @@ -34,7 +35,9 @@ pub struct RecursiveFileIterator { stack: Vec, } +/// 递归获取所有文件 impl RecursiveFileIterator { + #[allow(unused)] pub fn new(start_dir: &str) -> Self { let start_dir = PathBuf::from_str(start_dir).unwrap(); RecursiveFileIterator { @@ -45,7 +48,6 @@ impl RecursiveFileIterator { impl Iterator for RecursiveFileIterator { type Item = PathBuf; - fn next(&mut self) -> Option { while let Some(path) = self.stack.pop() { if path.is_dir() { @@ -60,4 +62,80 @@ impl Iterator for RecursiveFileIterator { } None } +} + + +/// 计算文件大小，单位 KB +pub fn file_size_in_kb(path: &str) -> Result { + // 获取文件元数据 + let metadata = fs::metadata(path)?; + + // 获取文件大小（以字节为单位） + let file_size_bytes = metadata.len(); + + // 将文件大小转换为 KB，并确保结果在 u32 范围内 + let file_size_kb = (file_size_bytes / 1024) as u64; + + Ok(file_size_kb) +} + +pub fn get_last_modified_time(path: &str) -> Result { + // 获取文件元数据 + let metadata = fs::metadata(path)?; + + // 获取文件的最近修改时间 + let modified_time = metadata.modified()?; + + let duration = modified_time.duration_since(UNIX_EPOCH).map_err( + |e| std::io::Error::new(std::io::ErrorKind::Other, e) + )?; + + let seconds = duration.as_secs(); + + Ok(seconds) +} + +/// 序列化结构体到二进制文件 +pub fn k_serialize(path: &PathBuf, data: T) -> Result<(), std::io::Error> +where T: Serialize + Debug { + let buffer: Vec = bincode::serialize(&data).unwrap(); + match File::create(path) { + Ok(mut file) => { + match file.write_all(&buffer) { + Ok(_) => Ok(()), + Err(err) => Err(err) + } + } + Err(err) => Err(err) + } +} + +/// 反序列化二进制文件到结构体 +pub fn k_deserialize(path: &PathBuf) -> Result +where T: for<'de> Deserialize<'de> + Debug { + match File::open(path) { + Ok(mut file) => { + let mut buffer = Vec::::new(); + match file.read_to_end(&mut buffer) { + Ok(_) => { + let meta_map: Result> = bincode::deserialize(&buffer); + if meta_map.is_ok() { + Ok(meta_map.unwrap()) + } else { + let err: Box = meta_map.unwrap_err(); + info!("error happen when deserialize index.cache from \"{path:?}\": {err:?}"); + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, err)) + } + } + Err(err) => { + info!("error happen when load bytes to buffer from \"{path:?}\": {err:?}"); + Err(err) + } + } + } + Err(err) => { + info!("error happen when read index.cache from \"{path:?}\": {err:?}"); + Err(err) + } + } } \ No newline at end of file