THE BIG MERGE v0.1

This commit is contained in:
Noah Santschi-Cooney 2020-08-03 01:57:19 +01:00
parent d80adf83a3
commit 88f165dd33
No known key found for this signature in database
GPG key ID: 3B22282472C8AE48
6 changed files with 235 additions and 97 deletions

View file

@ -7,4 +7,7 @@ test:
RUST_LIB_BACKTRACE=0 RUST_BACKTRACE=0 cargo test
build:
cargo build
build-release:
cargo build --release

View file

@ -54,6 +54,10 @@ impl CachedStableGraph {
&self.graph[node]
}
pub fn get_edge_meta(&self, parent: NodeIndex, child: NodeIndex) -> &IncludePosition {
self.graph.edge_weight(self.graph.find_edge(parent, child).unwrap()).unwrap()
}
pub fn remove_node(&mut self, name: impl Into<String>) {
let idx = self.cache.remove(&name.into());
if let Some(idx) = idx {
@ -69,7 +73,7 @@ impl CachedStableGraph {
idx
}
pub fn add_edge(&mut self, parent: NodeIndex, child: NodeIndex, line: u64, start: u64, end: u64) -> EdgeIndex {
pub fn add_edge(&mut self, parent: NodeIndex, child: NodeIndex, line: usize, start: usize, end: usize) -> EdgeIndex {
let child_path = self.reverse_index.get(&child).unwrap().clone();
self.graph.add_edge(parent, child, IncludePosition{filepath: child_path, line, start, end})
}

View file

@ -1,3 +1,6 @@
#![feature(write_all_vectored)]
#![feature(iter_map_while)]
use rust_lsp::jsonrpc::{*, method_types::*};
use rust_lsp::lsp::*;
use rust_lsp::lsp_types::{*, notification::*};
@ -15,6 +18,7 @@ use std::ops::Add;
use std::process;
use std::rc::Rc;
use std::fs;
use std::iter::Extend;
use anyhow::Result;
@ -30,6 +34,7 @@ mod graph;
mod provider;
mod lsp_ext;
mod dfs;
mod merge_views;
#[cfg(test)]
mod test;
@ -65,7 +70,7 @@ fn main() {
langserver.command_provider = Some(provider::CustomCommandProvider::new(vec![(
"graphDot",
Box::new(provider::GraphDotCommand {
graph: langserver.graph.clone(),
graph: Rc::clone(&langserver.graph),
}),
)]));
@ -96,9 +101,9 @@ impl Default for Configuration {
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct IncludePosition {
filepath: String,
line: u64,
start: u64,
end: u64,
line: usize,
start: usize,
end: usize,
}
impl Display for IncludePosition {
@ -112,11 +117,6 @@ struct GLSLFile {
includes: Vec<IncludePosition>,
}
struct MergeMeta {
node: NodeIndex,
last_slice: u64
}
impl MinecraftShaderLanguageServer {
pub fn error_not_available<DATA>(data: DATA) -> MethodError<DATA> {
let msg = "Functionality not implemented.".to_string();
@ -215,8 +215,8 @@ impl MinecraftShaderLanguageServer {
.unwrap();
//eprintln!("{:?}", caps);
let start = u64::try_from(cap.start()).unwrap();
let end = u64::try_from(cap.end()).unwrap();
let start = cap.start();
let end = cap.end();
let mut path: String = cap.as_str().into();
if !path.starts_with('/') {
path.insert(0, '/');
@ -224,7 +224,7 @@ impl MinecraftShaderLanguageServer {
let full_include = String::from(root).add("/shaders").add(path.as_str());
includes.push(IncludePosition {
filepath: full_include,
line: u64::try_from(line.0).unwrap(),
line: line.0,
start,
end,
});
@ -234,7 +234,7 @@ impl MinecraftShaderLanguageServer {
includes
}
pub fn lint(&self, uri: &str, source: impl Into<String>) -> Result<Vec<Diagnostic>> {
pub fn lint(&self, uri: &str) -> Result<Vec<Diagnostic>> {
// get all top level ancestors of this file
let file_ancestors = match self.get_file_toplevel_ancestors(uri) {
Ok(opt) => match opt {
@ -244,19 +244,22 @@ impl MinecraftShaderLanguageServer {
Err(e) => return Err(e),
};
let source: Rc<String> = Rc::new(source.into());
let mut all_sources: HashMap<String, String> = HashMap::new();
eprintln!("ancestors for {}:\n{:?}", uri, file_ancestors.iter().map(|e| self.graph.borrow().graph.node_weight(*e).unwrap().clone()).collect::<Vec<String>>());
let merge_list: Vec<LinkedList<&str>> = if file_ancestors.is_empty() {
let source = String::from_utf8(fs::read(uri)?)?;
let mut list = LinkedList::new();
list.push_back(source.as_str());
all_sources.insert(String::from(uri), source);
list.push_back(&all_sources.iter().next().unwrap().1.as_str()[..]);
vec![list]
} else {
let mut lists = Vec::with_capacity(file_ancestors.len());
for root in file_ancestors {
match self.generate_merge_list(root) {
Ok((sources, views)) => lists.push(views),
let mut all_trees = Vec::new();
for root in &file_ancestors {
let nodes = match self.get_dfs_for_node(*root) {
Ok(nodes) => nodes,
Err(e) => {
let e = e.downcast::<dfs::CycleError>().unwrap();
return Ok(vec![Diagnostic{
@ -269,22 +272,34 @@ impl MinecraftShaderLanguageServer {
related_information: None,
}])
}
}
};
let sources = self.load_sources(&nodes)?;
all_trees.push(nodes);
all_sources.extend(sources);
}
for (i, root) in file_ancestors.iter().enumerate() {
//self.generate_merge_list(*root, all_trees.get(i).unwrap(), RefCell::new(&all_sources));
}
lists
};
let mut diagnostics = Vec::new();
// run merged source through validator
let stdout = self.invoke_validator(source.as_ref())?;
let stdout = stdout.trim();
// run merged sources through validator
for root in merge_list {
let stdout = self.invoke_validator(root)?;
let stdout = stdout.trim();
eprintln!("glslangValidator output: {}\n", stdout);
diagnostics.extend(self.parse_validator_stdout(stdout, ""));
}
eprintln!("glslangValidator output: {}\n", stdout);
Ok(diagnostics)
}
fn parse_validator_stdout(&self, stdout: &str, source: &str) -> Vec<Diagnostic> {
let mut diagnostics: Vec<Diagnostic> = vec![];
let source_lines: Vec<&str> = source.split('\n').collect();
stdout.split('\n').for_each(|line| {
let diagnostic_capture = match RE_DIAGNOSTIC.captures(line) {
Some(d) => d,
@ -335,7 +350,33 @@ impl MinecraftShaderLanguageServer {
diagnostics.push(diagnostic);
});
Ok(diagnostics)
diagnostics
}
pub fn get_dfs_for_node(&self, root: NodeIndex) -> Result<Vec<(NodeIndex, Option<NodeIndex>)>> {
let graph_ref = self.graph.borrow();
let dfs = dfs::Dfs::new(&graph_ref, root);
dfs.collect::<Result<Vec<_>, _>>()
}
pub fn load_sources(&self, nodes: &[(NodeIndex, Option<NodeIndex>)]) -> Result<HashMap<String, String>> {
let mut sources = HashMap::new();
for node in nodes {
let graph = self.graph.borrow();
let path = graph.get_node(node.0);
if sources.contains_key(path) {
continue;
}
let source = String::from_utf8(fs::read(path)?)?;
sources.insert(path.clone(), source);
}
Ok(sources)
}
fn get_file_toplevel_ancestors(&self, uri: &str) -> Result<Option<Vec<petgraph::stable_graph::NodeIndex>>> {
@ -350,41 +391,7 @@ impl MinecraftShaderLanguageServer {
Ok(Some(roots))
}
pub fn generate_merge_list(&self, root: NodeIndex) -> Result<(LinkedList<String>, LinkedList<&str>)> {
let mut merge_list = LinkedList::new();
// need to return all sources along with the views to appease the lifetime god
let mut all_sources = LinkedList::new();
let graph_ref = self.graph.borrow();
let dfs = dfs::Dfs::new(&graph_ref, root);
//let slice_stack = Vec::new();
let iteration_order: Vec<_> = dfs.collect::<Result<Vec<_>, _>>()?;
/* for n in dfs {
if n.is_err() {
return Err(n.err().unwrap());
}
/* let path = self.graph.borrow().get_node(n);
let file_content = String::from_utf8(std::fs::read(path).unwrap()); */
} */
/* let children = self.graph.borrow().child_node_indexes(root);
// include positions sorted by earliest in file
let mut all_edges: Vec<IncludePosition> = self.graph.borrow().edge_weights(root);
all_edges.sort();
eprintln!("include positions for {:?}: {:?} {:?}", self.graph.borrow().graph.node_weight(root).unwrap(), all_edges, children); */
Ok((all_sources, merge_list))
}
fn invoke_validator(&self, source: &str) -> Result<String> {
let source: String = source.into();
fn invoke_validator(&self, source: LinkedList<&str>) -> Result<String> {
eprintln!("validator bin path: {}", self.config.glslang_validator_path);
let cmd = process::Command::new(&self.config.glslang_validator_path)
.args(&["--stdin", "-S", "frag"])
@ -394,7 +401,10 @@ impl MinecraftShaderLanguageServer {
let mut child = cmd?;//.expect("glslangValidator failed to spawn");
let stdin = child.stdin.as_mut().expect("no stdin handle found");
stdin.write_all(source.as_bytes())?;//.expect("failed to write to stdin");
let mut io_slices: Vec<std::io::IoSlice> = source.iter().map(|s| std::io::IoSlice::new(s.as_bytes())).collect();
stdin.write_all_vectored(&mut io_slices[..])?;//.expect("failed to write to stdin");
let output = child.wait_with_output()?;//.expect("expected output");
let stdout = String::from_utf8(output.stdout).unwrap();
@ -510,7 +520,7 @@ impl LanguageServerHandling for MinecraftShaderLanguageServer {
fn did_open_text_document(&mut self, params: DidOpenTextDocumentParams) {
eprintln!("opened doc {}", params.text_document.uri);
match self.lint(params.text_document.uri.path(), params.text_document.text) {
match self.lint(params.text_document.uri.path()/* , params.text_document.text */) {
Ok(diagnostics) => self.publish_diagnostic(diagnostics, params.text_document.uri, None),
Err(e) => eprintln!("error linting: {}", e),
}
@ -531,8 +541,8 @@ impl LanguageServerHandling for MinecraftShaderLanguageServer {
let path: String = percent_encoding::percent_decode_str(params.text_document.uri.path()).decode_utf8().unwrap().into();
let file_content = fs::read(path).unwrap();
match self.lint(params.text_document.uri.path(), String::from_utf8(file_content).unwrap()) {
/*let file_content = fs::read(path).unwrap(); */
match self.lint(path.as_str()/* , String::from_utf8(file_content).unwrap() */) {
Ok(diagnostics) => self.publish_diagnostic(diagnostics, params.text_document.uri, None),
Err(e) => eprintln!("error linting: {}", e),
}
@ -664,8 +674,12 @@ impl LanguageServerHandling for MinecraftShaderLanguageServer {
Some(DocumentLink {
range: Range::new(
Position::new(value.line, value.start),
Position::new(value.line, value.end),
Position::new(
u64::try_from(value.line).unwrap(),
u64::try_from(value.start).unwrap()),
Position::new(
u64::try_from(value.line).unwrap(),
u64::try_from(value.end).unwrap()),
),
target: Some(url),
//tooltip: Some(url.path().to_string().strip_prefix(self.root.clone().unwrap().as_str()).unwrap().to_string()),

102
server/src/merge_views.rs Normal file
View file

@ -0,0 +1,102 @@
use std::collections::{HashMap, LinkedList};
use std::cell::RefCell;
use std::iter::Peekable;
use core::slice::Iter;
use petgraph::stable_graph::NodeIndex;
use crate::graph::CachedStableGraph;
pub struct MergeViewGenerator<'a> {
sources: &'a mut HashMap<String, String>,
graph: &'a CachedStableGraph,
line_directives: RefCell<Vec<String>>,
}
impl <'a> MergeViewGenerator<'a> {
pub fn new(sources: &'a mut HashMap<String, String>, graph: &'a CachedStableGraph) -> Self {
Self { sources, graph, line_directives: RefCell::new(Vec::new()) }
}
pub fn generate_merge_list(&'a mut self, nodes: &'a [(NodeIndex, Option<NodeIndex>)]) -> LinkedList<&'a str> {
// list of source code views onto the below sources
let mut merge_list: LinkedList<&'a str> = LinkedList::new();
self.line_directives.borrow_mut().reserve(nodes.len() * 2);
let mut last_offset_set: HashMap<String, usize> = HashMap::new();
let mut last_offset: Vec<String> = Vec::new();
let mut nodes_iter = nodes.iter().peekable();
nodes_iter.next();
self.create_merge_views(nodes_iter, &mut merge_list, &mut last_offset_set, &mut last_offset);
for file in last_offset.iter().rev() {
let offset = *last_offset_set.get(file).unwrap();
merge_list.push_back(&self.sources.get(file).unwrap().as_str()[offset..]);
}
merge_list
}
fn create_merge_views(
&'a self,
mut nodes: Peekable<Iter<(NodeIndex, Option<NodeIndex>)>>,
merge_list: &mut LinkedList<&'a str>,
last_offset_set: &mut HashMap<String, usize>,
last_offset: &mut Vec<String>,
) {
let n = match nodes.next() {
Some(n) => n,
None => return,
};
let parent = n.1.unwrap();
let child = n.0;
let edge = self.graph.get_edge_meta(parent, child);
let parent_path = self.graph.get_node(parent).clone();
let child_path = self.graph.get_node(child).clone();
if !last_offset_set.contains_key(&parent_path) {
last_offset.push(parent_path.clone());
}
let offset = *last_offset_set.insert(parent_path.clone(), edge.line).get_or_insert(0);
let source = self.sources.get(&parent_path).unwrap();
let mut char_for_line: usize = 0;
let mut char_following_line: usize = 0;
for (n, line) in source.as_str().lines().enumerate() {//.map(|s| s.len()).sum();
if n == edge.line {
char_following_line += line.len()+1;
break;
}
char_for_line += line.len()+1;
char_following_line = char_for_line;
}
// TODO: update after finding offset? why set it to 0 on L71 then
last_offset_set.insert(parent_path.clone(), char_following_line);
merge_list.push_back(&source.as_str()[offset..char_for_line]);
merge_list.push_back(&"#line 1\n"[..]);
match nodes.peek() {
Some(_) => self.create_merge_views(nodes, merge_list, last_offset_set, last_offset),
None => {
let source = self.sources.get(&child_path).unwrap();
merge_list.push_back(&source.as_str()[..]);
}
}
// +2 because edge.line is 0 indexed but #line is 1 indexed and references the *following* line
let line_directive = format!("\n#line {}\n", edge.line+2);
self.line_directives.borrow_mut().push(line_directive);
unsafe {
// :^)
let vec_ptr_offset = self.line_directives.borrow().as_ptr().add(self.line_directives.borrow().len()-1);
merge_list.push_back(&vec_ptr_offset.as_ref().unwrap().as_str()[..]);
}
}
}

View file

@ -52,6 +52,26 @@ fn copy_files(files: &str, dest: &TempDir) {
copy_items(&files, dest.path().join("shaders"), opts).unwrap();
}
fn copy_to_and_set_root(test_path: &str, server: &mut MinecraftShaderLanguageServer) -> (Rc<TempDir>, String) {
let (tmp_dir, tmp_path) = copy_to_tmp_dir(test_path);
server.root = Some(format!("{}{}", "file://", tmp_path));
(tmp_dir, tmp_path)
}
fn copy_to_tmp_dir(test_path: &str) -> (Rc<TempDir>, String) {
let tmp_dir = Rc::new(TempDir::new("mcshader").unwrap());
fs::create_dir(tmp_dir.path().join("shaders")).unwrap();
copy_files(test_path, &tmp_dir);
let tmp_clone = tmp_dir.clone();
let tmp_path = tmp_clone.path().as_os_str().to_str().unwrap();
(tmp_dir, tmp_path.into())
}
#[allow(deprecated)]
#[test]
fn test_empty_initialize() {
@ -92,7 +112,6 @@ fn test_empty_initialize() {
assert_eq!(format!("{:?}", server.wait), "WaitGroup { count: 1 }");
server.endpoint.request_shutdown();
//std::thread::sleep(std::time::Duration::from_secs(1));
}
#[allow(deprecated)]
@ -100,18 +119,12 @@ fn test_empty_initialize() {
fn test_01_initialize() {
let mut server = new_temp_server();
let tmp_dir = TempDir::new("mcshader").unwrap();
fs::create_dir(tmp_dir.path().join("shaders")).unwrap();
copy_files("./testdata/01", &tmp_dir);
let tmp_path = tmp_dir.path().as_os_str().to_str().unwrap();
let tmp_uri = format!("{}{}", "file://", tmp_path);
let (tmp_dir, tmp_path) = copy_to_tmp_dir("./testdata/01");
let initialize_params = InitializeParams{
process_id: None,
root_path: None,
root_uri: Some(Url::parse(tmp_uri.as_str()).unwrap()),
root_uri: Some(Url::parse(format!("{}{}", "file://", tmp_path).as_str()).unwrap()),
client_info: None,
initialization_options: None,
capabilities: ClientCapabilities{workspace: None, text_document: None, experimental: None, window: None},
@ -138,8 +151,10 @@ fn test_01_initialize() {
let (node1, node2) = server.graph.borrow().graph.edge_endpoints(edge).unwrap();
// Assert the values of the two nodes in the tree
assert_eq!(server.graph.borrow().graph[node1].as_str(), tmp_dir.path().join("shaders").join("final.fsh").as_os_str().to_str().unwrap());
assert_eq!(server.graph.borrow().graph[node2].as_str(), tmp_dir.path().join("shaders").join("common.glsl").as_os_str().to_str().unwrap());
assert_eq!(server.graph.borrow().graph[node1], format!("{}/{}/{}", tmp_path, "shaders", "final.fsh"));
assert_eq!(server.graph.borrow().graph[node2], format!("{}/{}/{}", tmp_path, "shaders", "common.glsl"));
assert_eq!(server.graph.borrow().graph.edge_weight(edge).unwrap().line, 2);
server.endpoint.request_shutdown();
}
@ -361,33 +376,33 @@ fn test_graph_dfs_cycle() {
fn test_generate_merge_list() {
let mut server = new_temp_server();
let tmp_dir = TempDir::new("mcshader").unwrap();
fs::create_dir(tmp_dir.path().join("shaders")).unwrap();
let (tmp_dir, tmp_path) = copy_to_and_set_root("./testdata/01", &mut server);
copy_files("./testdata/01", &tmp_dir);
let tmp_path = tmp_dir.path().as_os_str().to_str().unwrap();
let tmp_uri = format!("{}{}/shaders", "file://", tmp_path);
server.root = Some(tmp_uri);
let final_idx = server.graph.borrow_mut().add_node(format!("{}/{}", tmp_path, "final.fsh"));
let common_idx = server.graph.borrow_mut().add_node(format!("{}/{}", tmp_path, "common.glsl"));
let final_idx = server.graph.borrow_mut().add_node(format!("{}/shaders/{}", tmp_path, "final.fsh"));
let common_idx = server.graph.borrow_mut().add_node(format!("{}/shaders/{}", tmp_path, "common.glsl"));
server.graph.borrow_mut().add_edge(final_idx, common_idx, 2, 0, 0);
let nodes = server.get_dfs_for_node(final_idx).unwrap();
let mut sources = server.load_sources(&nodes).unwrap();
let result = server.generate_merge_list(final_idx);
println!("{:?}", nodes);
let graph_borrow = server.graph.borrow();
let mut merger = merge_views::MergeViewGenerator::new(&mut sources, &graph_borrow);
assert_that!(&result, ok());
let result = merger.generate_merge_list(&nodes);
//let result: LinkedList<&str> = LinkedList::new();
//assert_that!(&result, ok());
let total: String = result
.unwrap().1
.iter()
.map(|s| &**s)
.collect::<Vec<&str>>()
.join("");
let merge_file = String::from(tmp_path) + "/shaders/final.fsh.merge";
let merge_file = tmp_path + "/shaders/final.fsh.merge";
let truth = String::from_utf8(fs::read::<String>(merge_file).unwrap()).unwrap();

View file

@ -1,11 +1,11 @@
#version 120
#line 1 1
#line 1
float test() {
return 0.5;
}
#line 4 0
#line 4
void main() {
gl_FragColor = vec4(0.0);
gl_FragColor[0] = vec4(0.0);
}