Merge #678

678: WIP: automatically collect garbage r=matklad a=matklad



Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/ra_ide_api/src/db.rs

@@ -1,4 +1,7 @@
-use std::sync::Arc;
+use std::{
+    sync::Arc,
+    time,
+};
 
 use ra_db::{
     CheckCanceled, FileId, Canceled, SourceDatabase,
@@ -17,6 +20,8 @@ use crate::{LineIndex, symbol_index::{self, SymbolsDatabase}};
 pub(crate) struct RootDatabase {
     runtime: salsa::Runtime<RootDatabase>,
     interner: Arc<hir::HirInterner>,
+    pub(crate) last_gc: time::Instant,
+    pub(crate) last_gc_check: time::Instant,
 }
 
 impl salsa::Database for RootDatabase {
@@ -33,6 +38,8 @@ impl Default for RootDatabase {
         let mut db = RootDatabase {
             runtime: salsa::Runtime::default(),
             interner: Default::default(),
+            last_gc: time::Instant::now(),
+            last_gc_check: time::Instant::now(),
         };
         db.set_crate_graph(Default::default());
         db.set_local_roots(Default::default());
@@ -46,6 +53,8 @@ impl salsa::ParallelDatabase for RootDatabase {
         salsa::Snapshot::new(RootDatabase {
             runtime: self.runtime.snapshot(self),
             interner: Arc::clone(&self.interner),
+            last_gc: self.last_gc.clone(),
+            last_gc_check: self.last_gc_check.clone(),
         })
     }
 }

crates/ra_ide_api/src/imp.rs

@@ -1,4 +1,7 @@
-use std::sync::Arc;
+use std::{
+    sync::Arc,
+    time,
+};
 
 use hir::{
     self, Problem, source_binder
@@ -19,12 +22,14 @@ use crate::{
     CrateId, db, Diagnostic, FileId, FilePosition, FileRange, FileSystemEdit,
     Query, RootChange, SourceChange, SourceFileEdit,
     symbol_index::{FileSymbol, SymbolsDatabase},
+    status::syntax_tree_stats
 };
 
+const GC_COOLDOWN: time::Duration = time::Duration::from_millis(100);
+
 impl db::RootDatabase {
     pub(crate) fn apply_change(&mut self, change: AnalysisChange) {
         log::info!("apply_change {:?}", change);
-        // self.gc_syntax_trees();
         if !change.new_roots.is_empty() {
             let mut local_roots = Vec::clone(&self.local_roots());
             for (root_id, is_local) in change.new_roots {
@@ -72,18 +77,36 @@ impl db::RootDatabase {
         self.set_source_root(root_id, Arc::new(source_root));
     }
 
-    /// Ideally, we should call this function from time to time to collect heavy
-    /// syntax trees. However, if we actually do that, everything is recomputed
-    /// for some reason. Needs investigation.
+    pub(crate) fn maybe_collect_garbage(&mut self) {
+        if self.last_gc_check.elapsed() > GC_COOLDOWN {
+            self.last_gc_check = time::Instant::now();
+            let retained_trees = syntax_tree_stats(self).retained;
+            if retained_trees > 100 {
+                log::info!(
+                    "automatic garbadge collection, {} retained trees",
+                    retained_trees
+                );
+                self.collect_garbage();
+            }
+        }
+    }
+
     pub(crate) fn collect_garbage(&mut self) {
-        self.query(ra_db::ParseQuery)
-            .sweep(SweepStrategy::default().discard_values());
-        self.query(hir::db::HirParseQuery)
-            .sweep(SweepStrategy::default().discard_values());
-        self.query(hir::db::FileItemsQuery)
-            .sweep(SweepStrategy::default().discard_values());
-        self.query(hir::db::FileItemQuery)
-            .sweep(SweepStrategy::default().discard_values());
+        self.last_gc = time::Instant::now();
+
+        let sweep = SweepStrategy::default()
+            .discard_values()
+            .sweep_all_revisions();
+
+        self.query(ra_db::ParseQuery).sweep(sweep);
+
+        self.query(hir::db::HirParseQuery).sweep(sweep);
+        self.query(hir::db::FileItemsQuery).sweep(sweep);
+        self.query(hir::db::FileItemQuery).sweep(sweep);
+
+        self.query(hir::db::LowerModuleQuery).sweep(sweep);
+        self.query(hir::db::LowerModuleSourceMapQuery).sweep(sweep);
+        self.query(hir::db::BodySyntaxMappingQuery).sweep(sweep);
     }
 }
 

crates/ra_ide_api/src/lib.rs

@@ -59,6 +59,11 @@ pub use ra_db::{
     Canceled, CrateGraph, CrateId, FileId, FilePosition, FileRange, SourceRootId
 };
 
+// We use jemalloc mainly to get heap usage statistics, actual performance
+// differnece is not measures.
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
 pub type Cancelable<T> = Result<T, Canceled>;
 
 #[derive(Default)]
@@ -286,6 +291,10 @@ impl AnalysisHost {
         self.db.apply_change(change)
     }
 
+    pub fn maybe_collect_garbage(&mut self) {
+        self.db.maybe_collect_garbage();
+    }
+
     pub fn collect_garbage(&mut self) {
         self.db.collect_garbage();
     }

crates/ra_ide_api/src/status.rs

@@ -15,9 +15,13 @@ use crate::{
     symbol_index::{SymbolIndex, LibrarySymbolsQuery},
 };
 
+pub(crate) fn syntax_tree_stats(db: &RootDatabase) -> SyntaxTreeStats {
+    db.query(ParseQuery).entries::<SyntaxTreeStats>()
+}
+
 pub(crate) fn status(db: &RootDatabase) -> String {
     let files_stats = db.query(FileTextQuery).entries::<FilesStats>();
-    let syntax_tree_stats = db.query(ParseQuery).entries::<SyntaxTreeStats>();
+    let syntax_tree_stats = syntax_tree_stats(db);
     let symbols_stats = db
         .query(LibrarySymbolsQuery)
         .entries::<LibrarySymbolsStats>();
@@ -26,8 +30,13 @@ pub(crate) fn status(db: &RootDatabase) -> String {
         interner.len()
     };
     format!(
-        "{}\n{}\n{}\nn_defs {}\n",
-        files_stats, symbols_stats, syntax_tree_stats, n_defs
+        "{}\n{}\n{}\n{} defs\n\nmemory:\n{}\ngc {:?} seconds ago",
+        files_stats,
+        symbols_stats,
+        syntax_tree_stats,
+        n_defs,
+        MemoryStats::current(),
+        db.last_gc.elapsed().as_secs(),
     )
 }
 
@@ -58,9 +67,9 @@ impl FromIterator<TableEntry<FileId, Arc<String>>> for FilesStats {
 }
 
 #[derive(Default)]
-struct SyntaxTreeStats {
+pub(crate) struct SyntaxTreeStats {
     total: usize,
-    retained: usize,
+    pub(crate) retained: usize,
     retained_size: Bytes,
 }
 
@@ -118,6 +127,31 @@ impl FromIterator<TableEntry<SourceRootId, Arc<SymbolIndex>>> for LibrarySymbols
     }
 }
 
+struct MemoryStats {
+    allocated: Bytes,
+    resident: Bytes,
+}
+
+impl MemoryStats {
+    fn current() -> MemoryStats {
+        jemalloc_ctl::epoch().unwrap();
+        MemoryStats {
+            allocated: Bytes(jemalloc_ctl::stats::allocated().unwrap()),
+            resident: Bytes(jemalloc_ctl::stats::resident().unwrap()),
+        }
+    }
+}
+
+impl fmt::Display for MemoryStats {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            fmt,
+            "{} allocated {} resident",
+            self.allocated, self.resident,
+        )
+    }
+}
+
 #[derive(Default)]
 struct Bytes(usize);