mirror of
https://github.com/salsa-rs/salsa.git
synced 2025-07-09 22:35:22 +00:00
292 lines
24 KiB
HTML
292 lines
24 KiB
HTML
<!DOCTYPE HTML>
|
|
<html lang="en" class="sidebar-visible no-js light">
|
|
<head>
|
|
<!-- Book generated using mdBook -->
|
|
<meta charset="UTF-8">
|
|
<title>Defining the parser: memoized functions and inputs - Salsa</title>
|
|
|
|
|
|
<!-- Custom HTML head -->
|
|
|
|
|
|
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
|
|
<meta name="description" content="">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="theme-color" content="#ffffff" />
|
|
|
|
<link rel="icon" href="../favicon.svg">
|
|
<link rel="shortcut icon" href="../favicon.png">
|
|
<link rel="stylesheet" href="../css/variables.css">
|
|
<link rel="stylesheet" href="../css/general.css">
|
|
<link rel="stylesheet" href="../css/chrome.css">
|
|
<link rel="stylesheet" href="../css/print.css" media="print">
|
|
|
|
<!-- Fonts -->
|
|
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
|
|
<link rel="stylesheet" href="../fonts/fonts.css">
|
|
|
|
<!-- Highlight.js Stylesheets -->
|
|
<link rel="stylesheet" href="../highlight.css">
|
|
<link rel="stylesheet" href="../tomorrow-night.css">
|
|
<link rel="stylesheet" href="../ayu-highlight.css">
|
|
|
|
<!-- Custom theme stylesheets -->
|
|
<link rel="stylesheet" href="../mermaid.css">
|
|
|
|
</head>
|
|
<body>
|
|
<!-- Provide site root to javascript -->
|
|
<script type="text/javascript">
|
|
var path_to_root = "../";
|
|
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
|
|
</script>
|
|
|
|
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
|
<script type="text/javascript">
|
|
try {
|
|
var theme = localStorage.getItem('mdbook-theme');
|
|
var sidebar = localStorage.getItem('mdbook-sidebar');
|
|
|
|
if (theme.startsWith('"') && theme.endsWith('"')) {
|
|
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
|
}
|
|
|
|
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
|
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
|
}
|
|
} catch (e) { }
|
|
</script>
|
|
|
|
<!-- Set the theme before any content is loaded, prevents flash -->
|
|
<script type="text/javascript">
|
|
var theme;
|
|
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
|
if (theme === null || theme === undefined) { theme = default_theme; }
|
|
var html = document.querySelector('html');
|
|
html.classList.remove('no-js')
|
|
html.classList.remove('light')
|
|
html.classList.add(theme);
|
|
html.classList.add('js');
|
|
</script>
|
|
|
|
<!-- Hide / unhide sidebar before it is displayed -->
|
|
<script type="text/javascript">
|
|
var html = document.querySelector('html');
|
|
var sidebar = 'hidden';
|
|
if (document.body.clientWidth >= 1080) {
|
|
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
|
sidebar = sidebar || 'visible';
|
|
}
|
|
html.classList.remove('sidebar-visible');
|
|
html.classList.add("sidebar-" + sidebar);
|
|
</script>
|
|
|
|
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
|
<div class="sidebar-scrollbox">
|
|
<ol class="chapter"><li class="chapter-item expanded "><a href="../about_salsa.html"><strong aria-hidden="true">1.</strong> About salsa</a></li><li class="chapter-item expanded affix "><li class="part-title">How to use Salsa</li><li class="chapter-item expanded "><a href="../overview.html"><strong aria-hidden="true">2.</strong> Overview</a></li><li class="chapter-item expanded "><a href="../tutorial.html"><strong aria-hidden="true">3.</strong> Tutorial: calc language</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../tutorial/structure.html"><strong aria-hidden="true">3.1.</strong> Basic structure</a></li><li class="chapter-item expanded "><a href="../tutorial/jar.html"><strong aria-hidden="true">3.2.</strong> Jars and databases</a></li><li class="chapter-item expanded "><a href="../tutorial/db.html"><strong aria-hidden="true">3.3.</strong> Defining the database struct</a></li><li class="chapter-item expanded "><a href="../tutorial/ir.html"><strong aria-hidden="true">3.4.</strong> Defining the IR: the various "salsa structs"</a></li><li class="chapter-item expanded "><a href="../tutorial/parser.html" class="active"><strong aria-hidden="true">3.5.</strong> Defining the parser: memoized functions and inputs</a></li><li class="chapter-item expanded "><a href="../tutorial/accumulators.html"><strong aria-hidden="true">3.6.</strong> Defining the parser: reporting errors</a></li><li class="chapter-item expanded "><a href="../tutorial/debug.html"><strong aria-hidden="true">3.7.</strong> Defining the parser: debug impls and testing</a></li><li class="chapter-item expanded "><a href="../tutorial/checker.html"><strong aria-hidden="true">3.8.</strong> Defining the checker</a></li><li class="chapter-item expanded "><a href="../tutorial/interpreter.html"><strong aria-hidden="true">3.9.</strong> Defining the interpreter</a></li></ol></li><li class="chapter-item expanded "><a href="../reference.html"><strong aria-hidden="true">4.</strong> Reference</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../reference/durability.html"><strong aria-hidden="true">4.1.</strong> Durability</a></li><li class="chapter-item expanded "><a href="../reference/algorithm.html"><strong aria-hidden="true">4.2.</strong> Algorithm</a></li></ol></li><li class="chapter-item expanded "><a href="../common_patterns.html"><strong aria-hidden="true">5.</strong> Common patterns</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../common_patterns/on_demand_inputs.html"><strong aria-hidden="true">5.1.</strong> On-demand (Lazy) inputs</a></li></ol></li><li class="chapter-item expanded "><a href="../tuning.html"><strong aria-hidden="true">6.</strong> Tuning</a></li><li class="chapter-item expanded "><a href="../cycles.html"><strong aria-hidden="true">7.</strong> Cycle handling</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../cycles/fallback.html"><strong aria-hidden="true">7.1.</strong> Recovering via fallback</a></li></ol></li><li class="chapter-item expanded "><li class="part-title">How Salsa works internally</li><li class="chapter-item expanded "><a href="../how_salsa_works.html"><strong aria-hidden="true">8.</strong> How Salsa works</a></li><li class="chapter-item expanded "><a href="../videos.html"><strong aria-hidden="true">9.</strong> Videos</a></li><li class="chapter-item expanded "><a href="../plumbing.html"><strong aria-hidden="true">10.</strong> Plumbing</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../plumbing/jars_and_ingredients.html"><strong aria-hidden="true">10.1.</strong> Jars and ingredients</a></li><li class="chapter-item expanded "><a href="../plumbing/database_and_runtime.html"><strong aria-hidden="true">10.2.</strong> Databases and runtime</a></li><li class="chapter-item expanded "><a href="../plumbing/db_lifetime.html"><strong aria-hidden="true">10.3.</strong> The db lifetime on tracked/interned structs</a></li><li class="chapter-item expanded "><a href="../plumbing/tracked_structs.html"><strong aria-hidden="true">10.4.</strong> Tracked structures</a></li><li class="chapter-item expanded "><a href="../plumbing/query_ops.html"><strong aria-hidden="true">10.5.</strong> Query operations</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../plumbing/maybe_changed_after.html"><strong aria-hidden="true">10.5.1.</strong> maybe changed after</a></li><li class="chapter-item expanded "><a href="../plumbing/fetch.html"><strong aria-hidden="true">10.5.2.</strong> Fetch</a></li><li class="chapter-item expanded "><a href="../plumbing/derived_flowchart.html"><strong aria-hidden="true">10.5.3.</strong> Derived queries flowchart</a></li><li class="chapter-item expanded "><a href="../plumbing/cycles.html"><strong aria-hidden="true">10.5.4.</strong> Cycle handling</a></li></ol></li><li class="chapter-item expanded "><a href="../plumbing/terminology.html"><strong aria-hidden="true">10.6.</strong> Terminology</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="../plumbing/terminology/backdate.html"><strong aria-hidden="true">10.6.1.</strong> Backdate</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/changed_at.html"><strong aria-hidden="true">10.6.2.</strong> Changed at</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/dependency.html"><strong aria-hidden="true">10.6.3.</strong> Dependency</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/derived_query.html"><strong aria-hidden="true">10.6.4.</strong> Derived query</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/durability.html"><strong aria-hidden="true">10.6.5.</strong> Durability</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/input_query.html"><strong aria-hidden="true">10.6.6.</strong> Input query</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/ingredient.html"><strong aria-hidden="true">10.6.7.</strong> Ingredient</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/LRU.html"><strong aria-hidden="true">10.6.8.</strong> LRU</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/memo.html"><strong aria-hidden="true">10.6.9.</strong> Memo</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/query.html"><strong aria-hidden="true">10.6.10.</strong> Query</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/query_function.html"><strong aria-hidden="true">10.6.11.</strong> Query function</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/revision.html"><strong aria-hidden="true">10.6.12.</strong> Revision</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/salsa_item.html"><strong aria-hidden="true">10.6.13.</strong> Salsa item</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/salsa_struct.html"><strong aria-hidden="true">10.6.14.</strong> Salsa struct</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/untracked.html"><strong aria-hidden="true">10.6.15.</strong> Untracked dependency</a></li><li class="chapter-item expanded "><a href="../plumbing/terminology/verified.html"><strong aria-hidden="true">10.6.16.</strong> Verified</a></li></ol></li></ol></li><li class="chapter-item expanded "><li class="part-title">Appendices</li><li class="chapter-item expanded "><a href="../meta.html"><strong aria-hidden="true">11.</strong> Meta: about the book itself</a></li></ol> </div>
|
|
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
|
|
</nav>
|
|
|
|
<div id="page-wrapper" class="page-wrapper">
|
|
|
|
<div class="page">
|
|
|
|
<div id="menu-bar-hover-placeholder"></div>
|
|
<div id="menu-bar" class="menu-bar sticky bordered">
|
|
<div class="left-buttons">
|
|
<button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
|
<i class="fa fa-bars"></i>
|
|
</button>
|
|
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
|
<i class="fa fa-paint-brush"></i>
|
|
</button>
|
|
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
|
<li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
|
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
|
</ul>
|
|
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
|
|
<i class="fa fa-search"></i>
|
|
</button>
|
|
</div>
|
|
|
|
<h1 class="menu-title">Salsa</h1>
|
|
|
|
<div class="right-buttons">
|
|
<a href="../print.html" title="Print this book" aria-label="Print this book">
|
|
<i id="print-button" class="fa fa-print"></i>
|
|
</a>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div id="search-wrapper" class="hidden">
|
|
<form id="searchbar-outer" class="searchbar-outer">
|
|
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
|
</form>
|
|
<div id="searchresults-outer" class="searchresults-outer hidden">
|
|
<div id="searchresults-header" class="searchresults-header"></div>
|
|
<ul id="searchresults">
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
|
<script type="text/javascript">
|
|
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
|
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
|
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
|
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
|
});
|
|
</script>
|
|
|
|
<div id="content" class="content">
|
|
<main>
|
|
<h1 id="defining-the-parser-memoized-functions-and-inputs"><a class="header" href="#defining-the-parser-memoized-functions-and-inputs">Defining the parser: memoized functions and inputs</a></h1>
|
|
<p>The next step in the <code>calc</code> compiler is to define the parser.
|
|
The role of the parser will be to take the <code>ProgramSource</code> input,
|
|
read the string from the <code>text</code> field,
|
|
and create the <code>Statement</code>, <code>Function</code>, and <code>Expression</code> structures that <a href="./ir.html">we defined in the <code>ir</code> module</a>.</p>
|
|
<p>To minimize dependencies, we are going to write a <a href="https://en.wikipedia.org/wiki/Recursive_descent_parser">recursive descent parser</a>.
|
|
Another option would be to use a <a href="https://rustrepo.com/catalog/rust-parsing_newest_1">Rust parsing framework</a>.
|
|
We won't cover the parsing itself in this tutorial -- you can read the code if you want to see how it works.
|
|
We're going to focus only on the Salsa-related aspects.</p>
|
|
<h2 id="the-parse_statements-function"><a class="header" href="#the-parse_statements-function">The <code>parse_statements</code> function</a></h2>
|
|
<p>The starting point for the parser is the <code>parse_statements</code> function:</p>
|
|
<pre><pre class="playground"><code class="language-rust">
|
|
<span class="boring">#![allow(unused)]
|
|
</span><span class="boring">fn main() {
|
|
</span>#[salsa::tracked]
|
|
pub fn parse_statements<'db>(db: &'db dyn crate::Db, source: SourceProgram) -> Program<'db> {
|
|
// Get the source text from the database
|
|
let source_text = source.text(db);
|
|
|
|
// Create the parser
|
|
let mut parser = Parser {
|
|
db,
|
|
source_text,
|
|
position: 0,
|
|
};
|
|
|
|
// Read in statements until we reach the end of the input
|
|
let mut result = vec![];
|
|
loop {
|
|
// Skip over any whitespace
|
|
parser.skip_whitespace();
|
|
|
|
// If there are no more tokens, break
|
|
if parser.peek().is_none() {
|
|
break;
|
|
}
|
|
|
|
// Otherwise, there is more input, so parse a statement.
|
|
if let Some(statement) = parser.parse_statement() {
|
|
result.push(statement);
|
|
} else {
|
|
// If we failed, report an error at whatever position the parser
|
|
// got stuck. We could recover here by skipping to the end of the line
|
|
// or something like that. But we leave that as an exercise for the reader!
|
|
parser.report_error();
|
|
break;
|
|
}
|
|
}
|
|
|
|
Program::new(db, result)
|
|
}
|
|
<span class="boring">}
|
|
</span></code></pre></pre>
|
|
<p>This function is annotated as <code>#[salsa::tracked]</code>.
|
|
That means that, when it is called, Salsa will track what inputs it reads as well as what value it returns.
|
|
The return value is <em>memoized</em>,
|
|
which means that if you call this function again without changing the inputs,
|
|
Salsa will just clone the result rather than re-execute it.</p>
|
|
<h3 id="tracked-functions-are-the-unit-of-reuse"><a class="header" href="#tracked-functions-are-the-unit-of-reuse">Tracked functions are the unit of reuse</a></h3>
|
|
<p>Tracked functions are the core part of how Salsa enables incremental reuse.
|
|
The goal of the framework is to avoid re-executing tracked functions and instead to clone their result.
|
|
Salsa uses the <a href="../reference/algorithm.html">red-green algorithm</a> to decide when to re-execute a function.
|
|
The short version is that a tracked function is re-executed if either (a) it directly reads an input, and that input has changed,
|
|
or (b) it directly invokes another tracked function and that function's return value has changed.
|
|
In the case of <code>parse_statements</code>, it directly reads <code>ProgramSource::text</code>, so if the text changes, then the parser will re-execute.</p>
|
|
<p>By choosing which functions to mark as <code>#[tracked]</code>, you control how much reuse you get.
|
|
In our case, we're opting to mark the outermost parsing function as tracked, but not the inner ones.
|
|
This means that if the input changes, we will always re-parse the entire input and re-create the resulting statements and so forth.
|
|
We'll see later that this <em>doesn't</em> mean we will always re-run the type checker and other parts of the compiler.</p>
|
|
<p>This trade-off makes sense because (a) parsing is very cheap, so the overhead of tracking and enabling finer-grained reuse doesn't pay off
|
|
and because (b) since strings are just a big blob-o-bytes without any structure, it's rather hard to identify which parts of the IR need to be reparsed.
|
|
Some systems do choose to do more granular reparsing, often by doing a "first pass" over the string to give it a bit of structure,
|
|
e.g. to identify the functions,
|
|
but deferring the parsing of the body of each function until later.
|
|
Setting up a scheme like this is relatively easy in Salsa and uses the same principles that we will use later to avoid re-executing the type checker.</p>
|
|
<h3 id="parameters-to-a-tracked-function"><a class="header" href="#parameters-to-a-tracked-function">Parameters to a tracked function</a></h3>
|
|
<p>The <strong>first</strong> parameter to a tracked function is <strong>always</strong> the database, <code>db: &dyn crate::Db</code>.
|
|
It must be a <code>dyn</code> value of whatever database is associated with the jar.</p>
|
|
<p>The <strong>second</strong> parameter to a tracked function is <strong>always</strong> some kind of Salsa struct.
|
|
The first parameter to a memoized function is always the database,
|
|
which should be a <code>dyn Trait</code> value for the database trait associated with the jar
|
|
(the default jar is <code>crate::Jar</code>).</p>
|
|
<p>Tracked functions may take other arguments as well, though our examples here do not.
|
|
Functions that take additional arguments are less efficient and flexible.
|
|
It's generally better to structure tracked functions as functions of a single Salsa struct if possible.</p>
|
|
<h3 id="the-return_ref-annotation"><a class="header" href="#the-return_ref-annotation">The <code>return_ref</code> annotation</a></h3>
|
|
<p>You may have noticed that <code>parse_statements</code> is tagged with <code>#[salsa::tracked(return_ref)]</code>.
|
|
Ordinarily, when you call a tracked function, the result you get back is cloned out of the database.
|
|
The <code>return_ref</code> attribute means that a reference into the database is returned instead.
|
|
So, when called, <code>parse_statements</code> will return an <code>&Vec<Statement></code> rather than cloning the <code>Vec</code>.
|
|
This is useful as a performance optimization.
|
|
(You may recall the <code>return_ref</code> annotation from the <a href="./ir.html">ir</a> section of the tutorial,
|
|
where it was placed on struct fields, with roughly the same meaning.)</p>
|
|
|
|
</main>
|
|
|
|
<nav class="nav-wrapper" aria-label="Page navigation">
|
|
<!-- Mobile navigation buttons -->
|
|
<a rel="prev" href="../tutorial/ir.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next" href="../tutorial/accumulators.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
|
|
<div style="clear: both"></div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
|
<a rel="prev" href="../tutorial/ir.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
|
|
<i class="fa fa-angle-left"></i>
|
|
</a>
|
|
|
|
<a rel="next" href="../tutorial/accumulators.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
|
|
<i class="fa fa-angle-right"></i>
|
|
</a>
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
window.playground_copyable = true;
|
|
</script>
|
|
|
|
|
|
<script src="../elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
|
|
<script src="../mark.min.js" type="text/javascript" charset="utf-8"></script>
|
|
<script src="../searcher.js" type="text/javascript" charset="utf-8"></script>
|
|
|
|
<script src="../clipboard.min.js" type="text/javascript" charset="utf-8"></script>
|
|
<script src="../highlight.js" type="text/javascript" charset="utf-8"></script>
|
|
<script src="../book.js" type="text/javascript" charset="utf-8"></script>
|
|
|
|
<!-- Custom JS scripts -->
|
|
<script type="text/javascript" src="../mermaid.min.js"></script>
|
|
<script type="text/javascript" src="../mermaid-init.js"></script>
|
|
|
|
|
|
</body>
|
|
</html>
|