Switch to always encoding package names / paths as strings

This will simplify parsing and make it possible to have a uniform lexer for the language. Previously unquoted package names were allowed to include '-'s, which aren't valid identifiers.

In the future, we'll distinguish local paths from packages in the package-manager by looking for a ".roc" suffix, which should only be present in local paths.
This commit is contained in:
Joshua Warner 2021-12-20 18:23:41 -08:00
parent f680b83b9e
commit 8b58d5cbc7
31 changed files with 120 additions and 293 deletions

View file

@ -6,7 +6,7 @@ use crate::Buf;
use roc_parse::ast::{Collection, Module, Spaced};
use roc_parse::header::{
AppHeader, Effects, ExposedName, ImportsEntry, InterfaceHeader, ModuleName, PackageEntry,
PackageName, PackageOrPath, PlatformHeader, PlatformRequires, PlatformRigid, To, TypedIdent,
PackageName, PlatformHeader, PlatformRequires, PlatformRigid, To, TypedIdent,
};
use roc_region::all::Loc;
@ -91,7 +91,7 @@ pub fn fmt_platform_header<'a, 'buf>(buf: &mut Buf<'buf>, header: &'a PlatformHe
buf.push_str("platform");
fmt_default_spaces(buf, header.after_platform_keyword, indent);
fmt_package_name(buf, header.name.value);
fmt_package_name(buf, header.name.value, indent);
// requires
fmt_default_spaces(buf, header.before_requires, indent);
@ -171,10 +171,10 @@ impl<'a> Formattable for TypedIdent<'a> {
}
}
fn fmt_package_name<'buf>(buf: &mut Buf<'buf>, name: PackageName) {
buf.push_str(name.account);
buf.push('/');
buf.push_str(name.pkg);
fn fmt_package_name<'buf>(buf: &mut Buf<'buf>, name: PackageName, _indent: u16) {
buf.push('"');
buf.push_str_allow_spaces(name.0);
buf.push('"');
}
impl<'a, T: Formattable> Formattable for Spaced<'a, T> {
@ -239,7 +239,7 @@ fn fmt_to<'buf>(buf: &mut Buf<'buf>, to: To, indent: u16) {
To::ExistingPackage(name) => {
buf.push_str(name);
}
To::NewPackage(package_or_path) => fmt_package_or_path(buf, &package_or_path, indent),
To::NewPackage(package_name) => fmt_package_name(buf, package_name, indent),
}
}
@ -324,20 +324,7 @@ fn fmt_packages_entry<'a, 'buf>(buf: &mut Buf<'buf>, entry: &PackageEntry<'a>, i
buf.push_str(entry.shorthand);
buf.push(':');
fmt_default_spaces(buf, entry.spaces_after_shorthand, indent);
fmt_package_or_path(buf, &entry.package_or_path.value, indent);
}
fn fmt_package_or_path<'a, 'buf>(
buf: &mut Buf<'buf>,
package_or_path: &PackageOrPath<'a>,
indent: u16,
) {
match package_or_path {
PackageOrPath::Package(_name, _version) => {
todo!("format package");
}
PackageOrPath::Path(str_literal) => fmt_str_literal(buf, *str_literal, indent),
}
fmt_package_name(buf, entry.package_name.value, indent);
}
fn fmt_imports_entry<'a, 'buf>(buf: &mut Buf<'buf>, entry: &ImportsEntry<'a>, indent: u16) {

View file

@ -2651,7 +2651,7 @@ mod test_fmt {
#[test]
fn single_line_platform() {
module_formats_same(
"platform folkertdev/foo \
"platform \"folkertdev/foo\" \
requires { model=>Model, msg=>Msg } { main : Effect {} } \
exposes [] \
packages {} \

View file

@ -24,9 +24,8 @@ use roc_mono::ir::{
};
use roc_mono::layout::{Layout, LayoutCache, LayoutProblem};
use roc_parse::ast::{self, ExtractSpaces, Spaced, StrLiteral, TypeAnnotation};
use roc_parse::header::{
ExposedName, ImportsEntry, PackageEntry, PackageOrPath, PlatformHeader, To, TypedIdent,
};
use roc_parse::header::PackageName;
use roc_parse::header::{ExposedName, ImportsEntry, PackageEntry, PlatformHeader, To, TypedIdent};
use roc_parse::module::module_defs;
use roc_parse::parser::{ParseProblem, Parser, SyntaxError};
use roc_region::all::{Loc, Region};
@ -668,7 +667,7 @@ struct ModuleHeader<'a> {
is_root_module: bool,
exposed_ident_ids: IdentIds,
deps_by_name: MutMap<PQModuleName<'a>, ModuleId>,
packages: MutMap<&'a str, PackageOrPath<'a>>,
packages: MutMap<&'a str, PackageName<'a>>,
imported_modules: MutMap<ModuleId, Region>,
package_qualified_imported_modules: MutSet<PackageQualified<'a, ModuleId>>,
exposes: Vec<Symbol>,
@ -887,7 +886,7 @@ struct State<'a> {
/// From now on, these will be used by multiple threads; time to make an Arc<Mutex<_>>!
pub arc_modules: Arc<Mutex<PackageModuleIds<'a>>>,
pub arc_shorthands: Arc<Mutex<MutMap<&'a str, PackageOrPath<'a>>>>,
pub arc_shorthands: Arc<Mutex<MutMap<&'a str, PackageName<'a>>>>,
pub ident_ids_by_module: Arc<Mutex<MutMap<ModuleId, IdentIds>>>,
@ -980,7 +979,7 @@ enum BuildTask<'a> {
LoadModule {
module_name: PQModuleName<'a>,
module_ids: Arc<Mutex<PackageModuleIds<'a>>>,
shorthands: Arc<Mutex<MutMap<&'a str, PackageOrPath<'a>>>>,
shorthands: Arc<Mutex<MutMap<&'a str, PackageName<'a>>>>,
ident_ids_by_module: Arc<Mutex<MutMap<ModuleId, IdentIds>>>,
},
Parse {
@ -1680,8 +1679,8 @@ fn update<'a>(
{
let mut shorthands = (*state.arc_shorthands).lock();
for (shorthand, package_or_path) in header.packages.iter() {
shorthands.insert(shorthand, *package_or_path);
for (shorthand, package_name) in header.packages.iter() {
shorthands.insert(shorthand, *package_name);
}
if let PkgConfig {
@ -2215,7 +2214,7 @@ fn finish_specialization(
let path_to_platform = {
use PlatformPath::*;
let package_or_path = match platform_path {
let package_name = match platform_path {
Valid(To::ExistingPackage(shorthand)) => {
match (*state.arc_shorthands).lock().get(shorthand) {
Some(p_or_p) => *p_or_p,
@ -2229,11 +2228,7 @@ fn finish_specialization(
}
};
match package_or_path {
PackageOrPath::Path(StrLiteral::PlainLine(path)) => path,
PackageOrPath::Path(_) => unreachable!("invalid"),
_ => todo!("packages"),
}
package_name.0
};
let platform_path = path_to_platform.into();
@ -2434,7 +2429,7 @@ fn load_module<'a>(
src_dir: &Path,
module_name: PQModuleName<'a>,
module_ids: Arc<Mutex<PackageModuleIds<'a>>>,
arc_shorthands: Arc<Mutex<MutMap<&'a str, PackageOrPath<'a>>>>,
arc_shorthands: Arc<Mutex<MutMap<&'a str, PackageName<'a>>>>,
ident_ids_by_module: Arc<Mutex<MutMap<ModuleId, IdentIds>>>,
) -> Result<(ModuleId, Msg<'a>), LoadingProblem<'a>> {
let module_start_time = SystemTime::now();
@ -2456,13 +2451,9 @@ fn load_module<'a>(
let shorthands = arc_shorthands.lock();
match shorthands.get(shorthand) {
Some(PackageOrPath::Path(StrLiteral::PlainLine(path))) => {
Some(PackageName(path)) => {
filename.push(path);
}
Some(PackageOrPath::Path(_str_liteal)) => {
unreachable!("invalid structure for path")
}
Some(PackageOrPath::Package(_name, _version)) => todo!("packages"),
None => unreachable!("there is no shorthand named {:?}", shorthand),
}
@ -2614,58 +2605,44 @@ fn parse_header<'a>(
if let Some(PackageEntry {
shorthand,
package_or_path:
package_name:
Loc {
value: package_or_path,
value: package_name,
..
},
..
}) = opt_base_package
{
match package_or_path {
PackageOrPath::Path(StrLiteral::PlainLine(package)) => {
// check whether we can find a Package-Config.roc file
let mut pkg_config_roc = pkg_config_dir;
pkg_config_roc.push(package);
pkg_config_roc.push(PKG_CONFIG_FILE_NAME);
pkg_config_roc.set_extension(ROC_FILE_EXTENSION);
let package = package_name.0;
if pkg_config_roc.as_path().exists() {
let load_pkg_config_msg = load_pkg_config(
arena,
&pkg_config_roc,
shorthand,
module_id,
module_ids,
ident_ids_by_module,
)?;
// check whether we can find a Package-Config.roc file
let mut pkg_config_roc = pkg_config_dir;
pkg_config_roc.push(package);
pkg_config_roc.push(PKG_CONFIG_FILE_NAME);
pkg_config_roc.set_extension(ROC_FILE_EXTENSION);
Ok((
module_id,
Msg::Many(vec![app_module_header_msg, load_pkg_config_msg]),
))
} else {
Ok((module_id, app_module_header_msg))
}
}
_ => unreachable!(),
if pkg_config_roc.as_path().exists() {
let load_pkg_config_msg = load_pkg_config(
arena,
&pkg_config_roc,
shorthand,
module_id,
module_ids,
ident_ids_by_module,
)?;
Ok((
module_id,
Msg::Many(vec![app_module_header_msg, load_pkg_config_msg]),
))
} else {
Ok((module_id, app_module_header_msg))
}
} else {
panic!("could not find base")
}
}
To::NewPackage(package_or_path) => match package_or_path {
PackageOrPath::Package(_, _) => panic!("TODO implement packages"),
PackageOrPath::Path(StrLiteral::PlainLine(_package)) => {
Ok((module_id, app_module_header_msg))
}
PackageOrPath::Path(StrLiteral::Block(_)) => {
panic!("TODO implement block package path")
}
PackageOrPath::Path(StrLiteral::Line(_)) => {
panic!("TODO implement line package path")
}
},
To::NewPackage(_package_name) => Ok((module_id, app_module_header_msg)),
}
}
Ok((ast::Module::Platform { header }, _parse_state)) => Ok(fabricate_effects_module(
@ -2910,7 +2887,7 @@ fn send_header<'a>(
.iter()
.map(|pkg| {
let pkg = pkg.value;
(pkg.shorthand, pkg.package_or_path.value)
(pkg.shorthand, pkg.package_name.value)
})
.collect::<MutMap<_, _>>();
@ -3131,7 +3108,7 @@ fn send_header_two<'a>(
let package_entries = packages
.iter()
.map(|pkg| (pkg.value.shorthand, pkg.value.package_or_path.value))
.map(|pkg| (pkg.value.shorthand, pkg.value.package_name.value))
.collect::<MutMap<_, _>>();
// Send the deps to the coordinator thread for processing,

View file

@ -1,19 +1,13 @@
use crate::ast::{Collection, CommentOrNewline, Spaced, StrLiteral, TypeAnnotation};
use crate::blankspace::space0_e;
use crate::ident::lowercase_ident;
use crate::parser::Progress::{self, *};
use crate::parser::{specialize, word1, EPackageEntry, EPackageName, EPackageOrPath, Parser};
use crate::parser::Progress::*;
use crate::parser::{specialize, word1, EPackageEntry, EPackageName, Parser};
use crate::state::State;
use crate::string_literal;
use bumpalo::collections::Vec;
use roc_region::all::Loc;
#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
pub struct PackageName<'a> {
pub account: &'a str,
pub pkg: &'a str,
}
#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
pub enum Version<'a> {
Exact(&'a str),
@ -32,10 +26,7 @@ pub enum VersionComparison {
}
#[derive(Copy, Clone, PartialEq, Debug)]
pub enum PackageOrPath<'a> {
Package(PackageName<'a>, Version<'a>),
Path(StrLiteral<'a>),
}
pub struct PackageName<'a>(pub &'a str);
#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
pub struct ModuleName<'a>(&'a str);
@ -93,7 +84,7 @@ pub struct InterfaceHeader<'a> {
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum To<'a> {
ExistingPackage(&'a str),
NewPackage(PackageOrPath<'a>),
NewPackage(PackageName<'a>),
}
#[derive(Clone, Debug, PartialEq)]
@ -121,7 +112,7 @@ pub struct AppHeader<'a> {
pub struct PackageHeader<'a> {
pub name: Loc<PackageName<'a>>,
pub exposes: Vec<'a, Loc<Spaced<'a, ExposedName<'a>>>>,
pub packages: Vec<'a, (Loc<&'a str>, Loc<PackageOrPath<'a>>)>,
pub packages: Vec<'a, (Loc<&'a str>, Loc<PackageName<'a>>)>,
pub imports: Vec<'a, Loc<ImportsEntry<'a>>>,
// Potential comments and newlines - these will typically all be empty.
@ -213,7 +204,7 @@ pub struct TypedIdent<'a> {
pub struct PackageEntry<'a> {
pub shorthand: &'a str,
pub spaces_after_shorthand: &'a [CommentOrNewline<'a>],
pub package_or_path: Loc<PackageOrPath<'a>>,
pub package_name: Loc<PackageName<'a>>,
}
pub fn package_entry<'a>() -> impl Parser<'a, Spaced<'a, PackageEntry<'a>>, EPackageEntry<'a>> {
@ -232,27 +223,24 @@ pub fn package_entry<'a>() -> impl Parser<'a, Spaced<'a, PackageEntry<'a>>, EPac
space0_e(
min_indent,
EPackageEntry::Space,
EPackageEntry::IndentPackageOrPath
EPackageEntry::IndentPackage
)
))
.parse(arena, state)?;
let (_, package_or_path, state) = loc!(specialize(
EPackageEntry::BadPackageOrPath,
package_or_path()
))
.parse(arena, state)?;
let (_, package_or_path, state) =
loc!(specialize(EPackageEntry::BadPackage, package_name())).parse(arena, state)?;
let entry = match opt_shorthand {
Some((shorthand, spaces_after_shorthand)) => PackageEntry {
shorthand,
spaces_after_shorthand,
package_or_path,
package_name: package_or_path,
},
None => PackageEntry {
shorthand: "",
spaces_after_shorthand: &[],
package_or_path,
package_name: package_or_path,
},
};
@ -260,113 +248,15 @@ pub fn package_entry<'a>() -> impl Parser<'a, Spaced<'a, PackageEntry<'a>>, EPac
}
}
pub fn package_or_path<'a>() -> impl Parser<'a, PackageOrPath<'a>, EPackageOrPath<'a>> {
one_of![
map!(
specialize(EPackageOrPath::BadPath, string_literal::parse()),
PackageOrPath::Path
),
map!(
and!(
specialize(EPackageOrPath::BadPackage, package_name()),
skip_first!(skip_spaces(), package_version())
),
|(name, version)| { PackageOrPath::Package(name, version) }
)
]
}
fn skip_spaces<'a, T>() -> impl Parser<'a, (), T>
where
T: 'a,
{
|_, mut state: State<'a>| {
let mut chomped = 0usize;
let mut it = state.bytes().iter();
while let Some(b' ') = it.next() {
chomped += 1;
}
if chomped == 0 {
Ok((NoProgress, (), state))
} else {
state.pos.column += chomped as u16;
state = state.advance(chomped);
Ok((MadeProgress, (), state))
}
}
}
fn package_version<'a, T>() -> impl Parser<'a, Version<'a>, T>
where
T: 'a,
{
move |_, _| todo!("TODO parse package version")
}
#[inline(always)]
pub fn package_name<'a>() -> impl Parser<'a, PackageName<'a>, EPackageName> {
use encode_unicode::CharExt;
// e.g. rtfeldman/blah
//
// Package names and accounts can be capitalized and can contain dashes.
// They cannot contain underscores or other special characters.
// They must be ASCII.
|_, mut state: State<'a>| match chomp_package_part(state.bytes()) {
Err(progress) => Err((progress, EPackageName::Account(state.pos), state)),
Ok(account) => {
let mut chomped = account.len();
if let Ok(('/', width)) = char::from_utf8_slice_start(&state.bytes()[chomped..]) {
chomped += width;
match chomp_package_part(&state.bytes()[chomped..]) {
Err(progress) => Err((
progress,
EPackageName::Pkg(state.pos.bump_column(chomped as u16)),
state,
)),
Ok(pkg) => {
chomped += pkg.len();
state.pos.column += chomped as u16;
state = state.advance(chomped);
let value = PackageName { account, pkg };
Ok((MadeProgress, value, state))
}
}
} else {
Err((
MadeProgress,
EPackageName::MissingSlash(state.pos.bump_column(chomped as u16)),
state,
))
}
}
}
}
fn chomp_package_part(buffer: &[u8]) -> Result<&str, Progress> {
use encode_unicode::CharExt;
let mut chomped = 0;
while let Ok((ch, width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
if ch == '-' || ch.is_ascii_alphanumeric() {
chomped += width;
} else {
// we're done
break;
}
}
if chomped == 0 {
Err(Progress::NoProgress)
} else {
let name = unsafe { std::str::from_utf8_unchecked(&buffer[..chomped]) };
Ok(name)
pub fn package_name<'a>() -> impl Parser<'a, PackageName<'a>, EPackageName<'a>> {
move |arena, state: State<'a>| {
let pos = state.pos;
specialize(EPackageName::BadPath, string_literal::parse())
.parse(arena, state)
.and_then(|(progress, text, next_state)| match text {
StrLiteral::PlainLine(text) => Ok((progress, PackageName(text), next_state)),
StrLiteral::Line(_) => Err((progress, EPackageName::Escapes(pos), next_state)),
StrLiteral::Block(_) => Err((progress, EPackageName::Multiline(pos), next_state)),
})
}
}

View file

@ -1,9 +1,8 @@
use crate::ast::{Collection, CommentOrNewline, Def, Module, Spaced};
use crate::blankspace::{space0_around_ee, space0_before_e, space0_e};
use crate::header::{
package_entry, package_name, package_or_path, AppHeader, Effects, ExposedName, ImportsEntry,
InterfaceHeader, ModuleName, PackageEntry, PlatformHeader, PlatformRequires, PlatformRigid, To,
TypedIdent,
package_entry, package_name, AppHeader, Effects, ExposedName, ImportsEntry, InterfaceHeader,
ModuleName, PackageEntry, PlatformHeader, PlatformRequires, PlatformRigid, To, TypedIdent,
};
use crate::ident::{lowercase_ident, unqualified_ident, uppercase_ident};
use crate::parser::Progress::{self, *};
@ -312,7 +311,7 @@ fn provides_to_package<'a>() -> impl Parser<'a, To<'a>, EProvides<'a>> {
|_, pos| EProvides::Identifier(pos),
map!(lowercase_ident(), To::ExistingPackage)
),
specialize(EProvides::Package, map!(package_or_path(), To::NewPackage))
specialize(EProvides::Package, map!(package_name(), To::NewPackage))
]
}

View file

@ -79,7 +79,7 @@ pub enum EHeader<'a> {
Start(Position),
ModuleName(Position),
AppName(EString<'a>, Position),
PlatformName(EPackageName, Position),
PlatformName(EPackageName<'a>, Position),
IndentStart(Position),
}
@ -96,7 +96,7 @@ pub enum EProvides<'a> {
ListStart(Position),
ListEnd(Position),
Identifier(Position),
Package(EPackageOrPath<'a>, Position),
Package(EPackageName<'a>, Position),
Space(BadInputError, Position),
}
@ -151,25 +151,19 @@ pub enum EPackages<'a> {
PackageEntry(EPackageEntry<'a>, Position),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EPackageName {
MissingSlash(Position),
Account(Position),
Pkg(Position),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum EPackageOrPath<'a> {
pub enum EPackageName<'a> {
BadPath(EString<'a>, Position),
BadPackage(EPackageName, Position),
Escapes(Position),
Multiline(Position),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum EPackageEntry<'a> {
BadPackageOrPath(EPackageOrPath<'a>, Position),
BadPackage(EPackageName<'a>, Position),
Shorthand(Position),
Colon(Position),
IndentPackageOrPath(Position),
IndentPackage(Position),
Space(BadInputError, Position),
}

View file

@ -1,15 +1,14 @@
Platform {
header: PlatformHeader {
name: |L 0-0, C 9-23| PackageName {
account: "rtfeldman",
pkg: "blah",
},
name: |L 0-0, C 9-25| PackageName(
"rtfeldman/blah",
),
requires: PlatformRequires {
rigids: [],
signature: |L 0-0, C 38-47| TypedIdent {
ident: |L 0-0, C 38-42| "main",
signature: |L 0-0, C 40-49| TypedIdent {
ident: |L 0-0, C 40-44| "main",
spaces_before_colon: [],
ann: |L 0-0, C 45-47| Record {
ann: |L 0-0, C 47-49| Record {
fields: [],
ext: None,
},

View file

@ -1 +1 @@
platform rtfeldman/blah requires {} { main : {} } exposes [] packages {} imports [] provides [] effects fx.Blah {}
platform "rtfeldman/blah" requires {} { main : {} } exposes [] packages {} imports [] provides [] effects fx.Blah {}

View file

@ -7,10 +7,8 @@ App {
|L 1-1, C 15-31| PackageEntry {
shorthand: "pf",
spaces_after_shorthand: [],
package_or_path: |L 1-1, C 19-31| Path(
PlainLine(
"./platform",
),
package_name: |L 1-1, C 19-31| PackageName(
"./platform",
),
},
],

View file

@ -7,10 +7,8 @@ App {
|L 1-1, C 15-31| PackageEntry {
shorthand: "pf",
spaces_after_shorthand: [],
package_or_path: |L 1-1, C 19-31| Path(
PlainLine(
"./platform",
),
package_name: |L 1-1, C 19-31| PackageName(
"./platform",
),
},
],

View file

@ -1,9 +1,8 @@
Platform {
header: PlatformHeader {
name: |L 0-0, C 9-21| PackageName {
account: "examples",
pkg: "cli",
},
name: |L 0-0, C 9-23| PackageName(
"examples/cli",
),
requires: PlatformRequires {
rigids: [],
signature: |L 1-1, C 17-34| TypedIdent {

View file

@ -1,4 +1,4 @@
platform examples/cli
platform "examples/cli"
requires {}{ main : Task {} [] } # TODO FIXME
exposes []
packages {}

View file

@ -7,10 +7,8 @@ App {
imports: [],
provides: [],
to: |L 0-0, C 30-38| NewPackage(
Path(
PlainLine(
"./blah",
),
PackageName(
"./blah",
),
),
before_header: [],

View file

@ -1,9 +1,8 @@
Platform {
header: PlatformHeader {
name: |L 0-0, C 9-19| PackageName {
account: "foo",
pkg: "barbaz",
},
name: |L 0-0, C 9-21| PackageName(
"foo/barbaz",
),
requires: PlatformRequires {
rigids: [
|L 1-1, C 14-26| PlatformRigid {
@ -25,10 +24,8 @@ Platform {
|L 3-3, C 15-27| PackageEntry {
shorthand: "foo",
spaces_after_shorthand: [],
package_or_path: |L 3-3, C 20-27| Path(
PlainLine(
"./foo",
),
package_name: |L 3-3, C 20-27| PackageName(
"./foo",
),
},
],

View file

@ -1,4 +1,4 @@
platform foo/barbaz
platform "foo/barbaz"
requires {model=>Model} { main : {} }
exposes []
packages { foo: "./foo" }