cmounce/marzipan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement a basic parser, with snapshot testing

Browse Source
This commit is contained in:
Chris Mounce
2024-06-01 11:01:01 -07:00
parent fe21c1a792
commit 4730f11f31
6 changed files with 327 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+135
View File
@@ -2,11 +2,19 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "anyhow"
version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
[[package]]
name = "assembler"
version = "0.1.0"
dependencies = [
"anyhow",
"codepage-437",
"insta",
"nom",
]
@@ -19,6 +27,18 @@ dependencies = [
"csv",
]
[[package]]
name = "console"
version = "0.15.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"windows-sys",
]
[[package]]
name = "csv"
version = "1.3.0"
@@ -40,12 +60,48 @@ dependencies = [
"memchr",
]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "insta"
version = "1.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "810ae6042d48e2c9e9215043563a58a80b877bc863228a74cf10c49d4620a6f5"
dependencies = [
"console",
"lazy_static",
"linked-hash-map",
"similar",
]
[[package]]
name = "itoa"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "memchr"
version = "2.7.1"
@@ -112,6 +168,12 @@ dependencies = [
"syn",
]
[[package]]
name = "similar"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640"
[[package]]
name = "syn"
version = "2.0.60"
@@ -128,3 +190,76 @@ name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "windows-sys"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
[[package]]
name = "windows_i686_gnu"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
[[package]]
name = "windows_i686_msvc"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
Cargo.toml
+9
View File
@@ -6,5 +6,14 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = "1.0.86"
codepage-437 = "0.1.0"
insta = "1.39.0"
nom = "7.1.3"
[dev-dependencies]
insta = "1.39.0"
[profile.dev.package]
insta.opt-level = 3
similar.opt-level = 3
src/main.rs
+2 -2
View File
@@ -1,10 +1,10 @@
mod lang;
mod preprocess;
mod world;
use std::{env, error::Error, fs, process::exit};
use world::World;
use crate::lang::scan;
use crate::preprocess::scan::scan;
fn to_latin1(bytes: &[u8]) -> String {
bytes.iter().map(|&x| x as char).collect()
src/preprocess/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod parse;
pub mod scan;
src/preprocess/parse.rs
+171
View File
@@ -0,0 +1,171 @@
use anyhow::{anyhow, Result};
use super::scan::Token;
pub fn parse(input: Vec<Token>) -> Result<Vec<Expr>> {
// TODO: Should this take ownership of the tokens?
let mut parser = Parser::new(input);
parser.parse()
}
struct Parser {
pos: usize,
tokens: Vec<Token>,
}
#[derive(Debug)]
pub enum Expr {
ZztOop(String),
Macro(String, Vec<Box<Expr>>),
String(String),
}
impl Parser {
fn new(input: Vec<Token>) -> Self {
Self {
pos: 0,
tokens: input,
}
}
fn parse(&mut self) -> Result<Vec<Expr>> {
let mut result = vec![];
while let Some(token) = self.peek() {
match token {
Token::Newline | Token::RawText(_) => result.push(self.parse_zzt_oop()),
Token::Percent => result.push(self.parse_macro()?),
_ => return Err(anyhow!("Unexpected token {:?}", token)),
}
}
Ok(result)
}
fn parse_zzt_oop(&mut self) -> Expr {
let mut parts: Vec<String> = vec![];
while let Some(token) = self.peek() {
match token {
Token::RawText(s) => {
parts.push(s.into());
self.advance();
}
Token::Newline => {
parts.push("\n".into());
self.advance();
}
_ => break,
}
}
Expr::ZztOop(parts.concat())
}
fn parse_macro(&mut self) -> anyhow::Result<Expr> {
self.advance().unwrap(); // consume %
let ident = self.advance();
let name = match ident {
Some(Token::Identifier(name)) => name.into(),
_ => return Err(anyhow!("Expected identifier")),
};
let mut args = vec![];
while let Some(token) = self.peek() {
match token {
Token::String(s) => {
let val = s.into();
self.advance();
args.push(Box::new(Expr::String(val)));
}
Token::Newline => {
self.advance();
break;
}
_ => return Err(anyhow!("Unexpected token {:?}", token)),
}
}
Ok(Expr::Macro(name, args))
}
fn peek(&self) -> Option<&Token> {
self.tokens.get(self.pos)
}
fn advance(&mut self) -> Option<&Token> {
let result = self.tokens.get(self.pos);
self.pos = (self.pos + 1).min(self.tokens.len());
result
}
// TODO: a consume helper that takes a list of expr variants?
}
#[cfg(test)]
mod tests {
use crate::preprocess::scan::scan;
use insta::assert_debug_snapshot;
use super::*;
fn parse_str(input: &str) -> Vec<Expr> {
let tokens = scan(input).0;
parse(tokens).expect("Failed to parse code string")
}
#[test]
fn raw_text() {
assert_debug_snapshot!(parse_str("This is\nsome text."), @r###"
[
ZztOop(
"This is\nsome text.",
),
]
"###);
}
#[test]
fn single_macro() {
assert_debug_snapshot!(parse_str("%foo"), @r###"
[
Macro(
"foo",
[],
),
]
"###);
}
#[test]
fn macro_with_args() {
assert_debug_snapshot!(parse_str(r#"%foo "bar" "baz""#), @r###"
[
Macro(
"foo",
[
String(
"bar",
),
String(
"baz",
),
],
),
]
"###)
}
#[test]
fn macro_with_literals() {
assert_debug_snapshot!(parse_str("Foo.\n%bar\nBaz."), @r###"
[
ZztOop(
"Foo.\n",
),
Macro(
"bar",
[],
),
ZztOop(
"Baz.",
),
]
"###)
}
}
src/lang.rs → src/preprocess/scan.rs
+8
View File
@@ -207,6 +207,14 @@ mod tests {
assert_eq!(vec![raw_text("Hello, world!")], scan("Hello, world!"));
}
#[test]
fn scan_raw_multiline() {
assert_eq!(
vec![raw_text("foo"), Newline, raw_text("bar")],
scan("foo\nbar")
);
}
#[test]
fn scan_mixed() {
assert_eq!(