Implement a basic parser, with snapshot testing

Cargo.lock

@@ -2,11 +2,19 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "anyhow"
+version = "1.0.86"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
+
 [[package]]
 name = "assembler"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "codepage-437",
+ "insta",
  "nom",
 ]
 
@@ -19,6 +27,18 @@ dependencies = [
  "csv",
 ]
 
+[[package]]
+name = "console"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "windows-sys",
+]
+
 [[package]]
 name = "csv"
 version = "1.3.0"
@@ -40,12 +60,48 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
+[[package]]
+name = "insta"
+version = "1.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "810ae6042d48e2c9e9215043563a58a80b877bc863228a74cf10c49d4620a6f5"
+dependencies = [
+ "console",
+ "lazy_static",
+ "linked-hash-map",
+ "similar",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.155"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
 [[package]]
 name = "memchr"
 version = "2.7.1"
@@ -112,6 +168,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "similar"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640"
+
 [[package]]
 name = "syn"
 version = "2.0.60"
@@ -128,3 +190,76 @@ name = "unicode-ident"
 version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"

Cargo.toml

@@ -6,5 +6,14 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1.0.86"
 codepage-437 = "0.1.0"
+insta = "1.39.0"
 nom = "7.1.3"
+
+[dev-dependencies]
+insta = "1.39.0"
+
+[profile.dev.package]
+insta.opt-level = 3
+similar.opt-level = 3

src/main.rs

@@ -1,10 +1,10 @@
-mod lang;
+mod preprocess;
 mod world;
 
 use std::{env, error::Error, fs, process::exit};
 use world::World;
 
-use crate::lang::scan;
+use crate::preprocess::scan::scan;
 
 fn to_latin1(bytes: &[u8]) -> String {
     bytes.iter().map(|&x| x as char).collect()

src/preprocess/mod.rs

@@ -0,0 +1,2 @@
+pub mod parse;
+pub mod scan;

src/preprocess/parse.rs

@@ -0,0 +1,171 @@
+use anyhow::{anyhow, Result};
+
+use super::scan::Token;
+
+pub fn parse(input: Vec<Token>) -> Result<Vec<Expr>> {
+    // TODO: Should this take ownership of the tokens?
+    let mut parser = Parser::new(input);
+    parser.parse()
+}
+
+struct Parser {
+    pos: usize,
+    tokens: Vec<Token>,
+}
+
+#[derive(Debug)]
+pub enum Expr {
+    ZztOop(String),
+    Macro(String, Vec<Box<Expr>>),
+    String(String),
+}
+
+impl Parser {
+    fn new(input: Vec<Token>) -> Self {
+        Self {
+            pos: 0,
+            tokens: input,
+        }
+    }
+
+    fn parse(&mut self) -> Result<Vec<Expr>> {
+        let mut result = vec![];
+        while let Some(token) = self.peek() {
+            match token {
+                Token::Newline | Token::RawText(_) => result.push(self.parse_zzt_oop()),
+                Token::Percent => result.push(self.parse_macro()?),
+                _ => return Err(anyhow!("Unexpected token {:?}", token)),
+            }
+        }
+        Ok(result)
+    }
+
+    fn parse_zzt_oop(&mut self) -> Expr {
+        let mut parts: Vec<String> = vec![];
+        while let Some(token) = self.peek() {
+            match token {
+                Token::RawText(s) => {
+                    parts.push(s.into());
+                    self.advance();
+                }
+                Token::Newline => {
+                    parts.push("\n".into());
+                    self.advance();
+                }
+                _ => break,
+            }
+        }
+        Expr::ZztOop(parts.concat())
+    }
+
+    fn parse_macro(&mut self) -> anyhow::Result<Expr> {
+        self.advance().unwrap(); // consume %
+
+        let ident = self.advance();
+        let name = match ident {
+            Some(Token::Identifier(name)) => name.into(),
+            _ => return Err(anyhow!("Expected identifier")),
+        };
+
+        let mut args = vec![];
+        while let Some(token) = self.peek() {
+            match token {
+                Token::String(s) => {
+                    let val = s.into();
+                    self.advance();
+                    args.push(Box::new(Expr::String(val)));
+                }
+                Token::Newline => {
+                    self.advance();
+                    break;
+                }
+                _ => return Err(anyhow!("Unexpected token {:?}", token)),
+            }
+        }
+        Ok(Expr::Macro(name, args))
+    }
+
+    fn peek(&self) -> Option<&Token> {
+        self.tokens.get(self.pos)
+    }
+
+    fn advance(&mut self) -> Option<&Token> {
+        let result = self.tokens.get(self.pos);
+        self.pos = (self.pos + 1).min(self.tokens.len());
+        result
+    }
+    // TODO: a consume helper that takes a list of expr variants?
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::preprocess::scan::scan;
+    use insta::assert_debug_snapshot;
+
+    use super::*;
+
+    fn parse_str(input: &str) -> Vec<Expr> {
+        let tokens = scan(input).0;
+        parse(tokens).expect("Failed to parse code string")
+    }
+
+    #[test]
+    fn raw_text() {
+        assert_debug_snapshot!(parse_str("This is\nsome text."), @r###"
+        [
+            ZztOop(
+                "This is\nsome text.",
+            ),
+        ]
+        "###);
+    }
+
+    #[test]
+    fn single_macro() {
+        assert_debug_snapshot!(parse_str("%foo"), @r###"
+        [
+            Macro(
+                "foo",
+                [],
+            ),
+        ]
+        "###);
+    }
+
+    #[test]
+    fn macro_with_args() {
+        assert_debug_snapshot!(parse_str(r#"%foo "bar" "baz""#), @r###"
+        [
+            Macro(
+                "foo",
+                [
+                    String(
+                        "bar",
+                    ),
+                    String(
+                        "baz",
+                    ),
+                ],
+            ),
+        ]
+        "###)
+    }
+
+    #[test]
+    fn macro_with_literals() {
+        assert_debug_snapshot!(parse_str("Foo.\n%bar\nBaz."), @r###"
+        [
+            ZztOop(
+                "Foo.\n",
+            ),
+            Macro(
+                "bar",
+                [],
+            ),
+            ZztOop(
+                "Baz.",
+            ),
+        ]
+        "###)
+    }
+}

src/preprocess/scan.rs

@@ -207,6 +207,14 @@ mod tests {
         assert_eq!(vec![raw_text("Hello, world!")], scan("Hello, world!"));
     }
 
+    #[test]
+    fn scan_raw_multiline() {
+        assert_eq!(
+            vec![raw_text("foo"), Newline, raw_text("bar")],
+            scan("foo\nbar")
+        );
+    }
+
     #[test]
     fn scan_mixed() {
         assert_eq!(