Start work on lexer for macro language

Cargo.lock

@@ -6,9 +6,46 @@ version = 3
 name = "assembler"
 version = "0.1.0"
 dependencies = [
+ "codepage-437",
  "nom",
 ]
 
+[[package]]
+name = "codepage-437"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e40c1169585d8d08e5675a39f2fc056cd19a258fc4cba5e3bbf4a9c1026de535"
+dependencies = [
+ "csv",
+]
+
+[[package]]
+name = "csv"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+
 [[package]]
 name = "memchr"
 version = "2.7.1"
@@ -30,3 +67,64 @@ dependencies = [
  "memchr",
  "minimal-lexical",
 ]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
+
+[[package]]
+name = "serde"
+version = "1.0.199"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.199"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

Cargo.toml

@@ -6,4 +6,5 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+codepage-437 = "0.1.0"
 nom = "7.1.3"

src/lang.rs

@@ -0,0 +1,280 @@
+/*
+    What are we parsing?
+
+    1. Evaluate macro invocations first.
+        - Resolve escape sequences, e.g., long lines.
+        - Replace top-level invocations with their output.
+        - Eventually, we'll replace in-line `${...}` expressions.
+        - Output: a string representing a ZZT-OOP program.
+    2. Parse ZZT-OOP next.
+        - Output: looks like Vec<Statement>.
+        - Statements are ZZT-OOP statements: text, centered, command, move, etc.
+
+    From there, we can implement things like label rewriting.
+*/
+
+use std::{
+    iter::{Fuse, Peekable},
+    str::Chars,
+};
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum Token {
+    Identifier(String),
+    Newline,
+    Percent,
+    RawText(String),
+    String(String),
+}
+
+pub fn scan(input: &str) -> Vec<Token> {
+    let mut scanner = Scanner::new(input);
+    scanner.scan();
+    scanner.tokens
+}
+
+struct Scanner<'a> {
+    text: Peekable<Fuse<Chars<'a>>>,
+    tokens: Vec<Token>,
+}
+
+impl<'a> Scanner<'a> {
+    fn new(input: &'a str) -> Self {
+        Scanner {
+            text: input.chars().fuse().peekable(),
+            tokens: vec![],
+        }
+    }
+
+    fn scan(&mut self) {
+        while let Some(next) = self.peek() {
+            match next {
+                '\n' => {
+                    self.tokens.push(Token::Newline);
+                    self.advance();
+                }
+                '%' => {
+                    self.tokens.push(Token::Percent);
+                    self.advance();
+                    self.scan_macro()
+                }
+                _ => self.raw_text(),
+            }
+        }
+    }
+
+    fn scan_macro(&mut self) {
+        while let Some(next) = self.peek() {
+            match next {
+                '\n' => break,
+                ' ' => {
+                    self.advance();
+                }
+                '"' => {
+                    self.string();
+                }
+                'A'..='Z' | 'a'..='z' | '_' => {
+                    self.identifier();
+                }
+                _ => {
+                    todo!("unused chars")
+                }
+            }
+        }
+    }
+
+    fn string(&mut self) {
+        self.advance(); // discard leading quote
+        let mut result = String::new();
+        while let Some(next) = self.peek() {
+            match next {
+                '"' => break,
+                '\\' => {
+                    self.advance();
+                    let char = self.get_escaped_char();
+                    if char != '\n' {
+                        result.push(char)
+                    }
+                }
+                _ => {
+                    result.push(next);
+                    self.advance();
+                }
+            }
+        }
+        if !self.consume('"') {
+            todo!("Handle premature EOF")
+        }
+        self.tokens.push(Token::String(result));
+    }
+
+    fn identifier(&mut self) {
+        let mut result = String::new();
+        while let Some(next) = self.peek() {
+            match next {
+                '0'..='9' | 'A'..='Z' | 'a'..='z' | '_' => {
+                    result.push(next);
+                    self.advance();
+                }
+                _ => break,
+            }
+        }
+        self.tokens.push(Token::Identifier(result));
+    }
+
+    fn raw_text(&mut self) {
+        let mut result = String::new();
+        while let Some(next) = self.peek() {
+            match next {
+                '\n' => break,
+                '\\' => {
+                    self.advance();
+                    let char = self.get_escaped_char();
+                    if char != '\n' {
+                        result.push(char);
+                    }
+                }
+                _ => {
+                    result.push(next);
+                    self.advance();
+                }
+            }
+        }
+        self.tokens.push(Token::RawText(result));
+    }
+
+    fn get_escaped_char(&mut self) -> char {
+        if let Some(next) = self.advance() {
+            next
+        } else {
+            '\\'
+        }
+    }
+
+    fn peek(&mut self) -> Option<char> {
+        self.text.peek().copied()
+    }
+
+    fn advance(&mut self) -> Option<char> {
+        self.text.next()
+    }
+
+    fn consume(&mut self, c: char) -> bool {
+        if let Some(next) = self.peek() {
+            if next == c {
+                self.advance();
+                return true;
+            }
+        }
+        false
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::vec;
+
+    use super::*;
+    use Token::*;
+    macro_rules! token_helper {
+        ($fn:ident, $type:ident) => {
+            fn $fn(s: &str) -> Token {
+                Token::$type(s.to_string())
+            }
+        };
+    }
+    token_helper!(identifier, Identifier);
+    token_helper!(raw_text, RawText);
+    token_helper!(string, String);
+
+    #[test]
+    fn scan_empty() {
+        assert_eq!(vec![] as Vec<Token>, scan(""));
+    }
+
+    #[test]
+    fn scan_raw() {
+        assert_eq!(vec![raw_text("Hello, world!")], scan("Hello, world!"));
+    }
+
+    #[test]
+    fn scan_mixed() {
+        assert_eq!(
+            vec![
+                raw_text("foo"),
+                Newline,
+                Percent,
+                identifier("bar"),
+                Newline,
+                raw_text("baz")
+            ],
+            scan("foo\n%bar\nbaz")
+        );
+    }
+
+    #[test]
+    fn scan_multiline() {
+        assert_eq!(
+            vec![raw_text("foo"), Newline, raw_text("bar")],
+            scan("foo\nbar")
+        );
+    }
+
+    #[test]
+    fn scan_raw_escapes() {
+        assert_eq!(
+            vec![raw_text(r#"a long-line of raw text with \escapes"#)],
+            scan(concat!(
+                r#"a long\-line of raw text with\"#,
+                "\n",
+                r#" \\escapes"#
+            ))
+        );
+    }
+
+    #[test]
+    fn scan_directive() {
+        assert_eq!(vec![Percent, identifier("foo")], scan("%foo"));
+    }
+
+    #[test]
+    fn scan_multiple_identifiers() {
+        assert_eq!(
+            vec![Percent, identifier("foo"), identifier("bar")],
+            scan("%foo bar")
+        );
+    }
+
+    #[test]
+    fn scan_directive_string() {
+        assert_eq!(
+            vec![Percent, identifier("foo"), string("bar")],
+            scan("%foo \"bar\"")
+        );
+    }
+
+    #[test]
+    fn scan_directive_string_escapes() {
+        assert_eq!(
+            vec![
+                Percent,
+                identifier("foo"),
+                string(r#"a long-line string with "quotes" and \escapes"#)
+            ],
+            scan(concat!(
+                r#"%foo "a long\-line string with\"#,
+                "\n",
+                r#" \"quotes\" and \\escapes""#
+            ))
+        );
+    }
+
+    #[ignore = "not implemented yet"]
+    #[test]
+    fn scan_premature_end_of_string() {
+        assert_eq!(
+            vec![Percent, identifier("foo"), string("bar")],
+            scan("%foo \"bar")
+        );
+    }
+}

src/main.rs

@@ -1,8 +1,11 @@
+mod lang;
 mod world;
 
 use std::{env, error::Error, fs, process::exit};
 use world::World;
 
+use crate::lang::scan;
+
 fn to_latin1(bytes: &[u8]) -> String {
     bytes.iter().map(|&x| x as char).collect()
 }
@@ -32,6 +35,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                 board.terrain[(x - 1) + (y - 1) * 60],
                 to_latin1(&stat.code)
             );
+            scan("todo: convert [u8] to strings and scan here");
         }
     }