diff --git a/Cargo.lock b/Cargo.lock index 978aa7b..7b74b54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,9 +6,46 @@ version = 3 name = "assembler" version = "0.1.0" dependencies = [ + "codepage-437", "nom", ] +[[package]] +name = "codepage-437" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40c1169585d8d08e5675a39f2fc056cd19a258fc4cba5e3bbf4a9c1026de535" +dependencies = [ + "csv", +] + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + [[package]] name = "memchr" version = "2.7.1" @@ -30,3 +67,64 @@ dependencies = [ "memchr", "minimal-lexical", ] + +[[package]] +name = "proc-macro2" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "serde" +version = "1.0.199" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.199" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/Cargo.toml b/Cargo.toml index b05aa2b..64470ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,4 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +codepage-437 = "0.1.0" nom = "7.1.3" diff --git a/src/lang.rs b/src/lang.rs new file mode 100644 index 0000000..420fd87 --- /dev/null +++ b/src/lang.rs @@ -0,0 +1,280 @@ +/* + What are we parsing? + + 1. Evaluate macro invocations first. + - Resolve escape sequences, e.g., long lines. + - Replace top-level invocations with their output. + - Eventually, we'll replace in-line `${...}` expressions. + - Output: a string representing a ZZT-OOP program. + 2. Parse ZZT-OOP next. + - Output: looks like Vec. + - Statements are ZZT-OOP statements: text, centered, command, move, etc. + + From there, we can implement things like label rewriting. +*/ + +use std::{ + iter::{Fuse, Peekable}, + str::Chars, +}; + +#[derive(Debug, Eq, PartialEq)] +pub enum Token { + Identifier(String), + Newline, + Percent, + RawText(String), + String(String), +} + +pub fn scan(input: &str) -> Vec { + let mut scanner = Scanner::new(input); + scanner.scan(); + scanner.tokens +} + +struct Scanner<'a> { + text: Peekable>>, + tokens: Vec, +} + +impl<'a> Scanner<'a> { + fn new(input: &'a str) -> Self { + Scanner { + text: input.chars().fuse().peekable(), + tokens: vec![], + } + } + + fn scan(&mut self) { + while let Some(next) = self.peek() { + match next { + '\n' => { + self.tokens.push(Token::Newline); + self.advance(); + } + '%' => { + self.tokens.push(Token::Percent); + self.advance(); + self.scan_macro() + } + _ => self.raw_text(), + } + } + } + + fn scan_macro(&mut self) { + while let Some(next) = self.peek() { + match next { + '\n' => break, + ' ' => { + self.advance(); + } + '"' => { + self.string(); + } + 'A'..='Z' | 'a'..='z' | '_' => { + self.identifier(); + } + _ => { + todo!("unused chars") + } + } + } + } + + fn string(&mut self) { + self.advance(); // discard leading quote + let mut result = String::new(); + while let Some(next) = self.peek() { + match next { + '"' => break, + '\\' => { + self.advance(); + let char = self.get_escaped_char(); + if char != '\n' { + result.push(char) + } + } + _ => { + result.push(next); + self.advance(); + } + } + } + if !self.consume('"') { + todo!("Handle premature EOF") + } + self.tokens.push(Token::String(result)); + } + + fn identifier(&mut self) { + let mut result = String::new(); + while let Some(next) = self.peek() { + match next { + '0'..='9' | 'A'..='Z' | 'a'..='z' | '_' => { + result.push(next); + self.advance(); + } + _ => break, + } + } + self.tokens.push(Token::Identifier(result)); + } + + fn raw_text(&mut self) { + let mut result = String::new(); + while let Some(next) = self.peek() { + match next { + '\n' => break, + '\\' => { + self.advance(); + let char = self.get_escaped_char(); + if char != '\n' { + result.push(char); + } + } + _ => { + result.push(next); + self.advance(); + } + } + } + self.tokens.push(Token::RawText(result)); + } + + fn get_escaped_char(&mut self) -> char { + if let Some(next) = self.advance() { + next + } else { + '\\' + } + } + + fn peek(&mut self) -> Option { + self.text.peek().copied() + } + + fn advance(&mut self) -> Option { + self.text.next() + } + + fn consume(&mut self, c: char) -> bool { + if let Some(next) = self.peek() { + if next == c { + self.advance(); + return true; + } + } + false + } +} + +#[cfg(test)] +mod tests { + use std::vec; + + use super::*; + use Token::*; + macro_rules! token_helper { + ($fn:ident, $type:ident) => { + fn $fn(s: &str) -> Token { + Token::$type(s.to_string()) + } + }; + } + token_helper!(identifier, Identifier); + token_helper!(raw_text, RawText); + token_helper!(string, String); + + #[test] + fn scan_empty() { + assert_eq!(vec![] as Vec, scan("")); + } + + #[test] + fn scan_raw() { + assert_eq!(vec![raw_text("Hello, world!")], scan("Hello, world!")); + } + + #[test] + fn scan_mixed() { + assert_eq!( + vec![ + raw_text("foo"), + Newline, + Percent, + identifier("bar"), + Newline, + raw_text("baz") + ], + scan("foo\n%bar\nbaz") + ); + } + + #[test] + fn scan_multiline() { + assert_eq!( + vec![raw_text("foo"), Newline, raw_text("bar")], + scan("foo\nbar") + ); + } + + #[test] + fn scan_raw_escapes() { + assert_eq!( + vec![raw_text(r#"a long-line of raw text with \escapes"#)], + scan(concat!( + r#"a long\-line of raw text with\"#, + "\n", + r#" \\escapes"# + )) + ); + } + + #[test] + fn scan_directive() { + assert_eq!(vec![Percent, identifier("foo")], scan("%foo")); + } + + #[test] + fn scan_multiple_identifiers() { + assert_eq!( + vec![Percent, identifier("foo"), identifier("bar")], + scan("%foo bar") + ); + } + + #[test] + fn scan_directive_string() { + assert_eq!( + vec![Percent, identifier("foo"), string("bar")], + scan("%foo \"bar\"") + ); + } + + #[test] + fn scan_directive_string_escapes() { + assert_eq!( + vec![ + Percent, + identifier("foo"), + string(r#"a long-line string with "quotes" and \escapes"#) + ], + scan(concat!( + r#"%foo "a long\-line string with\"#, + "\n", + r#" \"quotes\" and \\escapes""# + )) + ); + } + + #[ignore = "not implemented yet"] + #[test] + fn scan_premature_end_of_string() { + assert_eq!( + vec![Percent, identifier("foo"), string("bar")], + scan("%foo \"bar") + ); + } +} diff --git a/src/main.rs b/src/main.rs index 5f600ff..0a9f901 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,11 @@ +mod lang; mod world; use std::{env, error::Error, fs, process::exit}; use world::World; +use crate::lang::scan; + fn to_latin1(bytes: &[u8]) -> String { bytes.iter().map(|&x| x as char).collect() } @@ -32,6 +35,7 @@ fn main() -> Result<(), Box> { board.terrain[(x - 1) + (y - 1) * 60], to_latin1(&stat.code) ); + scan("todo: convert [u8] to strings and scan here"); } }