mclang/mclangc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MCLang, now with testing!

Browse Source
This commit is contained in:
Gvidas Juknevičius
2024-12-21 05:10:03 +02:00
parent 54b6df5862
commit debcf6ad6c
26 changed files with 1210 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
src/tokeniser/mod.rs
View File

@@ -33,8 +33,8 @@ impl Token {
}
pub fn tokenise(s: &str) -> anyhow::Result<Vec<Token>> {
let mut loc = Loc::default();
pub fn tokenise(s: &str, file_p: &str) -> anyhow::Result<Vec<Token>> {
let mut loc = Loc::new(file_p, 1, 1);
let mut tokens = Vec::new();
let chars: Vec<_> = s.chars().collect();
let mut chars = chars.iter().peekable();
@@ -70,44 +70,52 @@ pub fn tokenise(s: &str) -> anyhow::Result<Vec<Token>> {
// tokens.push(Token::new(TokenType::Comment(Comment::Line(buf.clone())), &loc));
}
'\n' => loc.inc_line(),
'"' | '\'' |
'c' if *c != 'c' || chars.peek() == Some(&&'"') => {
let str_typ = *c;
let mut sc = *c;
if *c == 'c' {
sc = '"';
chars.peek();
}
'"' => {
let mut last = '\0';
let mut buf = String::new();
while let Some(c) = chars.next_if(|v| **v != '\n') {
loc.inc_col();
if *c == sc && last != '\\' {
if *c == '"' && last != '\\' {
break;
}
buf.push(*c);
last = *c;
}
match str_typ {
'"' => {
tokens.push(Token::new(TokenType::string(&buf, false), &loc));
tokens.push(Token::new(TokenType::string(&buf, false), &loc));
}
'\'' => {
let mut last = '\0';
let mut buf = String::new();
while let Some(c) = chars.next_if(|v| **v != '\n') {
loc.inc_col();
if *c == '\'' && last != '\\' {
break;
}
'c' => {
tokens.push(Token::new(TokenType::string(&buf, true), &loc));
}
'\'' => {
let buf = buf
.replace("\\n", "\n")
.replace("\\r", "\r");
if buf.len() > 1 {
lerror!(&loc, "Chars can only have 1 byte");
bail!("")
}
tokens.push(Token::new(TokenType::char(buf.chars().nth(0).unwrap()), &loc));
}
_ => unreachable!()
buf.push(*c);
last = *c;
}
let buf = buf
.replace("\\n", "\n")
.replace("\\r", "\r");
if buf.len() > 1 {
lerror!(&loc, "Chars can only have 1 byte");
bail!("")
}
tokens.push(Token::new(TokenType::char(buf.chars().nth(0).unwrap()), &loc));
}
'c' if chars.peek() == Some(&&'"') => {
chars.next();
let mut last = '\0';
let mut buf = String::new();
while let Some(c) = chars.next_if(|v| **v != '\n') {
loc.inc_col();
if *c == '"' && last != '\\' {
break;
}
buf.push(*c);
last = *c;
}
tokens.push(Token::new(TokenType::string(&buf, true), &loc));
}
'a'..='z' | 'A'..='Z' | '_' => {
let mut buf = String::new();
@@ -139,12 +147,13 @@ pub fn tokenise(s: &str) -> anyhow::Result<Vec<Token>> {
'o' => radix = 8,
_ => (),
}
},
None => {
tokens.push(Token::new(TokenType::number(parse(&buf).unwrap(), radix, signed), &loc));
}
}
while let Some(c) = chars.next_if(|v| matches!(**v, '0'..='9' | '.' | 'a'..='f' | 'A'..='F')) {
while let Some(c) = chars.next_if(|v| matches!(**v, '0'..='9' | '.' | 'a'..='f' | 'A'..='F' | 'x' | 'o')) {
loc.inc_col();
buf.push(*c);
}
@@ -178,7 +187,7 @@ pub fn tokenise(s: &str) -> anyhow::Result<Vec<Token>> {
}
tokens.push(Token::new(TokenType::number(parse(&buf).unwrap(), radix, signed), &loc));
}
16 => {
16 => {
if buf.strip_prefix("0x").expect("Unreachable")
.chars().filter(|v| !matches!(v, '0'..='9' | 'a'..='f' | 'A'..='F')).collect::<Vec<_>>().len() > 0 {
lerror!(&loc, "Invalid character in hex number");
@@ -268,8 +277,8 @@ lazy_static::lazy_static!(
("|", TokenType::Punct(Punctuation::Or)),
(">", TokenType::Punct(Punctuation::Gt)),
("<", TokenType::Punct(Punctuation::Lt)),
(">=", TokenType::Punct(Punctuation::Ge)),
("<=", TokenType::Punct(Punctuation::Le)),
(">=", TokenType::Punct(Punctuation::Ge)),
("<=", TokenType::Punct(Punctuation::Le)),
("^", TokenType::Punct(Punctuation::Xor)),
("+=", TokenType::Punct(Punctuation::AddEq)),
("-=", TokenType::Punct(Punctuation::SubEq)),