amethyst
/
basic-to-mindustry
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Working and tested expression parser

Browse Source
main
Shad Amethyst 1 year ago
parent ab0380cd79
commit f47b89b20b
3 changed files with 355 additions and 50 deletions
Show Stats Download Patch File Download Diff File

112
src/cursor.rs
Unescape View File

@ -0,0 +1,112 @@
#[derive(Clone, Debug)]
pub struct Cursor<'a, T> {
data: &'a [T],
offset: usize,
}
impl<'a, T> PartialEq for Cursor<'a, T> {
fn eq(&self, other: &Self) -> bool {
self.data.as_ptr() == other.data.as_ptr()
&& self.data.len() == other.data.len()
&& self.offset == other.offset
}
}
impl<'a, T: 'a> std::ops::Index<usize> for Cursor<'a, T> {
type Output = T;
fn index(&self, index: usize) -> &Self::Output {
&self.data[self.offset + index]
}
}
macro_rules! impl_range {
( $range:ty ) => {
impl<'a, T> std::ops::Index<$range> for Cursor<'a, T> {
type Output = [T];
fn index(&self, range: $range) -> &Self::Output {
&self.data[self.offset..][range]
}
}
};
}
impl_range!(std::ops::Range<usize>);
impl_range!(std::ops::RangeFrom<usize>);
impl_range!(std::ops::RangeFull);
impl_range!(std::ops::RangeInclusive<usize>);
impl_range!(std::ops::RangeTo<usize>);
impl_range!(std::ops::RangeToInclusive<usize>);
impl<'a, T> From<Cursor<'a, T>> for &'a [T] {
fn from(cursor: Cursor<'a, T>) -> Self {
&cursor.data[cursor.offset..]
}
}
impl<'a, T> From<&'_ Cursor<'a, T>> for &'a [T] {
fn from(cursor: &'_ Cursor<'a, T>) -> Self {
&cursor.data[cursor.offset..]
}
}
impl<'a, T> From<&'a [T]> for Cursor<'a, T> {
fn from(data: &'a [T]) -> Self {
Self { data, offset: 0 }
}
}
impl<'a, T, const N: usize> From<&'a [T; N]> for Cursor<'a, T> {
fn from(data: &'a [T; N]) -> Self {
Self { data, offset: 0 }
}
}
impl<'a, T> std::ops::Deref for Cursor<'a, T> {
type Target = [T];
fn deref(&self) -> &Self::Target {
&self.data[self.offset..]
}
}
impl<'a, T> Cursor<'a, T> {
pub fn new(data: &'a [T], offset: usize) -> Self {
Self { data, offset }
}
pub fn take(&mut self, amount: usize) -> &'a [T] {
let head = self.peek(amount);
self.offset += amount;
head
}
pub fn peek(&self, amount: usize) -> &'a [T] {
if self.offset >= self.data.len() {
return &[];
}
let max = (self.offset + amount).min(self.data.len());
&self.data[self.offset..max]
}
pub fn get(&self, at: usize) -> Option<&'a T> {
self.data.get(self.offset + at)
}
pub fn iter<'b>(&'b self) -> impl Iterator<Item = &'a T> + 'b {
self.data[self.offset..].iter()
}
pub fn len(&self) -> usize {
self.data.len().saturating_sub(self.offset)
}
pub fn range<R: std::slice::SliceIndex<[T], Output = [T]>>(&self, range: R) -> Self {
Self {
data: &self.data[self.offset..][range],
offset: 0,
}
}
}

1
src/lib.rs
Unescape View File

@ -1 +1,2 @@
pub mod cursor;
pub mod parse;

292
src/parse/mod.rs
Unescape View File

@ -1,3 +1,4 @@
use crate::cursor::Cursor;
use regex::Regex;
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
@ -22,11 +23,11 @@ impl Operator {
fn precedence(self) -> u8 {
use Operator as O;
match self {
O::Add | O::Sub => 5,
O::Add | O::Sub => 3,
O::RShift | O::LShift => 4,
O::Mod => 3,
O::Mul | O::Div => 0,
O::Eq | O::Neq | O::Gt | O::Lt | O::Gte | O::Lte => 10,
O::Mod => 5,
O::Mul | O::Div => 10,
O::Eq | O::Neq | O::Gt | O::Lt | O::Gte | O::Lte => 0,
}
}
}
@ -57,6 +58,7 @@ pub enum ParseError {
ExpectedOperand,
}
/// Transforms a raw string into a sequence of `BasicToken`s
pub fn tokenize(raw: &str) -> Result<Vec<BasicToken>, ParseError> {
macro_rules! match_token {
( $line:expr, $res:expr $(;)? ) => {};
@ -121,6 +123,7 @@ pub fn tokenize(raw: &str) -> Result<Vec<BasicToken>, ParseError> {
res.push(BasicToken::NewLine);
}
while line.len() > 0 {
// Main match clause for tokens
match_token!(line, res;
match_space => (),
match_let => (),
@ -158,6 +161,7 @@ pub fn tokenize(raw: &str) -> Result<Vec<BasicToken>, ParseError> {
}),
match_string(with_quotes) => (BasicToken::String(with_quotes[1..with_quotes.len() - 1].to_string())),
);
// If this line is reached, then none of the matches above matched
return Err(ParseError::InvalidToken(line.to_string()));
}
@ -166,6 +170,7 @@ pub fn tokenize(raw: &str) -> Result<Vec<BasicToken>, ParseError> {
Ok(res)
}
#[derive(Clone, Debug, PartialEq)]
pub enum BasicAstExpression {
Integer(i64),
Float(f64),
@ -173,22 +178,25 @@ pub enum BasicAstExpression {
Binary(Operator, Box<BasicAstExpression>, Box<BasicAstExpression>),
}
#[derive(Clone, Debug, PartialEq)]
pub enum BasicAstOperation {
Assign(String, BasicAstExpression),
Jump(String),
IfThenElse(BasicAstExpression, BasicAstBlock, BasicAstBlock),
}
#[derive(Clone, Debug, PartialEq)]
pub struct BasicAstInstruction {
pub label: Option<String>,
pub operation: BasicAstOperation,
}
#[derive(Default)]
#[derive(Clone, Debug, PartialEq, Default)]
pub struct BasicAstBlock {
pub instructions: Vec<BasicAstInstruction>,
}
/// Returns the index of the first token matching `needle`
fn find_token_index(tokens: &[BasicToken], needle: BasicToken) -> Result<usize, ParseError> {
tokens
.iter()
@ -198,40 +206,43 @@ fn find_token_index(tokens: &[BasicToken], needle: BasicToken) -> Result<usize,
.ok_or(ParseError::MissingToken(needle))
}
fn parse_expression(mut tokens: &[BasicToken]) -> Result<BasicAstExpression, ParseError> {
/// Advances `tokens` by `by` tokens, skipping the first newline tokens if present
fn advance(tokens: &mut &[BasicToken], by: usize) {
while let Some(BasicToken::NewLine) = tokens.get(0) {
*tokens = &(*tokens)[1..];
}
*tokens = &(*tokens)[by..];
}
fn parse_expression(tokens: &mut Cursor<'_, BasicToken>) -> Result<BasicAstExpression, ParseError> {
/// Returns the first non-newline token in `tokens`
fn peek<'a>(tokens: &'a &[BasicToken]) -> Option<&'a BasicToken> {
fn peek<'a>(tokens: &'a [BasicToken]) -> Option<&'a BasicToken> {
tokens.iter().find(|t| !matches!(t, BasicToken::NewLine))
}
/// Parses a single expression item
fn parse_expression_item(tokens: &mut &[BasicToken]) -> Result<BasicAstExpression, ParseError> {
match *tokens {
fn parse_expression_item(
tokens: &mut Cursor<'_, BasicToken>,
) -> Result<BasicAstExpression, ParseError> {
match tokens.peek(2) {
[BasicToken::Integer(int), ..] => {
advance(tokens, 1);
tokens.take(1);
Ok(BasicAstExpression::Integer(*int))
},
}
[BasicToken::Float(float), ..] => {
advance(tokens, 1);
tokens.take(1);
Ok(BasicAstExpression::Float(*float))
},
}
[BasicToken::Name(_fn_name), BasicToken::OpenParen, ..] => {
unimplemented!("Function calls are not yet supported");
},
}
[BasicToken::Name(name), ..] => {
advance(tokens, 1);
tokens.take(1);
Ok(BasicAstExpression::Variable(name.clone()))
},
}
[BasicToken::OpenParen, ..] => {
tokens.take(1);
let res = parse_expression(tokens)?;
if let Some(BasicToken::CloseParen) = tokens.take(1).get(0) {
Ok(res)
} else {
Err(ParseError::MissingToken(BasicToken::CloseParen))
}
}
[first, ..] => Err(ParseError::UnexpectedToken(first.clone())),
[] => Err(ParseError::ExpectedOperand),
_ => Err(ParseError::UnexpectedToken(tokens[0].clone())),
}
}
@ -239,17 +250,23 @@ fn parse_expression(mut tokens: &[BasicToken]) -> Result<BasicAstExpression, Par
/// recursively calling itself when an operator with a higher precedence is encountered.
///
/// See https://en.wikipedia.org/wiki/Operator-precedence_parser for more information
fn parse_expression_main(tokens: &mut &[BasicToken], lhs: BasicAstExpression, min_precedence: u8) -> Result<BasicAstExpression, ParseError> {
fn parse_expression_main(
tokens: &mut Cursor<'_, BasicToken>,
lhs: BasicAstExpression,
min_precedence: u8,
) -> Result<BasicAstExpression, ParseError> {
let mut ast = lhs;
while let Some(&BasicToken::Operator(operator)) = peek(tokens) {
if operator.precedence() < min_precedence {
break
break;
}
advance(tokens, 1);
tokens.take(1);
let mut rhs = parse_expression_item(tokens)?;
while let Some(&BasicToken::Operator(sub_operator)) = peek(tokens) {
if sub_operator.precedence() > operator.precedence() {
rhs = parse_expression_main(tokens, rhs, operator.precedence() + 1)?;
} else {
break;
}
}
@ -260,65 +277,70 @@ fn parse_expression(mut tokens: &[BasicToken]) -> Result<BasicAstExpression, Par
}
// Remove starting newlines
let lhs = parse_expression_item(&mut tokens)?;
advance(&mut tokens, 1);
let res = parse_expression_main(&mut tokens, lhs, 0)?;
assert_eq!(tokens, []);
let lhs = parse_expression_item(tokens)?;
let res = parse_expression_main(tokens, lhs, 0)?;
Ok(res)
}
pub fn build_ast(mut tokens: &[BasicToken]) -> Result<BasicAstBlock, ParseError> {
pub fn build_ast(tokens: &[BasicToken]) -> Result<BasicAstBlock, ParseError> {
let mut tokens = Cursor::from(tokens);
let mut instructions = Vec::new();
let mut current_label: Option<String> = None;
while tokens.len() > 0 {
match &tokens[..] {
match tokens.peek(2) {
[BasicToken::NewLine, BasicToken::Integer(label), ..] => {
tokens = &tokens[2..];
tokens.take(2);
current_label = Some(label.to_string());
}
[BasicToken::NewLine, BasicToken::Name(label), ..] => {
tokens = &tokens[2..];
tokens.take(2);
current_label = Some(label.clone());
}
[BasicToken::NewLine, ..] => {
tokens = &tokens[1..];
tokens.take(1);
current_label = None;
}
[BasicToken::Name(variable_name), BasicToken::Assign, ..] => {
tokens = &tokens[2..];
let expression = parse_expression(tokens)?;
// TODO: advance `tokens`
tokens.take(2);
let expression = parse_expression(&mut tokens)?;
instructions.push(BasicAstInstruction {
label: current_label.take(),
operation: BasicAstOperation::Assign(variable_name.clone(), expression)
operation: BasicAstOperation::Assign(variable_name.clone(), expression),
});
}
[BasicToken::If, ..] => {
tokens = &tokens[1..];
let then_index = find_token_index(tokens, BasicToken::Then)?;
let end_index = find_token_index(tokens, BasicToken::EndIf)?;
tokens.take(1);
let then_index = find_token_index(&tokens, BasicToken::Then)?;
let end_index = find_token_index(&tokens, BasicToken::EndIf)?;
let condition = parse_expression(&tokens[0..then_index])?;
if let Ok(else_index) = find_token_index(tokens, BasicToken::Else) {
let condition = parse_expression(&mut tokens.range(0..then_index))?;
if let Ok(else_index) = find_token_index(&tokens, BasicToken::Else) {
let true_branch = build_ast(&tokens[(then_index + 1)..else_index])?;
let false_branch = build_ast(&tokens[(else_index + 1)..end_index])?;
instructions.push(BasicAstInstruction {
label: current_label.take(),
operation: BasicAstOperation::IfThenElse(condition, true_branch, false_branch)
operation: BasicAstOperation::IfThenElse(
condition,
true_branch,
false_branch,
),
});
} else {
let true_branch = build_ast(&tokens[(then_index + 1)..end_index])?;
instructions.push(BasicAstInstruction {
label: current_label.take(),
operation: BasicAstOperation::IfThenElse(condition, true_branch, BasicAstBlock::default())
operation: BasicAstOperation::IfThenElse(
condition,
true_branch,
BasicAstBlock::default(),
),
});
}
tokens = &tokens[end_index..];
tokens.take(end_index);
}
_ => {
return Err(ParseError::UnexpectedToken(tokens[0].clone()));
@ -396,4 +418,174 @@ mod test {
],
);
}
#[test]
fn test_operator_precedence() {
fn test_parse<const N: usize>(list: [BasicToken; N]) -> BasicAstExpression {
parse_expression(&mut Cursor::from(&list)).unwrap()
}
fn test_err<const N: usize>(list: [BasicToken; N]) -> ParseError {
parse_expression(&mut Cursor::from(&list)).err().unwrap()
}
assert_eq!(
test_parse([BasicToken::Name(String::from("hello"))]),
BasicAstExpression::Variable(String::from("hello"))
);
assert_eq!(
test_parse([
BasicToken::Name(String::from("hello")),
BasicToken::Name(String::from("world")),
]),
BasicAstExpression::Variable(String::from("hello"))
);
assert_eq!(
test_parse([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
BasicToken::Integer(1),
]),
BasicAstExpression::Binary(
Operator::Add,
Box::new(BasicAstExpression::Variable(String::from("hello"))),
Box::new(BasicAstExpression::Integer(1)),
)
);
assert_eq!(
test_parse([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
BasicToken::Integer(2),
BasicToken::Operator(Operator::Mul),
BasicToken::Name(String::from("world")),
]),
BasicAstExpression::Binary(
Operator::Add,
Box::new(BasicAstExpression::Variable(String::from("hello"))),
Box::new(BasicAstExpression::Binary(
Operator::Mul,
Box::new(BasicAstExpression::Integer(2)),
Box::new(BasicAstExpression::Variable(String::from("world"))),
)),
)
);
assert_eq!(
test_parse([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Mul),
BasicToken::Integer(2),
BasicToken::Operator(Operator::Add),
BasicToken::Name(String::from("world")),
]),
BasicAstExpression::Binary(
Operator::Add,
Box::new(BasicAstExpression::Binary(
Operator::Mul,
Box::new(BasicAstExpression::Variable(String::from("hello"))),
Box::new(BasicAstExpression::Integer(2)),
)),
Box::new(BasicAstExpression::Variable(String::from("world"))),
)
);
assert_eq!(
test_parse([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Mul),
BasicToken::OpenParen,
BasicToken::Integer(2),
BasicToken::Operator(Operator::Add),
BasicToken::Name(String::from("world")),
BasicToken::CloseParen,
]),
BasicAstExpression::Binary(
Operator::Mul,
Box::new(BasicAstExpression::Variable(String::from("hello"))),
Box::new(BasicAstExpression::Binary(
Operator::Add,
Box::new(BasicAstExpression::Integer(2)),
Box::new(BasicAstExpression::Variable(String::from("world"))),
)),
)
);
assert_eq!(
test_parse([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
BasicToken::OpenParen,
BasicToken::Name(String::from("world")),
BasicToken::Operator(Operator::Mul),
BasicToken::Integer(2),
BasicToken::CloseParen,
]),
BasicAstExpression::Binary(
Operator::Add,
Box::new(BasicAstExpression::Variable(String::from("hello"))),
Box::new(BasicAstExpression::Binary(
Operator::Mul,
Box::new(BasicAstExpression::Variable(String::from("world"))),
Box::new(BasicAstExpression::Integer(2)),
)),
)
);
assert_eq!(
test_err([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
]),
ParseError::ExpectedOperand
);
assert_eq!(
test_err([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
BasicToken::OpenParen,
BasicToken::Name(String::from("world")),
BasicToken::Operator(Operator::Mul),
BasicToken::Integer(2),
]),
ParseError::MissingToken(BasicToken::CloseParen)
);
assert_eq!(
test_err([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
BasicToken::Operator(Operator::Mul),
]),
ParseError::UnexpectedToken(BasicToken::Operator(Operator::Mul))
);
assert!(matches!(
test_err([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
BasicToken::OpenParen,
]),
ParseError::ExpectedOperand | ParseError::MissingToken(BasicToken::CloseParen)
));
assert!(matches!(
test_err([
BasicToken::Name(String::from("hello")),
BasicToken::Operator(Operator::Add),
BasicToken::OpenParen,
BasicToken::CloseParen
]),
ParseError::ExpectedOperand | ParseError::UnexpectedToken(BasicToken::CloseParen)
));
assert_eq!(
test_err([BasicToken::Operator(Operator::Add), BasicToken::Integer(2)]),
ParseError::UnexpectedToken(BasicToken::Operator(Operator::Add))
);
}
}

Write Preview
Loading…
Cancel
Save