From 3478da43a3528947c69586261184c800b83bdfae Mon Sep 17 00:00:00 2001 From: LunarAkai Date: Thu, 7 Aug 2025 21:44:41 +0200 Subject: [PATCH] guess i'm stupid, var works now :3 --- src/language_frontend/abstract_syntax_tree/parser.rs | 4 ++-- src/language_frontend/lexer/tokens.rs | 6 ++++-- src/main.rs | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/language_frontend/abstract_syntax_tree/parser.rs b/src/language_frontend/abstract_syntax_tree/parser.rs index acc319c..7b700da 100644 --- a/src/language_frontend/abstract_syntax_tree/parser.rs +++ b/src/language_frontend/abstract_syntax_tree/parser.rs @@ -1,5 +1,5 @@ use chumsky::{ - combinator::Or, error::Rich, extra, input::ValueInput, prelude::{choice, end, just, nested_delimiters, recursive, via_parser}, primitive::select, recursive, select, select_ref, span::{self, SimpleSpan}, text::{self, ascii::{ident, keyword}, whitespace}, Boxed, ConfigIterParser, IterParser, Parser + combinator::Or, error::Rich, extra, input::ValueInput, prelude::{choice, end, just, nested_delimiters, recursive, via_parser}, primitive::select, recursive, select, select_ref, span::{self, SimpleSpan}, text::{self, ascii::{ident, keyword}, newline, whitespace}, Boxed, ConfigIterParser, IterParser, Parser }; use crate::language_frontend::{abstract_syntax_tree::ast::{BinaryOp, Expr}, lexer::tokens::Token}; @@ -69,7 +69,7 @@ where .ignore_then(ident) .then_ignore(just(Token::Assign)) .then(expr.clone()) - .then_ignore(just(Token::NewLine).or_not()) + .then_ignore(just(Token::NewLine)) .map(|(name, rhs)| Expr::Assignment { target: Box::new(Expr::Ident(name)), value: Box::new(rhs), diff --git a/src/language_frontend/lexer/tokens.rs b/src/language_frontend/lexer/tokens.rs index 294cc35..ee0fad9 100644 --- a/src/language_frontend/lexer/tokens.rs +++ b/src/language_frontend/lexer/tokens.rs @@ -3,6 +3,7 @@ use std::fmt; use logos::{Logos}; #[derive(Logos, Debug, Clone, PartialEq)] +#[regex(r"[ \t\f]+", logos::skip)] pub enum Token { // Identifier #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_owned())] @@ -138,6 +139,8 @@ pub enum Token { // Special + #[regex(r"\n")] + NewLine, #[regex(r"//[^\r]*", logos::skip)] #[regex(r"/\*([^*]|\*[^/])*\*/", logos::skip)] @@ -146,8 +149,7 @@ pub enum Token { #[regex(r"[ \t\f]+", logos::skip)] Whitespace, - #[token(r"[ \n]")] - NewLine, + Eof, diff --git a/src/main.rs b/src/main.rs index bd92e7b..e127dd7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,6 +32,11 @@ fn main() { Err(()) => (Token::Error, span.into()), }); + println!("Token Stream:"); + for (token, span) in token_iter.clone() { + println!("{:?} at {:?}", token, span); + } + // Turn the token iterator into a stream that chumsky can use for things like backtracking let token_stream = Stream::from_iter(token_iter) // Tell chumsky to split the (Token, SimpleSpan) stream into its parts so that it can handle the spans for us