better ast

src/language_frontend/lexer/tokens.rs

@@ -1,26 +1,96 @@
 use std::fmt;
 
-use logos::{Lexer, Logos};
+use logos::{Logos};
 
 #[derive(Logos, Debug, Clone, PartialEq)]
-#[logos(skip r"[ \r\f]+")] // Skip whitespace
-pub enum Token<'src> {
+pub enum Token {
     Error,
-    Null,
 
-    Indent,
-    NewLine,
-    Dedent,
+    // Identifier
+    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |lex| lex.slice().to_owned())]
+    Identifier(String),
+
+    // Keywords
+    #[token("fun")]
+    Fun,
+    
+    #[token("class")]
+    Class,
+
+    #[token("var")]
+    Var,
+
+    #[token("interface")]
+    Interface,
+
+    #[token("derive")]
+    Derive,
+
+    #[token("impl")]
+    Impl,
+
+    #[token("if")]
+    If,
+
+    #[token("else")]
+    Else,
+
+    #[token("->")]
+    Return,
+
+    #[token("enum")]
+    Enum,
+
+
+    // Types
+    #[token("int")]
+    IntType,     
+
+    #[token("float")]
+    FloatType, 
+
+    #[token("bool")]
+    BoolType, 
+
+    #[token("String")]
+    StringType, 
+
+
+    // Literals
+    #[regex(r#""([^"\\]|\\.)*""#, |lex| lex.slice().to_owned())]
+    StringLiteral(String),
+
+    #[regex(r"[0-9]+", |lex| lex.slice().parse::<i64>().ok())]
+    IntLiteral(i64),
+
+    #[regex(r"[0-9]+\.[0-9]+", |lex| lex.slice().parse::<f64>().ok())]
+    FloatLiteral(f64),
 
-    #[token("false", |_| false)]
     #[token("true", |_| true)]
-    Bool(bool),
+    #[token("false", |_| false)]
+    BoolLiteral(bool),
 
-    #[token("+")]
-    Add,
+    // Operators
+    #[token("=")]
+    Assign,
+    
+    #[token("==")]
+    Equals,
 
-    #[token("-")]
-    Substract,
+    #[token("!=")]
+    NotEquals,
+
+    #[token("<", priority = 2)]
+    Less,
+
+    #[token("<=")]
+    LessEquals,
+
+    #[token(">", priority = 2)]
+    Greater,
+
+    #[token(">=")]
+    GreaterEquals,
 
     #[token("*")]
     Multiply,
@@ -28,70 +98,98 @@ pub enum Token<'src> {
     #[token("/")]
     Divide,
 
-    #[token("=")]
-    Equals,
+    #[token("+")]
+    Add,
+
+    #[token("-")]
+    Substract,
+
+    #[token("&&")]
+    And,
+
+    #[token("||")]
+    Or,
+
+
+    // Punctiuation
+    #[token("(")]
+    LParen,
+
+    #[token(")")]
+    RParen,
+
+    #[token("{")]
+    LBrace,
+
+    #[token("}")]
+    RBrace,
+
+    #[token("[")]
+    LBracket,
+
+    #[token("]")]
+    RBracket,
 
     #[token(":")]
     Colon,
 
-    #[token("(")]
-    ParenBegin,
-
-    #[token(")")]
-    ParenEnd,
-
-    #[token("{")]
-    BraceBegin,
-
-    #[token("}")]
-    BraceEnd,
-
-    #[regex(r"[+-]?[0-9]+", |lex| lex.slice().parse::<i64>().unwrap(), priority = 3)]
-    Integer(i64),
-
-    #[regex(r"[+-]?([0-9]*[.])?[0-9]+", |lex| lex.slice().parse::<f64>().unwrap())]
-    Float(f64),
-
-    #[regex(r"[_a-zA-Z][_0-9a-zA-Z]*")]
-    Ident(&'src str),
-
-    #[regex(r#""([^"\\\x00-\x1F]|\\(["\\bnfrt/]|u[a-fA-F0-9]{4}))*""#, |lex| lex.slice().to_owned())]
-    String(String),
-
-    #[token("class")]
-    #[token("fun")]
-    #[token("var")]
-    #[token("if")]
-    #[token("else")]
-    Keyword(&'src str),
+    // Special
+    #[regex(r"//[^\n\r]*", logos::skip)]
+    #[regex(r"/\*([^*]|\*[^/])*\*/", logos::skip)]
+    Comment,
+    
+    #[regex(r"[ \t\n\f]+", logos::skip)]
+    Whitespace,
+    
+    Eof,
 }
 
-
-impl fmt::Display for Token<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+impl fmt::Display for Token {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            Token::Float(s) => write!(f, "{s}"),
-            Token::Null => write!(f, "<null>"),
-            Token::Indent => write!(f, "<indent>"),
-            Token::NewLine => write!(f, "<new_line>"),
-            Token::Dedent => write!(f, "<dedent>"),
-            Token::Add => write!(f, "+"),
-            Token::Bool(_) => write!(f, "+"),
-            Token::Substract => write!(f, "-"),
+            Token::Identifier(ident) => write!(f, "{ident}"),
+            Token::Fun => write!(f, "fun"),
+            Token::Class => write!(f, "class"),
+            Token::Var => write!(f, "var"),
+            Token::Interface => write!(f, "interface"),
+            Token::Derive => write!(f, "derive"),
+            Token::Impl => write!(f, "impl"),
+            Token::If => write!(f, "if"),
+            Token::Else => write!(f, "else"),
+            Token::Return => write!(f, "->"),
+            Token::Enum => write!(f, "enum"),
+            Token::IntType => write!(f, "int"),
+            Token::FloatType => write!(f, "float"),
+            Token::BoolType => write!(f, "bool"),
+            Token::StringType => write!(f, "String"),
+            Token::StringLiteral(s) => write!(f, "{s}"),
+            Token::IntLiteral(i) => write!(f, "{i}"),
+            Token::FloatLiteral(fl) => write!(f, "{fl}"),
+            Token::BoolLiteral(b) => write!(f, "{b}"),
+            Token::Assign => write!(f, "="),
+            Token::Equals => write!(f, "=="),
+            Token::NotEquals => write!(f, "!="),
+            Token::Less => write!(f, "<"),
+            Token::LessEquals => write!(f, "<="),
+            Token::Greater => write!(f, ">"),
+            Token::GreaterEquals => write!(f, ">="),
             Token::Multiply => write!(f, "*"),
             Token::Divide => write!(f, "/"),
-            Token::Equals => write!(f, "="),
+            Token::Add => write!(f, "+"),
+            Token::Substract => write!(f, "-"),
+            Token::And => write!(f, "&&"),
+            Token::Or => write!(f, "||"),
+            Token::LParen => write!(f, "("),
+            Token::RParen => write!(f, ")"),
+            Token::LBrace => write!(f, "{{"),
+            Token::RBrace => write!(f, "}}"),
+            Token::LBracket => write!(f, "["),
+            Token::RBracket => write!(f, "]"),
             Token::Colon => write!(f, ":"),
-            Token::ParenBegin => write!(f, "("),
-            Token::ParenEnd => write!(f, ")"),
-            Token::BraceBegin => write!(f, "{{"),
-            Token::BraceEnd => write!(f, "}}"),
-            Token::Integer(s) => write!(f, "{s}"),
-            Token::Ident(s) => write!(f, "{s}"),
-            Token::String(s) => write!(f, "{s}"),
-            Token::Keyword(s) => write!(f, "{s}"),
+            Token::Comment => write!(f, ""),
+            Token::Whitespace => write!(f, ""),
+            Token::Eof => write!(f, ""),
             Token::Error => write!(f, "<error>"),
-           
         }
     }
-}
+}