update

include/Lexer.hpp

@@ -1,30 +1,58 @@
 #pragma once
 
-#include <array>
 #include <cstddef>
-#include <exception>
 #include <functional>
+#include <memory>
+#include <ostream>
 #include <stdexcept>
 #include <string>
-#include <variant>
+#include <type_traits>
 #include <vector>
-#include "Lexer.hpp"
+#include <cxxabi.h>
 #include "Tools.hpp"
 
 namespace ztl{
     namespace lexer{
-        struct Token{std::string data;};
+        inline std::string demangle(const char* mangled_name) {
+        int status = 0;
+        // 调用 __cxa_demangle 解 mangling
+        char* demangled = abi::__cxa_demangle(
+            mangled_name,
+            nullptr,   // 输出缓冲区（nullptr 表示自动分配）
+            nullptr,   // 缓冲区大小（nullptr 表示自动计算）
+            &status    // 输出状态（0 表示成功）
+        );
+
+        // 用智能指针管理动态分配的内存，避免泄漏
+        std::unique_ptr<char, decltype(&std::free)> demangled_ptr(demangled, std::free);
+
+        if (status == 0) {
+            return std::string(demangled_ptr.get());  // 解 mangling 成功
+        } else {
+            return std::string(mangled_name);  // 失败时返回原始名称
+        }
+    }
+        struct Token{
+            std::string data;
+            // std::string got_token_type() const {
+            //     return demangle(typeid(*this).name());
+            // }
+        };
         struct Keywords:Token{};
         struct Identifiers:Token{};
         struct Literals:Token{};
         struct Operators:Token{};
         struct Separators:Token{};
         struct Comments:Token{};
+        struct EndSeparators:Token{};
         struct EOF_Token:Token{};
 
+        const std::vector<char> white_spaces{'\n','\t',' '};
+        const std::vector<char> symbols{'(',')','"'};
+
         struct CharProvider{
             std::string buffer;
-            std::vector<char> white_spaces{'\n','\t',' '};
+            
             size_t idx=0;
             Result<char> peek(){
                 if (idx<buffer.size()) {
@@ -61,7 +89,8 @@ namespace ztl{
                 if(idx-start_idx==0){
                     return Result<std::string>(Err("don't hava any string"));
                 }
-                return Result<std::string>(buffer.substr(start_idx,idx-start_idx+1));
+                idx++; 
+                return Result<std::string>(buffer.substr(start_idx,idx-start_idx-1));
             }
             void skip_whitespace(){
                 while(1) {
@@ -96,6 +125,17 @@ namespace ztl{
         inline bool is_alphanumeric(char c){
             return is_digit(c) || is_alpha(c);
         }
+
+        inline bool is_symbol(char c){
+            bool is_symbol = false;
+            for(char i:symbols){
+                if(c==i){
+                    is_symbol=true;
+                    break;
+                }
+            }
+            return is_symbol;
+        }
         
         inline std::vector<Token> lexer(const std::string&s){
             CharProvider cp(s);
@@ -106,6 +146,30 @@ namespace ztl{
                 match(cp.peek(),[&cp,&tokens](char c){
                     if (is_alpha(c)){
                         tokens.push_back(Keywords{cp.consume_until(is_alphanumeric).unwrap()});
+                    }else if(c=='('){
+                        tokens.push_back(Separators{std::string()+cp.consume().unwrap()});
+                        match(cp.match(')'),[&cp](bool matched){
+                            if(!matched){
+                                throw std::runtime_error("unmatched '('");
+                            }
+                        },[](const Err &e){
+                            throw std::runtime_error(e);
+                        });
+                    }else if(c=='"'){
+                        cp.consume().unwrap();
+                        tokens.push_back(Literals{cp.consume_until([](char c)->bool{return c!='"';}).unwrap()});
+                        match(cp.match('"'),[&cp](bool matched){
+                            if(!matched){
+                                throw std::runtime_error("unmatched '\"'");
+                            }
+                        },[](const Err &e){
+                            throw std::runtime_error(e);
+                        });
+                    }else if(c==';'){
+                        tokens.push_back(EndSeparators{std::string()+cp.consume().unwrap()});
+                    }
+                    else{
+                        throw std::runtime_error(std::string("unknown token named :'")+c+"'");
                     }
                 },[&should_break](const Err &e){
                     should_break=true;
@@ -115,4 +179,32 @@ namespace ztl{
             return tokens;
         }
     }
+    
+}
+
+namespace std {
+    template <typename T>
+    concept DerivedFromToken = std::is_base_of_v<ztl::lexer::Token, T>;
+
+    template<DerivedFromToken T>
+    std::ostream&operator<<(std::ostream &os, const T &t){
+        string name = ztl::get_T_name<T>();
+        // ztl::logger(ztl::demangle(typeid(t).name()));
+        // string name = ztl::demangle(typeid(t).name());
+        return os<<name<<" {data: '"<<t.data<<"'}";
+    }
+
+    template<class T>
+    std::ostream&operator<<(std::ostream &os, const std::vector<T> &v){
+        os<<"std::vector<"<<ztl::get_T_name<T>()<<"> "<<"{";
+        if(v.size()>0){
+            for(size_t i=0;i<v.size()-1;i++){
+                os<<" "<<v[i]<<",";
+            }
+            os<<" "<<v[v.size()-1];
+        }else{
+            os<<"}";
+        }
+        return os;
+    }
 }

include/Tools.hpp

@@ -1,11 +1,43 @@
+#pragma once
+#include <cstddef>
 #include <iostream>
 #include <stdexcept>
 #include <string>
 #include <utility>
 #include <variant>
-
+#include <vector>
 
 namespace ztl{
+    template<class ...Ts>
+    void logger(Ts&&...v){
+        #ifndef NDEBUG
+        std::cout<<"log[";
+        (std::cout<<...<<v);
+        std::cout<<"]\n";
+        #endif
+    }
+
+    template<class T>
+    std::string get_type_detail(){
+        return __PRETTY_FUNCTION__;
+    }
+
+    template<class T>
+    std::string get_T_name(){
+        std::string type_detail = get_type_detail<T>();
+        // logger(type_detail);
+        size_t start = type_detail.find("T = ");
+        if(start == std::string::npos) {
+            throw std::runtime_error("Failed to get type name");
+        }
+        size_t end = type_detail.find(';', start);
+        if(end == std::string::npos) {
+            end = type_detail.size()-1;
+        }
+        // logger(type_detail.substr(start + 4, end - start - 4));
+        return type_detail.substr(start + 4, end - start - 4);
+    }
+
     template<class ...Ts>
     struct overloaded:Ts...{using Ts::operator()...;};
 
@@ -44,12 +76,5 @@ namespace ztl{
         }
     };
 
-    template<class ...Ts>
+}
-    void logger(Ts&&...v){
+
-        #ifndef NDEBUG
-        std::cout<<"log[";
-        (std::cout<<...<<v);
-        std::cout<<"]\n";
-        #endif
-    }
-}

tests/test.cpp

@@ -1,14 +1,20 @@
 #include "Lexer.hpp"
+#include "Tools.hpp"
 #include <iostream>
 #include <string>
+#include <vector>
 
 #define NV(v) {std::cout<<#v<<" : '"<<(v)<<"'\n";}
 
+
+
 int main(){
+    ztl::get_T_name<ztl::lexer::Keywords>();
     std::string hello_world = R"(
         print("hello ztl!");
     )";
     std::cout<<"This is a ztl's test\n";
     NV(hello_world)
-    ztl::lexer::lexer(hello_world);
+    std::vector<ztl::lexer::Token> lexer_hello_world = ztl::lexer::lexer(hello_world);
+    ztl::logger(lexer_hello_world);
 }