From 839cf3eccfeb0e1b9bbba8a109a4f6e3caa1224a Mon Sep 17 00:00:00 2001
From: Zengtudor <Zengtudor@outlook.com>
Date: Sat, 12 Jul 2025 00:35:49 +0800
Subject: [PATCH] update

---
 CMakeLists.txt    |   5 +-
 include/Lexer.hpp | 116 ++++++++++++++++++++++++++++++++++++++++++++++
 include/Tools.hpp |  55 ++++++++++++++++++++++
 src/main.cpp      |   1 +
 tests/test.cpp    |  14 ++++++
 5 files changed, 190 insertions(+), 1 deletion(-)
 create mode 100644 include/Tools.hpp
 create mode 100644 tests/test.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9615e46..4221130 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -9,4 +9,7 @@ set(CMAKE_CXX_STANDARD 26
 file(GLOB_RECURSE SRC_CPP ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp)
 
 add_executable(${PROJECT_NAME} ${SRC_CPP})
-target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include)
\ No newline at end of file
+
+target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include)
+add_executable(${PROJECT_NAME}_test ${CMAKE_CURRENT_LIST_DIR}/tests/test.cpp)
+target_include_directories(${PROJECT_NAME}_test PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include)
\ No newline at end of file
diff --git a/include/Lexer.hpp b/include/Lexer.hpp
index 3f59c93..acc992e 100644
--- a/include/Lexer.hpp
+++ b/include/Lexer.hpp
@@ -1,2 +1,118 @@
 #pragma once
 
+#include <array>
+#include <cstddef>
+#include <exception>
+#include <functional>
+#include <stdexcept>
+#include <string>
+#include <variant>
+#include <vector>
+#include "Lexer.hpp"
+#include "Tools.hpp"
+
+namespace ztl{
+    namespace lexer{
+        struct Token{std::string data;};
+        struct Keywords:Token{};
+        struct Identifiers:Token{};
+        struct Literals:Token{};
+        struct Operators:Token{};
+        struct Separators:Token{};
+        struct Comments:Token{};
+        struct EOF_Token:Token{};
+
+        struct CharProvider{
+            std::string buffer;
+            std::vector<char> white_spaces{'\n','\t',' '};
+            size_t idx=0;
+            Result<char> peek(){
+                if (idx<buffer.size()) {
+                    return Result<char>(buffer[idx]);
+                }else{
+                    return Result<char>(Err("got EOF"));
+                }
+            }
+            Result<char> consume(){
+                return ztl::match(peek(), [this](char c){
+                    idx++;
+                    return Result<char>(c);
+                },[](const Err &e){
+                    return Result<char>(e);
+                });
+            }
+            Result<bool> match(char c){
+                return ztl::match(peek(),[this](char c){
+                    if(c==buffer[idx]){
+                        idx++;
+                        return Result<bool>(true);
+                    }else{
+                        return Result<bool>(false);
+                    }
+                },[this](const Err &e){
+                    return Result<bool>(e);
+                });
+            }
+            Result<std::string> consume_until(std::function<bool(char)> func){
+                size_t start_idx = idx;
+                while(idx<buffer.size()&&func(buffer[idx])){
+                    idx++;
+                }
+                if(idx-start_idx==0){
+                    return Result<std::string>(Err("don't hava any string"));
+                }
+                return Result<std::string>(buffer.substr(start_idx,idx-start_idx+1));
+            }
+            void skip_whitespace(){
+                while(1) {
+                    if(
+                        !ztl::match(peek(),[this](char c){
+                            bool is_ws = false;
+                            for(char ws:white_spaces){
+                                if(c==ws){
+                                    consume();
+                                    is_ws=true;
+                                }
+                            }
+                            return is_ws;
+                        },[](const Err &e){
+                            return false;
+                        })
+                    ){
+                        return;
+                    }
+                }
+            }
+        };
+
+        inline bool is_digit(char c){
+            return '0'<=c&&c<='9';
+        }
+
+        inline bool is_alpha(char c){
+            return 'A'<=c&&c<='z';
+        }
+
+        inline bool is_alphanumeric(char c){
+            return is_digit(c) || is_alpha(c);
+        }
+        
+        inline std::vector<Token> lexer(const std::string&s){
+            CharProvider cp(s);
+            std::vector<Token> tokens;
+            while(1){
+                bool should_break=false;
+                cp.skip_whitespace();
+                match(cp.peek(),[&cp,&tokens](char c){
+                    if (is_alpha(c)){
+                        tokens.push_back(Keywords{cp.consume_until(is_alphanumeric).unwrap()});
+                    }
+                },[&should_break](const Err &e){
+                    should_break=true;
+                });
+                if(should_break)break;
+            }
+            return tokens;
+        }
+    }
+}
\ No newline at end of file
diff --git a/include/Tools.hpp b/include/Tools.hpp
new file mode 100644
index 0000000..73404c0
--- /dev/null
+++ b/include/Tools.hpp
@@ -0,0 +1,55 @@
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <variant>
+
+
+namespace ztl{
+    template<class ...Ts>
+    struct overloaded:Ts...{using Ts::operator()...;};
+
+    template<class ...Ts>
+    overloaded(Ts...)->overloaded<Ts...>;
+
+    template<class ...Ts, class ...Fs>
+    auto match(std::variant<Ts...>& v, Fs&&... func) {
+        return std::visit(ztl::overloaded{std::forward<Fs>(func)...}, v);
+    }
+    template<class ...Ts, class ...Fs>
+    auto match(const std::variant<Ts...>& v, Fs&&... func) {
+        return std::visit(ztl::overloaded{std::forward<Fs>(func)...}, v);
+    }
+    template<class ...Ts, class ...Fs>
+    auto match(std::variant<Ts...>&& v, Fs&&... func) {
+        return std::visit(ztl::overloaded{std::forward<Fs>(func)...}, std::move(v));
+    }
+    struct Err:std::string{};
+
+    // template<class T>
+    // struct Ok{
+    //     T data;
+    // };
+
+    template<class T>
+    struct Result:std::variant<T,Err>{
+        T unwrap(){
+            return match<T>(*this,[](const T&t){
+                return t;
+            },[](const Err &e){
+                throw std::runtime_error(e);
+                T t;
+                return t;
+            });
+        }
+    };
+
+    template<class ...Ts>
+    void logger(Ts&&...v){
+        #ifndef NDEBUG
+        std::cout<<"log[";
+        (std::cout<<...<<v);
+        std::cout<<"]\n";
+        #endif
+    }
+}
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
index 5f27d60..6ce65bd 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1,4 +1,5 @@
 #include "Lexer.hpp"
+#include <iostream>
 
 int main(){
     
diff --git a/tests/test.cpp b/tests/test.cpp
new file mode 100644
index 0000000..8cc7099
--- /dev/null
+++ b/tests/test.cpp
@@ -0,0 +1,14 @@
+#include "Lexer.hpp"
+#include <iostream>
+#include <string>
+
+#define NV(v) {std::cout<<#v<<" : '"<<(v)<<"'\n";}
+
+int main(){
+    std::string hello_world = R"(
+        print("hello ztl!");
+    )";
+    std::cout<<"This is a ztl's test\n";
+    NV(hello_world)
+    ztl::lexer::lexer(hello_world);
+}
\ No newline at end of file