#pragma once #include #include #include #include #include #include #include #include #include "Lexer.hpp" #include "Tools.hpp" namespace ztl{ namespace lexer{ struct Token{std::string data;}; struct Keywords:Token{}; struct Identifiers:Token{}; struct Literals:Token{}; struct Operators:Token{}; struct Separators:Token{}; struct Comments:Token{}; struct EOF_Token:Token{}; struct CharProvider{ std::string buffer; std::vector white_spaces{'\n','\t',' '}; size_t idx=0; Result peek(){ if (idx(buffer[idx]); }else{ return Result(Err("got EOF")); } } Result consume(){ return ztl::match(peek(), [this](char c){ idx++; return Result(c); },[](const Err &e){ return Result(e); }); } Result match(char c){ return ztl::match(peek(),[this](char c){ if(c==buffer[idx]){ idx++; return Result(true); }else{ return Result(false); } },[this](const Err &e){ return Result(e); }); } Result consume_until(std::function func){ size_t start_idx = idx; while(idx(Err("don't hava any string")); } return Result(buffer.substr(start_idx,idx-start_idx+1)); } void skip_whitespace(){ while(1) { if( !ztl::match(peek(),[this](char c){ bool is_ws = false; for(char ws:white_spaces){ if(c==ws){ consume(); is_ws=true; } } return is_ws; },[](const Err &e){ return false; }) ){ return; } } } }; inline bool is_digit(char c){ return '0'<=c&&c<='9'; } inline bool is_alpha(char c){ return 'A'<=c&&c<='z'; } inline bool is_alphanumeric(char c){ return is_digit(c) || is_alpha(c); } inline std::vector lexer(const std::string&s){ CharProvider cp(s); std::vector tokens; while(1){ bool should_break=false; cp.skip_whitespace(); match(cp.peek(),[&cp,&tokens](char c){ if (is_alpha(c)){ tokens.push_back(Keywords{cp.consume_until(is_alphanumeric).unwrap()}); } },[&should_break](const Err &e){ should_break=true; }); if(should_break)break; } return tokens; } } }