update
This commit is contained in:
parent
839cf3eccf
commit
65bc648860
@ -1,30 +1,58 @@
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <exception>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <ostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <variant>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
#include "Lexer.hpp"
|
||||
#include <cxxabi.h>
|
||||
#include "Tools.hpp"
|
||||
|
||||
namespace ztl{
|
||||
namespace lexer{
|
||||
struct Token{std::string data;};
|
||||
inline std::string demangle(const char* mangled_name) {
|
||||
int status = 0;
|
||||
// 调用 __cxa_demangle 解 mangling
|
||||
char* demangled = abi::__cxa_demangle(
|
||||
mangled_name,
|
||||
nullptr, // 输出缓冲区（nullptr 表示自动分配）
|
||||
nullptr, // 缓冲区大小（nullptr 表示自动计算）
|
||||
&status // 输出状态（0 表示成功）
|
||||
);
|
||||
|
||||
// 用智能指针管理动态分配的内存，避免泄漏
|
||||
std::unique_ptr<char, decltype(&std::free)> demangled_ptr(demangled, std::free);
|
||||
|
||||
if (status == 0) {
|
||||
return std::string(demangled_ptr.get()); // 解 mangling 成功
|
||||
} else {
|
||||
return std::string(mangled_name); // 失败时返回原始名称
|
||||
}
|
||||
}
|
||||
struct Token{
|
||||
std::string data;
|
||||
// std::string got_token_type() const {
|
||||
// return demangle(typeid(*this).name());
|
||||
// }
|
||||
};
|
||||
struct Keywords:Token{};
|
||||
struct Identifiers:Token{};
|
||||
struct Literals:Token{};
|
||||
struct Operators:Token{};
|
||||
struct Separators:Token{};
|
||||
struct Comments:Token{};
|
||||
struct EndSeparators:Token{};
|
||||
struct EOF_Token:Token{};
|
||||
|
||||
const std::vector<char> white_spaces{'\n','\t',' '};
|
||||
const std::vector<char> symbols{'(',')','"'};
|
||||
|
||||
struct CharProvider{
|
||||
std::string buffer;
|
||||
std::vector<char> white_spaces{'\n','\t',' '};
|
||||
|
||||
size_t idx=0;
|
||||
Result<char> peek(){
|
||||
if (idx<buffer.size()) {
|
||||
@ -61,7 +89,8 @@ namespace ztl{
|
||||
if(idx-start_idx==0){
|
||||
return Result<std::string>(Err("don't hava any string"));
|
||||
}
|
||||
return Result<std::string>(buffer.substr(start_idx,idx-start_idx+1));
|
||||
idx++;
|
||||
return Result<std::string>(buffer.substr(start_idx,idx-start_idx-1));
|
||||
}
|
||||
void skip_whitespace(){
|
||||
while(1) {
|
||||
@ -96,6 +125,17 @@ namespace ztl{
|
||||
inline bool is_alphanumeric(char c){
|
||||
return is_digit(c) || is_alpha(c);
|
||||
}
|
||||
|
||||
inline bool is_symbol(char c){
|
||||
bool is_symbol = false;
|
||||
for(char i:symbols){
|
||||
if(c==i){
|
||||
is_symbol=true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return is_symbol;
|
||||
}
|
||||
|
||||
inline std::vector<Token> lexer(const std::string&s){
|
||||
CharProvider cp(s);
|
||||
@ -106,6 +146,30 @@ namespace ztl{
|
||||
match(cp.peek(),[&cp,&tokens](char c){
|
||||
if (is_alpha(c)){
|
||||
tokens.push_back(Keywords{cp.consume_until(is_alphanumeric).unwrap()});
|
||||
}else if(c=='('){
|
||||
tokens.push_back(Separators{std::string()+cp.consume().unwrap()});
|
||||
match(cp.match(')'),[&cp](bool matched){
|
||||
if(!matched){
|
||||
throw std::runtime_error("unmatched '('");
|
||||
}
|
||||
},[](const Err &e){
|
||||
throw std::runtime_error(e);
|
||||
});
|
||||
}else if(c=='"'){
|
||||
cp.consume().unwrap();
|
||||
tokens.push_back(Literals{cp.consume_until([](char c)->bool{return c!='"';}).unwrap()});
|
||||
match(cp.match('"'),[&cp](bool matched){
|
||||
if(!matched){
|
||||
throw std::runtime_error("unmatched '\"'");
|
||||
}
|
||||
},[](const Err &e){
|
||||
throw std::runtime_error(e);
|
||||
});
|
||||
}else if(c==';'){
|
||||
tokens.push_back(EndSeparators{std::string()+cp.consume().unwrap()});
|
||||
}
|
||||
else{
|
||||
throw std::runtime_error(std::string("unknown token named :'")+c+"'");
|
||||
}
|
||||
},[&should_break](const Err &e){
|
||||
should_break=true;
|
||||
@ -115,4 +179,32 @@ namespace ztl{
|
||||
return tokens;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace std {
|
||||
template <typename T>
|
||||
concept DerivedFromToken = std::is_base_of_v<ztl::lexer::Token, T>;
|
||||
|
||||
template<DerivedFromToken T>
|
||||
std::ostream&operator<<(std::ostream &os, const T &t){
|
||||
string name = ztl::get_T_name<T>();
|
||||
// ztl::logger(ztl::demangle(typeid(t).name()));
|
||||
// string name = ztl::demangle(typeid(t).name());
|
||||
return os<<name<<" {data: '"<<t.data<<"'}";
|
||||
}
|
||||
|
||||
template<class T>
|
||||
std::ostream&operator<<(std::ostream &os, const std::vector<T> &v){
|
||||
os<<"std::vector<"<<ztl::get_T_name<T>()<<"> "<<"{";
|
||||
if(v.size()>0){
|
||||
for(size_t i=0;i<v.size()-1;i++){
|
||||
os<<" "<<v[i]<<",";
|
||||
}
|
||||
os<<" "<<v[v.size()-1];
|
||||
}else{
|
||||
os<<"}";
|
||||
}
|
||||
return os;
|
||||
}
|
||||
}
|
||||
@ -1,11 +1,43 @@
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace ztl{
|
||||
template<class ...Ts>
|
||||
void logger(Ts&&...v){
|
||||
#ifndef NDEBUG
|
||||
std::cout<<"log[";
|
||||
(std::cout<<...<<v);
|
||||
std::cout<<"]\n";
|
||||
#endif
|
||||
}
|
||||
|
||||
template<class T>
|
||||
std::string get_type_detail(){
|
||||
return __PRETTY_FUNCTION__;
|
||||
}
|
||||
|
||||
template<class T>
|
||||
std::string get_T_name(){
|
||||
std::string type_detail = get_type_detail<T>();
|
||||
// logger(type_detail);
|
||||
size_t start = type_detail.find("T = ");
|
||||
if(start == std::string::npos) {
|
||||
throw std::runtime_error("Failed to get type name");
|
||||
}
|
||||
size_t end = type_detail.find(';', start);
|
||||
if(end == std::string::npos) {
|
||||
end = type_detail.size()-1;
|
||||
}
|
||||
// logger(type_detail.substr(start + 4, end - start - 4));
|
||||
return type_detail.substr(start + 4, end - start - 4);
|
||||
}
|
||||
|
||||
template<class ...Ts>
|
||||
struct overloaded:Ts...{using Ts::operator()...;};
|
||||
|
||||
@ -44,12 +76,5 @@ namespace ztl{
|
||||
}
|
||||
};
|
||||
|
||||
template<class ...Ts>
|
||||
void logger(Ts&&...v){
|
||||
#ifndef NDEBUG
|
||||
std::cout<<"log[";
|
||||
(std::cout<<...<<v);
|
||||
std::cout<<"]\n";
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,14 +1,20 @@
|
||||
#include "Lexer.hpp"
|
||||
#include "Tools.hpp"
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#define NV(v) {std::cout<<#v<<" : '"<<(v)<<"'\n";}
|
||||
|
||||
|
||||
|
||||
int main(){
|
||||
ztl::get_T_name<ztl::lexer::Keywords>();
|
||||
std::string hello_world = R"(
|
||||
print("hello ztl!");
|
||||
)";
|
||||
std::cout<<"This is a ztl's test\n";
|
||||
NV(hello_world)
|
||||
ztl::lexer::lexer(hello_world);
|
||||
std::vector<ztl::lexer::Token> lexer_hello_world = ztl::lexer::lexer(hello_world);
|
||||
ztl::logger(lexer_hello_world);
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user