diff --git a/indexer.cpp b/indexer.cpp index 87115b27..6639bd61 100644 --- a/indexer.cpp +++ b/indexer.cpp @@ -71,7 +71,26 @@ std::string IndexedFile::ToString() { return output.GetString(); } +IndexedTypeDef::IndexedTypeDef(TypeId id, const std::string& usr) : id(id), usr(usr) { + assert(usr.size() > 0); + //std::cout << "Creating type with usr " << usr << std::endl; +} +void IndexedTypeDef::AddUsage(Location loc, bool insert_if_not_present = true) { + if (is_system_def) + return; + + for (int i = uses.size() - 1; i >= 0; --i) { + if (uses[i].IsEqualTo(loc)) { + if (loc.interesting) + uses[i].interesting = true; + return; + } + } + + if (insert_if_not_present) + uses.push_back(loc); +} template @@ -725,7 +744,7 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re // the same, this is most likely an implicit ctors. clang::Cursor ref_cursor = ref->cursor; if (ref->referencedEntity->kind == CXIdxEntity_CXXConstructor || - ref->referencedEntity->kind == CXIdxEntity_CXXDestructor) { + ref->referencedEntity->kind == CXIdxEntity_CXXDestructor) { Location parent_loc = db->file_db.Resolve(ref->parentEntity->cursor, true /*interesting*/); Location our_loc = db->file_db.Resolve(ref->loc, true /*is_interesting*/); diff --git a/indexer.h b/indexer.h index 9d8e03cf..ab19cc68 100644 --- a/indexer.h +++ b/indexer.h @@ -19,14 +19,13 @@ #include #include -struct TypeDef; -struct FuncDef; -struct VarDef; +struct IndexedTypeDef; +struct IndexedFuncDef; +struct IndexedVarDef; using FileId = int64_t; using namespace std::experimental; - // TODO: Move off of this weird wrapper, use struct with custom wrappers // directly. BEGIN_BITFIELD_TYPE(Location, uint64_t) @@ -145,9 +144,9 @@ bool operator==(const LocalId& a, const LocalId& b) { return a.local_id == b.local_id; } -using TypeId = LocalId; -using FuncId = LocalId; -using VarId = LocalId; +using TypeId = LocalId; +using FuncId = LocalId; +using VarId = LocalId; template @@ -157,9 +156,9 @@ struct Ref { Ref(LocalId id, Location loc) : id(id), loc(loc) {} }; -using TypeRef = Ref; -using FuncRef = Ref; -using VarRef = Ref; +using TypeRef = Ref; +using FuncRef = Ref; +using VarRef = Ref; // TODO: skip as much forward-processing as possible when |is_system_def| is @@ -204,26 +203,8 @@ struct IndexedTypeDef { // NOTE: Do not insert directly! Use AddUsage instead. std::vector uses; - IndexedTypeDef(TypeId id, const std::string& usr) : id(id), usr(usr) { - assert(usr.size() > 0); - //std::cout << "Creating type with usr " << usr << std::endl; - } - - void AddUsage(Location loc, bool insert_if_not_present = true) { - if (is_system_def) - return; - - for (int i = uses.size() - 1; i >= 0; --i) { - if (uses[i].IsEqualTo(loc)) { - if (loc.interesting) - uses[i].interesting = true; - return; - } - } - - if (insert_if_not_present) - uses.push_back(loc); - } + IndexedTypeDef(TypeId id, const std::string& usr); + void AddUsage(Location loc, bool insert_if_not_present = true); }; struct IndexedFuncDef { @@ -322,4 +303,30 @@ struct IndexedFile { std::string ToString(); }; + + +// TODO: Maybe instead of clearing/adding diffs, we should just clear out the +// entire previous index and readd the new one? That would be simpler. +// TODO: ^^^ I don't think we can do this. It will probably stall the main +// indexer for far too long since we will have to iterate over tons of +// data. +struct IndexedTypeDefDiff {}; +struct IndexedFuncDefDiff {}; +struct IndexedVarDefDiff {}; + +struct IndexedFileDiff { + std::vector removed_types; + std::vector removed_funcs; + std::vector removed_vars; + + std::vector added_types; + std::vector added_funcs; + std::vector added_vars; + + // TODO: Instead of change, maybe we just remove and then add again? not sure. + std::vector changed_types; + std::vector changed_funcs; + std::vector changed_vars; +}; + IndexedFile Parse(std::string filename, std::vector args); \ No newline at end of file diff --git a/query.cc b/query.cc index 65ce5606..96521d79 100644 --- a/query.cc +++ b/query.cc @@ -24,6 +24,55 @@ struct SymbolIdx { }; }; +template +struct TrackContributors { + std::vector values; + std::vector contributors; +}; + +// See comments in IndexedTypeDef for variable descriptions. +struct QueryableTypeDef { + TypeId id; + std::string short_name; + std::string qualified_name; + optional definition; + optional alias_of; + std::vector parents; + TrackContributors derived; + std::vector types; + std::vector funcs; + std::vector vars; + TrackContributors uses; +}; + +// See comments in IndexedFuncDef for variable descriptions. +struct QueryableFuncDef { + FuncId id; + std::string short_name; + std::string qualified_name; + TrackContributors declarations; + optional definition; + optional declaring_type; + optional base; + TrackContributors derived; + std::vector locals; + TrackContributors callers; + std::vector callees; + TrackContributors uses; +}; + +// See comments in IndexedVarDef for variable descriptions. +struct QueryableVarDef { + VarId id; + std::string short_name; + std::string qualified_name; + TrackContributors declaration; + optional definition; + optional variable_type; + optional declaring_type; + TrackContributors uses; +}; + struct QueryableFile { // Symbols declared in the file. std::vector declared_symbols; @@ -44,9 +93,9 @@ struct QueryableDatabase { std::vector symbols; // Raw data storage. - std::vector types; - std::vector funcs; - std::vector vars; + std::vector types; + std::vector funcs; + std::vector vars; // |files| is indexed by FileId. Retrieve a FileId from a path using // |file_locator|. diff --git a/serializer.cc b/serializer.cc index 7414c832..87ffe055 100644 --- a/serializer.cc +++ b/serializer.cc @@ -2,6 +2,115 @@ #include "indexer.h" +#if false +template +void Emit(Reader& a, const char* key, T& v) { + static_assert(false); // Must be specialized. +} +template +void Emit(Writer& a, const char* key, T& v) { + static_assert(false); // Must be specialized. +} + +template<> +void Emit(Reader& r, const char* key, int& v) { + v = r[key].GetInt(); +} + +template<> +void Emit(Writer& w, const char* key, int &v) { + w.Key(key); + w.Int(v); +} + +void StartObject(Reader& r) {} +void StartObject(Writer& w) { + w.StartObject(); +} + +void EndObject(Reader& r) {} +void EndObject(Writer& w) { + w.EndObject(); +} + +void StartArray(Reader& r) {} +void StartArray(Writer& w) { + w.StartArray(); +} + +void EndArray(Reader& r) {} +void EndArray(Writer& w) { + w.EndArray(); +} + +struct Object { + //Location l; + int a = 0, b = 0, c = 0; +}; + +/* +void EmitKey(Reader& r, const char* key) { + w.Key(key); +} +void EmitKey(Writer& w, const char* key) { + w = w[key]; +} +*/ + +template +void Serialize(S& stream, Object& obj) { + StartObject(stream); + Emit(stream, "a", obj.a); + Emit(stream, "b", obj.b); + Emit(stream, "b", obj.c); + EndObject(stream); +} + +/* +template +C& operator&(C& stream, T& t) { +t.serialize(stream); +} +*/ + +int main(int argc, char** argv) { + + rapidjson::StringBuffer output; + rapidjson::PrettyWriter writer(output); + writer.SetFormatOptions( + rapidjson::PrettyFormatOptions::kFormatSingleLineArray); + writer.SetIndent(' ', 2); + + Object foo; + foo.a = 10; + Serialize(writer, foo); + std::cout << output.GetString() << std::endl; + + std::cout << "----" << std::endl; + + rapidjson::Document doc; + //doc = doc["foo"]; + doc.Parse(output.GetString()); + Object foo2; + Serialize(doc, foo2); + + std::cin.get(); + //Reader r; + //foo.Serialize(r); + + return 0; +} +#endif + + + + + + + + + + void Serialize(Writer& writer, const char* key, Location location) { if (key) writer.Key(key); std::string s = location.ToString(); diff --git a/serializer.h b/serializer.h index 120e191d..1703099f 100644 --- a/serializer.h +++ b/serializer.h @@ -3,5 +3,6 @@ struct IndexedFile; using Writer = rapidjson::PrettyWriter; +using Reader = rapidjson::Document; void Serialize(Writer& writer, IndexedFile* file); \ No newline at end of file diff --git a/task.cc b/task.cc index 431f6425..e7c22c0d 100644 --- a/task.cc +++ b/task.cc @@ -8,82 +8,107 @@ #include "third_party/tiny-process-library/process.hpp" -struct BaseTask { +#include +#include +#include + +// A threadsafe-queue. http://stackoverflow.com/a/16075550 +template +class SafeQueue { +public: + // Add an element to the queue. + void enqueue(T t) { + std::lock_guard lock(mutex_); + queue_.push(t); + cv_.notify_one(); + } + + // Get the "front"-element. + // If the queue is empty, wait till a element is avaiable. + T dequeue() { + std::unique_lock lock(mutex_); + while (queue_.empty()) { + // release lock as long as the wait and reaquire it afterwards. + cv_.wait(lock); + } + T val = queue_.front(); + queue_.pop(); + return val; + } + +private: + std::queue queue_; + mutable std::mutex mutex_; + std::condition_variable cv_; +}; + +struct Task { int priority = 0; bool writes_to_index = false; -}; + bool should_exit = false; -// Task running in a separate process, parsing a file into something we can -// import. -struct IndexCreateTask : public BaseTask { - IndexCreateTask() { - writes_to_index = true; - } -}; - -// Completed parse task that wants to import content into the global database. -// Runs in main process, primary thread. Stops all other threads. -struct IndexImportTask : public BaseTask { - IndexImportTask() { - writes_to_index = true; - } -}; - -// Completed parse task that wants to update content previously imported into -// the global database. Runs in main process, primary thread. Stops all other -// threads. -// -// Note that this task just contains a set of operations to apply to the global -// database. The operations come from a diff based on the previously indexed -// state in comparison to the newly indexed state. -// -// TODO: We may be able to run multiple freshen and import tasks in parallel if -// we restrict what ranges of the db they may change. -struct IndexFreshenTask : public BaseTask { - IndexFreshenTask() { - writes_to_index = true; - } -}; - -// Task running a query against the global database. Run in main process, -// separate thread. -struct QueryTask : public BaseTask { - QueryTask() { - writes_to_index = false; + static Task MakeExit() { + Task task; + task.should_exit = true; + return task; } + // TODO: Create index task. + // Task running in a separate process, parsing a file into something we can + // import. + + // TODO: Index import task. + // Completed parse task that wants to import content into the global database. + // Runs in main process, primary thread. Stops all other threads. + + // TODO: Index fresh task. + // Completed parse task that wants to update content previously imported into + // the global database. Runs in main process, primary thread. Stops all other + // threads. + // + // Note that this task just contains a set of operations to apply to the global + // database. The operations come from a diff based on the previously indexed + // state in comparison to the newly indexed state. + // + // TODO: We may be able to run multiple freshen and import tasks in parallel if + // we restrict what ranges of the db they may change. + + // TODO: QueryTask + // Task running a query against the global database. Run in main process, + // separate thread. Command query; Location location; std::string argument; }; - // NOTE: When something enters a value into master db, it will have to have a // ref count, since multiple parsings could enter it (unless we require // that it be defined in that declaration unit!) struct TaskManager { - // Tasks that are currently executing. - std::vector running; - std::vector pending; + SafeQueue queued_tasks; // Available threads. std::vector threads; - std::condition_variable wakeup_thread; - std::mutex mutex; TaskManager(int num_threads); }; -static void ThreadMain(int id, std::condition_variable* waiter, std::mutex* mutex) { - std::unique_lock lock(*mutex); - waiter->wait(lock); +static void ThreadMain(int id, TaskManager* tm) { + while (true) { + Task task = tm->queued_tasks.dequeue(); + if (task.should_exit) { + std::cout << id << ": Exiting" << std::endl; + return; + } + + std::cout << id << ": waking" << std::endl; + } - std::cout << id << ": running in thread main" << std::endl; } TaskManager::TaskManager(int num_threads) { for (int i = 0; i < num_threads; ++i) { - threads.push_back(std::thread(&ThreadMain, i, &wakeup_thread, &mutex)); + threads.push_back(std::thread(&ThreadMain, i, this)); } } @@ -91,8 +116,8 @@ void Pump(TaskManager* tm) { //tm->threads[0]. } -int main(int argc, char** argv) { - TaskManager tm(10); +int main4(int argc, char** argv) { + TaskManager tm(5); // TODO: looks like we will have to write shared memory support. @@ -100,8 +125,8 @@ int main(int argc, char** argv) { // Repeat until we encounter a writer, wait for all threads to signal // they are done. // TODO: Let's use a thread safe queue/vector/etc instead. - tm.wakeup_thread.notify_one(); - tm.wakeup_thread.notify_one(); + for (int i = 0; i < 10; ++i) + tm.queued_tasks.enqueue(Task::MakeExit()); for (std::thread& thread : tm.threads) thread.join();