Rework some of the command line flag parsing logic. Hopefully make it more robust.

src/code_completion.cc

@@ -324,22 +324,17 @@ void CompletionMain(CompletionManager* completion_manager) {
 
 }  // namespace
 
-CompletionSession::CompletionSession(const Project::Entry& file, IndexerConfig* config, WorkingFiles* working_files) : file(file) {
+CompletionSession::CompletionSession(const Project::Entry& file, WorkingFiles* working_files) : file(file) {
   std::vector<CXUnsavedFile> unsaved = working_files->AsUnsavedFiles();
 
   std::vector<std::string> args = file.args;
-  args.push_back("-x");
-  args.push_back("c++");
   args.push_back("-fparse-all-comments");
 
-  std::string sent_args = "";
+  std::cerr << "Creating completion session with arguments " << StringJoin(args) << std::endl;
-  for (auto& arg : args)
-    sent_args += arg + ", ";
-  std::cerr << "Creating completion session with arguments " << sent_args << std::endl;
 
   // TODO: I think we crash when there are syntax errors.
   active_index = MakeUnique<clang::Index>(0 /*excludeDeclarationsFromPCH*/, 0 /*displayDiagnostics*/);
-  active = MakeUnique<clang::TranslationUnit>(config, *active_index, file.filename, args, unsaved, Flags());
+  active = MakeUnique<clang::TranslationUnit>(*active_index, file.filename, args, unsaved, Flags());
   std::cerr << "Done creating active; did_fail=" << active->did_fail << std::endl;
   //if (active->did_fail) {
   //  std::cerr << "Failed to create translation unit; trying again..." << std::endl;
@@ -398,6 +393,6 @@ CompletionSession* CompletionManager::GetOrOpenSession(const std::string& filena
   else {
     std::cerr << "Found compilation entry" << std::endl;
   }
-  sessions.push_back(MakeUnique<CompletionSession>(*entry, config, working_files));
+  sessions.push_back(MakeUnique<CompletionSession>(*entry, working_files));
   return sessions[sessions.size() - 1].get();
 }

src/code_completion.h

@@ -23,7 +23,7 @@ struct CompletionSession {
   //std::unique_ptr<clang::TranslationUnit> updated;
   //std::unique_ptr<clang::Index> updated_index;
 
-  CompletionSession(const Project::Entry& file, IndexerConfig* config, WorkingFiles* working_files);
+  CompletionSession(const Project::Entry& file, WorkingFiles* working_files);
   ~CompletionSession();
 
   // Refresh file index.

src/command_line.cc

@@ -1282,7 +1282,7 @@ bool QueryDbMainLoop(
           }
 
           // Open up / load the project.
-          project->Load(project_path);
+          project->Load(config->extraClangArguments, project_path);
           std::cerr << "Loaded compilation entries (" << project->entries.size() << " files)" << std::endl;
 
           project->ForAllFilteredFiles(config, [&](int i, const Project::Entry& entry) {

src/indexer.cc

@@ -1355,7 +1355,7 @@ std::vector<std::unique_ptr<IndexedFile>> Parse(
   clang::Index index(0 /*excludeDeclarationsFromPCH*/,
                      0 /*displayDiagnostics*/);
   std::vector<CXUnsavedFile> unsaved_files;
-  clang::TranslationUnit tu(config, index, file, args, unsaved_files, CXTranslationUnit_KeepGoing);
+  clang::TranslationUnit tu(index, file, args, unsaved_files, CXTranslationUnit_KeepGoing);
 
   if (dump_ast)
     Dump(tu.document_cursor());

src/libclangmm/TranslationUnit.cc

@@ -10,8 +10,7 @@
 
 namespace clang {
 
-TranslationUnit::TranslationUnit(IndexerConfig* config,
+TranslationUnit::TranslationUnit(Index& index,
-                                 Index& index,
                                  const std::string& filepath,
                                  const std::vector<std::string>& arguments,
                                  std::vector<CXUnsavedFile> unsaved_files,
@@ -24,13 +23,7 @@ TranslationUnit::TranslationUnit(IndexerConfig* config,
   for (const auto& arg : platform_args)
     args.push_back(arg.c_str());
 
-  for (const std::string& arg : config->extraClangArguments)
+  std::cerr << "Parsing " << filepath << " with args " << StringJoin(args) << std::endl;
-    args.push_back(arg.c_str());
-
-  std::cerr << "Parsing " << filepath << " with args ";
-  for (const auto& arg : args)
-    std::cerr << arg << " ";
-  std::cerr << std::endl;
 
   CXErrorCode error_code = clang_parseTranslationUnit2FullArgv(
       index.cx_index, filepath.c_str(), args.data(), args.size(),

src/libclangmm/TranslationUnit.h

@@ -12,8 +12,7 @@
 namespace clang {
 class TranslationUnit {
  public:
-  TranslationUnit(IndexerConfig* config,
+  TranslationUnit(Index& index,
-                  Index& index,
                   const std::string& filepath,
                   const std::vector<std::string>& arguments,
                   std::vector<CXUnsavedFile> unsaved_files,

src/project.cc

@@ -11,7 +11,6 @@
 #include <iostream>
 #include <vector>
 
-
 struct CompileCommandsEntry {
   std::string directory;
   std::string file;
@@ -22,137 +21,83 @@ MAKE_REFLECT_STRUCT(CompileCommandsEntry, directory, file, command, args);
 
 namespace {
 
-
+// Blacklisted flags which are always removed from the command line.
-
-// https://github.com/Andersbakken/rtags/blob/6b16b81ea93aeff4a58930b44b2a0a207b456192/src/Source.cpp
-static const char *kValueArgs[] = {
-  "--param",
-  "-G",
-  "-MF",
-  "-MQ",
-  "-MT",
-  "-T",
-  "-V",
-  "-Xanalyzer",
-  "-Xassembler",
-  "-Xclang",
-  "-Xlinker",
-  "-Xpreprocessor",
-  "-arch",
-  "-b",
-  "-gcc-toolchain",
-  //"-imacros",
-  "-imultilib",
-  //"-include",
-  //"-iprefix",
-  //"-isysroot",
-  "-ivfsoverlay",
-  "-iwithprefix",
-  "-iwithprefixbefore",
-  "-o",
-  "-target",
-  "-x"
-};
 static const char *kBlacklist[] = {
-  "--param",
+  "-stdlib=libc++"
-  "-M",
-  "-MD",
-  "-MF",
-  "-MG",
-  "-MM",
-  "-MMD",
-  "-MP",
-  "-MQ",
-  "-MT",
-  "-Og",
-  "-Wa,--32",
-  "-Wa,--64",
-  "-Wl,--incremental-full",
-  "-Wl,--incremental-patch,1",
-  "-Wl,--no-incremental",
-  "-fbuild-session-file=",
-  "-fbuild-session-timestamp=",
-  "-fembed-bitcode",
-  "-fembed-bitcode-marker",
-  "-fmodules-validate-once-per-build-session",
-  "-fno-delete-null-pointer-checks",
-  "-fno-use-linker-plugin"
-  "-fno-var-tracking",
-  "-fno-var-tracking-assignments",
-  "-fno-enforce-eh-specs",
-  "-fvar-tracking",
-  "-fvar-tracking-assignments",
-  "-fvar-tracking-assignments-toggle",
-  "-gcc-toolchain",
-  "-march=",
-  "-masm=",
-  "-mcpu=",
-  "-mfpmath=",
-  "-mtune=",
-  "-s",
-
-  //"-B",
-  //"-f",
-  //"-pipe",
-  //"-W",
-  // TODO
-  "-Wno-unused-lambda-capture",
-  "/",
-  "..",
 };
 
-Project::Entry GetCompilationEntryFromCompileCommandEntry(const CompileCommandsEntry& entry) {
+// Arguments which are followed by a potentially relative path. We need to make
+// all relative paths absolute, otherwise libclang will not resolve them.
+const char* kPathArgs[] = {
+  "-isystem",
+  "-I",
+  "-iquote",
+  "--sysroot="
+};
+
+Project::Entry GetCompilationEntryFromCompileCommandEntry(const std::vector<std::string>& extra_flags, const CompileCommandsEntry& entry) {
   Project::Entry result;
   result.filename = NormalizePath(entry.file);
 
-  size_t num_args = entry.args.size();
+  bool make_next_flag_absolute = false;
-  result.args.reserve(num_args);
-  for (size_t j = 0; j < num_args; ++j) {
-    std::string arg = entry.args[j];
 
+  result.args.reserve(entry.args.size() + extra_flags.size());
+  for (size_t i = 0; i < entry.args.size(); ++i) {
+    std::string arg = entry.args[i];
     
-    bool bad = false;
+    // If blacklist skip.
-    for (auto& entry : kValueArgs) {
+    if (std::any_of(std::begin(kBlacklist), std::end(kBlacklist), [&arg](const char* value) {
-      if (StartsWith(arg, entry)) {
+      return StartsWith(arg, value);
-        bad = true;
+    })) {
-        continue;
-      }
-    }
-    if (bad) {
-      ++j;
       continue;
     }
 
+    // Cleanup path for previous argument.
+    if (make_next_flag_absolute) {
+      if (arg.size() > 0 && arg[0] != '/')
+        arg = NormalizePath(entry.directory + arg);
+      make_next_flag_absolute = false;
+    }
 
-    for (auto& entry : kBlacklist) {
+    // Update arg if it is a path.
-      if (StartsWith(arg, entry)) {
+    for (const char* flag_type : kPathArgs) {
-        bad = true;
+      if (arg == flag_type) {
+        make_next_flag_absolute = true;
+        break;
+      }
+
+      if (StartsWith(arg, flag_type)) {
+        std::string path = arg.substr(strlen(flag_type));
+        if (path.size() > 0 && path[0] != '/') {
+          path = NormalizePath(entry.directory + "/" + path);
+          arg = flag_type + path;
+        }
+        break;
+      }
+    }
+    if (make_next_flag_absolute)
       continue;
-      }
-    }
-    if (bad) {
-      continue;
-    }
-
-
-    if (StartsWith(arg, "-I")) {
-      std::string path = entry.directory + "/" + arg.substr(2);
-      path = NormalizePath(path);
-      arg = "-I" + path;
-    }
 
     result.args.push_back(arg);
-    //if (StartsWith(arg, "-I") || StartsWith(arg, "-D") || StartsWith(arg, "-std"))
   }
 
-  // TODO/fixme
+  // We don't do any special processing on user-given extra flags.
+  for (const auto& flag : extra_flags)
+    result.args.push_back(flag);
+
+  // Clang does not have good hueristics for determining source language. We
+  // default to C++11 if the user has not specified.
+  if (!StartsWithAny(entry.args, "-x"))
     result.args.push_back("-xc++");
+  if (!StartsWithAny(entry.args, "-std="))
     result.args.push_back("-std=c++11");
 
   return result;
 }
 
-std::vector<Project::Entry> LoadFromCompileCommandsJson(const std::string& project_directory) {
+std::vector<Project::Entry> LoadFromCompileCommandsJson(const std::vector<std::string>& extra_flags, const std::string& project_directory) {
+  // TODO: Fix this function, it may be way faster than libclang's implementation.
+
   optional<std::string> compile_commands_content = ReadContent(project_directory + "/compile_commands.json");
   if (!compile_commands_content)
     return {};
@@ -168,11 +113,11 @@ std::vector<Project::Entry> LoadFromCompileCommandsJson(const std::string& proje
   std::vector<Project::Entry> result;
   result.reserve(entries.size());
   for (const auto& entry : entries)
-    result.push_back(GetCompilationEntryFromCompileCommandEntry(entry));
+    result.push_back(GetCompilationEntryFromCompileCommandEntry(extra_flags, entry));
   return result;
 }
 
-std::vector<Project::Entry> LoadFromDirectoryListing(const std::string& project_directory) {
+std::vector<Project::Entry> LoadFromDirectoryListing(const std::vector<std::string>& extra_flags, const std::string& project_directory) {
   std::vector<Project::Entry> result;
 
   std::vector<std::string> args;
@@ -185,6 +130,10 @@ std::vector<Project::Entry> LoadFromDirectoryListing(const std::string& project_
     std::cerr << line;
     args.push_back(line);
   }
+  for (const std::string& flag : extra_flags) {
+    std::cerr << flag << std::endl;
+    args.push_back(flag);
+  }
   std::cerr << std::endl;
 
 
@@ -201,7 +150,7 @@ std::vector<Project::Entry> LoadFromDirectoryListing(const std::string& project_
   return result;
 }
 
-std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(const std::string& project_directory) {
+std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(const std::vector<std::string>& extra_flags, const std::string& project_directory) {
   // TODO: Figure out if this function or the clang one is faster.
   //return LoadFromCompileCommandsJson(project_directory);
 
@@ -210,7 +159,7 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(const std::strin
   CXCompilationDatabase cx_db = clang_CompilationDatabase_fromDirectory(project_directory.c_str(), &cx_db_load_error);
   if (cx_db_load_error == CXCompilationDatabase_CanNotLoadDatabase) {
     std::cerr << "Unable to load compile_commands.json located at \"" << project_directory << "\"; using directory listing instead." << std::endl;
-    return LoadFromDirectoryListing(project_directory);
+    return LoadFromDirectoryListing(extra_flags, project_directory);
   }
 
   CXCompileCommands cx_commands = clang_CompilationDatabase_getAllCompileCommands(cx_db);
@@ -233,7 +182,7 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(const std::strin
     for (unsigned i = 0; i < num_args; ++i)
       entry.args.push_back(clang::ToString(clang_CompileCommand_getArg(cx_command, i)));
 
-    result.push_back(GetCompilationEntryFromCompileCommandEntry(entry));
+    result.push_back(GetCompilationEntryFromCompileCommandEntry(extra_flags, entry));
   }
 
   clang_CompileCommands_dispose(cx_commands);
@@ -243,8 +192,8 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(const std::strin
 }
 }  // namespace
 
-void Project::Load(const std::string& directory) {
+void Project::Load(const std::vector<std::string>& extra_flags, const std::string& directory) {
-  entries = LoadCompilationEntriesFromDirectory(directory);
+  entries = LoadCompilationEntriesFromDirectory(extra_flags, directory);
 
   absolute_path_to_entry_index_.resize(entries.size());
   for (int i = 0; i < entries.size(); ++i)

src/project.h

@@ -28,7 +28,7 @@ struct Project {
   // discover all files and args. Otherwise, a recursive directory listing of
   // all *.cpp, *.cc, *.h, and *.hpp files will be used. clang arguments can be
   // specified in a clang_args file located inside of |directory|.
-  void Load(const std::string& directory);
+  void Load(const std::vector<std::string>& extra_flags, const std::string& directory);
 
   // Lookup the CompilationEntry for |filename|.
   optional<Entry> FindCompilationEntryForFile(const std::string& filename);

src/utils.cc

@@ -66,18 +66,6 @@ std::string ReplaceAll(const std::string& source, const std::string& from, const
   return result;
 }
 
-std::string StringJoin(const std::vector<std::string>& values) {
-  std::string result;
-  bool first = true;
-  for (auto& entry : values) {
-    if (!first)
-      result += ", ";
-    first = false;
-    result += entry;
-  }
-  return result;
-}
-
 static std::vector<std::string> GetFilesInFolderHelper(std::string folder, bool recursive, std::string output_prefix) {
   std::vector<std::string> result;
 

src/utils.h

@@ -18,7 +18,25 @@ bool StartsWith(const std::string& value, const std::string& start);
 bool EndsWith(const std::string& value, const std::string& ending);
 std::string ReplaceAll(const std::string& source, const std::string& from, const std::string& to);
 
-std::string StringJoin(const std::vector<std::string>& values);
+template <typename TValues>
+bool StartsWithAny(const TValues& values, const std::string& start) {
+  return std::any_of(std::begin(values), std::end(values), [&start](const auto& value) {
+    return StartsWith(value, start);
+  });
+}
+
+template <typename TValues>
+std::string StringJoin(const TValues& values) {
+  std::string result;
+  bool first = true;
+  for (auto& entry : values) {
+    if (!first)
+      result += ", ";
+    first = false;
+    result += entry;
+  }
+  return result;
+}
 
 // Finds all files in the given folder. This is recursive.
 std::vector<std::string> GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path);