Add TEXT_REPLACE in index tests.

This allows USRs to vary across platforms in index tests, which is required to
make them pass on all platforms.

src/lex_utils.cc

@@ -154,7 +154,7 @@ void LexFunctionDeclaration(const std::string& buffer_content,
   if (type_name && !type_name->empty())
     result += *type_name + "::";
   result += buffer_content.substr(name_start, end - name_start);
-  TrimEnd(result);
+  TrimEndInPlace(result);
   result += " {\n}";
   *insert_text = result;
 }

src/project.cc

@@ -227,7 +227,7 @@ std::vector<Project::Entry> LoadFromDirectoryListing(ProjectConfig* config) {
   std::cerr << "Using arguments: ";
   for (std::string line :
        ReadLinesWithEnding(config->project_dir + "/.cquery")) {
-    Trim(line);
+    TrimInPlace(line);
     if (line.empty() || StartsWith(line, "#"))
       continue;
     if (!args.empty())

src/test.cc

@@ -124,34 +124,36 @@ void RunIndexTests(const std::string& filter_path) {
         continue;
 
       // Parse expected output from the test, parse it into JSON document.
+      std::vector<std::string> lines_with_endings = ReadLinesWithEnding(path);
+      TextReplacer text_replacer;
       std::vector<std::string> flags;
-      std::unordered_map<std::string, std::string> all_expected_output =
+      std::unordered_map<std::string, std::string> all_expected_output;
-          ParseTestExpectation(path, &flags);
+      ParseTestExpectation(path, lines_with_endings, &text_replacer, &flags,
+                           &all_expected_output);
+
+      // Build flags.
       bool had_extra_flags = !flags.empty();
       if (!AnyStartsWith(flags, "-x"))
         flags.push_back("-xc++");
       if (!AnyStartsWith(flags, "-std"))
         flags.push_back("-std=c++11");
       flags.push_back("-resource_dir=" + GetDefaultResourceDirectory());
-
       if (had_extra_flags) {
         std::cout << "For " << path << std::endl;
         std::cout << "  flags: " << StringJoin(flags) << std::endl;
       }
 
+      // Run test.
       Config config;
       FileConsumer::SharedState file_consumer_shared;
-
-      // Run test.
-      // std::cout << "[START] " << path << std::endl;
       PerformanceImportFile perf;
       std::vector<std::unique_ptr<IndexFile>> dbs =
-          Parse(&config, &file_consumer_shared, path, flags,
+          Parse(&config, &file_consumer_shared, path, flags, {}, &perf, &index,
-                {}, &perf, &index, false /*dump_ast*/);
+                false /*dump_ast*/);
 
-      for (auto& entry : all_expected_output) {
+      for (const auto& entry : all_expected_output) {
         const std::string& expected_path = entry.first;
-        const std::string& expected_output = entry.second;
+        std::string expected_output = text_replacer.Apply(entry.second);
 
         // FIXME: promote to utils, find and remove duplicates (ie,
         // cquery_call_tree.cc, maybe something in project.cc).
@@ -195,6 +197,7 @@ void RunIndexTests(const std::string& filter_path) {
           VerifySerializeToFrom(db);
           actual_output = db->ToString();
         }
+        actual_output = text_replacer.Apply(actual_output);
 
         // Compare output via rapidjson::Document to ignore any formatting
         // differences.

src/utils.cc

@@ -2,6 +2,7 @@
 
 #include "platform.h"
 
+#include <doctest/doctest.h>
 #include <tinydir.h>
 #include <loguru/loguru.hpp>
 
@@ -21,7 +22,7 @@
 
 // DEFAULT_RESOURCE_DIRECTORY is passed with quotes for non-MSVC compilers, ie,
 // foo vs "foo".
-#if defined(_MSC_VER )
+#if defined(_MSC_VER)
 #define _STRINGIFY(x) #x
 #define ENSURE_STRING_MACRO_ARGUMENT(x) _STRINGIFY(x)
 #else
@@ -29,20 +30,24 @@
 #endif
 
 // See http://stackoverflow.com/a/217605
-void TrimStart(std::string& s) {
+void TrimStartInPlace(std::string& s) {
   s.erase(s.begin(),
           std::find_if(s.begin(), s.end(),
                        std::not1(std::ptr_fun<int, int>(std::isspace))));
 }
-void TrimEnd(std::string& s) {
+void TrimEndInPlace(std::string& s) {
   s.erase(std::find_if(s.rbegin(), s.rend(),
                        std::not1(std::ptr_fun<int, int>(std::isspace)))
               .base(),
           s.end());
 }
-void Trim(std::string& s) {
+void TrimInPlace(std::string& s) {
-  TrimStart(s);
+  TrimStartInPlace(s);
-  TrimEnd(s);
+  TrimEndInPlace(s);
+}
+std::string Trim(std::string s) {
+  TrimInPlace(s);
+  return s;
 }
 
 // See http://stackoverflow.com/a/2072890
@@ -273,29 +278,64 @@ std::vector<std::string> ToLines(const std::string& content,
   std::string line;
   while (getline(lines, line)) {
     if (trim_whitespace)
-      Trim(line);
+      TrimInPlace(line);
     result.push_back(line);
   }
 
   return result;
 }
 
-std::unordered_map<std::string, std::string> ParseTestExpectation(
+std::string TextReplacer::Apply(const std::string& content) {
-    std::string filename, std::vector<std::string>* flags) {
+  std::string result = content;
+
+  for (const Replacement& replacement : replacements) {
+    while (true) {
+      size_t idx = result.find(replacement.from);
+      if (idx == std::string::npos)
+        break;
+
+      result.replace(result.begin() + idx,
+                     result.begin() + idx + replacement.from.size(),
+                     replacement.to);
+    }
+  }
+
+  return result;
+}
+
+void ParseTestExpectation(
+    const std::string& filename,
+    const std::vector<std::string>& lines_with_endings,
+    TextReplacer* replacer,
+    std::vector<std::string>* flags,
+    std::unordered_map<std::string, std::string>* output_sections) {
 #if false
 #include "bar.h"
 
   void foo();
 
   /*
-  // if no name is given assume to be this file name
+  // DOCS for TEXT_REPLACE:
-  // no output section means we don't check that index.
+  //  Each line under TEXT_REPLACE is a replacement, ie, the two entries will be
+  //  considered equivalent. This is useful for USRs which vary across files.
 
-  // EXTRA_FLAGS parses until the first newline.
+  // DOCS for EXTRA_FLAGS:
+  //  Additional flags to pass to clang.
+
+  // DOCS for OUTPUT:
+  //  If no name is given assume to be this file name. If there is not an output
+  //  section for a file it is not checked.
+
+  TEXT_REPLACE:
+  foo <===> bar
+  one <===> two
 
   EXTRA_FLAGS:
   -std=c++14
 
+  OUTPUT:
+  {}
+
   OUTPUT: bar.cc
   {}
 
@@ -304,57 +344,93 @@ std::unordered_map<std::string, std::string> ParseTestExpectation(
   */
 #endif
 
-  std::unordered_map<std::string, std::string> result;
+  // Scan for TEXT_REPLACE:
+  {
+    bool in_output = false;
+    for (std::string line : lines_with_endings) {
+      TrimInPlace(line);
 
-  std::string active_output_filename;
+      if (StartsWith(line, "TEXT_REPLACE:")) {
-  std::string active_output_contents;
+        assert(!in_output && "multiple TEXT_REPLACE sections");
+        in_output = true;
+        continue;
+      }
 
-  bool in_output = false;
+      if (in_output && line.empty())
-  for (std::string line_with_ending : ReadLinesWithEnding(filename)) {
+        break;
-    if (StartsWith(line_with_ending, "EXTRA_FLAGS:")) {
+
-      assert(!in_output && "multiple EXTRA_FLAGS sections");
+      if (in_output) {
-      in_output = true;
+        static const std::string kKey = " <===> ";
-      continue;
+        size_t index = line.find(kKey);
+        LOG_IF_S(FATAL, index == std::string::npos)
+            << " No '" << kKey << "' in replacement string '" << line << "'"
+            << ", index=" << index;
+
+        TextReplacer::Replacement replacement;
+        replacement.from = line.substr(0, index);
+        replacement.to = line.substr(index + kKey.size());
+        TrimInPlace(replacement.from);
+        TrimInPlace(replacement.to);
+        replacer->replacements.push_back(replacement);
+      }
+    }
+  }
+
+  // Scan for EXTRA_FLAGS:
+  {
+    bool in_output = false;
+    for (std::string line : lines_with_endings) {
+      TrimInPlace(line);
+
+      if (StartsWith(line, "EXTRA_FLAGS:")) {
+        assert(!in_output && "multiple EXTRA_FLAGS sections");
+        in_output = true;
+        continue;
+      }
+
+      if (in_output && line.empty())
+        break;
+
+      if (in_output)
+        flags->push_back(line);
+    }
+  }
+
+  // Scan for OUTPUT:
+  {
+    std::string active_output_filename;
+    std::string active_output_contents;
+
+    bool in_output = false;
+    for (std::string line_with_ending : lines_with_endings) {
+      if (StartsWith(line_with_ending, "*/"))
+        break;
+
+      if (StartsWith(line_with_ending, "OUTPUT:")) {
+        // Terminate the previous output section if we found a new one.
+        if (in_output) {
+          (*output_sections)[active_output_filename] = active_output_contents;
+        }
+
+        // Try to tokenize OUTPUT: based one whitespace. If there is more than
+        // one token assume it is a filename.
+        std::vector<std::string> tokens = SplitString(line_with_ending, " ");
+        if (tokens.size() > 1) {
+          active_output_filename = tokens[1];
+          TrimInPlace(active_output_filename);
+        } else {
+          active_output_filename = filename;
+        }
+        active_output_contents = "";
+
+        in_output = true;
+      } else if (in_output)
+        active_output_contents += line_with_ending;
     }
 
-    Trim(line_with_ending);
-    if (in_output && line_with_ending.empty())
-      break;
-
     if (in_output)
-      flags->push_back(line_with_ending);
+      (*output_sections)[active_output_filename] = active_output_contents;
   }
-
-  in_output = false;
-  for (std::string line_with_ending : ReadLinesWithEnding(filename)) {
-    if (StartsWith(line_with_ending, "*/"))
-      break;
-
-    if (StartsWith(line_with_ending, "OUTPUT:")) {
-      // Terminate the previous output section if we found a new one.
-      if (in_output) {
-        result[active_output_filename] = active_output_contents;
-      }
-
-      // Try to tokenize OUTPUT: based one whitespace. If there is more than one
-      // token assume it is a filename.
-      std::vector<std::string> tokens = SplitString(line_with_ending, " ");
-      if (tokens.size() > 1) {
-        active_output_filename = tokens[1];
-        Trim(active_output_filename);
-      } else {
-        active_output_filename = filename;
-      }
-      active_output_contents = "";
-
-      in_output = true;
-    } else if (in_output)
-      active_output_contents += line_with_ending;
-  }
-
-  if (in_output)
-    result[active_output_filename] = active_output_contents;
-  return result;
 }
 
 void UpdateTestExpectation(const std::string& filename,
@@ -412,7 +488,8 @@ std::string FormatMicroseconds(long long microseconds) {
 std::string GetDefaultResourceDirectory() {
   std::string result;
 
-  std::string resource_directory = std::string(ENSURE_STRING_MACRO_ARGUMENT(DEFAULT_RESOURCE_DIRECTORY));
+  std::string resource_directory =
+      std::string(ENSURE_STRING_MACRO_ARGUMENT(DEFAULT_RESOURCE_DIRECTORY));
   if (resource_directory.find("..") != std::string::npos) {
     std::string executable_path = GetExecutablePath();
     size_t pos = executable_path.find_last_of('/');
@@ -424,3 +501,54 @@ std::string GetDefaultResourceDirectory() {
 
   return NormalizePath(result);
 }
+
+TEST_SUITE("ParseTestExpectation") {
+  TEST_CASE("Parse TEXT_REPLACE") {
+    // clang-format off
+    std::vector<std::string> lines_with_endings = {
+        "/*\n",
+        "TEXT_REPLACE:\n",
+        "  foo   <===> \tbar  \n",
+        "01 <===> 2\n",
+        "\n",
+        "*/\n"};
+    // clang-format on
+
+    TextReplacer text_replacer;
+    std::vector<std::string> flags;
+    std::unordered_map<std::string, std::string> all_expected_output;
+    ParseTestExpectation("foo.cc", lines_with_endings, &text_replacer, &flags,
+                         &all_expected_output);
+
+    REQUIRE(text_replacer.replacements.size() == 2);
+    REQUIRE(text_replacer.replacements[0].from == "foo");
+    REQUIRE(text_replacer.replacements[0].to == "bar");
+    REQUIRE(text_replacer.replacements[1].from == "01");
+    REQUIRE(text_replacer.replacements[1].to == "2");
+  }
+
+  TEST_CASE("Apply TEXT_REPLACE") {
+    TextReplacer replacer;
+    replacer.replacements.push_back(TextReplacer::Replacement{"foo", "bar"});
+    replacer.replacements.push_back(TextReplacer::Replacement{"01", "2"});
+    replacer.replacements.push_back(TextReplacer::Replacement{"3", "456"});
+
+    // Equal-length.
+    REQUIRE(replacer.Apply("foo") == "bar");
+    REQUIRE(replacer.Apply("bar") == "bar");
+
+    // Shorter replacement.
+    REQUIRE(replacer.Apply("01") == "2");
+    REQUIRE(replacer.Apply("2") == "2");
+
+    // Longer replacement.
+    REQUIRE(replacer.Apply("3") == "456");
+    REQUIRE(replacer.Apply("456") == "456");
+
+    // Content before-after replacement.
+    REQUIRE(replacer.Apply("aaaa01bbbb") == "aaaa2bbbb");
+
+    // Multiple replacements.
+    REQUIRE(replacer.Apply("foofoobar0123") == "barbarbar22456");
+  }
+}

src/utils.h

@@ -14,11 +14,12 @@ using std::experimental::nullopt;
 using std::experimental::optional;
 
 // Trim from start (in place)
-void TrimStart(std::string& s);
+void TrimStartInPlace(std::string& s);
 // Trim from end (in place)
-void TrimEnd(std::string& s);
+void TrimEndInPlace(std::string& s);
 // Trim from both ends (in place)
-void Trim(std::string& s);
+void TrimInPlace(std::string& s);
+std::string Trim(std::string s);
 
 // Returns true if |value| starts/ends with |start| or |ending|.
 bool StartsWith(const std::string& value, const std::string& start);
@@ -78,9 +79,24 @@ std::vector<std::string> ReadLinesWithEnding(std::string filename);
 std::vector<std::string> ToLines(const std::string& content,
                                  bool trim_whitespace);
 
-std::unordered_map<std::string, std::string> ParseTestExpectation(
+struct TextReplacer {
-    std::string filename,
+  struct Replacement {
-    std::vector<std::string>* flags);
+    std::string from;
+    std::string to;
+  };
+
+  std::vector<Replacement> replacements;
+
+  std::string Apply(const std::string& content);
+};
+
+void ParseTestExpectation(
+    const std::string& filename,
+    const std::vector<std::string>& lines_with_endings,
+    TextReplacer* text_replacer,
+    std::vector<std::string>* flags,
+    std::unordered_map<std::string, std::string>* output_sections);
+
 void UpdateTestExpectation(const std::string& filename,
                            const std::string& expectation,
                            const std::string& actual);

tests/outline/outline.cc

@@ -7,6 +7,9 @@ struct MergeableUpdate {
 };
 
 /*
+TEXT_REPLACE:
+c:@N@std@ST>2#T#T@vector <===> c:@N@std@N@__1@ST>2#T#T@vector
+
 OUTPUT:
 {
   "includes": [{

tests/outline/outline2.cc

@@ -12,6 +12,11 @@ struct CompilationEntry {
 std::vector<CompilationEntry> LoadCompilationEntriesFromDirectory(const std::string& project_directory);
 
 /*
+TEXT_REPLACE:
+c:@N@std@T@string <===> c:@N@std@N@__1@T@string
+c:@N@std@ST>2#T#T@vector <===> c:@N@std@N@__1@ST>2#T#T@vector
+c:@F@LoadCompilationEntriesFromDirectory#&1$@N@std@N@__1@S@basic_string>#C#$@N@std@N@__1@S@char_traits>#C#$@N@std@N@__1@S@allocator>#C# <===> c:@F@LoadCompilationEntriesFromDirectory#&1$@N@std@S@basic_string>#C#$@N@std@S@char_traits>#C#$@N@std@S@allocator>#C#
+
 OUTPUT:
 {
   "includes": [{