Fix test hanging in MultiQueueWaiter destructor && allow it to run with clang >= 6.0.0

CMakeLists.txt

@@ -120,6 +120,10 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL FreeBSD)
   target_link_libraries(ccls PRIVATE thr)
 endif()
 
+if(LLVM_ENABLE_ZLIB)
+  find_package(ZLIB)
+endif()
+
 ### Definitions
 
 # Find Clang resource directory with Clang executable.

src/clang_tu.cc

@@ -17,15 +17,21 @@
 #include <llvm/Support/Path.h>
 
 using namespace clang;
+using namespace llvm;
 
 namespace ccls {
 std::string pathFromFileEntry(const FileEntry &file) {
-  StringRef name = file.tryGetRealPathName();
-  if (name.empty())
-    name = file.getName();
-  std::string ret = normalizePath(name);
-  // Resolve symlinks outside of workspace folders, e.g. /usr/include/c++/7.3.0
-  return normalizeFolder(ret) ? ret : realPath(ret);
+  // If getName() refers to a file within a workspace folder, we prefer it
+  // (which may be a symlink).
+  std::string ret = normalizePath(file.getName());
+  if (normalizeFolder(ret))
+    return ret;
+  // Resolve symlinks outside of working folders. This handles leading path
+  // components, e.g. (/lib -> /usr/lib) in
+  // /../lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/utility
+  ret = realPath(ret);
+  normalizeFolder(ret);
+  return ret;
 }
 
 bool isInsideMainFile(const SourceManager &sm, SourceLocation sl) {

src/lsp.hh

@@ -219,6 +219,17 @@ struct TextDocumentDidChangeParam {
   std::vector<TextDocumentContentChangeEvent> contentChanges;
 };
 
+struct WorkDoneProgress {
+  const char *kind;
+  std::optional<std::string> title;
+  std::optional<std::string> message;
+  std::optional<double> percentage;
+};
+struct WorkDoneProgressParam {
+  const char *token;
+  WorkDoneProgress value;
+};
+
 struct WorkspaceFolder {
   DocumentUri uri;
   std::string name;

src/message_handler.hh

@@ -191,6 +191,8 @@ REFLECT_UNDERLYING_B(SymbolKind);
 REFLECT_STRUCT(TextDocumentIdentifier, uri);
 REFLECT_STRUCT(TextDocumentItem, uri, languageId, version, text);
 REFLECT_STRUCT(TextEdit, range, newText);
+REFLECT_STRUCT(WorkDoneProgress, kind, title, message, percentage);
+REFLECT_STRUCT(WorkDoneProgressParam, token, value);
 REFLECT_STRUCT(DiagnosticRelatedInformation, location, message);
 REFLECT_STRUCT(Diagnostic, range, severity, code, source, message,
                relatedInformation);

src/messages/ccls_info.cc

@@ -17,14 +17,14 @@ struct Out_cclsInfo {
     int files, funcs, types, vars;
   } db;
   struct Pipeline {
-    int pendingIndexRequests;
+    int64_t lastIdle, completed, enqueued;
   } pipeline;
   struct Project {
     int entries;
   } project;
 };
 REFLECT_STRUCT(Out_cclsInfo::DB, files, funcs, types, vars);
-REFLECT_STRUCT(Out_cclsInfo::Pipeline, pendingIndexRequests);
+REFLECT_STRUCT(Out_cclsInfo::Pipeline, lastIdle, completed, enqueued);
 REFLECT_STRUCT(Out_cclsInfo::Project, entries);
 REFLECT_STRUCT(Out_cclsInfo, db, pipeline, project);
 } // namespace
@@ -35,7 +35,9 @@ void MessageHandler::ccls_info(EmptyParam &, ReplyOnce &reply) {
   result.db.funcs = db->funcs.size();
   result.db.types = db->types.size();
   result.db.vars = db->vars.size();
-  result.pipeline.pendingIndexRequests = pipeline::pending_index_requests;
+  result.pipeline.lastIdle = pipeline::stats.last_idle;
+  result.pipeline.completed = pipeline::stats.completed;
+  result.pipeline.enqueued = pipeline::stats.enqueued;
   result.project.entries = 0;
   for (auto &[_, folder] : project->root2folder)
     result.project.entries += folder.entries.size();

src/messages/initialize.cc

@@ -348,17 +348,16 @@ void do_initialize(MessageHandler *m, InitializeParam &param,
     std::string path = wf.uri.getPath();
     ensureEndsInSlash(path);
     std::string real = realPath(path) + '/';
-    workspaceFolders.emplace_back(path, path == real ? "" : real);
+    workspaceFolders.emplace_back(path, real);
   }
   if (workspaceFolders.empty()) {
     std::string real = realPath(project_path) + '/';
-    workspaceFolders.emplace_back(project_path,
-                                  project_path == real ? "" : real);
+    workspaceFolders.emplace_back(project_path, real);
   }
   std::sort(workspaceFolders.begin(), workspaceFolders.end(),
             [](auto &l, auto &r) { return l.first.size() > r.first.size(); });
   for (auto &[folder, real] : workspaceFolders)
-    if (real.empty())
+    if (real == folder)
       LOG_S(INFO) << "workspace folder: " << folder;
     else
       LOG_S(INFO) << "workspace folder: " << folder << " -> " << real;

src/messages/textDocument_did.cc

@@ -44,7 +44,7 @@ void MessageHandler::textDocument_didOpen(DidOpenTextDocumentParam &param) {
   // pending index request.
   auto [lang, header] = lookupExtension(path);
   if ((lang != LanguageId::Unknown && !header) ||
-      !pipeline::pending_index_requests)
+      pipeline::stats.completed == pipeline::stats.enqueued)
     pipeline::index(path, {}, IndexMode::Normal, false);
   if (header)
     project->indexRelated(path);

src/pipeline.cc

@@ -22,6 +22,7 @@
 #include <llvm/Support/Threading.h>
 
 #include <chrono>
+#include <inttypes.h>
 #include <mutex>
 #include <shared_mutex>
 #include <thread>
@@ -38,6 +39,12 @@ struct PublishDiagnosticParam {
   std::vector<Diagnostic> diagnostics;
 };
 REFLECT_STRUCT(PublishDiagnosticParam, uri, diagnostics);
+
+constexpr char index_progress_token[] = "index";
+struct WorkDoneProgressCreateParam {
+  const char *token = index_progress_token;
+};
+REFLECT_STRUCT(WorkDoneProgressCreateParam, token);
 } // namespace
 
 void VFS::clear() {
@@ -67,7 +74,8 @@ void standaloneInitialize(MessageHandler &, const std::string &root);
 namespace pipeline {
 
 std::atomic<bool> g_quit;
-std::atomic<int64_t> loaded_ts{0}, pending_index_requests{0}, request_id{0};
+std::atomic<int64_t> loaded_ts{0}, request_id{0};
+IndexStats stats;
 int64_t tick = 0;
 
 namespace {
@@ -195,9 +203,6 @@ bool indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
     return false;
   auto &request = *opt_request;
   bool loud = request.mode != IndexMode::OnChange;
-  struct RAII {
-    ~RAII() { pending_index_requests--; }
-  } raii;
 
   // Dummy one to trigger refresh semantic highlight.
   if (request.path.empty()) {
@@ -207,6 +212,9 @@ bool indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
     return false;
   }
 
+  struct RAII {
+    ~RAII() { stats.completed++; }
+  } raii;
   if (!matcher.matches(request.path)) {
     LOG_IF_S(INFO, loud) << "skip " << request.path;
     return false;
@@ -643,7 +651,9 @@ void mainLoop() {
   handler.manager = &manager;
   handler.include_complete = &include_complete;
 
+  bool work_done_created = false, in_progress = false;
   bool has_indexed = false;
+  int64_t last_completed = 0;
   std::deque<InMessage> backlog;
   StringMap<std::deque<InMessage *>> path2backlog;
   while (true) {
@@ -693,6 +703,44 @@ void mainLoop() {
       }
     }
 
+    int64_t completed = stats.completed.load(std::memory_order_relaxed);
+    if (completed != last_completed) {
+      if (!work_done_created) {
+        WorkDoneProgressCreateParam param;
+        request("window/workDoneProgress/create", param);
+        work_done_created = true;
+      }
+
+      int64_t enqueued = stats.enqueued.load(std::memory_order_relaxed);
+      if (completed != enqueued) {
+        if (!in_progress) {
+          WorkDoneProgressParam param;
+          param.token = index_progress_token;
+          param.value.kind = "begin";
+          param.value.title = "indexing";
+          notify("$/progress", param);
+          in_progress = true;
+        }
+        int64_t last_idle = stats.last_idle.load(std::memory_order_relaxed);
+        WorkDoneProgressParam param;
+        param.token = index_progress_token;
+        param.value.kind = "report";
+        param.value.message =
+            (Twine(completed - last_idle) + "/" + Twine(enqueued - last_idle))
+                .str();
+        param.value.percentage =
+            100.0 * (completed - last_idle) / (enqueued - last_idle);
+        notify("$/progress", param);
+      } else if (in_progress) {
+        stats.last_idle.store(enqueued, std::memory_order_relaxed);
+        WorkDoneProgressParam param;
+        param.token = index_progress_token;
+        param.value.kind = "end";
+        notify("$/progress", param);
+        in_progress = false;
+      }
+    }
+
     if (did_work) {
       has_indexed |= indexed;
       if (g_quit.load(std::memory_order_relaxed))
@@ -736,16 +784,16 @@ void standalone(const std::string &root) {
     int entries = 0;
     for (auto &[_, folder] : project.root2folder)
       entries += folder.entries.size();
-    printf("entries: %5d\n", entries);
+    printf("entries:   %4d\n", entries);
   }
   while (1) {
     (void)on_indexed->dequeueAll();
-    int pending = pending_index_requests;
+    int64_t enqueued = stats.enqueued, completed = stats.completed;
     if (tty) {
-      printf("\rpending: %5d", pending);
+      printf("\rcompleted: %4" PRId64 "/%" PRId64, completed, enqueued);
       fflush(stdout);
     }
-    if (!pending)
+    if (completed == enqueued)
       break;
     std::this_thread::sleep_for(std::chrono::milliseconds(100));
   }
@@ -756,7 +804,8 @@ void standalone(const std::string &root) {
 
 void index(const std::string &path, const std::vector<const char *> &args,
            IndexMode mode, bool must_exist, RequestId id) {
-  pending_index_requests++;
+  if (!path.empty())
+    stats.enqueued++;
   index_request->pushBack({path, args, mode, must_exist, std::move(id)},
                           mode != IndexMode::Background);
 }

src/pipeline.hh

@@ -39,9 +39,14 @@ enum class IndexMode {
   Normal,
 };
 
+struct IndexStats {
+  std::atomic<int64_t> last_idle, completed, enqueued;
+};
+
 namespace pipeline {
 extern std::atomic<bool> g_quit;
-extern std::atomic<int64_t> loaded_ts, pending_index_requests;
+extern std::atomic<int64_t> loaded_ts;
+extern IndexStats stats;
 extern int64_t tick;
 
 void threadEnter();

src/project.cc

@@ -433,7 +433,9 @@ void Project::loadDirectory(const std::string &root, Project::Folder &folder) {
       // If workspace folder is real/ but entries use symlink/, convert to
       // real/.
       entry.directory = realPath(cmd.Directory);
+      entry.directory.push_back('/');
       normalizeFolder(entry.directory);
+      entry.directory.pop_back();
       doPathMapping(entry.directory);
       entry.filename =
           realPath(resolveIfRelative(entry.directory, cmd.Filename));

src/test.cc

@@ -260,15 +260,16 @@ findDbForPathEnding(const std::string &path,
 bool runIndexTests(const std::string &filter_path, bool enable_update) {
   gTestOutputMode = true;
   std::string version = LLVM_VERSION_STRING;
+  const int majorVersion = LLVM_VERSION_MAJOR;
 
   // Index tests change based on the version of clang used.
-  static const char kRequiredClangVersion[] = "6.0.0";
-  if (version != kRequiredClangVersion &&
+  const int kMinRequiredClangVersion = 6;
+  if (majorVersion < kMinRequiredClangVersion &&
       version.find("svn") == std::string::npos) {
     fprintf(stderr,
-            "Index tests must be run using clang version %s, ccls is running "
+            "Index tests must be run using clang major version >= %d, ccls is running "
             "with %s\n",
-            kRequiredClangVersion, version.c_str());
+            kMinRequiredClangVersion, version.c_str());
     return false;
   }
 

src/threaded_queue.hh

@@ -46,6 +46,9 @@ private:
 
 struct MultiQueueWaiter {
   std::condition_variable_any cv;
+  ~MultiQueueWaiter() {
+	  cv.notify_all();
+  }
 
   static bool hasState(std::initializer_list<BaseThreadQueue *> queues) {
     for (BaseThreadQueue *queue : queues) {

src/utils.cc

@@ -138,11 +138,15 @@ std::string realPath(const std::string &path) {
 }
 
 bool normalizeFolder(std::string &path) {
-  for (auto &[root, real] : g_config->workspaceFolders)
-    if (real.size() && llvm::StringRef(path).startswith(real)) {
+  for (auto &[root, real] : g_config->workspaceFolders) {
+    StringRef p(path);
+    if (p.startswith(root))
+      return true;
+    if (p.startswith(real)) {
       path = root + path.substr(real.size());
       return true;
     }
+  }
   return false;
 }