cmLocalGenerator: Improve performance of source group lookups

Introduce an index from source file to source group in
`cmLocalGenerator` to make lookups significantly more efficient
during the source group generation, which could be particularly
slow when dealing with thousands of source files.
Falls back to the recursive lookup when the source file is not present
on the index, which is more likely to happen when dealing with
regex-based sources.

Fixes: #27359

Source/cmExtraEclipseCDT4Generator.cxx

@@ -461,21 +461,21 @@ void cmExtraEclipseCDT4Generator::CreateProjectFile()
 }
 
 void cmExtraEclipseCDT4Generator::WriteGroups(
-  std::vector<cmSourceGroup> const& sourceGroups, std::string& linkName,
+  SourceGroupVector const& sourceGroups, std::string& linkName,
   cmXMLWriter& xml)
 {
-  for (cmSourceGroup const& sg : sourceGroups) {
-    std::string linkName3 = cmStrCat(linkName, '/', sg.GetFullName());
+  for (auto const& sg : sourceGroups) {
+    std::string linkName3 = cmStrCat(linkName, '/', sg->GetFullName());
 
     std::replace(linkName3.begin(), linkName3.end(), '\\', '/');
 
     cmExtraEclipseCDT4Generator::AppendLinkedResource(
       xml, linkName3, "virtual:/virtual", VirtualFolder);
-    std::vector<cmSourceGroup> const& children = sg.GetGroupChildren();
+    SourceGroupVector const& children = sg->GetGroupChildren();
     if (!children.empty()) {
       this->WriteGroups(children, linkName, xml);
     }
-    std::vector<cmSourceFile const*> sFiles = sg.GetSourceFiles();
+    std::vector<cmSourceFile const*> sFiles = sg->GetSourceFiles();
     for (cmSourceFile const* file : sFiles) {
       std::string const& fullPath = file->GetFullPath();
 
@@ -529,9 +529,7 @@ void cmExtraEclipseCDT4Generator::CreateLinksForTargets(cmXMLWriter& xml)
             sourceGroup->AssignSource(sf);
           }
 
-          std::vector<cmSourceGroup> sourceGroups =
-            makefile->GetSourceGroups();
-          this->WriteGroups(sourceGroups, linkName2, xml);
+          this->WriteGroups(makefile->GetSourceGroups(), linkName2, xml);
         } break;
         // ignore all others:
         default:

Source/cmExtraEclipseCDT4Generator.h

@@ -10,10 +10,10 @@
 #include <vector>
 
 #include "cmExternalMakefileProjectGenerator.h"
+#include "cmSourceGroup.h"
 
 class cmLocalGenerator;
 class cmMakefile;
-class cmSourceGroup;
 class cmXMLWriter;
 
 /** \class cmExtraEclipseCDT4Generator
@@ -87,7 +87,7 @@ private:
   static void AddEnvVar(std::ostream& out, char const* envVar,
                         cmLocalGenerator& lg);
 
-  void WriteGroups(std::vector<cmSourceGroup> const& sourceGroups,
+  void WriteGroups(SourceGroupVector const& sourceGroups,
                    std::string& linkName, cmXMLWriter& xml);
   void CreateLinksToSubprojects(cmXMLWriter& xml, std::string const& baseDir);
   void CreateLinksForTargets(cmXMLWriter& xml);

Source/cmGeneratorTarget_Sources.cxx

@@ -122,7 +122,7 @@ void addFileSetEntry(cmGeneratorTarget const* headTarget,
       }
       bool found = false;
       for (auto const& sg : headTarget->Makefile->GetSourceGroups()) {
-        if (sg.MatchChildrenFiles(path)) {
+        if (sg->MatchChildrenFiles(path)) {
           found = true;
           break;
         }

Source/cmGlobalGenerator.cxx

@@ -1615,6 +1615,7 @@ bool cmGlobalGenerator::Compute()
 
   for (auto const& localGen : this->LocalGenerators) {
     localGen->ComputeHomeRelativeOutputPath();
+    localGen->ComputeSourceGroupSearchIndex();
   }
 
   return true;

Source/cmLocalGenerator.cxx

@@ -10,6 +10,7 @@
 #include <cstdlib>
 #include <initializer_list>
 #include <iterator>
+#include <queue>
 #include <sstream>
 #include <unordered_set>
 #include <utility>
@@ -4104,11 +4105,52 @@ std::string cmLocalGenerator::CreateSafeObjectFileName(
   return ssin;
 }
 
-cmSourceGroup* cmLocalGenerator::FindSourceGroup(
-  std::string const& source) const
+void cmLocalGenerator::ComputeSourceGroupSearchIndex()
 {
-  return cmSourceGroup::FindSourceGroup(source,
-                                        this->Makefile->GetSourceGroups());
+#if !defined(CMAKE_BOOTSTRAP)
+  SourceGroupVector const& sourceGroups = this->Makefile->GetSourceGroups();
+
+  // Build lookup index from sources to source groups
+  std::queue<cmSourceGroup*> sgToVisit;
+  for (auto const& group : sourceGroups) {
+    cmSourceGroup* cmSourceGroup = group.get();
+    sgToVisit.emplace(cmSourceGroup);
+  }
+
+  while (!sgToVisit.empty()) {
+    cmSourceGroup* sourceGroup = sgToVisit.front();
+    sgToVisit.pop();
+    for (auto const& sgChild : sourceGroup->GetGroupChildren()) {
+      sgToVisit.emplace(sgChild.get());
+    }
+    for (std::string const& source : sourceGroup->GetGroupFiles()) {
+      this->SourceGroupSearchIndex.emplace(source, sourceGroup);
+    }
+  }
+#endif
+}
+
+cmSourceGroup* cmLocalGenerator::FindSourceGroup(std::string const& source)
+{
+#if !defined(CMAKE_BOOTSTRAP)
+  auto const indexIt = SourceGroupSearchIndex.find(source);
+  if (indexIt != SourceGroupSearchIndex.cend()) {
+    if (cmSourceGroup* result = indexIt->second) {
+      return result;
+    }
+  }
+
+  cmSourceGroup* sourceGroup =
+    cmSourceGroup::FindSourceGroup(source, this->Makefile->GetSourceGroups());
+  if (sourceGroup) {
+    // Update index if we have a miss
+    SourceGroupSearchIndex.emplace(source, sourceGroup);
+  }
+  return sourceGroup;
+#else
+  static_cast<void>(source);
+  return nullptr;
+#endif
 }
 
 std::string& cmLocalGenerator::CreateSafeUniqueObjectFileName(

Source/cmLocalGenerator.h

@@ -574,10 +574,15 @@ public:
 
   std::string CreateSafeObjectFileName(std::string const& sin) const;
 
+  /**
+   * Build the search index from source files to source groups
+   */
+  void ComputeSourceGroupSearchIndex();
+
   /**
    * find what source group this source is in
    */
-  cmSourceGroup* FindSourceGroup(std::string const& source) const;
+  cmSourceGroup* FindSourceGroup(std::string const& source);
 
 protected:
   // The default implementation converts to a Windows shortpath to
@@ -637,6 +642,12 @@ protected:
 
   bool EmitUniversalBinaryFlags;
 
+#if !defined(CMAKE_BOOTSTRAP)
+  // Map from source file path to source group for lookup acceleration
+  using SourceGroupMap = std::unordered_map<std::string, cmSourceGroup*>;
+  SourceGroupMap SourceGroupSearchIndex;
+#endif
+
 private:
   /**
    * See LinearGetSourceFileWithOutput for background information

Source/cmLocalVisualStudio7Generator.cxx

@@ -1459,9 +1459,9 @@ void cmLocalVisualStudio7Generator::WriteVCProjFile(std::ostream& fout,
   fout << "\t<Files>\n";
 
   // Loop through every source group.
-  std::vector<cmSourceGroup> sourceGroups = this->Makefile->GetSourceGroups();
+  SourceGroupVector const& sourceGroups = this->Makefile->GetSourceGroups();
   for (auto const& sg : sourceGroups) {
-    this->WriteGroup(&sg, target, fout, libName, configs, sources);
+    this->WriteGroup(sg.get(), target, fout, libName, configs, sources);
   }
 
   fout << "\t</Files>\n";
@@ -1676,13 +1676,14 @@ bool cmLocalVisualStudio7Generator::WriteGroup(
   cmGlobalVisualStudio7Generator* gg =
     static_cast<cmGlobalVisualStudio7Generator*>(this->GlobalGenerator);
   std::vector<cmSourceFile const*> const& sourceFiles = sg->GetSourceFiles();
-  std::vector<cmSourceGroup> const& children = sg->GetGroupChildren();
+  SourceGroupVector const& children = sg->GetGroupChildren();
 
   // Write the children to temporary output.
   bool hasChildrenWithSources = false;
   std::ostringstream tmpOut;
   for (auto const& child : children) {
-    if (this->WriteGroup(&child, target, tmpOut, libName, configs, sources)) {
+    if (this->WriteGroup(child.get(), target, tmpOut, libName, configs,
+                         sources)) {
       hasChildrenWithSources = true;
     }
   }

Source/cmMakefile.cxx

@@ -195,7 +195,8 @@ cmMakefile::cmMakefile(cmGlobalGenerator* globalGenerator,
   this->AddSourceGroup("Object Files", "\\.(lo|o|obj)$");
 
   this->ObjectLibrariesSourceGroupIndex = this->SourceGroups.size();
-  this->SourceGroups.emplace_back("Object Libraries", "^MATCH_NO_SOURCES$");
+  this->SourceGroups.emplace_back(
+    cm::make_unique<cmSourceGroup>("Object Libraries", "^MATCH_NO_SOURCES$"));
 #endif
 }
 
@@ -2085,8 +2086,8 @@ namespace {
 
 void cmMakefile::ResolveSourceGroupGenex(cmLocalGenerator* lg)
 {
-  for (cmSourceGroup& sourceGroup : this->SourceGroups) {
-    sourceGroup.ResolveGenex(lg, {});
+  for (auto const& sourceGroup : this->SourceGroups) {
+    sourceGroup->ResolveGenex(lg, {});
   }
 }
 
@@ -2096,10 +2097,10 @@ cmSourceGroup* cmMakefile::GetSourceGroup(
   cmSourceGroup* sg = nullptr;
 
   // first look for source group starting with the same as the one we want
-  for (cmSourceGroup const& srcGroup : this->SourceGroups) {
-    std::string const& sgName = srcGroup.GetName();
+  for (auto const& srcGroup : this->SourceGroups) {
+    std::string const& sgName = srcGroup->GetName();
     if (sgName == name[0]) {
-      sg = const_cast<cmSourceGroup*>(&srcGroup);
+      sg = srcGroup.get();
       break;
     }
   }
@@ -2151,7 +2152,8 @@ void cmMakefile::AddSourceGroup(std::vector<std::string> const& name,
   if (i == -1) {
     // group does not exist nor belong to any existing group
     // add its first component
-    this->SourceGroups.emplace_back(name[0], regex);
+    this->SourceGroups.emplace_back(
+      cm::make_unique<cmSourceGroup>(name[0], regex));
     sg = this->GetSourceGroup(currentName);
     i = 0; // last component found
   }
@@ -2161,7 +2163,8 @@ void cmMakefile::AddSourceGroup(std::vector<std::string> const& name,
   }
   // build the whole source group path
   for (++i; i <= lastElement; ++i) {
-    sg->AddChild(cmSourceGroup(name[i], nullptr, sg->GetFullName().c_str()));
+    sg->AddChild(cm::make_unique<cmSourceGroup>(name[i], nullptr,
+                                                sg->GetFullName().c_str()));
     sg = sg->LookupChild(name[i]);
   }
 
@@ -3134,7 +3137,7 @@ void cmMakefile::AddTargetObject(std::string const& tgtName,
   // file that compiles to it. Needs a policy as it likely affects link
   // language selection if done unconditionally.
 #if !defined(CMAKE_BOOTSTRAP)
-  this->SourceGroups[this->ObjectLibrariesSourceGroupIndex].AddGroupFile(
+  this->SourceGroups[this->ObjectLibrariesSourceGroupIndex]->AddGroupFile(
     sf->ResolveFullPath());
 #endif
 }

Source/cmMakefile.h

@@ -40,6 +40,10 @@
 // will not compile without the complete type.
 #include "cmTarget.h" // IWYU pragma: keep
 
+#if !defined(CMAKE_BOOTSTRAP)
+#  include "cmSourceGroup.h"
+#endif
+
 enum class cmCustomCommandType;
 enum class cmObjectLibraryCommands;
 
@@ -60,10 +64,6 @@ class cmTestGenerator;
 class cmVariableWatch;
 class cmake;
 
-#if !defined(CMAKE_BOOTSTRAP)
-class cmSourceGroup;
-#endif
-
 /** A type-safe wrapper for a string representing a directory id.  */
 class cmDirectoryId
 {
@@ -621,7 +621,7 @@ public:
   /**
    * Get the vector source groups.
    */
-  std::vector<cmSourceGroup> const& GetSourceGroups() const
+  SourceGroupVector const& GetSourceGroups() const
   {
     return this->SourceGroups;
   }
@@ -1131,7 +1131,7 @@ protected:
   std::string DefineFlags;
 
 #if !defined(CMAKE_BOOTSTRAP)
-  std::vector<cmSourceGroup> SourceGroups;
+  SourceGroupVector SourceGroups;
   size_t ObjectLibrariesSourceGroupIndex;
 #endif
 

Source/cmSourceGroup.cxx

@@ -13,7 +13,7 @@
 class cmSourceGroupInternals
 {
 public:
-  std::vector<cmSourceGroup> GroupChildren;
+  SourceGroupVector GroupChildren;
 };
 
 cmSourceGroup::cmSourceGroup(std::string name, char const* regex,
@@ -30,28 +30,6 @@ cmSourceGroup::cmSourceGroup(std::string name, char const* regex,
 
 cmSourceGroup::~cmSourceGroup() = default;
 
-cmSourceGroup::cmSourceGroup(cmSourceGroup const& r)
-{
-  this->Name = r.Name;
-  this->FullName = r.FullName;
-  this->GroupRegex = r.GroupRegex;
-  this->GroupFiles = r.GroupFiles;
-  this->SourceFiles = r.SourceFiles;
-  this->Internal = cm::make_unique<cmSourceGroupInternals>(*r.Internal);
-}
-
-cmSourceGroup& cmSourceGroup::operator=(cmSourceGroup const& r)
-{
-  if (this != &r) {
-    this->Name = r.Name;
-    this->GroupRegex = r.GroupRegex;
-    this->GroupFiles = r.GroupFiles;
-    this->SourceFiles = r.SourceFiles;
-    *(this->Internal) = *(r.Internal);
-  }
-  return *this;
-}
-
 void cmSourceGroup::SetGroupRegex(char const* regex)
 {
   if (regex) {
@@ -76,8 +54,8 @@ void cmSourceGroup::ResolveGenex(cmLocalGenerator* lg,
     return;
   }
 
-  for (cmSourceGroup& group : this->Internal->GroupChildren) {
-    group.ResolveGenex(lg, config);
+  for (auto const& group : this->Internal->GroupChildren) {
+    group->ResolveGenex(lg, config);
   }
 }
 
@@ -112,22 +90,27 @@ void cmSourceGroup::AssignSource(cmSourceFile const* sf)
   this->SourceFiles.push_back(sf);
 }
 
+std::set<std::string> const& cmSourceGroup::GetGroupFiles() const
+{
+  return this->GroupFiles;
+}
+
 std::vector<cmSourceFile const*> const& cmSourceGroup::GetSourceFiles() const
 {
   return this->SourceFiles;
 }
 
-void cmSourceGroup::AddChild(cmSourceGroup const& child)
+void cmSourceGroup::AddChild(std::unique_ptr<cmSourceGroup> child)
 {
-  this->Internal->GroupChildren.push_back(child);
+  this->Internal->GroupChildren.push_back(std::move(child));
 }
 
 cmSourceGroup* cmSourceGroup::LookupChild(std::string const& name) const
 {
-  for (cmSourceGroup& group : this->Internal->GroupChildren) {
+  for (auto& group : this->Internal->GroupChildren) {
     // look if descendant is the one we're looking for
-    if (group.GetName() == name) {
-      return (&group); // if so return it
+    if (group->GetName() == name) {
+      return group.get(); // if so return it
     }
   }
 
@@ -140,8 +123,8 @@ cmSourceGroup* cmSourceGroup::MatchChildrenFiles(std::string const& name)
   if (this->MatchesFiles(name)) {
     return this;
   }
-  for (cmSourceGroup& group : this->Internal->GroupChildren) {
-    cmSourceGroup* result = group.MatchChildrenFiles(name);
+  for (auto& group : this->Internal->GroupChildren) {
+    cmSourceGroup* result = group->MatchChildrenFiles(name);
     if (result) {
       return result;
     }
@@ -155,8 +138,8 @@ cmSourceGroup const* cmSourceGroup::MatchChildrenFiles(
   if (this->MatchesFiles(name)) {
     return this;
   }
-  for (cmSourceGroup const& group : this->Internal->GroupChildren) {
-    cmSourceGroup const* result = group.MatchChildrenFiles(name);
+  for (auto const& group : this->Internal->GroupChildren) {
+    cmSourceGroup const* result = group->MatchChildrenFiles(name);
     if (result) {
       return result;
     }
@@ -166,8 +149,8 @@ cmSourceGroup const* cmSourceGroup::MatchChildrenFiles(
 
 cmSourceGroup* cmSourceGroup::MatchChildrenRegex(std::string const& name) const
 {
-  for (cmSourceGroup& group : this->Internal->GroupChildren) {
-    cmSourceGroup* result = group.MatchChildrenRegex(name);
+  for (auto& group : this->Internal->GroupChildren) {
+    cmSourceGroup* result = group->MatchChildrenRegex(name);
     if (result) {
       return result;
     }
@@ -179,7 +162,7 @@ cmSourceGroup* cmSourceGroup::MatchChildrenRegex(std::string const& name) const
   return nullptr;
 }
 
-std::vector<cmSourceGroup> const& cmSourceGroup::GetGroupChildren() const
+SourceGroupVector const& cmSourceGroup::GetGroupChildren() const
 {
   return this->Internal->GroupChildren;
 }
@@ -191,25 +174,25 @@ std::vector<cmSourceGroup> const& cmSourceGroup::GetGroupChildren() const
  * non-inherited SOURCE_GROUP commands will have precedence over
  * inherited ones.
  */
-cmSourceGroup* cmSourceGroup::FindSourceGroup(
-  std::string const& source, std::vector<cmSourceGroup> const& groups)
+cmSourceGroup* cmSourceGroup::FindSourceGroup(std::string const& source,
+                                              SourceGroupVector const& groups)
 {
   // First search for a group that lists the file explicitly.
   for (auto sg = groups.rbegin(); sg != groups.rend(); ++sg) {
-    cmSourceGroup const* result = sg->MatchChildrenFiles(source);
+    cmSourceGroup* result = (*sg)->MatchChildrenFiles(source);
     if (result) {
-      return const_cast<cmSourceGroup*>(result);
+      return result;
     }
   }
 
   // Now search for a group whose regex matches the file.
   for (auto sg = groups.rbegin(); sg != groups.rend(); ++sg) {
-    cmSourceGroup const* result = sg->MatchChildrenRegex(source);
+    cmSourceGroup* result = (*sg)->MatchChildrenRegex(source);
     if (result) {
-      return const_cast<cmSourceGroup*>(result);
+      return result;
     }
   }
 
   // Shouldn't get here, but just in case, return the default group.
-  return const_cast<cmSourceGroup*>(groups.data());
+  return groups.data()->get();
 }

Source/cmSourceGroup.h

@@ -13,8 +13,11 @@
 
 class cmLocalGenerator;
 class cmSourceFile;
+class cmSourceGroup;
 class cmSourceGroupInternals;
 
+using SourceGroupVector = std::vector<std::unique_ptr<cmSourceGroup>>;
+
 /** \class cmSourceGroup
  * \brief Hold a group of sources as specified by a SOURCE_GROUP command.
  *
@@ -30,9 +33,9 @@ class cmSourceGroup
 public:
   cmSourceGroup(std::string name, char const* regex,
                 char const* parentName = nullptr);
-  cmSourceGroup(cmSourceGroup const& r);
+  cmSourceGroup(cmSourceGroup const& r) = delete;
   ~cmSourceGroup();
-  cmSourceGroup& operator=(cmSourceGroup const&);
+  cmSourceGroup& operator=(cmSourceGroup const&) = delete;
 
   /**
    * Set the regular expression for this group.
@@ -52,7 +55,7 @@ public:
   /**
    * Add child to this sourcegroup
    */
-  void AddChild(cmSourceGroup const& child);
+  void AddChild(std::unique_ptr<cmSourceGroup> child);
 
   /**
    * Looks up child and returns it
@@ -102,19 +105,24 @@ public:
    */
   void AssignSource(cmSourceFile const* sf);
 
+  /**
+   * Get the set of file names explicitly added to this source group.
+   */
+  std::set<std::string> const& GetGroupFiles() const;
+
   /**
    * Get the list of the source files that have been assigned to this
    * source group.
    */
   std::vector<cmSourceFile const*> const& GetSourceFiles() const;
 
-  std::vector<cmSourceGroup> const& GetGroupChildren() const;
+  SourceGroupVector const& GetGroupChildren() const;
 
   /**
    * Given a source group collection, find the source group for a given source.
    */
-  static cmSourceGroup* FindSourceGroup(
-    std::string const& source, std::vector<cmSourceGroup> const& groups);
+  static cmSourceGroup* FindSourceGroup(std::string const& source,
+                                        SourceGroupVector const& groups);
 
 private:
   /**

Source/cmVisualStudio10TargetGenerator.cxx

@@ -2125,17 +2125,17 @@ void cmVisualStudio10TargetGenerator::WriteGroups()
 // Add to groupsUsed empty source groups that have non-empty children.
 void cmVisualStudio10TargetGenerator::AddMissingSourceGroups(
   std::set<cmSourceGroup const*>& groupsUsed,
-  std::vector<cmSourceGroup> const& allGroups)
+  SourceGroupVector const& allGroups)
 {
-  for (cmSourceGroup const& current : allGroups) {
-    std::vector<cmSourceGroup> const& children = current.GetGroupChildren();
+  for (auto const& current : allGroups) {
+    SourceGroupVector const& children = current->GetGroupChildren();
     if (children.empty()) {
       continue; // the group is really empty
     }
 
     this->AddMissingSourceGroups(groupsUsed, children);
 
-    if (groupsUsed.count(&current) > 0) {
+    if (groupsUsed.count(current.get()) > 0) {
       continue; // group has already been added to set
     }
 
@@ -2143,17 +2143,17 @@ void cmVisualStudio10TargetGenerator::AddMissingSourceGroups(
     // (at least one child must already have been added)
     auto child_it = children.begin();
     while (child_it != children.end()) {
-      if (groupsUsed.count(&(*child_it)) > 0) {
+      if (groupsUsed.count(child_it->get()) > 0) {
         break; // found a child that was already added => add current group too
       }
-      child_it++;
+      ++child_it;
     }
 
     if (child_it == children.end()) {
       continue; // no descendants have source files => ignore this group
     }
 
-    groupsUsed.insert(&current);
+    groupsUsed.insert(current.get());
   }
 }
 

Source/cmVisualStudio10TargetGenerator.h

@@ -14,6 +14,7 @@
 #include <vector>
 
 #include "cmGeneratorTarget.h"
+#include "cmSourceGroup.h"
 #include "cmVsProjectType.h"
 
 class cmComputeLinkInformation;
@@ -204,7 +205,7 @@ private:
   void WriteGroupSources(Elem& e0, std::string const& name,
                          ToolSources const& sources);
   void AddMissingSourceGroups(std::set<cmSourceGroup const*>& groupsUsed,
-                              std::vector<cmSourceGroup> const& allGroups);
+                              SourceGroupVector const& allGroups);
   bool IsResxHeader(std::string const& headerFile);
   bool IsXamlHeader(std::string const& headerFile);
   bool IsXamlSource(std::string const& headerFile);