[clang][modules] Lazily load by name lookups in module maps

Instead of eagerly populating the `clang::ModuleMap` when looking up
a module by name, this patch changes `HeaderSearch` to only load the
modules that are actually used.

This introduces `ModuleMap::findOrLoadModule` which will load modules
from parsed but not loaded module maps. This cannot be used anywhere
that the module loading code calls into as it can create infinite
recursion.

This currently just reparses module maps when looking up a module by
header. This is fine as redeclarations are allowed from the same file,
but future patches will also make looking up a module by header lazy.

This patch changes the shadow.m test to use explicitly built modules
and `#import`. This test and the shadow feature are very brittle and
do not work in general. The test relied on pcm files being left behind
by prior failing clang invocations that were then reused by the last
invocation. If you clean the cache then the last invocation will
always fail. This is because the input module map and the
`-fmodule-map-file=` module map are parsed in the same module scope,
and `-fmodule-map-file=` is forwarded to implicit module builds. That
means you are guaranteed to hit a module redeclaration error if the TU
actually imports the module it is trying to shadow.

This patch changes when we load A2's module map to after the `A`
module has been loaded, which sets the `IsFromModuleFile` bit on `A`.
This means that A2's `A` is skipped entirely instead of creating a
shadow module, and we get textual inclusion. It is possible to
construct a case where this would happen before this patch too.

An upcoming patch in this series will rework shadowing to work in the
general case, but that's only possible once header -> module lookup is
lazy too.

Author: Bigcheese

clang/include/clang/Basic/DiagnosticGroups.td

@@ -576,6 +576,7 @@ def ModuleImport : DiagGroup<"module-import">;
 def ModuleConflict : DiagGroup<"module-conflict">;
 def ModuleFileExtension : DiagGroup<"module-file-extension">;
 def ModuleIncludeDirectiveTranslation : DiagGroup<"module-include-translation">;
+def ModuleMap : DiagGroup<"module-map">;
 def RoundTripCC1Args : DiagGroup<"round-trip-cc1-args">;
 def NewlineEOF : DiagGroup<"newline-eof">;
 def Nullability : DiagGroup<"nullability">;

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -836,6 +836,12 @@ def warn_pp_date_time : Warning<
   ShowInSystemHeader, DefaultIgnore, InGroup<DiagGroup<"date-time">>;
 
 // Module map parsing
+def remark_mmap_parse : Remark<
+  "parsing modulemap '%0'">, ShowInSystemHeader, InGroup<ModuleMap>;
+def remark_mmap_load : Remark<
+  "loading modulemap '%0'">, ShowInSystemHeader, InGroup<ModuleMap>;
+def remark_mmap_load_module : Remark<
+  "loading parsed module '%0'">, ShowInSystemHeader, InGroup<ModuleMap>;
 def err_mmap_unknown_token : Error<"skipping stray token">;
 def err_mmap_expected_module : Error<"expected module declaration">;
 def err_mmap_expected_module_name : Error<"expected module name">;

clang/include/clang/Lex/HeaderSearch.h

@@ -332,13 +332,24 @@ class HeaderSearch {
   /// The mapping between modules and headers.
   mutable ModuleMap ModMap;
 
+  enum ModuleMapDirectoryState {
+    MMDS_Parsed,
+    MMDS_Loaded,
+    MMDS_Invalid,
+  };
+
   /// Describes whether a given directory has a module map in it.
-  llvm::DenseMap<const DirectoryEntry *, bool> DirectoryHasModuleMap;
+  llvm::DenseMap<const DirectoryEntry *, ModuleMapDirectoryState>
+      DirectoryHasModuleMap;
 
   /// Set of module map files we've already loaded, and a flag indicating
   /// whether they were valid or not.
   llvm::DenseMap<const FileEntry *, bool> LoadedModuleMaps;
 
+  /// Set of module map files we've already pre-parsed, and a flag indicating
+  /// whether they were valid or not.
+  llvm::DenseMap<const FileEntry *, bool> PreParsedModuleMaps;
+
   // A map of discovered headers with their associated include file name.
   llvm::DenseMap<const FileEntry *, llvm::SmallString<64>> IncludeNames;
 
@@ -435,7 +446,7 @@ class HeaderSearch {
 
   /// Consider modules when including files from this directory.
   void setDirectoryHasModuleMap(const DirectoryEntry* Dir) {
-    DirectoryHasModuleMap[Dir] = true;
+    DirectoryHasModuleMap[Dir] = MMDS_Loaded;
   }
 
   /// Forget everything we know about headers so far.
@@ -717,6 +728,23 @@ class HeaderSearch {
                          unsigned *Offset = nullptr,
                          StringRef OriginalModuleMapFile = StringRef());
 
+  /// Read the contents of the given module map file.
+  ///
+  /// \param File The module map file.
+  /// \param IsSystem Whether this file is in a system header directory.
+  /// \param ID If the module map file is already mapped (perhaps as part of
+  ///        processing a preprocessed module), the ID of the file.
+  /// \param Offset [inout] An offset within ID to start parsing. On exit,
+  ///        filled by the end of the parsed contents (either EOF or the
+  ///        location of an end-of-module-map pragma).
+  /// \param OriginalModuleMapFile The original path to the module map file,
+  ///        used to resolve paths within the module (this is required when
+  ///        building the module from preprocessed source).
+  /// \returns true if an error occurred, false otherwise.
+  bool preLoadModuleMapFile(FileEntryRef File, bool IsSystem,
+                            FileID ID = FileID(),
+                            StringRef OriginalModuleMapFile = StringRef());
+
   /// Collect the set of all known, top-level modules.
   ///
   /// \param Modules Will be filled with the set of known, top-level modules.
@@ -936,6 +964,10 @@ class HeaderSearch {
                                             FileID ID = FileID(),
                                             unsigned *Offset = nullptr);
 
+  LoadModuleMapResult parseModuleMapFileImpl(FileEntryRef File, bool IsSystem,
+                                             DirectoryEntryRef Dir,
+                                             FileID ID = FileID());
+
   /// Try to load the module map file in the given directory.
   ///
   /// \param DirName The name of the directory where we will look for a module
@@ -958,6 +990,11 @@ class HeaderSearch {
   /// named directory.
   LoadModuleMapResult loadModuleMapFile(DirectoryEntryRef Dir, bool IsSystem,
                                         bool IsFramework);
+
+  LoadModuleMapResult parseModuleMapFile(StringRef DirName, bool IsSystem,
+                                         bool IsFramework);
+  LoadModuleMapResult parseModuleMapFile(DirectoryEntryRef Dir, bool IsSystem,
+                                         bool IsFramework);
 };
 
 /// Apply the header search options to get given HeaderSearch object.

clang/include/clang/Lex/ModuleMap.h

@@ -18,6 +18,7 @@
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Lex/ModuleMapFile.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
@@ -267,6 +268,18 @@ class ModuleMap {
   /// Describes whether we haved parsed a particular file as a module
   /// map.
   llvm::DenseMap<const FileEntry *, bool> ParsedModuleMap;
+  llvm::DenseMap<const FileEntry *, const modulemap::ModuleMapFile *>
+      PreParsedModuleMap;
+
+  std::vector<std::unique_ptr<modulemap::ModuleMapFile>> PreParsedModuleMaps;
+
+  /// Map from top level module name to a list of ModuleDecls in the order they
+  /// were discovered. This allows handling shadowing correctly and diagnosing
+  /// redefinitions.
+  llvm::StringMap<SmallVector<std::pair<const modulemap::ModuleMapFile *,
+                                        const modulemap::ModuleDecl *>,
+                              1>>
+      PreParsedModules;
 
   /// Resolve the given export declaration into an actual export
   /// declaration.
@@ -483,6 +496,8 @@ class ModuleMap {
   /// \returns The named module, if known; otherwise, returns null.
   Module *findModule(StringRef Name) const;
 
+  Module *findOrLoadModule(StringRef Name);
+
   Module *findOrInferSubmodule(Module *Parent, StringRef Name);
 
   /// Retrieve a module with the given name using lexical name lookup,
@@ -698,6 +713,11 @@ class ModuleMap {
   void addHeader(Module *Mod, Module::Header Header,
                  ModuleHeaderRole Role, bool Imported = false);
 
+  /// Parse a module map without creating `clang::Module` instances.
+  bool preParseModuleMapFile(FileEntryRef File, bool IsSystem,
+                             DirectoryEntryRef Dir, FileID ID = FileID(),
+                             SourceLocation ExternModuleLoc = SourceLocation());
+
   /// Parse the given module map file, and record any modules we
   /// encounter.
   ///

clang/include/clang/Lex/ModuleMapFile.h

@@ -133,8 +133,17 @@ using TopLevelDecl = std::variant<ModuleDecl, ExternModuleDecl>;
 /// This holds many reference types (StringRef, SourceLocation, etc.) whose
 /// lifetimes are bound by the SourceManager and FileManager used.
 struct ModuleMapFile {
+  /// The FileID used to parse this module map. This is always a local ID.
+  FileID ID;
+
+  /// The directory in which the module map was discovered. Declarations in
+  /// the module map are relative to this directory.
+  OptionalDirectoryEntryRef Dir;
+
   /// Beginning of the file, used for moduleMapFileRead callback.
   SourceLocation Start;
+
+  bool IsSystem;
   std::vector<TopLevelDecl> Decls;
 
   void dump(llvm::raw_ostream &out) const;

clang/lib/Frontend/CompilerInstance.cpp

@@ -579,13 +579,13 @@ struct ReadModuleNames : ASTReaderListener {
     ModuleMap &MM = PP.getHeaderSearchInfo().getModuleMap();
     for (const std::string &LoadedModule : LoadedModules)
       MM.cacheModuleLoad(*PP.getIdentifierInfo(LoadedModule),
-                         MM.findModule(LoadedModule));
+                         MM.findOrLoadModule(LoadedModule));
     LoadedModules.clear();
   }
 
   void markAllUnavailable() {
     for (const std::string &LoadedModule : LoadedModules) {
-      if (Module *M = PP.getHeaderSearchInfo().getModuleMap().findModule(
+      if (Module *M = PP.getHeaderSearchInfo().getModuleMap().findOrLoadModule(
               LoadedModule)) {
         M->HasIncompatibleModuleFile = true;
 

clang/lib/Lex/HeaderSearch.cpp

@@ -300,7 +300,7 @@ Module *HeaderSearch::lookupModule(StringRef ModuleName,
                                    SourceLocation ImportLoc, bool AllowSearch,
                                    bool AllowExtraModuleMapSearch) {
   // Look in the module map to determine if there is a module by this name.
-  Module *Module = ModMap.findModule(ModuleName);
+  Module *Module = ModMap.findOrLoadModule(ModuleName);
   if (Module || !AllowSearch || !HSOpts->ImplicitModuleMaps)
     return Module;
 
@@ -360,11 +360,11 @@ Module *HeaderSearch::lookupModule(StringRef ModuleName, StringRef SearchName,
     // checked
     DirectoryEntryRef NormalDir = *Dir.getDirRef();
     // Search for a module map file in this directory.
-    if (loadModuleMapFile(NormalDir, IsSystem,
-                          /*IsFramework*/false) == LMM_NewlyLoaded) {
+    if (parseModuleMapFile(NormalDir, IsSystem,
+                           /*IsFramework*/ false) == LMM_NewlyLoaded) {
       // We just loaded a module map file; check whether the module is
       // available now.
-      Module = ModMap.findModule(ModuleName);
+      Module = ModMap.findOrLoadModule(ModuleName);
       if (Module)
         break;
     }
@@ -374,10 +374,10 @@ Module *HeaderSearch::lookupModule(StringRef ModuleName, StringRef SearchName,
     SmallString<128> NestedModuleMapDirName;
     NestedModuleMapDirName = Dir.getDirRef()->getName();
     llvm::sys::path::append(NestedModuleMapDirName, ModuleName);
-    if (loadModuleMapFile(NestedModuleMapDirName, IsSystem,
-                          /*IsFramework*/false) == LMM_NewlyLoaded){
+    if (parseModuleMapFile(NestedModuleMapDirName, IsSystem,
+                           /*IsFramework*/ false) == LMM_NewlyLoaded) {
       // If we just loaded a module map file, look for the module again.
-      Module = ModMap.findModule(ModuleName);
+      Module = ModMap.findOrLoadModule(ModuleName);
       if (Module)
         break;
     }
@@ -394,7 +394,7 @@ Module *HeaderSearch::lookupModule(StringRef ModuleName, StringRef SearchName,
         loadSubdirectoryModuleMaps(Dir);
 
       // Look again for the module.
-      Module = ModMap.findModule(ModuleName);
+      Module = ModMap.findOrLoadModule(ModuleName);
       if (Module)
         break;
     }
@@ -1583,7 +1583,7 @@ bool HeaderSearch::hasModuleMap(StringRef FileName,
       // Success. All of the directories we stepped through inherit this module
       // map file.
       for (unsigned I = 0, N = FixUpDirectories.size(); I != N; ++I)
-        DirectoryHasModuleMap[FixUpDirectories[I]] = true;
+        DirectoryHasModuleMap[FixUpDirectories[I]] = MMDS_Loaded;
       return true;
 
     case LMM_NoDirectory:
@@ -1801,6 +1801,33 @@ HeaderSearch::loadModuleMapFileImpl(FileEntryRef File, bool IsSystem,
   return LMM_NewlyLoaded;
 }
 
+HeaderSearch::LoadModuleMapResult
+HeaderSearch::parseModuleMapFileImpl(FileEntryRef File, bool IsSystem,
+                                     DirectoryEntryRef Dir, FileID ID) {
+  // Check whether we've already parsed this module map, and mark it as being
+  // parsed in case we recursively try to parse it from itself.
+  auto AddResult = PreParsedModuleMaps.insert(std::make_pair(File, true));
+  if (!AddResult.second)
+    return AddResult.first->second ? LMM_AlreadyLoaded : LMM_InvalidModuleMap;
+
+  if (ModMap.preParseModuleMapFile(File, IsSystem, Dir, ID)) {
+    PreParsedModuleMaps[File] = false;
+    return LMM_InvalidModuleMap;
+  }
+
+  // Try to load a corresponding private module map.
+  if (OptionalFileEntryRef PMMFile =
+          getPrivateModuleMap(File, FileMgr, Diags)) {
+    if (ModMap.preParseModuleMapFile(*PMMFile, IsSystem, Dir)) {
+      PreParsedModuleMaps[File] = false;
+      return LMM_InvalidModuleMap;
+    }
+  }
+
+  // This directory has a module map.
+  return LMM_NewlyLoaded;
+}
+
 OptionalFileEntryRef
 HeaderSearch::lookupModuleMapFile(DirectoryEntryRef Dir, bool IsFramework) {
   if (!HSOpts->ImplicitModuleMaps)
@@ -1853,7 +1880,7 @@ Module *HeaderSearch::loadFrameworkModule(StringRef Name, DirectoryEntryRef Dir,
     break;
   }
 
-  return ModMap.findModule(Name);
+  return ModMap.findOrLoadModule(Name);
 }
 
 HeaderSearch::LoadModuleMapResult
@@ -1869,8 +1896,16 @@ HeaderSearch::LoadModuleMapResult
 HeaderSearch::loadModuleMapFile(DirectoryEntryRef Dir, bool IsSystem,
                                 bool IsFramework) {
   auto KnownDir = DirectoryHasModuleMap.find(Dir);
-  if (KnownDir != DirectoryHasModuleMap.end())
-    return KnownDir->second ? LMM_AlreadyLoaded : LMM_InvalidModuleMap;
+  if (KnownDir != DirectoryHasModuleMap.end()) {
+    switch (KnownDir->second) {
+    case MMDS_Parsed:
+      break;
+    case MMDS_Loaded:
+      return LMM_AlreadyLoaded;
+    case MMDS_Invalid:
+      return LMM_InvalidModuleMap;
+    };
+  }
 
   if (OptionalFileEntryRef ModuleMapFile =
           lookupModuleMapFile(Dir, IsFramework)) {
@@ -1880,9 +1915,49 @@ HeaderSearch::loadModuleMapFile(DirectoryEntryRef Dir, bool IsSystem,
     // E.g. Foo.framework/Modules/module.modulemap
     //      ^Dir                  ^ModuleMapFile
     if (Result == LMM_NewlyLoaded)
-      DirectoryHasModuleMap[Dir] = true;
+      DirectoryHasModuleMap[Dir] = MMDS_Loaded;
+    else if (Result == LMM_InvalidModuleMap)
+      DirectoryHasModuleMap[Dir] = MMDS_Invalid;
+    return Result;
+  }
+  return LMM_InvalidModuleMap;
+}
+
+HeaderSearch::LoadModuleMapResult
+HeaderSearch::parseModuleMapFile(StringRef DirName, bool IsSystem,
+                                 bool IsFramework) {
+  if (auto Dir = FileMgr.getOptionalDirectoryRef(DirName))
+    return parseModuleMapFile(*Dir, IsSystem, IsFramework);
+
+  return LMM_NoDirectory;
+}
+
+HeaderSearch::LoadModuleMapResult
+HeaderSearch::parseModuleMapFile(DirectoryEntryRef Dir, bool IsSystem,
+                                 bool IsFramework) {
+  // Check if this modulemap has already been fully loaded, if so skip it.
+  auto KnownDir = DirectoryHasModuleMap.find(Dir);
+  if (KnownDir != DirectoryHasModuleMap.end()) {
+    switch (KnownDir->second) {
+    case MMDS_Parsed:
+    case MMDS_Loaded:
+      return LMM_AlreadyLoaded;
+    case MMDS_Invalid:
+      return LMM_InvalidModuleMap;
+    };
+  }
+
+  if (OptionalFileEntryRef ModuleMapFile =
+          lookupModuleMapFile(Dir, IsFramework)) {
+    LoadModuleMapResult Result =
+        parseModuleMapFileImpl(*ModuleMapFile, IsSystem, Dir);
+    // Add Dir explicitly in case ModuleMapFile is in a subdirectory.
+    // E.g. Foo.framework/Modules/module.modulemap
+    //      ^Dir                  ^ModuleMapFile
+    if (Result == LMM_NewlyLoaded)
+      DirectoryHasModuleMap[Dir] = MMDS_Parsed;
     else if (Result == LMM_InvalidModuleMap)
-      DirectoryHasModuleMap[Dir] = false;
+      DirectoryHasModuleMap[Dir] = MMDS_Invalid;
     return Result;
   }
   return LMM_InvalidModuleMap;