Skip to content

Commit

Permalink
Address comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChuanqiXu9
ChuanqiXu9 committed Nov 12, 2024
1 parent 370b01f commit 4e0f1e3
Showing 1 changed file with 109 additions and 157 deletions.
266 changes: 109 additions & 157 deletions clang-tools-extra/clangd/ModulesBuilder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,47 +95,9 @@ class FailedPrerequisiteModules : public PrerequisiteModules {
}
};

struct ModuleFile;

// ReusablePrerequisiteModules - stands for PrerequisiteModules for which all
// the required modules are built successfully. All the module files
// are owned by the modules builder.
class ReusablePrerequisiteModules : public PrerequisiteModules {
public:
ReusablePrerequisiteModules() = default;

ReusablePrerequisiteModules(const ReusablePrerequisiteModules &Other) =
default;
ReusablePrerequisiteModules &
operator=(const ReusablePrerequisiteModules &) = default;
ReusablePrerequisiteModules(ReusablePrerequisiteModules &&) = delete;
ReusablePrerequisiteModules
operator=(ReusablePrerequisiteModules &&) = delete;

~ReusablePrerequisiteModules() override = default;

void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override;

bool canReuse(const CompilerInvocation &CI,
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override;

bool isModuleUnitBuilt(llvm::StringRef ModuleName) const {
return BuiltModuleNames.contains(ModuleName);
}

void addModuleFile(std::shared_ptr<const ModuleFile> ModuleFile);

private:
llvm::SmallVector<std::shared_ptr<const ModuleFile>, 8> RequiredModules;
// A helper class to speedup the query if a module is built.
llvm::StringSet<> BuiltModuleNames;
};

struct ModuleFile {
ModuleFile(StringRef ModuleName, PathRef ModuleFilePath,
const ReusablePrerequisiteModules &RequiredModuleFiles)
: ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()),
RequiredModuleFiles(RequiredModuleFiles) {}
ModuleFile(StringRef ModuleName, PathRef ModuleFilePath)
: ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {}

ModuleFile() = delete;

Expand Down Expand Up @@ -171,26 +133,50 @@ struct ModuleFile {
private:
std::string ModuleName;
std::string ModuleFilePath;

// The required module files. We need to share the ownership for required
// module files.
ReusablePrerequisiteModules RequiredModuleFiles;
};

void ReusablePrerequisiteModules::adjustHeaderSearchOptions(
HeaderSearchOptions &Options) const {
// Appending all built module files.
for (const auto &RequiredModule : RequiredModules)
Options.PrebuiltModuleFiles.insert_or_assign(
RequiredModule->getModuleName().str(),
RequiredModule->getModuleFilePath().str());
}
// ReusablePrerequisiteModules - stands for PrerequisiteModules for which all
// the required modules are built successfully. All the module files
// are owned by the modules builder.
class ReusablePrerequisiteModules : public PrerequisiteModules {
public:
ReusablePrerequisiteModules() = default;

void ReusablePrerequisiteModules::addModuleFile(
std::shared_ptr<const ModuleFile> ModuleFile) {
BuiltModuleNames.insert(ModuleFile->getModuleName());
RequiredModules.emplace_back(std::move(ModuleFile));
}
ReusablePrerequisiteModules(const ReusablePrerequisiteModules &Other) =
default;
ReusablePrerequisiteModules &
operator=(const ReusablePrerequisiteModules &) = default;
ReusablePrerequisiteModules(ReusablePrerequisiteModules &&) = delete;
ReusablePrerequisiteModules
operator=(ReusablePrerequisiteModules &&) = delete;

~ReusablePrerequisiteModules() override = default;

void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
// Appending all built module files.
for (const auto &RequiredModule : RequiredModules)
Options.PrebuiltModuleFiles.insert_or_assign(
RequiredModule->getModuleName().str(),
RequiredModule->getModuleFilePath().str());
}

bool canReuse(const CompilerInvocation &CI,
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override;

bool isModuleUnitBuilt(llvm::StringRef ModuleName) const {
return BuiltModuleNames.contains(ModuleName);
}

void addModuleFile(std::shared_ptr<const ModuleFile> ModuleFile) {
BuiltModuleNames.insert(ModuleFile->getModuleName());
RequiredModules.emplace_back(std::move(ModuleFile));
}

private:
llvm::SmallVector<std::shared_ptr<const ModuleFile>, 8> RequiredModules;
// A helper class to speedup the query if a module is built.
llvm::StringSet<> BuiltModuleNames;
};

bool IsModuleFileUpToDate(PathRef ModuleFilePath,
const PrerequisiteModules &RequisiteModules,
Expand Down Expand Up @@ -309,7 +295,7 @@ buildModuleFile(llvm::StringRef ModuleName, PathRef ModuleUnitFileName,
if (Clang->getDiagnostics().hasErrorOccurred())
return llvm::createStringError("Compilation failed");

return ModuleFile{ModuleName, Inputs.CompileCommand.Output, BuiltModuleFiles};
return ModuleFile{ModuleName, Inputs.CompileCommand.Output};
}

bool ReusablePrerequisiteModules::canReuse(
Expand All @@ -327,24 +313,18 @@ bool ReusablePrerequisiteModules::canReuse(
class ModuleFileCache {
public:
ModuleFileCache(const GlobalCompilationDatabase &CDB) : CDB(CDB) {}

llvm::Error
getOrBuildModuleFile(StringRef ModuleName, const ThreadsafeFS &TFS,
ProjectModules &MDB,
ReusablePrerequisiteModules &RequiredModules);
const GlobalCompilationDatabase &getCDB() const { return CDB; }

std::shared_ptr<const ModuleFile>
getValidModuleFile(StringRef ModuleName, ProjectModules &MDB,
const ThreadsafeFS &TFS,
PrerequisiteModules &BuiltModuleFiles);
std::shared_ptr<const ModuleFile> getModule(StringRef ModuleName);

void add(StringRef ModuleName, std::shared_ptr<const ModuleFile> ModuleFile) {
std::lock_guard<std::mutex> Lock(ModuleFilesMutex);

ModuleFiles.insert_or_assign(ModuleName, ModuleFile);
}

void remove(StringRef ModuleName);

private:
const GlobalCompilationDatabase &CDB;

Expand All @@ -353,86 +333,55 @@ class ModuleFileCache {
std::mutex ModuleFilesMutex;
};

/// Collect the directly and indirectly required module names for \param
/// ModuleName. The \param ModuleName is guaranteed to be the first element in
/// \param ModuleNames.
llvm::SmallVector<StringRef> getAllRequiredModules(ProjectModules &MDB,
StringRef ModuleName) {
llvm::SmallVector<StringRef> ModuleNames;

std::queue<StringRef> Worklist;
llvm::StringSet<> ModuleNamesSet;
Worklist.push(ModuleName);

while (!Worklist.empty()) {
StringRef CurrentModule = Worklist.front();
Worklist.pop();

if (!ModuleNamesSet.insert(CurrentModule).second)
continue;

ModuleNames.push_back(CurrentModule);

for (StringRef RequiredModuleName :
MDB.getRequiredModules(MDB.getSourceForModuleName(CurrentModule)))
if (!ModuleNamesSet.contains(RequiredModuleName))
Worklist.push(RequiredModuleName);
}

return ModuleNames;
}

std::shared_ptr<const ModuleFile>
ModuleFileCache::getValidModuleFile(StringRef ModuleName, ProjectModules &MDB,
const ThreadsafeFS &TFS,
PrerequisiteModules &BuiltModuleFiles) {
{
std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
ModuleFileCache::getModule(StringRef ModuleName) {
std::lock_guard<std::mutex> Lock(ModuleFilesMutex);

auto Iter = ModuleFiles.find(ModuleName);
if (Iter == ModuleFiles.end())
return nullptr;
auto Iter = ModuleFiles.find(ModuleName);
if (Iter == ModuleFiles.end())
return nullptr;

if (Iter->second.expired()) {
ModuleFiles.erase(Iter);
return nullptr;
}
if (Iter->second.expired()) {
ModuleFiles.erase(Iter);
return nullptr;
}

llvm::SmallVector<StringRef> ModuleNames =
getAllRequiredModules(MDB, ModuleName);
return Iter->second.lock();
}

llvm::SmallVector<std::shared_ptr<const ModuleFile>> RequiredModuleFiles;
void ModuleFileCache::remove(StringRef ModuleName) {
std::lock_guard<std::mutex> Lock(ModuleFilesMutex);

{
std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
auto Iter = ModuleFiles.find(ModuleName);
if (Iter == ModuleFiles.end())
return;

for (StringRef ModuleName : ModuleNames) {
auto Iter = ModuleFiles.find(ModuleName);
if (Iter == ModuleFiles.end())
return nullptr;
ModuleFiles.erase(Iter);
}

if (Iter->second.expired()) {
ModuleFiles.erase(Iter);
return nullptr;
}
/// Collect the directly and indirectly required module names for \param
/// ModuleName in topological order. The \param ModuleName is guaranteed to
/// be the last element in \param ModuleNames.
llvm::SmallVector<StringRef> getAllRequiredModules(ProjectModules &MDB,
StringRef ModuleName) {
llvm::SmallVector<StringRef> ModuleNames;
llvm::StringSet<> ModuleNamesSet;

RequiredModuleFiles.push_back(Iter->second.lock());
}
}
auto traversal = [&](StringRef ModuleName, auto recursionHelper) -> void {
ModuleNamesSet.insert(ModuleName);

assert(!RequiredModuleFiles.empty());
for (StringRef RequiredModuleName :
MDB.getRequiredModules(MDB.getSourceForModuleName(ModuleName)))
if (ModuleNamesSet.insert(RequiredModuleName).second)
recursionHelper(RequiredModuleName, recursionHelper);

if (llvm::any_of(RequiredModuleFiles,
[&](std::shared_ptr<const ModuleFile> MF) {
return !IsModuleFileUpToDate(MF->getModuleFilePath(),
BuiltModuleFiles,
TFS.view(std::nullopt));
}))
return nullptr;
ModuleNames.push_back(ModuleName);
};
traversal(ModuleName, traversal);

return RequiredModuleFiles[0];
return ModuleNames;
}

} // namespace

class ModulesBuilder::ModulesBuilderImpl {
Expand Down Expand Up @@ -468,34 +417,37 @@ llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile(
return llvm::createStringError(
llvm::formatv("Don't get the module unit for module {0}", ModuleName));

for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName))
// Return early if there are errors building the module file.
if (!getOrBuildModuleFile(RequiredModuleName, TFS, MDB, BuiltModuleFiles))
return llvm::createStringError(
llvm::formatv("Failed to build module {0}", RequiredModuleName));

if (std::shared_ptr<const ModuleFile> Cached =
Cache.getValidModuleFile(ModuleName, MDB, TFS, BuiltModuleFiles)) {
log("Reusing module {0} from {1}", ModuleName, Cached->getModuleFilePath());
BuiltModuleFiles.addModuleFile(Cached);
return llvm::Error::success();
}
// Get Required modules in topological order.
auto ReqModuleNames = getAllRequiredModules(MDB, ModuleName);
for (llvm::StringRef ReqModuleName : ReqModuleNames) {
if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
continue;

log("Building module {0}", ModuleName);
if (auto Cached = Cache.getModule(ReqModuleName)) {
if (IsModuleFileUpToDate(Cached->getModuleFilePath(), BuiltModuleFiles,
TFS.view(std::nullopt))) {
log("Reusing module {0} from {1}", ModuleName,
Cached->getModuleFilePath());
BuiltModuleFiles.addModuleFile(std::move(Cached));
continue;
}
Cache.remove(ReqModuleName);
}

llvm::SmallString<256> ModuleFilesPrefix =
getUniqueModuleFilesPath(ModuleUnitFileName);
llvm::SmallString<256> ModuleFilesPrefix =
getUniqueModuleFilesPath(ModuleUnitFileName);

llvm::Expected<ModuleFile> MF =
buildModuleFile(ModuleName, ModuleUnitFileName, getCDB(), TFS,
ModuleFilesPrefix, BuiltModuleFiles);
if (llvm::Error Err = MF.takeError())
return Err;
llvm::Expected<ModuleFile> MF =
buildModuleFile(ModuleName, ModuleUnitFileName, getCDB(), TFS,
ModuleFilesPrefix, BuiltModuleFiles);
if (llvm::Error Err = MF.takeError())
return Err;

log("Built module {0} to {1}", ModuleName, MF->getModuleFilePath());
auto BuiltModuleFile = std::make_shared<const ModuleFile>(std::move(*MF));
Cache.add(ModuleName, BuiltModuleFile);
BuiltModuleFiles.addModuleFile(std::move(BuiltModuleFile));
log("Built module {0} to {1}", ModuleName, MF->getModuleFilePath());
auto BuiltModuleFile = std::make_shared<const ModuleFile>(std::move(*MF));
Cache.add(ModuleName, BuiltModuleFile);
BuiltModuleFiles.addModuleFile(std::move(BuiltModuleFile));
}

return llvm::Error::success();
}
Expand Down

0 comments on commit 4e0f1e3

Please sign in to comment.