Skip to content

[Database] For ReadTransaction::getProviderFileCodeReferences() make sure to filter the units properly #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 1, 2019
Merged

[Database] For ReadTransaction::getProviderFileCodeReferences() make sure to filter the units properly #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions include/IndexStoreDB/Database/ReadTransaction.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,14 @@ class INDEXSTOREDB_EXPORT ReadTransaction {
StringRef getModuleName(IDCode moduleName);
bool getProviderFileReferences(IDCode provider,
llvm::function_ref<bool(TimestampedPath path)> receiver);
/// `unitFilter` returns `true` if the unit should be included, `false` if it should be ignored.
bool getProviderFileCodeReferences(IDCode provider,
llvm::function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);
function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);
/// Returns all provider-file associations. Intended for debugging purposes.
bool foreachProviderAndFileCodeReference(llvm::function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);
/// `unitFilter` returns `true` if the unit should be included, `false` if it should be ignored.
bool foreachProviderAndFileCodeReference(function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);

/// Returns USR codes in batches.
bool foreachUSROfGlobalSymbolKind(SymbolKind symKind, llvm::function_ref<bool(ArrayRef<IDCode> usrCodes)> receiver);
Expand Down
66 changes: 44 additions & 22 deletions lib/Database/ReadTransaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ StringRef ReadTransaction::Implementation::getModuleName(IDCode moduleNameCode)

bool ReadTransaction::Implementation::getProviderFileReferences(IDCode provider,
llvm::function_ref<bool(TimestampedPath path)> receiver) {
return getProviderFileCodeReferences(provider, [&](IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
auto unitFilter = [](IDCode unitCode)->bool { return true; };
return getProviderFileCodeReferences(provider, unitFilter, [&](IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
std::string pathString;
llvm::raw_string_ostream OS(pathString);
if (!getFullFilePathFromCode(pathCode, OS)) {
Expand All @@ -138,9 +139,11 @@ bool ReadTransaction::Implementation::getProviderFileReferences(IDCode provider,
});
}

/// `unitFilter` returns `true` if the unit should be included, `false` if it should be ignored.
static bool passFileReferencesForProviderCursor(lmdb::val &key,
lmdb::val &value,
lmdb::cursor &cursor,
function_ref<bool(IDCode unitCode)> unitFilter,
llvm::function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
// Entries are sorted by file code and there can be multiple same file entries
// from different units. We want to pass each file only once with its most recent
Expand All @@ -159,32 +162,46 @@ static bool passFileReferencesForProviderCursor(lmdb::val &key,
const auto &entry = *(TimestampedFileForProviderData*)value.data();
llvm::sys::TimePoint<> modTime = llvm::sys::TimePoint<>(std::chrono::nanoseconds(entry.NanoTime));
if (!currFileCode) {
currFileCode = entry.FileCode;
currUnitCode = entry.UnitCode;
currModTime = modTime;
currModuleNameCode = entry.ModuleNameCode;
currIsSystem = entry.IsSystem;
if (unitFilter(entry.UnitCode)) {
currFileCode = entry.FileCode;
currUnitCode = entry.UnitCode;
currModTime = modTime;
currModuleNameCode = entry.ModuleNameCode;
currIsSystem = entry.IsSystem;
}
} else if (currFileCode.getValue() == entry.FileCode) {
if (currModTime < modTime) {
currModTime = modTime;
currUnitCode = entry.UnitCode;
if (unitFilter(entry.UnitCode)) {
currModTime = modTime;
currUnitCode = entry.UnitCode;
}
}
} else {
if (!passCurrFile())
return false;
currFileCode = entry.FileCode;
currUnitCode = entry.UnitCode;
currModTime = modTime;
currModuleNameCode = entry.ModuleNameCode;
currIsSystem = entry.IsSystem;
if (unitFilter(entry.UnitCode)) {
currFileCode = entry.FileCode;
currUnitCode = entry.UnitCode;
currModTime = modTime;
currModuleNameCode = entry.ModuleNameCode;
currIsSystem = entry.IsSystem;
} else {
currFileCode.reset();
currUnitCode.reset();
}
}
} while (cursor.get(key, value, MDB_NEXT_DUP));

return passCurrFile();
if (currFileCode) {
return passCurrFile();
} else {
return true;
}
}

bool ReadTransaction::Implementation::getProviderFileCodeReferences(IDCode provider,
llvm::function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
auto &db = DBase->impl();
auto &dbiFilesByProvider = db.getDBITimestampedFilesByProvider();
auto cursor = lmdb::cursor::open(Txn, dbiFilesByProvider);
Expand All @@ -195,10 +212,12 @@ bool ReadTransaction::Implementation::getProviderFileCodeReferences(IDCode provi
if (!found)
return true;

return passFileReferencesForProviderCursor(key, value, cursor, std::move(receiver));
return passFileReferencesForProviderCursor(key, value, cursor, std::move(unitFilter), std::move(receiver));
}

bool ReadTransaction::Implementation::foreachProviderAndFileCodeReference(llvm::function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
bool ReadTransaction::Implementation::foreachProviderAndFileCodeReference(
function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
auto &db = DBase->impl();
auto &dbiFilesByProvider = db.getDBITimestampedFilesByProvider();
auto cursor = lmdb::cursor::open(Txn, dbiFilesByProvider);
Expand All @@ -207,7 +226,7 @@ bool ReadTransaction::Implementation::foreachProviderAndFileCodeReference(llvm::
lmdb::val value{};
while (cursor.get(key, value, MDB_NEXT_NODUP)) {
IDCode providerCode = *(IDCode*)key.data();
bool cont = passFileReferencesForProviderCursor(key, value, cursor, [&](IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
bool cont = passFileReferencesForProviderCursor(key, value, cursor, unitFilter, [&](IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
return receiver(providerCode, pathCode, unitCode, modTime, moduleNameCode, isSystem);
});
if (!cont)
Expand Down Expand Up @@ -635,12 +654,15 @@ bool ReadTransaction::getProviderFileReferences(IDCode provider,
}

bool ReadTransaction::getProviderFileCodeReferences(IDCode provider,
llvm::function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
return Impl->getProviderFileCodeReferences(provider, std::move(receiver));
function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
return Impl->getProviderFileCodeReferences(provider, std::move(unitFilter), std::move(receiver));
}

bool ReadTransaction::foreachProviderAndFileCodeReference(llvm::function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
return Impl->foreachProviderAndFileCodeReference(std::move(receiver));
bool ReadTransaction::foreachProviderAndFileCodeReference(
function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver) {
return Impl->foreachProviderAndFileCodeReference(std::move(unitFilter), std::move(receiver));
}

bool ReadTransaction::foreachUSROfGlobalSymbolKind(SymbolKind symKind, llvm::function_ref<bool(ArrayRef<IDCode> usrCodes)> receiver) {
Expand Down
8 changes: 6 additions & 2 deletions lib/Database/ReadTransactionImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,13 @@ class ReadTransaction::Implementation {
StringRef getModuleName(IDCode moduleName);
bool getProviderFileReferences(IDCode provider,
llvm::function_ref<bool(TimestampedPath path)> receiver);
/// `unitFilter` returns `true` if the unit should be included, `false` if it should be ignored.
bool getProviderFileCodeReferences(IDCode provider,
llvm::function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);
bool foreachProviderAndFileCodeReference(llvm::function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);
function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);
/// `unitFilter` returns `true` if the unit should be included, `false` if it should be ignored.
bool foreachProviderAndFileCodeReference(function_ref<bool(IDCode unitCode)> unitFilter,
function_ref<bool(IDCode provider, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem)> receiver);

bool foreachUSROfGlobalSymbolKind(SymbolKind symKind, llvm::function_ref<bool(ArrayRef<IDCode> usrCodes)> receiver);
bool foreachUSROfGlobalUnitTestSymbol(llvm::function_ref<bool(ArrayRef<IDCode> usrCodes)> receiver);
Expand Down
22 changes: 13 additions & 9 deletions lib/Index/SymbolIndex.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ class SymbolIndexImpl {
std::vector<SymbolDataProviderRef> lookupProvidersForUSR(StringRef USR, SymbolRoleSet roles, SymbolRoleSet relatedRoles);
std::vector<std::pair<SymbolDataProviderRef, bool>> findCanonicalProvidersForUSR(IDCode usrCode);
SymbolDataProviderRef createVisibleProviderForCode(IDCode providerCode, ReadTransaction &reader);
SymbolDataProviderRef createProviderForCode(IDCode providerCode, ReadTransaction &reader, function_ref<bool(ReadTransaction &, const UnitInfo &)> unitFilter);
SymbolDataProviderRef createProviderForCode(IDCode providerCode, ReadTransaction &reader, function_ref<bool(const UnitInfo &)> unitFilter);
};

} // anonymous namespace
Expand Down Expand Up @@ -151,7 +151,8 @@ void SymbolIndexImpl::printStats(raw_ostream &OS) {
void SymbolIndexImpl::dumpProviderFileAssociations(raw_ostream &OS) {
ReadTransaction reader(DBase);
Optional<IDCode> prevProvCode;
reader.foreachProviderAndFileCodeReference([&](IDCode providerCode, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
auto unitFilter = [](IDCode unitCode)->bool { return true; };
reader.foreachProviderAndFileCodeReference(unitFilter, [&](IDCode providerCode, IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
if (!prevProvCode.hasValue() || prevProvCode.getValue() != providerCode) {
OS << reader.getProviderName(providerCode) << '\n';
prevProvCode = providerCode;
Expand All @@ -166,12 +167,12 @@ void SymbolIndexImpl::dumpProviderFileAssociations(raw_ostream &OS) {
}

SymbolDataProviderRef SymbolIndexImpl::createVisibleProviderForCode(IDCode providerCode, ReadTransaction &reader) {
return createProviderForCode(providerCode, reader, [&](ReadTransaction &reader, const UnitInfo &unitInfo) -> bool {
return createProviderForCode(providerCode, reader, [&](const UnitInfo &unitInfo) -> bool {
return VisibilityChecker->isUnitVisible(unitInfo, reader);
});
}

SymbolDataProviderRef SymbolIndexImpl::createProviderForCode(IDCode providerCode, ReadTransaction &reader, function_ref<bool(ReadTransaction &, const UnitInfo &)> unitFilter) {
SymbolDataProviderRef SymbolIndexImpl::createProviderForCode(IDCode providerCode, ReadTransaction &reader, function_ref<bool(const UnitInfo &)> unitFilter) {
StringRef recordName = reader.getProviderName(providerCode);
if (recordName.empty()) {
++NumMissingProvidersLookedUp;
Expand All @@ -180,12 +181,15 @@ SymbolDataProviderRef SymbolIndexImpl::createProviderForCode(IDCode providerCode

Optional<SymbolProviderKind> providerKind;
SmallVector<FileAndTarget, 8> fileRefs;
reader.getProviderFileCodeReferences(providerCode, [&](IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
auto unitCodeFilter = [&reader, unitFilter](IDCode unitCode) -> bool {
auto unitInfo = reader.getUnitInfo(unitCode);
if (unitInfo.isInvalid())
return true;
if (!unitFilter(reader, unitInfo))
return true;
return false;
return unitFilter(reader.getUnitInfo(unitCode));
};
reader.getProviderFileCodeReferences(providerCode, unitCodeFilter, [&](IDCode pathCode, IDCode unitCode, llvm::sys::TimePoint<> modTime, IDCode moduleNameCode, bool isSystem) -> bool {
auto unitInfo = reader.getUnitInfo(unitCode);
assert(!unitInfo.isInvalid());

if (!providerKind.hasValue()) {
providerKind = unitInfo.SymProviderKind;
Expand Down Expand Up @@ -483,7 +487,7 @@ bool SymbolIndexImpl::foreachUnitTestSymbolReferencedByOutputPaths(ArrayRef<Cano
outFileCodes.insert(reader.getFilePathCode(path));
}
for (IDCode providerCode : providerCodes) {
auto provider = createProviderForCode(providerCode, reader, [&](ReadTransaction &reader, const UnitInfo &unitInfo) -> bool {
auto provider = createProviderForCode(providerCode, reader, [&](const UnitInfo &unitInfo) -> bool {
return outFileCodes.count(unitInfo.OutFileCode);
});
if (provider) {
Expand Down