From c24d0dd84dea92e05346b6c13b286ddd2f23ccf9 Mon Sep 17 00:00:00 2001 From: mr-forust Date: Sun, 26 Jul 2026 16:27:33 +0200 Subject: [PATCH] feat(search): optimize duplicate detection --- CHANGELOG.md | 16 ++ CMakeLists.txt | 37 +++ README.md | 1 + VERSION | 2 +- src/main_window.cpp | 4 + src/options_dialog.cpp | 10 + src/options_dialog.h | 1 + src/results_model.cpp | 2 +- src/search_cache.cpp | 54 ++++- src/search_cache.h | 9 +- src/search_core.cpp | 188 ++++++++++++++++ src/search_core.h | 19 ++ src/search_engine.cpp | 256 ++++++++++++++++----- src/search_engine.h | 5 +- tests/options_dialog_test.cpp | 30 +++ tests/search_cache_test.cpp | 112 ++++++++++ tests/search_core_test.cpp | 202 +++++++++++++++++ tests/search_engine_test.cpp | 408 ++++++++++++++++++++++++++++++++++ 18 files changed, 1285 insertions(+), 71 deletions(-) create mode 100644 tests/options_dialog_test.cpp create mode 100644 tests/search_cache_test.cpp create mode 100644 tests/search_engine_test.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index f28352b..0f84fe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to FolderScope are documented here. The project uses [Semantic Versioning](https://semver.org/). Release tags are formatted as `vMAJOR.MINOR.PATCH`. +## [0.2.5] - 2026-07-26 + +### Added + +- Added an optional strict byte-by-byte verification mode for duplicate files. + +### Changed + +- Accelerated duplicate detection with size grouping, sampled block hashes, and + full SHA-256 hashing only for remaining candidates. +- Normalized overlapping search roots and deduplicated file candidates so a + path cannot be compared with itself. +- Strengthened persistent cache invalidation with nanosecond timestamps and + filesystem identity. + ## [0.2.4] - 2026-07-25 ### Fixed @@ -66,3 +81,4 @@ formatted as `vMAJOR.MINOR.PATCH`. [0.2.2]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.2 [0.2.3]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.3 [0.2.4]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.4 +[0.2.5]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.5 diff --git a/CMakeLists.txt b/CMakeLists.txt index 45e946f..4eba91b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,43 @@ if (BUILD_TESTING) target_compile_options(search_core_tests PRIVATE -Wall -Wextra -Wpedantic) add_test(NAME search-core COMMAND search_core_tests) + qt_add_executable(search_cache_tests + src/search_cache.cpp + src/search_cache.h + tests/search_cache_test.cpp + ) + target_link_libraries(search_cache_tests PRIVATE folderscope_core Qt6::Test) + target_compile_options(search_cache_tests PRIVATE -Wall -Wextra -Wpedantic) + target_include_directories(search_cache_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + add_test(NAME search-cache COMMAND search_cache_tests) + + qt_add_executable(search_engine_tests + src/search_cache.cpp + src/search_cache.h + src/search_engine.cpp + src/search_engine.h + tests/search_engine_test.cpp + ) + target_link_libraries(search_engine_tests PRIVATE + folderscope_core Qt6::Concurrent Qt6::Test) + target_compile_options(search_engine_tests PRIVATE -Wall -Wextra -Wpedantic) + target_include_directories(search_engine_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + add_test(NAME search-engine COMMAND search_engine_tests) + + qt_add_executable(options_dialog_tests + src/long_long_spin_box.cpp + src/long_long_spin_box.h + src/options_dialog.cpp + src/options_dialog.h + tests/options_dialog_test.cpp + ) + target_link_libraries(options_dialog_tests PRIVATE + folderscope_core Qt6::Widgets Qt6::Test) + target_compile_options(options_dialog_tests PRIVATE -Wall -Wextra -Wpedantic) + target_include_directories(options_dialog_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + add_test(NAME options-dialog COMMAND options_dialog_tests) + set_tests_properties(options-dialog PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen") + qt_add_executable(long_long_spin_box_tests src/long_long_spin_box.cpp src/long_long_spin_box.h diff --git a/README.md b/README.md index b6e6831..b7f5cd5 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ single window. - search across multiple root folders; - filter by name patterns, size, timestamps, and file attributes; - inspect duplicate files and files with matching names; +- optionally verify duplicate matches byte by byte in strict mode; - search inside file contents, including text and hex; - reuse a persistent, automatically invalidated cache for content matches and duplicate hashes; diff --git a/VERSION b/VERSION index abd4105..3a4036f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.4 +0.2.5 diff --git a/src/main_window.cpp b/src/main_window.cpp index a8d27c0..20acac5 100644 --- a/src/main_window.cpp +++ b/src/main_window.cpp @@ -977,6 +977,8 @@ void MainWindow::saveOptions() settings.setValue(QStringLiteral("mode"), options_.mode); settings.setValue(QStringLiteral("duplicateNameMode"), options_.duplicateNameMode); settings.setValue(QStringLiteral("duplicateNameWithoutExtension"), options_.duplicateNameWithoutExtension); + settings.setValue(QStringLiteral("strictDuplicateComparison"), + options_.strictDuplicateComparison); settings.setValue(QStringLiteral("showDuplicateCopiesOnly"), options_.showDuplicateCopiesOnly); settings.setValue(QStringLiteral("includeSubfoldersInSummary"), options_.includeSubfoldersInSummary); settings.setValue(QStringLiteral("hidden"), options_.hidden); @@ -1023,6 +1025,8 @@ void MainWindow::loadOptions() options_.mode = s.value(QStringLiteral("mode"), options_.mode).toString(); options_.duplicateNameMode = s.value(QStringLiteral("duplicateNameMode"), options_.duplicateNameMode).toString(); options_.duplicateNameWithoutExtension = s.value(QStringLiteral("duplicateNameWithoutExtension"), false).toBool(); + options_.strictDuplicateComparison = + s.value(QStringLiteral("strictDuplicateComparison"), false).toBool(); options_.showDuplicateCopiesOnly = s.value(QStringLiteral("showDuplicateCopiesOnly"), s.value(QStringLiteral("showOnlyDuplicateFiles"), true)).toBool(); diff --git a/src/options_dialog.cpp b/src/options_dialog.cpp index 8ba5c89..91338f5 100644 --- a/src/options_dialog.cpp +++ b/src/options_dialog.cpp @@ -112,6 +112,11 @@ void OptionsDialog::setupUi(const SearchOptions &o) accurateProgress_->setChecked(o.accurateProgress); useCache_ = new QCheckBox(tr("Use persistent cache for hashes and content searches")); useCache_->setChecked(o.useCache); + strictDuplicateComparison_ = + new QCheckBox(tr("Strict duplicate comparison (byte by byte)")); + strictDuplicateComparison_->setObjectName( + QStringLiteral("strictDuplicateComparison")); + strictDuplicateComparison_->setChecked(o.strictDuplicateComparison); maxDepth_ = new QSpinBox; maxDepth_->setRange(0, 1024); maxDepth_->setValue(o.maxDepth); maxDepth_->setSpecialValueText(tr("Unlimited")); maxResults_ = new QSpinBox; maxResults_->setRange(0, 100'000'000); maxResults_->setValue(o.maxResults); @@ -124,6 +129,7 @@ void OptionsDialog::setupUi(const SearchOptions &o) filesForm->addRow(retrieveOwner_); filesForm->addRow(accurateProgress_); filesForm->addRow(useCache_); + filesForm->addRow(strictDuplicateComparison_); filesForm->addRow(tr("Stop after finding:"), maxResults_); tabs->addTab(filesPage, tr("Files & folders")); @@ -221,6 +227,7 @@ SearchOptions OptionsDialog::options() const o.retrieveOwner = retrieveOwner_->isChecked(); o.accurateProgress = accurateProgress_->isChecked(); o.useCache = useCache_->isChecked(); + o.strictDuplicateComparison = strictDuplicateComparison_->isChecked(); o.maxDepth = maxDepth_->value(); o.maxResults = maxResults_->value(); o.mode = mode_->currentText(); @@ -360,6 +367,7 @@ void OptionsDialog::saveProfile() saveBool("accurateProgress", opt.accurateProgress); saveBool("useCache", opt.useCache); saveBool("duplicateNameWithoutExtension", opt.duplicateNameWithoutExtension); + saveBool("strictDuplicateComparison", opt.strictDuplicateComparison); saveInt("lastMinutes", opt.lastMinutes); saveInt("maxDepth", opt.maxDepth); @@ -419,6 +427,8 @@ void OptionsDialog::loadProfile() retrieveOwner_->setChecked(loadBool("retrieveOwner", false)); accurateProgress_->setChecked(loadBool("accurateProgress", true)); useCache_->setChecked(loadBool("useCache", true)); + strictDuplicateComparison_->setChecked( + loadBool("strictDuplicateComparison", false)); maxDepth_->setValue(loadInt("maxDepth", 0)); maxResults_->setValue(loadInt("maxResults", 0)); mode_->setCurrentText(load("mode")); diff --git a/src/options_dialog.h b/src/options_dialog.h index 19f0fb6..8f8c70e 100644 --- a/src/options_dialog.h +++ b/src/options_dialog.h @@ -66,6 +66,7 @@ private: QCheckBox *retrieveOwner_{}; QCheckBox *accurateProgress_{}; QCheckBox *useCache_{}; + QCheckBox *strictDuplicateComparison_{}; QSpinBox *maxDepth_{}; QSpinBox *maxResults_{}; QComboBox *hidden_{}; diff --git a/src/results_model.cpp b/src/results_model.cpp index 62eae89..7999e33 100644 --- a/src/results_model.cpp +++ b/src/results_model.cpp @@ -23,7 +23,7 @@ QVariant ResultsModel::headerData(int section, Qt::Orientation orientation, int if (section == DuplicateNumber) return tr("Identifies one set of identical files"); if (section == DuplicateGroup) - return tr("Order based on the base-folder list; 1 = preferred copy to keep"); + return tr("Most-specific base folder wins; ties use newer files, then path. 1 = preferred copy to keep"); return {}; } if (role != Qt::DisplayRole) diff --git a/src/search_cache.cpp b/src/search_cache.cpp index 71d4990..b93b1ce 100644 --- a/src/search_cache.cpp +++ b/src/search_cache.cpp @@ -10,14 +10,41 @@ #include #include -SearchCache::SearchCache() +SearchCache::SearchCache(QString path) { - const QString cacheRoot = QStandardPaths::writableLocation(QStandardPaths::CacheLocation); - QDir().mkpath(cacheRoot); - path_ = QDir(cacheRoot).filePath(QStringLiteral("search-cache-v1.dat")); + if (path.isEmpty()) { + const QString cacheRoot = QStandardPaths::writableLocation(QStandardPaths::CacheLocation); + QDir().mkpath(cacheRoot); + path_ = QDir(cacheRoot).filePath(QStringLiteral("search-cache-v1.dat")); + } else { + path_ = std::move(path); + } load(); } +bool SearchCache::findSampleHash(const FileRecord &record, QByteArray &digest) +{ + if (!enabled_) + return false; + QMutexLocker lock(&mutex_); + auto it = entries_.find(record.path); + if (it == entries_.end() || !signatureMatches(*it, record) + || it->sampleSha256.isEmpty()) + return false; + it->lastUsed = QDateTime::currentSecsSinceEpoch(); + digest = it->sampleSha256; + return true; +} + +void SearchCache::storeSampleHash(const FileRecord &record, const QByteArray &digest) +{ + if (!enabled_ || digest.isEmpty()) + return; + QMutexLocker lock(&mutex_); + SearchCacheEntry &entry = currentEntry(record); + entry.sampleSha256 = digest; +} + bool SearchCache::findHash(const FileRecord &record, QByteArray &digest) { if (!enabled_) @@ -75,9 +102,10 @@ void SearchCache::save() return; QDataStream stream(&file); stream.setVersion(QDataStream::Qt_6_0); - stream << quint32(0x46534348) << quint32(1) << quint32(entries_.size()); + stream << quint32(0x46534348) << quint32(2) << quint32(entries_.size()); for (auto it = entries_.cbegin(); it != entries_.cend(); ++it) { - stream << it.key() << it->size << it->modified << it->lastUsed << it->sha256; + stream << it.key() << it->size << it->modifiedNs << it->device << it->inode + << it->lastUsed << it->sampleSha256 << it->sha256; stream << quint32(it->contentMatches.size()); for (auto match = it->contentMatches.cbegin(); match != it->contentMatches.cend(); ++match) stream << match.key() << match.value(); @@ -95,7 +123,10 @@ void SearchCache::clear() bool SearchCache::signatureMatches(const SearchCacheEntry &entry, const FileRecord &record) { - return entry.size == record.size && entry.modified == record.modified; + return entry.size == record.size + && entry.modifiedNs == record.modifiedNs + && entry.device == record.device + && entry.inode == record.inode; } SearchCacheEntry &SearchCache::currentEntry(const FileRecord &record) @@ -104,7 +135,9 @@ SearchCacheEntry &SearchCache::currentEntry(const FileRecord &record) if (!signatureMatches(entry, record)) { entry = {}; entry.size = record.size; - entry.modified = record.modified; + entry.modifiedNs = record.modifiedNs; + entry.device = record.device; + entry.inode = record.inode; } entry.lastUsed = QDateTime::currentSecsSinceEpoch(); return entry; @@ -121,13 +154,14 @@ void SearchCache::load() quint32 version = 0; quint32 count = 0; stream >> magic >> version >> count; - if (magic != 0x46534348 || version != 1 || count > 200'000) + if (magic != 0x46534348 || version != 2 || count > 200'000) return; for (quint32 index = 0; index < count && stream.status() == QDataStream::Ok; ++index) { QString path; SearchCacheEntry entry; quint32 contentCount = 0; - stream >> path >> entry.size >> entry.modified >> entry.lastUsed >> entry.sha256; + stream >> path >> entry.size >> entry.modifiedNs >> entry.device >> entry.inode + >> entry.lastUsed >> entry.sampleSha256 >> entry.sha256; stream >> contentCount; if (contentCount > 256) return; diff --git a/src/search_cache.h b/src/search_cache.h index cfaa566..67b7bef 100644 --- a/src/search_cache.h +++ b/src/search_cache.h @@ -9,19 +9,24 @@ struct SearchCacheEntry { qint64 size = -1; - qint64 modified = -1; + qint64 modifiedNs = -1; + quint64 device = 0; + quint64 inode = 0; qint64 lastUsed = 0; + QByteArray sampleSha256; QByteArray sha256; QHash contentMatches; }; class SearchCache { public: - SearchCache(); + explicit SearchCache(QString path = {}); void setEnabled(bool enabled) { enabled_ = enabled; } bool enabled() const { return enabled_; } + bool findSampleHash(const FileRecord &record, QByteArray &digest); + void storeSampleHash(const FileRecord &record, const QByteArray &digest); bool findHash(const FileRecord &record, QByteArray &digest); void storeHash(const FileRecord &record, const QByteArray &digest); std::optional findContent(const FileRecord &record, const QByteArray &queryKey); diff --git a/src/search_core.cpp b/src/search_core.cpp index 1f32747..2d26958 100644 --- a/src/search_core.cpp +++ b/src/search_core.cpp @@ -2,11 +2,16 @@ #include #include +#include #include +#include #include +#include #include +#include #include +#include QStringList patterns(const QString &text) { @@ -54,6 +59,129 @@ bool wildcardMatch(const QString &value, const QStringList &items, Qt::CaseSensi return false; } +namespace { +struct RootCandidate { + QString path; +}; + +bool isDescendantPath(const QString &parent, const QString &child, + QString *relativePath = nullptr) +{ + const QString relative = QDir(parent).relativeFilePath(child); + const bool descendant = relative != QStringLiteral(".") + && relative != QStringLiteral("..") + && !relative.startsWith(QStringLiteral("../")) + && !QDir::isAbsolutePath(relative); + if (descendant && relativePath) + *relativePath = relative; + return descendant; +} + +bool canCollapseInto(const RootCandidate &parent, const RootCandidate &child, + const SearchOptions &options, const QStringList &excludedFolders, + Qt::CaseSensitivity cs) +{ + if (!QFileInfo(parent.path).isDir()) + return false; + + QString relative; + if (!isDescendantPath(parent.path, child.path, &relative)) + return false; + + QString current = parent.path; + const QStringList segments = relative.split(u'/', Qt::SkipEmptyParts); + for (const QString &segment : segments) { + current = QDir(current).filePath(segment); + const QFileInfo info(current); + if ((!options.followLinks && info.isSymLink()) + || (!excludedFolders.isEmpty() + && (wildcardMatch(info.fileName(), excludedFolders, cs) + || wildcardMatch(QDir::cleanPath(current), excludedFolders, cs)))) { + return false; + } + } + return true; +} +} + +EffectiveRoots effectiveSearchRoots(const SearchOptions &options) +{ + QVector unique; + QSet identities; + int skipped = 0; + for (const QString &rootText : patterns(options.roots)) { + const QFileInfo info(rootText); + const QString path = QDir::cleanPath(info.absoluteFilePath()); + const QString canonical = info.canonicalFilePath(); + const QString identity = canonical.isEmpty() + ? path : QDir::cleanPath(canonical); + if (identities.contains(identity)) { + ++skipped; + continue; + } + identities.insert(identity); + unique.push_back({path}); + } + + const QStringList subfolderMasks = patterns(options.subfolderWildcards).isEmpty() + ? QStringList{QStringLiteral("*")} : patterns(options.subfolderWildcards); + const bool universalSubfolderMask = + subfolderMasks.contains(QStringLiteral("*")); + if (!options.recursive || options.maxDepth > 0 || options.maxResults > 0 + || !universalSubfolderMask) { + EffectiveRoots result; + result.skipped = skipped; + for (const RootCandidate &root : unique) { + result.paths.push_back(root.path); + result.priorityPaths.push_back(root.path); + } + return result; + } + + const Qt::CaseSensitivity cs = + options.caseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive; + const QStringList excludedFolders = patterns(options.excludeFolders); + EffectiveRoots result; + result.skipped = skipped; + for (const RootCandidate &root : unique) + result.priorityPaths.push_back(root.path); + for (qsizetype childIndex = 0; childIndex < unique.size(); ++childIndex) { + bool redundant = false; + for (qsizetype parentIndex = 0; parentIndex < unique.size(); ++parentIndex) { + if (parentIndex == childIndex) + continue; + if (canCollapseInto(unique[parentIndex], unique[childIndex], options, + excludedFolders, cs)) { + redundant = true; + break; + } + } + if (redundant) { + ++result.skipped; + } else { + result.paths.push_back(unique[childIndex].path); + } + } + return result; +} + +int preferredRootIndex(const QString &path, const QStringList &roots) +{ + const QString normalizedPath = QDir::cleanPath(QFileInfo(path).absoluteFilePath()); + int bestIndex = -1; + qsizetype bestLength = 0; + for (qsizetype index = 0; index < roots.size(); ++index) { + const QString root = QDir::cleanPath(QFileInfo(roots[index]).absoluteFilePath()); + if (normalizedPath != root && !isDescendantPath(root, normalizedPath)) + continue; + if (root.size() > bestLength) { + bestIndex = int(index); + bestLength = root.size(); + } + } + return bestIndex; +} + bool shouldShowDuplicateResult(bool copiesOnly, int keeperPriority) { if (keeperPriority < 1) @@ -99,6 +227,26 @@ bool isCancelled(const std::atomic_bool *cancelled) { return cancelled && cancelled->load(std::memory_order_relaxed); } + +qint64 modifiedNanoseconds(const struct stat &st) +{ + return qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec; +} + +bool signatureMatches(const FileRecord &record, const struct stat &st) +{ + return record.size == st.st_size + && record.modifiedNs == modifiedNanoseconds(st) + && record.device == quint64(st.st_dev) + && record.inode == quint64(st.st_ino); +} + +bool statMatches(const FileRecord &record) +{ + struct stat st {}; + const QByteArray nativePath = QFile::encodeName(record.path); + return ::stat(nativePath.constData(), &st) == 0 && signatureMatches(record, st); +} } bool fileContains(const QString &path, const SearchOptions &options, @@ -187,6 +335,46 @@ QByteArray sha256(const QString &path, const std::atomic_bool *cancelled) return isCancelled(cancelled) ? QByteArray{} : hash.result(); } +bool fileSignatureMatches(const FileRecord &record) +{ + return statMatches(record); +} + +QByteArray sha256(const FileRecord &record, const std::atomic_bool *cancelled) +{ + if (isCancelled(cancelled) || !statMatches(record)) + return {}; + const QByteArray digest = sha256(record.path, cancelled); + return !digest.isEmpty() && statMatches(record) ? digest : QByteArray{}; +} + +QByteArray sampleSha256(const FileRecord &record, const std::atomic_bool *cancelled) +{ + if (isCancelled(cancelled) || record.size <= duplicateSampleThreshold + || !statMatches(record)) + return {}; + + QFile file(record.path); + if (!file.open(QIODevice::ReadOnly)) + return {}; + + const qint64 middleOffset = (record.size - duplicateSampleBlockSize) / 2; + const std::array offsets{ + 0, middleOffset, record.size - duplicateSampleBlockSize + }; + QCryptographicHash hash(QCryptographicHash::Sha256); + for (const qint64 offset : offsets) { + if (isCancelled(cancelled) || !file.seek(offset)) + return {}; + const QByteArray block = file.read(duplicateSampleBlockSize); + if (block.size() != duplicateSampleBlockSize) + return {}; + hash.addData(block); + } + const QByteArray digest = hash.result(); + return !isCancelled(cancelled) && statMatches(record) ? digest : QByteArray{}; +} + bool parseOptionalIsoDate(const QString &text, qint64 &seconds) { const QString trimmed = text.trimmed(); diff --git a/src/search_core.h b/src/search_core.h index e737379..0186a09 100644 --- a/src/search_core.h +++ b/src/search_core.h @@ -11,6 +11,9 @@ #include #include +inline constexpr qint64 duplicateSampleBlockSize = 64LL * 1024; +inline constexpr qint64 duplicateSampleThreshold = 4LL * 1024 * 1024; + struct SearchOptions { QString roots = QDir::homePath(); QString fileWildcards = QStringLiteral("*"); @@ -41,6 +44,7 @@ struct SearchOptions { QString mode = QStringLiteral("Standard search"); QString duplicateNameMode = QStringLiteral("All files and folders"); bool duplicateNameWithoutExtension = false; + bool strictDuplicateComparison = false; bool showDuplicateCopiesOnly = true; bool includeSubfoldersInSummary = false; QString hidden = QStringLiteral("Any"); @@ -60,6 +64,9 @@ struct FileRecord { qint64 created = 0; qint64 accessed = 0; qint64 changed = 0; + qint64 modifiedNs = 0; + quint64 device = 0; + quint64 inode = 0; QString type; QString owner; QString attributes; @@ -67,6 +74,12 @@ struct FileRecord { int duplicateCopy = 0; }; +struct EffectiveRoots { + QStringList paths; + QStringList priorityPaths; + int skipped = 0; +}; + Q_DECLARE_METATYPE(SearchOptions) Q_DECLARE_METATYPE(FileRecord) Q_DECLARE_METATYPE(QVector) @@ -74,6 +87,8 @@ Q_DECLARE_METATYPE(QVector) QStringList patterns(const QString &text); QStringList exclusionPatterns(const QString &text); bool wildcardMatch(const QString &value, const QStringList &items, Qt::CaseSensitivity cs); +EffectiveRoots effectiveSearchRoots(const SearchOptions &options); +int preferredRootIndex(const QString &path, const QStringList &roots); QString humanSize(qint64 bytes); QString ownerName(uid_t uid); std::optional hexNeedle(QString text); @@ -82,5 +97,9 @@ bool fileContains(const QString &path, const SearchOptions &options, bool filesEqual(const QString &leftPath, const QString &rightPath, const std::atomic_bool *cancelled = nullptr); QByteArray sha256(const QString &path, const std::atomic_bool *cancelled = nullptr); +QByteArray sha256(const FileRecord &record, const std::atomic_bool *cancelled = nullptr); +QByteArray sampleSha256(const FileRecord &record, + const std::atomic_bool *cancelled = nullptr); +bool fileSignatureMatches(const FileRecord &record); bool parseOptionalIsoDate(const QString &text, qint64 &seconds); bool shouldShowDuplicateResult(bool copiesOnly, int keeperPriority); diff --git a/src/search_engine.cpp b/src/search_engine.cpp index e4b1efe..63b21a7 100644 --- a/src/search_engine.cpp +++ b/src/search_engine.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,27 @@ namespace fs = std::filesystem; +namespace { +struct HashedRecord { + QByteArray digest; + FileRecord record; +}; + +bool keepFirst(const FileRecord &left, const FileRecord &right, + const QStringList &priorityRoots) +{ + const int leftRoot = preferredRootIndex(left.path, priorityRoots); + const int rightRoot = preferredRootIndex(right.path, priorityRoots); + const int leftPriority = leftRoot < 0 ? priorityRoots.size() : leftRoot; + const int rightPriority = rightRoot < 0 ? priorityRoots.size() : rightRoot; + if (leftPriority != rightPriority) + return leftPriority < rightPriority; + if (left.modified != right.modified) + return left.modified > right.modified; + return left.path < right.path; +} +} + void SearchEngine::search(const SearchOptions &o) { cancelled_.store(false, std::memory_order_relaxed); @@ -33,6 +55,8 @@ void SearchEngine::search(const SearchOptions &o) cacheMisses_.store(0, std::memory_order_relaxed); const auto started = std::chrono::steady_clock::now(); QVector candidates; + QSet candidatePaths; + const EffectiveRoots roots = effectiveSearchRoots(o); const Qt::CaseSensitivity cs = o.caseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive; const QStringList fileMasks = patterns(o.fileWildcards).isEmpty() ? QStringList{QStringLiteral("*")} : patterns(o.fileWildcards); @@ -46,7 +70,7 @@ void SearchEngine::search(const SearchOptions &o) if (o.accurateProgress) { emit phaseProgress(tr("Counting files"), 0, 0); - for (const QString &rootText : patterns(o.roots)) { + for (const QString &rootText : roots.paths) { if (cancelled_.load(std::memory_order_relaxed)) break; std::error_code ec; @@ -84,7 +108,7 @@ void SearchEngine::search(const SearchOptions &o) emit phaseProgress(tr("Scanning files"), 0, totalEntries); } - for (const QString &rootText : patterns(o.roots)) { + for (const QString &rootText : roots.paths) { if (cancelled_.load(std::memory_order_relaxed)) break; std::error_code ec; @@ -96,7 +120,8 @@ void SearchEngine::search(const SearchOptions &o) auto process = [&](const fs::directory_entry &entry, const fs::path &baseRoot) { if (cancelled_.load(std::memory_order_relaxed)) return; - const QString path = QString::fromStdString(entry.path().string()); + const QString path = QDir::cleanPath(QFileInfo( + QString::fromStdString(entry.path().string())).absoluteFilePath()); const QString name = QString::fromStdString(entry.path().filename().string()); const bool isDir = entry.is_directory(ec); if ((isDir && !o.findFolders) || (!isDir && !o.findFiles)) @@ -147,6 +172,9 @@ void SearchEngine::search(const SearchOptions &o) || !attrMatches(o.readonly, readOnly) || !attrMatches(o.executable, executable)) return; + if (candidatePaths.contains(path)) + return; + candidatePaths.insert(path); QString attrs; attrs += isDir ? u'd' : u'-'; @@ -160,6 +188,8 @@ void SearchEngine::search(const SearchOptions &o) st.st_mtime, QFileInfo(path).birthTime().isValid() ? QFileInfo(path).birthTime().toSecsSinceEpoch() : 0, st.st_atime, st.st_ctime, + qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec, + quint64(st.st_dev), quint64(st.st_ino), isDir ? QStringLiteral("Folder") : QStringLiteral("File"), o.retrieveOwner ? ownerName(st.st_uid) : QString{}, attrs, 0, 0 }); @@ -237,68 +267,122 @@ void SearchEngine::search(const SearchOptions &o) candidates.resize(o.maxResults); } + QThreadPool hashPool; + const int idealThreads = QThread::idealThreadCount(); + hashPool.setMaxThreadCount(std::clamp( + idealThreads > 0 ? idealThreads / 2 : 1, 1, 4)); + QVector results; if (o.mode == QStringLiteral("Duplicates search") || o.mode == QStringLiteral("Non-Duplicates search")) { std::unordered_map> bySize; - for (const auto &record : candidates) - if (record.type == QStringLiteral("File")) + QSet duplicateInputPaths; + for (const auto &record : candidates) { + if (record.type == QStringLiteral("File") + && !duplicateInputPaths.contains(record.path)) { + duplicateInputPaths.insert(record.path); bySize[record.size].push_back(record); - quint64 hashTotal = 0; - for (const auto &[size, records] : bySize) { - Q_UNUSED(size); - if (records.size() > 1) - hashTotal += records.size(); + } } - std::atomic hashed = 0; - if (hashTotal) - emit phaseProgress(tr("Checking duplicates"), 0, hashTotal); - int group = 1; - QSet duplicatedPaths; - for (auto &[size, sameSize] : bySize) { - if (cancelled_.load(std::memory_order_relaxed)) - break; - Q_UNUSED(size); - if (sameSize.size() < 2) + + QVector sampleCandidates; + QVector fullHashCandidates; + for (const auto &[size, records] : bySize) { + if (records.size() < 2) continue; - auto hashes = QtConcurrent::blockingMapped(sameSize, [this, &hashed, hashTotal](const FileRecord &r) { - auto result = std::pair{cachedSha256(r), r}; - const quint64 done = hashed.fetch_add(1, std::memory_order_relaxed) + 1; - if (done == hashTotal || done % 16 == 0) - emit phaseProgress(tr("Checking duplicates"), done, hashTotal); + if (size > duplicateSampleThreshold) + sampleCandidates += records; + else + fullHashCandidates += records; + } + + std::atomic sampled = 0; + const qsizetype sampleTotal = sampleCandidates.size(); + if (sampleTotal) + emit phaseProgress(tr("Sampling duplicate candidates"), 0, sampleTotal); + const auto samples = QtConcurrent::blockingMapped( + &hashPool, sampleCandidates, + [this, &sampled, sampleTotal](const FileRecord &record) { + HashedRecord result{cachedSampleSha256(record), record}; + const qsizetype done = sampled.fetch_add(1, std::memory_order_relaxed) + 1; + if (done == sampleTotal || done % 16 == 0) + emit phaseProgress(tr("Sampling duplicate candidates"), done, sampleTotal); return result; }); - if (cancelled_.load(std::memory_order_relaxed)) - break; - QHash> sameHash; - for (auto &[hash, record] : hashes) - if (!hash.isEmpty()) - sameHash[hash].push_back(record); - for (const auto &hashCandidates : sameHash) { + if (finishIfCancelled()) + return; + + QHash>> bySample; + for (const auto &sample : samples) + if (!sample.digest.isEmpty()) + bySample[sample.record.size][sample.digest].push_back(sample.record); + for (const auto &sameSize : bySample) + for (const auto &sameSample : sameSize) + if (sameSample.size() > 1) + fullHashCandidates += sameSample; + + std::atomic hashed = 0; + const qsizetype hashTotal = fullHashCandidates.size(); + if (hashTotal) + emit phaseProgress(tr("Hashing duplicate candidates"), 0, hashTotal); + const auto hashes = QtConcurrent::blockingMapped( + &hashPool, fullHashCandidates, + [this, &hashed, hashTotal](const FileRecord &record) { + HashedRecord result{cachedSha256(record), record}; + const qsizetype done = hashed.fetch_add(1, std::memory_order_relaxed) + 1; + if (done == hashTotal || done % 16 == 0) + emit phaseProgress(tr("Hashing duplicate candidates"), done, hashTotal); + return result; + }); + if (finishIfCancelled()) + return; + + QHash>> byHash; + for (const auto &hash : hashes) + if (!hash.digest.isEmpty()) + byHash[hash.record.size][hash.digest].push_back(hash.record); + + int group = 1; + QSet duplicatedPaths; + for (const auto &sameSize : byHash) { + for (const auto &hashCandidates : sameSize) { if (cancelled_.load(std::memory_order_relaxed)) break; if (hashCandidates.size() < 2) continue; QVector> exactGroups; - for (const auto &record : hashCandidates) { - if (cancelled_.load(std::memory_order_relaxed)) - break; - bool placed = false; - for (auto &exact : exactGroups) { - if (filesEqual(exact.front().path, record.path, &cancelled_)) { - exact.push_back(record); - placed = true; + if (!o.strictDuplicateComparison) { + exactGroups.push_back(hashCandidates); + } else { + for (const auto &record : hashCandidates) { + if (cancelled_.load(std::memory_order_relaxed)) break; + bool placed = false; + for (auto &exact : exactGroups) { + if (fileSignatureMatches(exact.front()) + && fileSignatureMatches(record) + && filesEqual(exact.front().path, record.path, &cancelled_) + && fileSignatureMatches(exact.front()) + && fileSignatureMatches(record)) { + exact.push_back(record); + placed = true; + break; + } } + if (!placed) + exactGroups.push_back({record}); } - if (!placed) - exactGroups.push_back({record}); } for (auto &exact : exactGroups) { if (cancelled_.load(std::memory_order_relaxed)) break; if (exact.size() < 2) continue; + std::stable_sort(exact.begin(), exact.end(), + [&roots](const FileRecord &left, + const FileRecord &right) { + return keepFirst(left, right, roots.priorityPaths); + }); int duplicateCopy = 1; for (auto &record : exact) { record.group = group; @@ -343,13 +427,50 @@ void SearchEngine::search(const SearchOptions &o) [](const FileRecord &r) { return r.type != QStringLiteral("File"); }), sameName.end()); if (sameName.size() < 2) continue; + std::stable_sort(sameName.begin(), sameName.end(), + [&roots](const FileRecord &left, + const FileRecord &right) { + return keepFirst(left, right, roots.priorityPaths); + }); if (o.duplicateNameMode.contains(QStringLiteral("identical"), Qt::CaseInsensitive)) { - const QByteArray firstHash = cachedSha256(sameName.front()); - bool identical = !firstHash.isEmpty(); - for (qsizetype i = 1; identical && i < sameName.size(); ++i) { - identical = cachedSha256(sameName[i]) == firstHash - && filesEqual( - sameName.front().path, sameName[i].path, &cancelled_); + bool identical = std::all_of( + sameName.cbegin(), sameName.cend(), + [&sameName](const FileRecord &record) { + return record.size == sameName.front().size; + }); + if (identical && sameName.front().size > duplicateSampleThreshold) { + const auto samples = QtConcurrent::blockingMapped( + &hashPool, sameName, [this](const FileRecord &record) { + return cachedSampleSha256(record); + }); + identical = !samples.isEmpty() && !samples.front().isEmpty() + && std::all_of(samples.cbegin(), samples.cend(), + [&samples](const QByteArray &digest) { + return digest == samples.front(); + }); + } + QVector hashes; + if (identical) { + hashes = QtConcurrent::blockingMapped( + &hashPool, sameName, [this](const FileRecord &record) { + return cachedSha256(record); + }); + identical = !hashes.isEmpty() && !hashes.front().isEmpty() + && std::all_of(hashes.cbegin(), hashes.cend(), + [&hashes](const QByteArray &digest) { + return digest == hashes.front(); + }); + } + if (identical && o.strictDuplicateComparison) { + const FileRecord &first = sameName.front(); + for (qsizetype index = 1; identical && index < sameName.size(); ++index) { + const FileRecord &record = sameName[index]; + identical = fileSignatureMatches(first) + && fileSignatureMatches(record) + && filesEqual(first.path, record.path, &cancelled_) + && fileSignatureMatches(first) + && fileSignatureMatches(record); + } } const bool wantsNonIdentical = o.duplicateNameMode.startsWith(QStringLiteral("Only non")); if (identical == wantsNonIdentical) @@ -376,7 +497,7 @@ void SearchEngine::search(const SearchOptions &o) while (parent.cdUp()) { const QString ancestor = parent.absolutePath(); bool belongsToRoot = false; - for (const QString &root : patterns(o.roots)) { + for (const QString &root : roots.paths) { if (ancestor == QDir(root).absolutePath()) { belongsToRoot = true; break; @@ -401,8 +522,8 @@ void SearchEngine::search(const SearchOptions &o) } const QFileInfo info(it.key()); results.push_back({it.key(), info.fileName(), info.absolutePath(), total, totalOnDisk, - newest, 0, 0, 0, tr("%1 files").arg(it.value().size()), - {}, {}, 0, 0}); + newest, 0, 0, 0, 0, 0, 0, + tr("%1 files").arg(it.value().size()), {}, {}, 0, 0}); } if (finishIfCancelled()) return; @@ -415,21 +536,42 @@ void SearchEngine::search(const SearchOptions &o) const double seconds = std::chrono::duration( std::chrono::steady_clock::now() - started).count(); cache_.save(); - emit finished(results, tr("%1 results, %2 entries scanned in %3 seconds; cache: %4 hits") - .arg(results.size()).arg(scanned).arg(seconds, 0, 'f', 2) - .arg(cacheHits_.load(std::memory_order_relaxed))); + QString message = tr("%1 results, %2 entries scanned in %3 seconds; cache: %4 hits") + .arg(results.size()).arg(scanned).arg(seconds, 0, 'f', 2) + .arg(cacheHits_.load(std::memory_order_relaxed)); + if (roots.skipped > 0) + message += tr("; %1 redundant roots skipped").arg(roots.skipped); + emit finished(results, message); +} + +QByteArray SearchEngine::cachedSampleSha256(const FileRecord &record) +{ + QByteArray digest; + if (cache_.findSampleHash(record, digest)) { + if (!fileSignatureMatches(record)) + return {}; + cacheHits_.fetch_add(1, std::memory_order_relaxed); + return digest; + } + cacheMisses_.fetch_add(1, std::memory_order_relaxed); + digest = sampleSha256(record, &cancelled_); + if (!digest.isEmpty() && !cancelled_.load(std::memory_order_relaxed)) + cache_.storeSampleHash(record, digest); + return digest; } QByteArray SearchEngine::cachedSha256(const FileRecord &record) { QByteArray digest; if (cache_.findHash(record, digest)) { + if (!fileSignatureMatches(record)) + return {}; cacheHits_.fetch_add(1, std::memory_order_relaxed); return digest; } cacheMisses_.fetch_add(1, std::memory_order_relaxed); - digest = sha256(record.path, &cancelled_); - if (!cancelled_.load(std::memory_order_relaxed)) + digest = sha256(record, &cancelled_); + if (!digest.isEmpty() && !cancelled_.load(std::memory_order_relaxed)) cache_.storeHash(record, digest); return digest; } @@ -441,6 +583,8 @@ bool SearchEngine::cachedFileContains(const FileRecord &record, const SearchOpti stream << o.contains << o.binary << o.multipleValues << o.multipleAnd << o.caseSensitive; const QByteArray queryKey = QCryptographicHash::hash(queryData, QCryptographicHash::Sha256); if (const auto cached = cache_.findContent(record, queryKey)) { + if (!fileSignatureMatches(record)) + return false; cacheHits_.fetch_add(1, std::memory_order_relaxed); return *cached; } diff --git a/src/search_engine.h b/src/search_engine.h index 3898ed7..6a5decc 100644 --- a/src/search_engine.h +++ b/src/search_engine.h @@ -6,6 +6,7 @@ #include #include +#include #include "search_core.h" #include "search_cache.h" @@ -13,7 +14,8 @@ class SearchEngine final : public QObject { Q_OBJECT public: - explicit SearchEngine(QObject *parent = nullptr) : QObject(parent) {} + explicit SearchEngine(QObject *parent = nullptr, QString cachePath = {}) + : QObject(parent), cache_(std::move(cachePath)) {} public slots: void stop() { cancelled_.store(true, std::memory_order_relaxed); } @@ -32,6 +34,7 @@ signals: void finished(const QVector &results, const QString &message); private: + QByteArray cachedSampleSha256(const FileRecord &record); QByteArray cachedSha256(const FileRecord &record); bool cachedFileContains(const FileRecord &record, const SearchOptions &o); diff --git a/tests/options_dialog_test.cpp b/tests/options_dialog_test.cpp new file mode 100644 index 0000000..b1577a6 --- /dev/null +++ b/tests/options_dialog_test.cpp @@ -0,0 +1,30 @@ +#include "options_dialog.h" + +#include +#include + +class OptionsDialogTest final : public QObject { + Q_OBJECT + +private slots: + void exposesStrictDuplicateComparison() + { + SearchOptions options; + QVERIFY(!options.strictDuplicateComparison); + options.strictDuplicateComparison = true; + + OptionsDialog dialog(options); + auto *strict = dialog.findChild( + QStringLiteral("strictDuplicateComparison")); + QVERIFY(strict); + QVERIFY(strict->isChecked()); + QVERIFY(dialog.options().strictDuplicateComparison); + + strict->setChecked(false); + QVERIFY(!dialog.options().strictDuplicateComparison); + } +}; + +QTEST_MAIN(OptionsDialogTest) + +#include "options_dialog_test.moc" diff --git a/tests/search_cache_test.cpp b/tests/search_cache_test.cpp new file mode 100644 index 0000000..31e5d18 --- /dev/null +++ b/tests/search_cache_test.cpp @@ -0,0 +1,112 @@ +#include "search_cache.h" + +#include +#include +#include +#include + +class SearchCacheTest final : public QObject { + Q_OBJECT + +private: + static FileRecord record() + { + FileRecord value; + value.path = QStringLiteral("/tmp/cache-test.bin"); + value.size = 1234; + value.modifiedNs = 5678; + value.device = 9; + value.inode = 10; + return value; + } + +private slots: + void storesAndReloadsBothHashes() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + const QString path = directory.filePath(QStringLiteral("cache.dat")); + const FileRecord value = record(); + const QByteArray sample("sample"); + const QByteArray full("full"); + const QByteArray query("query"); + + { + SearchCache cache(path); + cache.storeSampleHash(value, sample); + cache.storeHash(value, full); + cache.storeContent(value, query, true); + cache.save(); + } + + SearchCache cache(path); + QByteArray digest; + QVERIFY(cache.findSampleHash(value, digest)); + QCOMPARE(digest, sample); + QVERIFY(cache.findHash(value, digest)); + QCOMPARE(digest, full); + QCOMPARE(cache.findContent(value, query), std::optional(true)); + } + + void invalidatesEverySignatureField() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + SearchCache cache(directory.filePath(QStringLiteral("cache.dat"))); + const FileRecord value = record(); + cache.storeSampleHash(value, QByteArray("sample")); + cache.storeHash(value, QByteArray("full")); + + const auto misses = [&cache](FileRecord changed) { + QByteArray digest; + return !cache.findSampleHash(changed, digest) + && !cache.findHash(changed, digest); + }; + + FileRecord changed = value; + ++changed.size; + QVERIFY(misses(changed)); + changed = value; + ++changed.modifiedNs; + QVERIFY(misses(changed)); + changed = value; + ++changed.device; + QVERIFY(misses(changed)); + changed = value; + ++changed.inode; + QVERIFY(misses(changed)); + } + + void ignoresVersionOneData() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + const QString path = directory.filePath(QStringLiteral("cache.dat")); + QFile file(path); + QVERIFY(file.open(QIODevice::WriteOnly)); + QDataStream stream(&file); + stream.setVersion(QDataStream::Qt_6_0); + stream << quint32(0x46534348) << quint32(1) << quint32(0); + file.close(); + + SearchCache cache(path); + QByteArray digest; + QVERIFY(!cache.findHash(record(), digest)); + cache.storeHash(record(), QByteArray("full")); + cache.save(); + + QFile saved(path); + QVERIFY(saved.open(QIODevice::ReadOnly)); + QDataStream savedStream(&saved); + savedStream.setVersion(QDataStream::Qt_6_0); + quint32 magic = 0; + quint32 version = 0; + savedStream >> magic >> version; + QCOMPARE(magic, quint32(0x46534348)); + QCOMPARE(version, quint32(2)); + } +}; + +QTEST_GUILESS_MAIN(SearchCacheTest) + +#include "search_cache_test.moc" diff --git a/tests/search_core_test.cpp b/tests/search_core_test.cpp index 6f97bba..d904a0d 100644 --- a/tests/search_core_test.cpp +++ b/tests/search_core_test.cpp @@ -5,6 +5,35 @@ #include #include +#include + +namespace { +FileRecord recordFor(const QString &path) +{ + struct stat st {}; + const QByteArray nativePath = QFile::encodeName(path); + if (::stat(nativePath.constData(), &st) != 0) + return {}; + FileRecord record; + record.path = path; + record.size = st.st_size; + record.modified = st.st_mtime; + record.modifiedNs = + qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec; + record.device = st.st_dev; + record.inode = st.st_ino; + record.type = QStringLiteral("File"); + return record; +} + +bool writeFile(const QString &path, const QByteArray &data) +{ + QFile file(path); + return file.open(QIODevice::WriteOnly) + && file.write(data) == data.size(); +} +} + class SearchCoreTest final : public QObject { Q_OBJECT @@ -23,6 +52,113 @@ private slots: Qt::CaseSensitive)); } + void collapsesRedundantSearchRoots() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + QDir root(directory.path()); + QVERIFY(root.mkpath(QStringLiteral("child"))); + QVERIFY(root.mkpath(QStringLiteral("misc"))); + + SearchOptions options; + const QString child = directory.filePath(QStringLiteral("child")); + const QString misc = directory.filePath(QStringLiteral("misc")); + options.roots = QStringLiteral("%1;%2/;%1;%3") + .arg(child, directory.path(), misc); + options.excludeFolders = QStringLiteral("unrelated"); + + const EffectiveRoots effective = effectiveSearchRoots(options); + QCOMPARE(effective.paths, QStringList({directory.path()})); + QCOMPARE(effective.skipped, 3); + } + + void preservesNestedRootsThatExpandCoverage() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + QDir root(directory.path()); + QVERIFY(root.mkpath(QStringLiteral("child"))); + const QString child = directory.filePath(QStringLiteral("child")); + + SearchOptions options; + options.roots = directory.path() + u';' + child; + + options.recursive = false; + QCOMPARE(effectiveSearchRoots(options).paths.size(), 2); + + options.recursive = true; + options.maxDepth = 1; + QCOMPARE(effectiveSearchRoots(options).paths.size(), 2); + + options.maxDepth = 0; + options.maxResults = 1; + QCOMPARE(effectiveSearchRoots(options).paths.size(), 2); + + options.maxResults = 0; + options.subfolderWildcards = QStringLiteral("matched-*"); + QCOMPARE(effectiveSearchRoots(options).paths.size(), 2); + + options.subfolderWildcards = QStringLiteral("*"); + options.excludeFolders = QStringLiteral("child"); + QCOMPARE(effectiveSearchRoots(options).paths.size(), 2); + + options.excludeFolders = QStringLiteral("unrelated"); + QCOMPARE(effectiveSearchRoots(options).paths, + QStringList({directory.path()})); + } + + void respectsSymlinksWhenCollapsingRoots() + { + QTemporaryDir directory; + QTemporaryDir external; + QVERIFY(directory.isValid()); + QVERIFY(external.isValid()); + QDir externalRoot(external.path()); + QVERIFY(externalRoot.mkpath(QStringLiteral("child"))); + const QString link = directory.filePath(QStringLiteral("link")); + QVERIFY(QFile::link(external.path(), link)); + const QString linkedChild = QDir(link).filePath(QStringLiteral("child")); + + SearchOptions options; + options.roots = directory.path() + u';' + linkedChild; + options.followLinks = false; + QCOMPARE(effectiveSearchRoots(options).paths.size(), 2); + + options.followLinks = true; + QCOMPARE(effectiveSearchRoots(options).paths, + QStringList({directory.path()})); + + options.roots = external.path() + u';' + link; + const EffectiveRoots aliases = effectiveSearchRoots(options); + QCOMPARE(aliases.paths.size(), 1); + QCOMPARE(aliases.skipped, 1); + } + + void prefersTheMostSpecificConfiguredRoot() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + QDir root(directory.path()); + QVERIFY(root.mkpath(QStringLiteral("Temp"))); + QVERIFY(root.mkpath(QStringLiteral("misc"))); + const QString temp = directory.filePath(QStringLiteral("Temp")); + const QString misc = directory.filePath(QStringLiteral("misc")); + const QStringList roots{directory.path(), temp, misc}; + + QCOMPARE(preferredRootIndex( + directory.filePath(QStringLiteral("base.mp4")), roots), 0); + QCOMPARE(preferredRootIndex( + QDir(temp).filePath(QStringLiteral("copy.mp4")), roots), 1); + QCOMPARE(preferredRootIndex( + QDir(misc).filePath(QStringLiteral("copy.mp4")), roots), 2); + QCOMPARE(preferredRootIndex( + directory.filePath(QStringLiteral("unrelated/file.mp4")), roots), 0); + + const QStringList similarRoots{directory.filePath(QStringLiteral("foo"))}; + QCOMPARE(preferredRootIndex( + directory.filePath(QStringLiteral("foobar/file.mp4")), similarRoots), -1); + } + void parsesOptionalIsoDates() { qint64 seconds = -1; @@ -108,6 +244,72 @@ private slots: QVERIFY(!filesEqual(path, path, &cancelled)); QVERIFY(sha256(path, &cancelled).isEmpty()); } + + void samplesLargeFilesAndValidatesSignatures() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + const QByteArray content(duplicateSampleThreshold + 1024 * 1024, 'x'); + const QString firstPath = directory.filePath(QStringLiteral("first.bin")); + const QString samePath = directory.filePath(QStringLiteral("same.bin")); + const QString outsidePath = directory.filePath(QStringLiteral("outside.bin")); + const QString startPath = directory.filePath(QStringLiteral("start.bin")); + const QString middlePath = directory.filePath(QStringLiteral("middle.bin")); + const QString endPath = directory.filePath(QStringLiteral("end.bin")); + + QVERIFY(writeFile(firstPath, content)); + QVERIFY(writeFile(samePath, content)); + + QByteArray outside = content; + outside[1024 * 1024] = 'y'; + QVERIFY(writeFile(outsidePath, outside)); + QByteArray start = content; + start[0] = 'y'; + QVERIFY(writeFile(startPath, start)); + QByteArray middle = content; + middle[(content.size() - duplicateSampleBlockSize) / 2] = 'y'; + QVERIFY(writeFile(middlePath, middle)); + QByteArray end = content; + end[end.size() - 1] = 'y'; + QVERIFY(writeFile(endPath, end)); + + const FileRecord first = recordFor(firstPath); + const QByteArray sample = sampleSha256(first); + QVERIFY(!sample.isEmpty()); + QCOMPARE(sampleSha256(recordFor(samePath)), sample); + QCOMPARE(sampleSha256(recordFor(outsidePath)), sample); + QVERIFY(sampleSha256(recordFor(startPath)) != sample); + QVERIFY(sampleSha256(recordFor(middlePath)) != sample); + QVERIFY(sampleSha256(recordFor(endPath)) != sample); + QVERIFY(sha256(recordFor(outsidePath)) != sha256(first)); + + std::atomic_bool cancelled = true; + QVERIFY(sampleSha256(first, &cancelled).isEmpty()); + + const QString shortPath = directory.filePath(QStringLiteral("short.bin")); + QVERIFY(writeFile(shortPath, QByteArray("short"))); + QVERIFY(sampleSha256(recordFor(shortPath)).isEmpty()); + QVERIFY(!sha256(recordFor(shortPath)).isEmpty()); + + const QString emptyPath = directory.filePath(QStringLiteral("empty.bin")); + QVERIFY(writeFile(emptyPath, {})); + QVERIFY(sampleSha256(recordFor(emptyPath)).isEmpty()); + QVERIFY(!sha256(recordFor(emptyPath)).isEmpty()); + + const QString replacedPath = directory.filePath(QStringLiteral("replaced.bin")); + QVERIFY(writeFile(replacedPath, content)); + const FileRecord replaced = recordFor(replacedPath); + QVERIFY(QFile::remove(replacedPath)); + QVERIFY(writeFile(replacedPath, content)); + QVERIFY(!fileSignatureMatches(replaced)); + QVERIFY(sha256(replaced).isEmpty()); + QVERIFY(sampleSha256(replaced).isEmpty()); + + FileRecord missing = first; + missing.path = directory.filePath(QStringLiteral("missing.bin")); + QVERIFY(sha256(missing).isEmpty()); + QVERIFY(sampleSha256(missing).isEmpty()); + } }; QTEST_GUILESS_MAIN(SearchCoreTest) diff --git a/tests/search_engine_test.cpp b/tests/search_engine_test.cpp new file mode 100644 index 0000000..f296f04 --- /dev/null +++ b/tests/search_engine_test.cpp @@ -0,0 +1,408 @@ +#include "search_engine.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace { +bool writeFile(const QString &path, const QByteArray &data) +{ + QFile file(path); + return file.open(QIODevice::WriteOnly) + && file.write(data) == data.size(); +} + +QVector runSearch(SearchEngine &engine, SearchOptions options) +{ + QSignalSpy finished(&engine, &SearchEngine::finished); + engine.search(options); + if (finished.size() != 1) + return {}; + return finished.takeFirst().at(0).value>(); +} + +QSet namesOf(const QVector &records) +{ + QSet names; + for (const auto &record : records) + names.insert(record.name); + return names; +} + +SearchOptions optionsFor(const QString &root, const QString &mode) +{ + SearchOptions options; + options.roots = root; + options.mode = mode; + options.recursive = true; + options.accurateProgress = false; + options.showDuplicateCopiesOnly = false; + options.useCache = false; + return options; +} +} + +class SearchEngineTest final : public QObject { + Q_OBJECT + +private slots: + void filtersBySampleThenFullHashAndSupportsStrictMode() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + QVERIFY(writeFile(directory.filePath(QStringLiteral("duplicate-a.bin")), + QByteArray("same content"))); + QVERIFY(writeFile(directory.filePath(QStringLiteral("duplicate-b.bin")), + QByteArray("same content"))); + QVERIFY(writeFile(directory.filePath(QStringLiteral("unique-size.bin")), + QByteArray("unique"))); + + const QByteArray large(duplicateSampleThreshold + 1024 * 1024, 'x'); + QByteArray different = large; + different[1024 * 1024] = 'y'; + QVERIFY(writeFile(directory.filePath(QStringLiteral("sample-a.bin")), large)); + QVERIFY(writeFile(directory.filePath(QStringLiteral("sample-b.bin")), different)); + + SearchEngine engine(nullptr, directory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + directory.path(), QStringLiteral("Duplicates search")); + QVector results = runSearch(engine, options); + QCOMPARE(namesOf(results), + QSet({QStringLiteral("duplicate-a.bin"), + QStringLiteral("duplicate-b.bin")})); + + options.strictDuplicateComparison = true; + results = runSearch(engine, options); + QCOMPARE(namesOf(results), + QSet({QStringLiteral("duplicate-a.bin"), + QStringLiteral("duplicate-b.bin")})); + + options.mode = QStringLiteral("Non-Duplicates search"); + results = runSearch(engine, options); + QCOMPARE(namesOf(results), + QSet({QStringLiteral("unique-size.bin"), + QStringLiteral("sample-a.bin"), + QStringLiteral("sample-b.bin")})); + } + + void appliesContentPipelineToDuplicateNames() + { + QTemporaryDir directory; + QVERIFY(directory.isValid()); + QDir root(directory.path()); + QVERIFY(root.mkpath(QStringLiteral("one"))); + QVERIFY(root.mkpath(QStringLiteral("two"))); + const QString first = + directory.filePath(QStringLiteral("one/report.bin")); + const QString second = + directory.filePath(QStringLiteral("two/report.bin")); + QVERIFY(writeFile(first, QByteArray("same"))); + QVERIFY(writeFile(second, QByteArray("same"))); + + SearchEngine engine(nullptr, directory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + directory.path(), QStringLiteral("Duplicate names search")); + options.duplicateNameMode = QStringLiteral("Only identical content"); + QVector results = runSearch(engine, options); + QCOMPARE(results.size(), 2); + + QVERIFY(writeFile(second, QByteArray("diff"))); + results = runSearch(engine, options); + QVERIFY(results.isEmpty()); + + options.duplicateNameMode = QStringLiteral("Only non-identical content"); + results = runSearch(engine, options); + QCOMPARE(results.size(), 2); + + options.strictDuplicateComparison = true; + results = runSearch(engine, options); + QCOMPARE(results.size(), 2); + } + + void assignsKeeperPriorityFromConfiguredRoots() + { + QTemporaryDir directory; + QTemporaryDir cacheDirectory; + QVERIFY(directory.isValid()); + QVERIFY(cacheDirectory.isValid()); + QDir root(directory.path()); + QVERIFY(root.mkpath(QStringLiteral("Temp"))); + QVERIFY(root.mkpath(QStringLiteral("misc"))); + const QString temp = directory.filePath(QStringLiteral("Temp")); + const QString misc = directory.filePath(QStringLiteral("misc")); + const QByteArray content("same"); + const QString basePath = directory.filePath(QStringLiteral("base.mp4")); + const QString tempPath = QDir(temp).filePath(QStringLiteral("temp.mp4")); + const QString miscPath = QDir(misc).filePath(QStringLiteral("misc.mp4")); + QVERIFY(writeFile(basePath, content)); + QVERIFY(writeFile(tempPath, content)); + QVERIFY(writeFile(miscPath, content)); + + SearchEngine engine(nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + QStringList({directory.path(), temp, misc}).join(u';'), + QStringLiteral("Duplicates search")); + QVector results = runSearch(engine, options); + QCOMPARE(results.size(), 3); + + QHash priorities; + for (const FileRecord &record : results) + priorities.insert(record.path, record.duplicateCopy); + QCOMPARE(priorities.value(basePath), 1); + QCOMPARE(priorities.value(tempPath), 2); + QCOMPARE(priorities.value(miscPath), 3); + } + + void prefersNewerFilesWhenRootPriorityTies() + { + QTemporaryDir directory; + QTemporaryDir cacheDirectory; + QVERIFY(directory.isValid()); + QVERIFY(cacheDirectory.isValid()); + QDir root(directory.path()); + QVERIFY(root.mkpath(QStringLiteral("child"))); + const QString older = QDir(directory.filePath(QStringLiteral("child"))) + .filePath(QStringLiteral("older.mp4")); + const QString newer = directory.filePath(QStringLiteral("newer.mp4")); + QVERIFY(writeFile(older, QByteArray("same"))); + QVERIFY(writeFile(newer, QByteArray("same"))); + QFile olderFile(older); + QVERIFY(olderFile.open(QIODevice::ReadOnly)); + QVERIFY(olderFile.setFileTime(QDateTime::fromSecsSinceEpoch(1), + QFileDevice::FileModificationTime)); + olderFile.close(); + + SearchEngine engine(nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + directory.path(), QStringLiteral("Duplicates search")); + const QVector results = runSearch(engine, options); + QCOMPARE(results.size(), 2); + QHash priorities; + for (const FileRecord &record : results) + priorities.insert(record.path, record.duplicateCopy); + QCOMPARE(priorities.value(newer), 1); + QCOMPARE(priorities.value(older), 2); + } + + void reusesSampleAndFullHashesOnWarmSearch() + { + QTemporaryDir directory; + QTemporaryDir cacheDirectory; + QVERIFY(directory.isValid()); + QVERIFY(cacheDirectory.isValid()); + const QByteArray content(duplicateSampleThreshold + 1024 * 1024, 'x'); + const QString first = directory.filePath(QStringLiteral("first.bin")); + const QString second = directory.filePath(QStringLiteral("second.bin")); + QVERIFY(writeFile(first, content)); + QVERIFY(writeFile(second, content)); + const QString cachePath = + cacheDirectory.filePath(QStringLiteral("duplicate-cache.dat")); + SearchEngine engine(nullptr, cachePath); + SearchOptions options = optionsFor( + directory.path(), QStringLiteral("Duplicates search")); + options.useCache = true; + + QSignalSpy finished(&engine, &SearchEngine::finished); + QElapsedTimer timer; + timer.start(); + engine.search(options); + const qint64 coldMilliseconds = timer.elapsed(); + QCOMPARE(finished.size(), 1); + finished.clear(); + + if (::geteuid() != 0) { + QVERIFY(QFile::setPermissions(first, {})); + QVERIFY(QFile::setPermissions(second, {})); + } + timer.restart(); + engine.search(options); + const qint64 warmMilliseconds = timer.elapsed(); + QCOMPARE(finished.size(), 1); + const QList arguments = finished.takeFirst(); + QCOMPARE(arguments.at(0).value>().size(), 2); + QVERIFY(arguments.at(1).toString().contains(QStringLiteral("cache: 4 hits"))); + qInfo().noquote() + << QStringLiteral("duplicate benchmark: cold %1 ms, warm %2 ms") + .arg(coldMilliseconds) + .arg(warmMilliseconds); + + if (::geteuid() != 0) { + options.strictDuplicateComparison = true; + finished.clear(); + engine.search(options); + QCOMPARE(finished.size(), 1); + QVERIFY(finished.takeFirst().at(0).value>().isEmpty()); + } + QFile::remove(cachePath); + } + + void doesNotFullyHashDifferentSamplesOrSizes() + { + QTemporaryDir directory; + QTemporaryDir cacheDirectory; + QVERIFY(directory.isValid()); + QVERIFY(cacheDirectory.isValid()); + const QByteArray firstContent( + duplicateSampleThreshold + 1024 * 1024, 'x'); + QByteArray secondContent = firstContent; + secondContent[0] = 'y'; + QVERIFY(writeFile(directory.filePath(QStringLiteral("large-a.bin")), + firstContent)); + QVERIFY(writeFile(directory.filePath(QStringLiteral("large-b.bin")), + secondContent)); + QVERIFY(writeFile(directory.filePath(QStringLiteral("small-a.bin")), + QByteArray("one"))); + QVERIFY(writeFile(directory.filePath(QStringLiteral("small-b.bin")), + QByteArray("different size"))); + + SearchEngine engine( + nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + directory.path(), QStringLiteral("Duplicates search")); + options.useCache = true; + QSignalSpy finished(&engine, &SearchEngine::finished); + engine.search(options); + QCOMPARE(finished.size(), 1); + finished.clear(); + + engine.search(options); + QCOMPARE(finished.size(), 1); + const QList arguments = finished.takeFirst(); + QVERIFY(arguments.at(0).value>().isEmpty()); + QVERIFY(arguments.at(1).toString().contains(QStringLiteral("cache: 2 hits"))); + } + + void collapsesCurrentOverlappingRootLayoutBeforeHashing() + { + QTemporaryDir directory; + QTemporaryDir cacheDirectory; + QVERIFY(directory.isValid()); + QVERIFY(cacheDirectory.isValid()); + QDir root(directory.path()); + const QStringList children{ + QStringLiteral(".Prt"), QStringLiteral("misc"), QStringLiteral("Music"), + QStringLiteral("Буфер"), QStringLiteral("Temp") + }; + QStringList roots{directory.path()}; + for (qsizetype index = 0; index < children.size(); ++index) { + QVERIFY(root.mkpath(children[index])); + const QString child = directory.filePath(children[index]); + roots.push_back(child); + QVERIFY(writeFile( + QDir(child).filePath(QStringLiteral("unique.bin")), + QByteArray(duplicateSampleThreshold + index + 1, + char('a' + index)))); + } + + SearchEngine engine( + nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + roots.join(u';'), QStringLiteral("Duplicates search")); + options.accurateProgress = true; + options.excludeFolders = + directory.filePath(QStringLiteral("Meta")); + QSignalSpy phases(&engine, &SearchEngine::phaseProgress); + QSignalSpy finished(&engine, &SearchEngine::finished); + QElapsedTimer timer; + timer.start(); + engine.search(options); + const qint64 elapsedMilliseconds = timer.elapsed(); + + QCOMPARE(finished.size(), 1); + const QList arguments = finished.takeFirst(); + QVERIFY(arguments.at(0).value>().isEmpty()); + QVERIFY(arguments.at(1).toString().contains( + QStringLiteral("5 redundant roots skipped"))); + quint64 scanningCompleted = 0; + quint64 scanningTotal = 0; + for (const QList &phase : phases) { + const QString name = phase.at(0).toString(); + QVERIFY(name != QStringLiteral("Sampling duplicate candidates")); + QVERIFY(name != QStringLiteral("Hashing duplicate candidates")); + if (name == QStringLiteral("Scanning files")) { + scanningCompleted = + std::max(scanningCompleted, phase.at(1).toULongLong()); + scanningTotal = phase.at(2).toULongLong(); + } + } + QVERIFY(scanningTotal > 0); + QCOMPARE(scanningCompleted, scanningTotal); + qInfo().noquote() + << QStringLiteral("overlapping-roots benchmark: %1 ms, 5 large files, 0 hash phases") + .arg(elapsedMilliseconds); + } + + void deduplicatesCandidatesWhenNestedRootsCannotBeCollapsed() + { + QTemporaryDir directory; + QTemporaryDir cacheDirectory; + QVERIFY(directory.isValid()); + QVERIFY(cacheDirectory.isValid()); + QDir root(directory.path()); + QVERIFY(root.mkpath(QStringLiteral("child"))); + const QString child = directory.filePath(QStringLiteral("child")); + QVERIFY(writeFile(QDir(child).filePath(QStringLiteral("only.bin")), + QByteArray(duplicateSampleThreshold + 1, 'x'))); + + SearchEngine engine( + nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + directory.path() + u';' + child, QStringLiteral("Standard search")); + options.maxDepth = 2; + QVector results = runSearch(engine, options); + QCOMPARE(results.size(), 1); + QCOMPARE(results.front().name, QStringLiteral("only.bin")); + + options.mode = QStringLiteral("Duplicates search"); + QSignalSpy phases(&engine, &SearchEngine::phaseProgress); + QSignalSpy finished(&engine, &SearchEngine::finished); + engine.search(options); + QCOMPARE(finished.size(), 1); + const QList arguments = finished.takeFirst(); + QVERIFY(arguments.at(0).value>().isEmpty()); + QVERIFY(!arguments.at(1).toString().contains( + QStringLiteral("redundant roots skipped"))); + for (const QList &phase : phases) { + const QString name = phase.at(0).toString(); + QVERIFY(name != QStringLiteral("Sampling duplicate candidates")); + QVERIFY(name != QStringLiteral("Hashing duplicate candidates")); + } + } + + void leavesUnreadableCandidatesAsNonDuplicates() + { + if (::geteuid() == 0) + QSKIP("Root can read files regardless of their permission bits"); + QTemporaryDir directory; + QTemporaryDir cacheDirectory; + QVERIFY(directory.isValid()); + QVERIFY(cacheDirectory.isValid()); + const QString first = directory.filePath(QStringLiteral("first.bin")); + const QString second = directory.filePath(QStringLiteral("second.bin")); + QVERIFY(writeFile(first, QByteArray("blocked"))); + QVERIFY(writeFile(second, QByteArray("blocked"))); + QVERIFY(QFile::setPermissions(first, {})); + QVERIFY(QFile::setPermissions(second, {})); + + SearchEngine engine( + nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat"))); + SearchOptions options = optionsFor( + directory.path(), QStringLiteral("Duplicates search")); + QVERIFY(runSearch(engine, options).isEmpty()); + options.mode = QStringLiteral("Non-Duplicates search"); + QCOMPARE(namesOf(runSearch(engine, options)), + QSet({QStringLiteral("first.bin"), + QStringLiteral("second.bin")})); + } +}; + +QTEST_GUILESS_MAIN(SearchEngineTest) + +#include "search_engine_test.moc"