feat(search): optimize duplicate detection

src/main_window.cpp

@@ -977,6 +977,8 @@ void MainWindow::saveOptions()
     settings.setValue(QStringLiteral("mode"), options_.mode);
     settings.setValue(QStringLiteral("duplicateNameMode"), options_.duplicateNameMode);
     settings.setValue(QStringLiteral("duplicateNameWithoutExtension"), options_.duplicateNameWithoutExtension);
+    settings.setValue(QStringLiteral("strictDuplicateComparison"),
+                      options_.strictDuplicateComparison);
     settings.setValue(QStringLiteral("showDuplicateCopiesOnly"), options_.showDuplicateCopiesOnly);
     settings.setValue(QStringLiteral("includeSubfoldersInSummary"), options_.includeSubfoldersInSummary);
     settings.setValue(QStringLiteral("hidden"), options_.hidden);
@@ -1023,6 +1025,8 @@ void MainWindow::loadOptions()
     options_.mode = s.value(QStringLiteral("mode"), options_.mode).toString();
     options_.duplicateNameMode = s.value(QStringLiteral("duplicateNameMode"), options_.duplicateNameMode).toString();
     options_.duplicateNameWithoutExtension = s.value(QStringLiteral("duplicateNameWithoutExtension"), false).toBool();
+    options_.strictDuplicateComparison =
+        s.value(QStringLiteral("strictDuplicateComparison"), false).toBool();
     options_.showDuplicateCopiesOnly =
         s.value(QStringLiteral("showDuplicateCopiesOnly"),
                 s.value(QStringLiteral("showOnlyDuplicateFiles"), true)).toBool();

src/options_dialog.cpp

@@ -112,6 +112,11 @@ void OptionsDialog::setupUi(const SearchOptions &o)
     accurateProgress_->setChecked(o.accurateProgress);
     useCache_ = new QCheckBox(tr("Use persistent cache for hashes and content searches"));
     useCache_->setChecked(o.useCache);
+    strictDuplicateComparison_ =
+        new QCheckBox(tr("Strict duplicate comparison (byte by byte)"));
+    strictDuplicateComparison_->setObjectName(
+        QStringLiteral("strictDuplicateComparison"));
+    strictDuplicateComparison_->setChecked(o.strictDuplicateComparison);
     maxDepth_ = new QSpinBox; maxDepth_->setRange(0, 1024); maxDepth_->setValue(o.maxDepth);
     maxDepth_->setSpecialValueText(tr("Unlimited"));
     maxResults_ = new QSpinBox; maxResults_->setRange(0, 100'000'000); maxResults_->setValue(o.maxResults);
@@ -124,6 +129,7 @@ void OptionsDialog::setupUi(const SearchOptions &o)
     filesForm->addRow(retrieveOwner_);
     filesForm->addRow(accurateProgress_);
     filesForm->addRow(useCache_);
+    filesForm->addRow(strictDuplicateComparison_);
     filesForm->addRow(tr("Stop after finding:"), maxResults_);
     tabs->addTab(filesPage, tr("Files & folders"));
 
@@ -221,6 +227,7 @@ SearchOptions OptionsDialog::options() const
     o.retrieveOwner = retrieveOwner_->isChecked();
     o.accurateProgress = accurateProgress_->isChecked();
     o.useCache = useCache_->isChecked();
+    o.strictDuplicateComparison = strictDuplicateComparison_->isChecked();
     o.maxDepth = maxDepth_->value();
     o.maxResults = maxResults_->value();
     o.mode = mode_->currentText();
@@ -360,6 +367,7 @@ void OptionsDialog::saveProfile()
     saveBool("accurateProgress", opt.accurateProgress);
     saveBool("useCache", opt.useCache);
     saveBool("duplicateNameWithoutExtension", opt.duplicateNameWithoutExtension);
+    saveBool("strictDuplicateComparison", opt.strictDuplicateComparison);
 
     saveInt("lastMinutes", opt.lastMinutes);
     saveInt("maxDepth", opt.maxDepth);
@@ -419,6 +427,8 @@ void OptionsDialog::loadProfile()
     retrieveOwner_->setChecked(loadBool("retrieveOwner", false));
     accurateProgress_->setChecked(loadBool("accurateProgress", true));
     useCache_->setChecked(loadBool("useCache", true));
+    strictDuplicateComparison_->setChecked(
+        loadBool("strictDuplicateComparison", false));
     maxDepth_->setValue(loadInt("maxDepth", 0));
     maxResults_->setValue(loadInt("maxResults", 0));
     mode_->setCurrentText(load("mode"));

src/options_dialog.h

@@ -66,6 +66,7 @@ private:
     QCheckBox *retrieveOwner_{};
     QCheckBox *accurateProgress_{};
     QCheckBox *useCache_{};
+    QCheckBox *strictDuplicateComparison_{};
     QSpinBox *maxDepth_{};
     QSpinBox *maxResults_{};
     QComboBox *hidden_{};

src/results_model.cpp

@@ -23,7 +23,7 @@ QVariant ResultsModel::headerData(int section, Qt::Orientation orientation, int
         if (section == DuplicateNumber)
             return tr("Identifies one set of identical files");
         if (section == DuplicateGroup)
-            return tr("Order based on the base-folder list; 1 = preferred copy to keep");
+            return tr("Most-specific base folder wins; ties use newer files, then path. 1 = preferred copy to keep");
         return {};
     }
     if (role != Qt::DisplayRole)

src/search_cache.cpp

@@ -10,14 +10,41 @@
 #include <algorithm>
 #include <utility>
 
-SearchCache::SearchCache()
+SearchCache::SearchCache(QString path)
 {
-    const QString cacheRoot = QStandardPaths::writableLocation(QStandardPaths::CacheLocation);
-    QDir().mkpath(cacheRoot);
-    path_ = QDir(cacheRoot).filePath(QStringLiteral("search-cache-v1.dat"));
+    if (path.isEmpty()) {
+        const QString cacheRoot = QStandardPaths::writableLocation(QStandardPaths::CacheLocation);
+        QDir().mkpath(cacheRoot);
+        path_ = QDir(cacheRoot).filePath(QStringLiteral("search-cache-v1.dat"));
+    } else {
+        path_ = std::move(path);
+    }
     load();
 }
 
+bool SearchCache::findSampleHash(const FileRecord &record, QByteArray &digest)
+{
+    if (!enabled_)
+        return false;
+    QMutexLocker lock(&mutex_);
+    auto it = entries_.find(record.path);
+    if (it == entries_.end() || !signatureMatches(*it, record)
+        || it->sampleSha256.isEmpty())
+        return false;
+    it->lastUsed = QDateTime::currentSecsSinceEpoch();
+    digest = it->sampleSha256;
+    return true;
+}
+
+void SearchCache::storeSampleHash(const FileRecord &record, const QByteArray &digest)
+{
+    if (!enabled_ || digest.isEmpty())
+        return;
+    QMutexLocker lock(&mutex_);
+    SearchCacheEntry &entry = currentEntry(record);
+    entry.sampleSha256 = digest;
+}
+
 bool SearchCache::findHash(const FileRecord &record, QByteArray &digest)
 {
     if (!enabled_)
@@ -75,9 +102,10 @@ void SearchCache::save()
         return;
     QDataStream stream(&file);
     stream.setVersion(QDataStream::Qt_6_0);
-    stream << quint32(0x46534348) << quint32(1) << quint32(entries_.size());
+    stream << quint32(0x46534348) << quint32(2) << quint32(entries_.size());
     for (auto it = entries_.cbegin(); it != entries_.cend(); ++it) {
-        stream << it.key() << it->size << it->modified << it->lastUsed << it->sha256;
+        stream << it.key() << it->size << it->modifiedNs << it->device << it->inode
+               << it->lastUsed << it->sampleSha256 << it->sha256;
         stream << quint32(it->contentMatches.size());
         for (auto match = it->contentMatches.cbegin(); match != it->contentMatches.cend(); ++match)
             stream << match.key() << match.value();
@@ -95,7 +123,10 @@ void SearchCache::clear()
 
 bool SearchCache::signatureMatches(const SearchCacheEntry &entry, const FileRecord &record)
 {
-    return entry.size == record.size && entry.modified == record.modified;
+    return entry.size == record.size
+        && entry.modifiedNs == record.modifiedNs
+        && entry.device == record.device
+        && entry.inode == record.inode;
 }
 
 SearchCacheEntry &SearchCache::currentEntry(const FileRecord &record)
@@ -104,7 +135,9 @@ SearchCacheEntry &SearchCache::currentEntry(const FileRecord &record)
     if (!signatureMatches(entry, record)) {
         entry = {};
         entry.size = record.size;
-        entry.modified = record.modified;
+        entry.modifiedNs = record.modifiedNs;
+        entry.device = record.device;
+        entry.inode = record.inode;
     }
     entry.lastUsed = QDateTime::currentSecsSinceEpoch();
     return entry;
@@ -121,13 +154,14 @@ void SearchCache::load()
     quint32 version = 0;
     quint32 count = 0;
     stream >> magic >> version >> count;
-    if (magic != 0x46534348 || version != 1 || count > 200'000)
+    if (magic != 0x46534348 || version != 2 || count > 200'000)
         return;
     for (quint32 index = 0; index < count && stream.status() == QDataStream::Ok; ++index) {
         QString path;
         SearchCacheEntry entry;
         quint32 contentCount = 0;
-        stream >> path >> entry.size >> entry.modified >> entry.lastUsed >> entry.sha256;
+        stream >> path >> entry.size >> entry.modifiedNs >> entry.device >> entry.inode
+               >> entry.lastUsed >> entry.sampleSha256 >> entry.sha256;
         stream >> contentCount;
         if (contentCount > 256)
             return;

src/search_cache.h

@@ -9,19 +9,24 @@
 
 struct SearchCacheEntry {
     qint64 size = -1;
-    qint64 modified = -1;
+    qint64 modifiedNs = -1;
+    quint64 device = 0;
+    quint64 inode = 0;
     qint64 lastUsed = 0;
+    QByteArray sampleSha256;
     QByteArray sha256;
     QHash<QByteArray, bool> contentMatches;
 };
 
 class SearchCache {
 public:
-    SearchCache();
+    explicit SearchCache(QString path = {});
 
     void setEnabled(bool enabled) { enabled_ = enabled; }
     bool enabled() const { return enabled_; }
 
+    bool findSampleHash(const FileRecord &record, QByteArray &digest);
+    void storeSampleHash(const FileRecord &record, const QByteArray &digest);
     bool findHash(const FileRecord &record, QByteArray &digest);
     void storeHash(const FileRecord &record, const QByteArray &digest);
     std::optional<bool> findContent(const FileRecord &record, const QByteArray &queryKey);

src/search_core.cpp

@@ -2,11 +2,16 @@
 
 #include <QCryptographicHash>
 #include <QDateTime>
+#include <QDir>
 #include <QFile>
+#include <QFileInfo>
 #include <QRegularExpression>
+#include <QSet>
 
 #include <algorithm>
+#include <array>
 #include <pwd.h>
+#include <sys/stat.h>
 
 QStringList patterns(const QString &text)
 {
@@ -54,6 +59,129 @@ bool wildcardMatch(const QString &value, const QStringList &items, Qt::CaseSensi
     return false;
 }
 
+namespace {
+struct RootCandidate {
+    QString path;
+};
+
+bool isDescendantPath(const QString &parent, const QString &child,
+                      QString *relativePath = nullptr)
+{
+    const QString relative = QDir(parent).relativeFilePath(child);
+    const bool descendant = relative != QStringLiteral(".")
+        && relative != QStringLiteral("..")
+        && !relative.startsWith(QStringLiteral("../"))
+        && !QDir::isAbsolutePath(relative);
+    if (descendant && relativePath)
+        *relativePath = relative;
+    return descendant;
+}
+
+bool canCollapseInto(const RootCandidate &parent, const RootCandidate &child,
+                     const SearchOptions &options, const QStringList &excludedFolders,
+                     Qt::CaseSensitivity cs)
+{
+    if (!QFileInfo(parent.path).isDir())
+        return false;
+
+    QString relative;
+    if (!isDescendantPath(parent.path, child.path, &relative))
+        return false;
+
+    QString current = parent.path;
+    const QStringList segments = relative.split(u'/', Qt::SkipEmptyParts);
+    for (const QString &segment : segments) {
+        current = QDir(current).filePath(segment);
+        const QFileInfo info(current);
+        if ((!options.followLinks && info.isSymLink())
+            || (!excludedFolders.isEmpty()
+                && (wildcardMatch(info.fileName(), excludedFolders, cs)
+                    || wildcardMatch(QDir::cleanPath(current), excludedFolders, cs)))) {
+            return false;
+        }
+    }
+    return true;
+}
+}
+
+EffectiveRoots effectiveSearchRoots(const SearchOptions &options)
+{
+    QVector<RootCandidate> unique;
+    QSet<QString> identities;
+    int skipped = 0;
+    for (const QString &rootText : patterns(options.roots)) {
+        const QFileInfo info(rootText);
+        const QString path = QDir::cleanPath(info.absoluteFilePath());
+        const QString canonical = info.canonicalFilePath();
+        const QString identity = canonical.isEmpty()
+            ? path : QDir::cleanPath(canonical);
+        if (identities.contains(identity)) {
+            ++skipped;
+            continue;
+        }
+        identities.insert(identity);
+        unique.push_back({path});
+    }
+
+    const QStringList subfolderMasks = patterns(options.subfolderWildcards).isEmpty()
+        ? QStringList{QStringLiteral("*")} : patterns(options.subfolderWildcards);
+    const bool universalSubfolderMask =
+        subfolderMasks.contains(QStringLiteral("*"));
+    if (!options.recursive || options.maxDepth > 0 || options.maxResults > 0
+        || !universalSubfolderMask) {
+        EffectiveRoots result;
+        result.skipped = skipped;
+        for (const RootCandidate &root : unique) {
+            result.paths.push_back(root.path);
+            result.priorityPaths.push_back(root.path);
+        }
+        return result;
+    }
+
+    const Qt::CaseSensitivity cs =
+        options.caseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive;
+    const QStringList excludedFolders = patterns(options.excludeFolders);
+    EffectiveRoots result;
+    result.skipped = skipped;
+    for (const RootCandidate &root : unique)
+        result.priorityPaths.push_back(root.path);
+    for (qsizetype childIndex = 0; childIndex < unique.size(); ++childIndex) {
+        bool redundant = false;
+        for (qsizetype parentIndex = 0; parentIndex < unique.size(); ++parentIndex) {
+            if (parentIndex == childIndex)
+                continue;
+            if (canCollapseInto(unique[parentIndex], unique[childIndex], options,
+                                excludedFolders, cs)) {
+                redundant = true;
+                break;
+            }
+        }
+        if (redundant) {
+            ++result.skipped;
+        } else {
+            result.paths.push_back(unique[childIndex].path);
+        }
+    }
+    return result;
+}
+
+int preferredRootIndex(const QString &path, const QStringList &roots)
+{
+    const QString normalizedPath = QDir::cleanPath(QFileInfo(path).absoluteFilePath());
+    int bestIndex = -1;
+    qsizetype bestLength = 0;
+    for (qsizetype index = 0; index < roots.size(); ++index) {
+        const QString root = QDir::cleanPath(QFileInfo(roots[index]).absoluteFilePath());
+        if (normalizedPath != root && !isDescendantPath(root, normalizedPath))
+            continue;
+        if (root.size() > bestLength) {
+            bestIndex = int(index);
+            bestLength = root.size();
+        }
+    }
+    return bestIndex;
+}
+
 bool shouldShowDuplicateResult(bool copiesOnly, int keeperPriority)
 {
     if (keeperPriority < 1)
@@ -99,6 +227,26 @@ bool isCancelled(const std::atomic_bool *cancelled)
 {
     return cancelled && cancelled->load(std::memory_order_relaxed);
 }
+
+qint64 modifiedNanoseconds(const struct stat &st)
+{
+    return qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec;
+}
+
+bool signatureMatches(const FileRecord &record, const struct stat &st)
+{
+    return record.size == st.st_size
+        && record.modifiedNs == modifiedNanoseconds(st)
+        && record.device == quint64(st.st_dev)
+        && record.inode == quint64(st.st_ino);
+}
+
+bool statMatches(const FileRecord &record)
+{
+    struct stat st {};
+    const QByteArray nativePath = QFile::encodeName(record.path);
+    return ::stat(nativePath.constData(), &st) == 0 && signatureMatches(record, st);
+}
 }
 
 bool fileContains(const QString &path, const SearchOptions &options,
@@ -187,6 +335,46 @@ QByteArray sha256(const QString &path, const std::atomic_bool *cancelled)
     return isCancelled(cancelled) ? QByteArray{} : hash.result();
 }
 
+bool fileSignatureMatches(const FileRecord &record)
+{
+    return statMatches(record);
+}
+
+QByteArray sha256(const FileRecord &record, const std::atomic_bool *cancelled)
+{
+    if (isCancelled(cancelled) || !statMatches(record))
+        return {};
+    const QByteArray digest = sha256(record.path, cancelled);
+    return !digest.isEmpty() && statMatches(record) ? digest : QByteArray{};
+}
+
+QByteArray sampleSha256(const FileRecord &record, const std::atomic_bool *cancelled)
+{
+    if (isCancelled(cancelled) || record.size <= duplicateSampleThreshold
+        || !statMatches(record))
+        return {};
+
+    QFile file(record.path);
+    if (!file.open(QIODevice::ReadOnly))
+        return {};
+
+    const qint64 middleOffset = (record.size - duplicateSampleBlockSize) / 2;
+    const std::array<qint64, 3> offsets{
+        0, middleOffset, record.size - duplicateSampleBlockSize
+    };
+    QCryptographicHash hash(QCryptographicHash::Sha256);
+    for (const qint64 offset : offsets) {
+        if (isCancelled(cancelled) || !file.seek(offset))
+            return {};
+        const QByteArray block = file.read(duplicateSampleBlockSize);
+        if (block.size() != duplicateSampleBlockSize)
+            return {};
+        hash.addData(block);
+    }
+    const QByteArray digest = hash.result();
+    return !isCancelled(cancelled) && statMatches(record) ? digest : QByteArray{};
+}
+
 bool parseOptionalIsoDate(const QString &text, qint64 &seconds)
 {
     const QString trimmed = text.trimmed();

src/search_core.h

@@ -11,6 +11,9 @@
 #include <optional>
 #include <sys/types.h>
 
+inline constexpr qint64 duplicateSampleBlockSize = 64LL * 1024;
+inline constexpr qint64 duplicateSampleThreshold = 4LL * 1024 * 1024;
+
 struct SearchOptions {
     QString roots = QDir::homePath();
     QString fileWildcards = QStringLiteral("*");
@@ -41,6 +44,7 @@ struct SearchOptions {
     QString mode = QStringLiteral("Standard search");
     QString duplicateNameMode = QStringLiteral("All files and folders");
     bool duplicateNameWithoutExtension = false;
+    bool strictDuplicateComparison = false;
     bool showDuplicateCopiesOnly = true;
     bool includeSubfoldersInSummary = false;
     QString hidden = QStringLiteral("Any");
@@ -60,6 +64,9 @@ struct FileRecord {
     qint64 created = 0;
     qint64 accessed = 0;
     qint64 changed = 0;
+    qint64 modifiedNs = 0;
+    quint64 device = 0;
+    quint64 inode = 0;
     QString type;
     QString owner;
     QString attributes;
@@ -67,6 +74,12 @@ struct FileRecord {
     int duplicateCopy = 0;
 };
 
+struct EffectiveRoots {
+    QStringList paths;
+    QStringList priorityPaths;
+    int skipped = 0;
+};
+
 Q_DECLARE_METATYPE(SearchOptions)
 Q_DECLARE_METATYPE(FileRecord)
 Q_DECLARE_METATYPE(QVector<FileRecord>)
@@ -74,6 +87,8 @@ Q_DECLARE_METATYPE(QVector<FileRecord>)
 QStringList patterns(const QString &text);
 QStringList exclusionPatterns(const QString &text);
 bool wildcardMatch(const QString &value, const QStringList &items, Qt::CaseSensitivity cs);
+EffectiveRoots effectiveSearchRoots(const SearchOptions &options);
+int preferredRootIndex(const QString &path, const QStringList &roots);
 QString humanSize(qint64 bytes);
 QString ownerName(uid_t uid);
 std::optional<QByteArray> hexNeedle(QString text);
@@ -82,5 +97,9 @@ bool fileContains(const QString &path, const SearchOptions &options,
 bool filesEqual(const QString &leftPath, const QString &rightPath,
                 const std::atomic_bool *cancelled = nullptr);
 QByteArray sha256(const QString &path, const std::atomic_bool *cancelled = nullptr);
+QByteArray sha256(const FileRecord &record, const std::atomic_bool *cancelled = nullptr);
+QByteArray sampleSha256(const FileRecord &record,
+                        const std::atomic_bool *cancelled = nullptr);
+bool fileSignatureMatches(const FileRecord &record);
 bool parseOptionalIsoDate(const QString &text, qint64 &seconds);
 bool shouldShowDuplicateResult(bool copiesOnly, int keeperPriority);

src/search_engine.cpp

@@ -6,6 +6,7 @@
 #include <QDir>
 #include <QFileInfo>
 #include <QThread>
+#include <QThreadPool>
 #include <QtConcurrent>
 
 #include <algorithm>
@@ -18,6 +19,27 @@
 
 namespace fs = std::filesystem;
 
+namespace {
+struct HashedRecord {
+    QByteArray digest;
+    FileRecord record;
+};
+
+bool keepFirst(const FileRecord &left, const FileRecord &right,
+               const QStringList &priorityRoots)
+{
+    const int leftRoot = preferredRootIndex(left.path, priorityRoots);
+    const int rightRoot = preferredRootIndex(right.path, priorityRoots);
+    const int leftPriority = leftRoot < 0 ? priorityRoots.size() : leftRoot;
+    const int rightPriority = rightRoot < 0 ? priorityRoots.size() : rightRoot;
+    if (leftPriority != rightPriority)
+        return leftPriority < rightPriority;
+    if (left.modified != right.modified)
+        return left.modified > right.modified;
+    return left.path < right.path;
+}
+}
+
 void SearchEngine::search(const SearchOptions &o)
 {
     cancelled_.store(false, std::memory_order_relaxed);
@@ -33,6 +55,8 @@ void SearchEngine::search(const SearchOptions &o)
     cacheMisses_.store(0, std::memory_order_relaxed);
     const auto started = std::chrono::steady_clock::now();
     QVector<FileRecord> candidates;
+    QSet<QString> candidatePaths;
+    const EffectiveRoots roots = effectiveSearchRoots(o);
     const Qt::CaseSensitivity cs = o.caseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive;
     const QStringList fileMasks = patterns(o.fileWildcards).isEmpty()
         ? QStringList{QStringLiteral("*")} : patterns(o.fileWildcards);
@@ -46,7 +70,7 @@ void SearchEngine::search(const SearchOptions &o)
 
     if (o.accurateProgress) {
         emit phaseProgress(tr("Counting files"), 0, 0);
-        for (const QString &rootText : patterns(o.roots)) {
+        for (const QString &rootText : roots.paths) {
             if (cancelled_.load(std::memory_order_relaxed))
                 break;
             std::error_code ec;
@@ -84,7 +108,7 @@ void SearchEngine::search(const SearchOptions &o)
         emit phaseProgress(tr("Scanning files"), 0, totalEntries);
     }
 
-    for (const QString &rootText : patterns(o.roots)) {
+    for (const QString &rootText : roots.paths) {
         if (cancelled_.load(std::memory_order_relaxed))
             break;
         std::error_code ec;
@@ -96,7 +120,8 @@ void SearchEngine::search(const SearchOptions &o)
         auto process = [&](const fs::directory_entry &entry, const fs::path &baseRoot) {
             if (cancelled_.load(std::memory_order_relaxed))
                 return;
-            const QString path = QString::fromStdString(entry.path().string());
+            const QString path = QDir::cleanPath(QFileInfo(
+                QString::fromStdString(entry.path().string())).absoluteFilePath());
             const QString name = QString::fromStdString(entry.path().filename().string());
             const bool isDir = entry.is_directory(ec);
             if ((isDir && !o.findFolders) || (!isDir && !o.findFiles))
@@ -147,6 +172,9 @@ void SearchEngine::search(const SearchOptions &o)
                 || !attrMatches(o.readonly, readOnly)
                 || !attrMatches(o.executable, executable))
                 return;
+            if (candidatePaths.contains(path))
+                return;
+            candidatePaths.insert(path);
 
             QString attrs;
             attrs += isDir ? u'd' : u'-';
@@ -160,6 +188,8 @@ void SearchEngine::search(const SearchOptions &o)
                 st.st_mtime,
                 QFileInfo(path).birthTime().isValid() ? QFileInfo(path).birthTime().toSecsSinceEpoch() : 0,
                 st.st_atime, st.st_ctime,
+                qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec,
+                quint64(st.st_dev), quint64(st.st_ino),
                 isDir ? QStringLiteral("Folder") : QStringLiteral("File"),
                 o.retrieveOwner ? ownerName(st.st_uid) : QString{}, attrs, 0, 0
             });
@@ -237,68 +267,122 @@ void SearchEngine::search(const SearchOptions &o)
             candidates.resize(o.maxResults);
     }
 
+    QThreadPool hashPool;
+    const int idealThreads = QThread::idealThreadCount();
+    hashPool.setMaxThreadCount(std::clamp(
+        idealThreads > 0 ? idealThreads / 2 : 1, 1, 4));
+
     QVector<FileRecord> results;
     if (o.mode == QStringLiteral("Duplicates search")
         || o.mode == QStringLiteral("Non-Duplicates search")) {
         std::unordered_map<qint64, QVector<FileRecord>> bySize;
-        for (const auto &record : candidates)
-            if (record.type == QStringLiteral("File"))
+        QSet<QString> duplicateInputPaths;
+        for (const auto &record : candidates) {
+            if (record.type == QStringLiteral("File")
+                && !duplicateInputPaths.contains(record.path)) {
+                duplicateInputPaths.insert(record.path);
                 bySize[record.size].push_back(record);
-        quint64 hashTotal = 0;
-        for (const auto &[size, records] : bySize) {
-            Q_UNUSED(size);
-            if (records.size() > 1)
-                hashTotal += records.size();
+            }
         }
-        std::atomic<quint64> hashed = 0;
-        if (hashTotal)
-            emit phaseProgress(tr("Checking duplicates"), 0, hashTotal);
-        int group = 1;
-        QSet<QString> duplicatedPaths;
-        for (auto &[size, sameSize] : bySize) {
-            if (cancelled_.load(std::memory_order_relaxed))
-                break;
-            Q_UNUSED(size);
-            if (sameSize.size() < 2)
+
+        QVector<FileRecord> sampleCandidates;
+        QVector<FileRecord> fullHashCandidates;
+        for (const auto &[size, records] : bySize) {
+            if (records.size() < 2)
                 continue;
-            auto hashes = QtConcurrent::blockingMapped(sameSize, [this, &hashed, hashTotal](const FileRecord &r) {
-                auto result = std::pair<QByteArray, FileRecord>{cachedSha256(r), r};
-                const quint64 done = hashed.fetch_add(1, std::memory_order_relaxed) + 1;
-                if (done == hashTotal || done % 16 == 0)
-                    emit phaseProgress(tr("Checking duplicates"), done, hashTotal);
+            if (size > duplicateSampleThreshold)
+                sampleCandidates += records;
+            else
+                fullHashCandidates += records;
+        }
+
+        std::atomic<qsizetype> sampled = 0;
+        const qsizetype sampleTotal = sampleCandidates.size();
+        if (sampleTotal)
+            emit phaseProgress(tr("Sampling duplicate candidates"), 0, sampleTotal);
+        const auto samples = QtConcurrent::blockingMapped(
+            &hashPool, sampleCandidates,
+            [this, &sampled, sampleTotal](const FileRecord &record) {
+                HashedRecord result{cachedSampleSha256(record), record};
+                const qsizetype done = sampled.fetch_add(1, std::memory_order_relaxed) + 1;
+                if (done == sampleTotal || done % 16 == 0)
+                    emit phaseProgress(tr("Sampling duplicate candidates"), done, sampleTotal);
                 return result;
             });
-            if (cancelled_.load(std::memory_order_relaxed))
-                break;
-            QHash<QByteArray, QVector<FileRecord>> sameHash;
-            for (auto &[hash, record] : hashes)
-                if (!hash.isEmpty())
-                    sameHash[hash].push_back(record);
-            for (const auto &hashCandidates : sameHash) {
+        if (finishIfCancelled())
+            return;
+
+        QHash<qint64, QHash<QByteArray, QVector<FileRecord>>> bySample;
+        for (const auto &sample : samples)
+            if (!sample.digest.isEmpty())
+                bySample[sample.record.size][sample.digest].push_back(sample.record);
+        for (const auto &sameSize : bySample)
+            for (const auto &sameSample : sameSize)
+                if (sameSample.size() > 1)
+                    fullHashCandidates += sameSample;
+
+        std::atomic<qsizetype> hashed = 0;
+        const qsizetype hashTotal = fullHashCandidates.size();
+        if (hashTotal)
+            emit phaseProgress(tr("Hashing duplicate candidates"), 0, hashTotal);
+        const auto hashes = QtConcurrent::blockingMapped(
+            &hashPool, fullHashCandidates,
+            [this, &hashed, hashTotal](const FileRecord &record) {
+                HashedRecord result{cachedSha256(record), record};
+                const qsizetype done = hashed.fetch_add(1, std::memory_order_relaxed) + 1;
+                if (done == hashTotal || done % 16 == 0)
+                    emit phaseProgress(tr("Hashing duplicate candidates"), done, hashTotal);
+                return result;
+            });
+        if (finishIfCancelled())
+            return;
+
+        QHash<qint64, QHash<QByteArray, QVector<FileRecord>>> byHash;
+        for (const auto &hash : hashes)
+            if (!hash.digest.isEmpty())
+                byHash[hash.record.size][hash.digest].push_back(hash.record);
+
+        int group = 1;
+        QSet<QString> duplicatedPaths;
+        for (const auto &sameSize : byHash) {
+            for (const auto &hashCandidates : sameSize) {
                 if (cancelled_.load(std::memory_order_relaxed))
                     break;
                 if (hashCandidates.size() < 2)
                     continue;
                 QVector<QVector<FileRecord>> exactGroups;
-                for (const auto &record : hashCandidates) {
-                    if (cancelled_.load(std::memory_order_relaxed))
-                        break;
-                    bool placed = false;
-                    for (auto &exact : exactGroups) {
-                        if (filesEqual(exact.front().path, record.path, &cancelled_)) {
-                            exact.push_back(record);
-                            placed = true;
+                if (!o.strictDuplicateComparison) {
+                    exactGroups.push_back(hashCandidates);
+                } else {
+                    for (const auto &record : hashCandidates) {
+                        if (cancelled_.load(std::memory_order_relaxed))
                             break;
+                        bool placed = false;
+                        for (auto &exact : exactGroups) {
+                            if (fileSignatureMatches(exact.front())
+                                && fileSignatureMatches(record)
+                                && filesEqual(exact.front().path, record.path, &cancelled_)
+                                && fileSignatureMatches(exact.front())
+                                && fileSignatureMatches(record)) {
+                                exact.push_back(record);
+                                placed = true;
+                                break;
+                            }
                         }
+                        if (!placed)
+                            exactGroups.push_back({record});
                     }
-                    if (!placed)
-                        exactGroups.push_back({record});
                 }
                 for (auto &exact : exactGroups) {
                     if (cancelled_.load(std::memory_order_relaxed))
                         break;
                     if (exact.size() < 2)
                         continue;
+                    std::stable_sort(exact.begin(), exact.end(),
+                                     [&roots](const FileRecord &left,
+                                              const FileRecord &right) {
+                        return keepFirst(left, right, roots.priorityPaths);
+                    });
                     int duplicateCopy = 1;
                     for (auto &record : exact) {
                         record.group = group;
@@ -343,13 +427,50 @@ void SearchEngine::search(const SearchOptions &o)
                     [](const FileRecord &r) { return r.type != QStringLiteral("File"); }), sameName.end());
             if (sameName.size() < 2)
                 continue;
+            std::stable_sort(sameName.begin(), sameName.end(),
+                             [&roots](const FileRecord &left,
+                                      const FileRecord &right) {
+                return keepFirst(left, right, roots.priorityPaths);
+            });
             if (o.duplicateNameMode.contains(QStringLiteral("identical"), Qt::CaseInsensitive)) {
-                const QByteArray firstHash = cachedSha256(sameName.front());
-                bool identical = !firstHash.isEmpty();
-                for (qsizetype i = 1; identical && i < sameName.size(); ++i) {
-                    identical = cachedSha256(sameName[i]) == firstHash
-                        && filesEqual(
-                            sameName.front().path, sameName[i].path, &cancelled_);
+                bool identical = std::all_of(
+                    sameName.cbegin(), sameName.cend(),
+                    [&sameName](const FileRecord &record) {
+                        return record.size == sameName.front().size;
+                    });
+                if (identical && sameName.front().size > duplicateSampleThreshold) {
+                    const auto samples = QtConcurrent::blockingMapped(
+                        &hashPool, sameName, [this](const FileRecord &record) {
+                            return cachedSampleSha256(record);
+                        });
+                    identical = !samples.isEmpty() && !samples.front().isEmpty()
+                        && std::all_of(samples.cbegin(), samples.cend(),
+                            [&samples](const QByteArray &digest) {
+                                return digest == samples.front();
+                            });
+                }
+                QVector<QByteArray> hashes;
+                if (identical) {
+                    hashes = QtConcurrent::blockingMapped(
+                        &hashPool, sameName, [this](const FileRecord &record) {
+                            return cachedSha256(record);
+                        });
+                    identical = !hashes.isEmpty() && !hashes.front().isEmpty()
+                        && std::all_of(hashes.cbegin(), hashes.cend(),
+                            [&hashes](const QByteArray &digest) {
+                                return digest == hashes.front();
+                            });
+                }
+                if (identical && o.strictDuplicateComparison) {
+                    const FileRecord &first = sameName.front();
+                    for (qsizetype index = 1; identical && index < sameName.size(); ++index) {
+                        const FileRecord &record = sameName[index];
+                        identical = fileSignatureMatches(first)
+                            && fileSignatureMatches(record)
+                            && filesEqual(first.path, record.path, &cancelled_)
+                            && fileSignatureMatches(first)
+                            && fileSignatureMatches(record);
+                    }
                 }
                 const bool wantsNonIdentical = o.duplicateNameMode.startsWith(QStringLiteral("Only non"));
                 if (identical == wantsNonIdentical)
@@ -376,7 +497,7 @@ void SearchEngine::search(const SearchOptions &o)
                 while (parent.cdUp()) {
                     const QString ancestor = parent.absolutePath();
                     bool belongsToRoot = false;
-                    for (const QString &root : patterns(o.roots)) {
+                    for (const QString &root : roots.paths) {
                         if (ancestor == QDir(root).absolutePath()) {
                             belongsToRoot = true;
                             break;
@@ -401,8 +522,8 @@ void SearchEngine::search(const SearchOptions &o)
             }
             const QFileInfo info(it.key());
             results.push_back({it.key(), info.fileName(), info.absolutePath(), total, totalOnDisk,
-                               newest, 0, 0, 0, tr("%1 files").arg(it.value().size()),
-                               {}, {}, 0, 0});
+                               newest, 0, 0, 0, 0, 0, 0,
+                               tr("%1 files").arg(it.value().size()), {}, {}, 0, 0});
         }
         if (finishIfCancelled())
             return;
@@ -415,21 +536,42 @@ void SearchEngine::search(const SearchOptions &o)
     const double seconds = std::chrono::duration<double>(
         std::chrono::steady_clock::now() - started).count();
     cache_.save();
-    emit finished(results, tr("%1 results, %2 entries scanned in %3 seconds; cache: %4 hits")
-                              .arg(results.size()).arg(scanned).arg(seconds, 0, 'f', 2)
-                              .arg(cacheHits_.load(std::memory_order_relaxed)));
+    QString message = tr("%1 results, %2 entries scanned in %3 seconds; cache: %4 hits")
+                          .arg(results.size()).arg(scanned).arg(seconds, 0, 'f', 2)
+                          .arg(cacheHits_.load(std::memory_order_relaxed));
+    if (roots.skipped > 0)
+        message += tr("; %1 redundant roots skipped").arg(roots.skipped);
+    emit finished(results, message);
+}
+
+QByteArray SearchEngine::cachedSampleSha256(const FileRecord &record)
+{
+    QByteArray digest;
+    if (cache_.findSampleHash(record, digest)) {
+        if (!fileSignatureMatches(record))
+            return {};
+        cacheHits_.fetch_add(1, std::memory_order_relaxed);
+        return digest;
+    }
+    cacheMisses_.fetch_add(1, std::memory_order_relaxed);
+    digest = sampleSha256(record, &cancelled_);
+    if (!digest.isEmpty() && !cancelled_.load(std::memory_order_relaxed))
+        cache_.storeSampleHash(record, digest);
+    return digest;
 }
 
 QByteArray SearchEngine::cachedSha256(const FileRecord &record)
 {
     QByteArray digest;
     if (cache_.findHash(record, digest)) {
+        if (!fileSignatureMatches(record))
+            return {};
         cacheHits_.fetch_add(1, std::memory_order_relaxed);
         return digest;
     }
     cacheMisses_.fetch_add(1, std::memory_order_relaxed);
-    digest = sha256(record.path, &cancelled_);
-    if (!cancelled_.load(std::memory_order_relaxed))
+    digest = sha256(record, &cancelled_);
+    if (!digest.isEmpty() && !cancelled_.load(std::memory_order_relaxed))
         cache_.storeHash(record, digest);
     return digest;
 }
@@ -441,6 +583,8 @@ bool SearchEngine::cachedFileContains(const FileRecord &record, const SearchOpti
     stream << o.contains << o.binary << o.multipleValues << o.multipleAnd << o.caseSensitive;
     const QByteArray queryKey = QCryptographicHash::hash(queryData, QCryptographicHash::Sha256);
     if (const auto cached = cache_.findContent(record, queryKey)) {
+        if (!fileSignatureMatches(record))
+            return false;
         cacheHits_.fetch_add(1, std::memory_order_relaxed);
         return *cached;
     }

src/search_engine.h

@@ -6,6 +6,7 @@
 
 #include <atomic>
 #include <chrono>
+#include <utility>
 
 #include "search_core.h"
 #include "search_cache.h"
@@ -13,7 +14,8 @@
 class SearchEngine final : public QObject {
     Q_OBJECT
 public:
-    explicit SearchEngine(QObject *parent = nullptr) : QObject(parent) {}
+    explicit SearchEngine(QObject *parent = nullptr, QString cachePath = {})
+        : QObject(parent), cache_(std::move(cachePath)) {}
 
 public slots:
     void stop() { cancelled_.store(true, std::memory_order_relaxed); }
@@ -32,6 +34,7 @@ signals:
     void finished(const QVector<FileRecord> &results, const QString &message);
 
 private:
+    QByteArray cachedSampleSha256(const FileRecord &record);
     QByteArray cachedSha256(const FileRecord &record);
     bool cachedFileContains(const FileRecord &record, const SearchOptions &o);