feat(search): optimize duplicate detection

2026-07-26 16:27:33 +02:00
parent 711536c74e
commit c24d0dd84d
18 changed files with 1285 additions and 71 deletions
@@ -5,6 +5,35 @@
 #include <QTemporaryDir>
 #include <QtTest>

+#include <sys/stat.h>
+
+namespace {
+FileRecord recordFor(const QString &path)
+{
+    struct stat st {};
+    const QByteArray nativePath = QFile::encodeName(path);
+    if (::stat(nativePath.constData(), &st) != 0)
+        return {};
+    FileRecord record;
+    record.path = path;
+    record.size = st.st_size;
+    record.modified = st.st_mtime;
+    record.modifiedNs =
+        qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec;
+    record.device = st.st_dev;
+    record.inode = st.st_ino;
+    record.type = QStringLiteral("File");
+    return record;
+}
+
+bool writeFile(const QString &path, const QByteArray &data)
+{
+    QFile file(path);
+    return file.open(QIODevice::WriteOnly)
+        && file.write(data) == data.size();
+}
+}
+
 class SearchCoreTest final : public QObject {
    Q_OBJECT

@@ -23,6 +52,113 @@ private slots:
                               Qt::CaseSensitive));
    }

+    void collapsesRedundantSearchRoots()
+    {
+        QTemporaryDir directory;
+        QVERIFY(directory.isValid());
+        QDir root(directory.path());
+        QVERIFY(root.mkpath(QStringLiteral("child")));
+        QVERIFY(root.mkpath(QStringLiteral("misc")));
+
+        SearchOptions options;
+        const QString child = directory.filePath(QStringLiteral("child"));
+        const QString misc = directory.filePath(QStringLiteral("misc"));
+        options.roots = QStringLiteral("%1;%2/;%1;%3")
+                            .arg(child, directory.path(), misc);
+        options.excludeFolders = QStringLiteral("unrelated");
+
+        const EffectiveRoots effective = effectiveSearchRoots(options);
+        QCOMPARE(effective.paths, QStringList({directory.path()}));
+        QCOMPARE(effective.skipped, 3);
+    }
+
+    void preservesNestedRootsThatExpandCoverage()
+    {
+        QTemporaryDir directory;
+        QVERIFY(directory.isValid());
+        QDir root(directory.path());
+        QVERIFY(root.mkpath(QStringLiteral("child")));
+        const QString child = directory.filePath(QStringLiteral("child"));
+
+        SearchOptions options;
+        options.roots = directory.path() + u';' + child;
+
+        options.recursive = false;
+        QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
+
+        options.recursive = true;
+        options.maxDepth = 1;
+        QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
+
+        options.maxDepth = 0;
+        options.maxResults = 1;
+        QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
+
+        options.maxResults = 0;
+        options.subfolderWildcards = QStringLiteral("matched-*");
+        QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
+
+        options.subfolderWildcards = QStringLiteral("*");
+        options.excludeFolders = QStringLiteral("child");
+        QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
+
+        options.excludeFolders = QStringLiteral("unrelated");
+        QCOMPARE(effectiveSearchRoots(options).paths,
+                 QStringList({directory.path()}));
+    }
+
+    void respectsSymlinksWhenCollapsingRoots()
+    {
+        QTemporaryDir directory;
+        QTemporaryDir external;
+        QVERIFY(directory.isValid());
+        QVERIFY(external.isValid());
+        QDir externalRoot(external.path());
+        QVERIFY(externalRoot.mkpath(QStringLiteral("child")));
+        const QString link = directory.filePath(QStringLiteral("link"));
+        QVERIFY(QFile::link(external.path(), link));
+        const QString linkedChild = QDir(link).filePath(QStringLiteral("child"));
+
+        SearchOptions options;
+        options.roots = directory.path() + u';' + linkedChild;
+        options.followLinks = false;
+        QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
+
+        options.followLinks = true;
+        QCOMPARE(effectiveSearchRoots(options).paths,
+                 QStringList({directory.path()}));
+
+        options.roots = external.path() + u';' + link;
+        const EffectiveRoots aliases = effectiveSearchRoots(options);
+        QCOMPARE(aliases.paths.size(), 1);
+        QCOMPARE(aliases.skipped, 1);
+    }
+
+    void prefersTheMostSpecificConfiguredRoot()
+    {
+        QTemporaryDir directory;
+        QVERIFY(directory.isValid());
+        QDir root(directory.path());
+        QVERIFY(root.mkpath(QStringLiteral("Temp")));
+        QVERIFY(root.mkpath(QStringLiteral("misc")));
+        const QString temp = directory.filePath(QStringLiteral("Temp"));
+        const QString misc = directory.filePath(QStringLiteral("misc"));
+        const QStringList roots{directory.path(), temp, misc};
+
+        QCOMPARE(preferredRootIndex(
+                     directory.filePath(QStringLiteral("base.mp4")), roots), 0);
+        QCOMPARE(preferredRootIndex(
+                     QDir(temp).filePath(QStringLiteral("copy.mp4")), roots), 1);
+        QCOMPARE(preferredRootIndex(
+                     QDir(misc).filePath(QStringLiteral("copy.mp4")), roots), 2);
+        QCOMPARE(preferredRootIndex(
+                     directory.filePath(QStringLiteral("unrelated/file.mp4")), roots), 0);
+
+        const QStringList similarRoots{directory.filePath(QStringLiteral("foo"))};
+        QCOMPARE(preferredRootIndex(
+                     directory.filePath(QStringLiteral("foobar/file.mp4")), similarRoots), -1);
+    }
+
    void parsesOptionalIsoDates()
    {
        qint64 seconds = -1;
@@ -108,6 +244,72 @@ private slots:
        QVERIFY(!filesEqual(path, path, &cancelled));
        QVERIFY(sha256(path, &cancelled).isEmpty());
    }
+
+    void samplesLargeFilesAndValidatesSignatures()
+    {
+        QTemporaryDir directory;
+        QVERIFY(directory.isValid());
+        const QByteArray content(duplicateSampleThreshold + 1024 * 1024, 'x');
+        const QString firstPath = directory.filePath(QStringLiteral("first.bin"));
+        const QString samePath = directory.filePath(QStringLiteral("same.bin"));
+        const QString outsidePath = directory.filePath(QStringLiteral("outside.bin"));
+        const QString startPath = directory.filePath(QStringLiteral("start.bin"));
+        const QString middlePath = directory.filePath(QStringLiteral("middle.bin"));
+        const QString endPath = directory.filePath(QStringLiteral("end.bin"));
+
+        QVERIFY(writeFile(firstPath, content));
+        QVERIFY(writeFile(samePath, content));
+
+        QByteArray outside = content;
+        outside[1024 * 1024] = 'y';
+        QVERIFY(writeFile(outsidePath, outside));
+        QByteArray start = content;
+        start[0] = 'y';
+        QVERIFY(writeFile(startPath, start));
+        QByteArray middle = content;
+        middle[(content.size() - duplicateSampleBlockSize) / 2] = 'y';
+        QVERIFY(writeFile(middlePath, middle));
+        QByteArray end = content;
+        end[end.size() - 1] = 'y';
+        QVERIFY(writeFile(endPath, end));
+
+        const FileRecord first = recordFor(firstPath);
+        const QByteArray sample = sampleSha256(first);
+        QVERIFY(!sample.isEmpty());
+        QCOMPARE(sampleSha256(recordFor(samePath)), sample);
+        QCOMPARE(sampleSha256(recordFor(outsidePath)), sample);
+        QVERIFY(sampleSha256(recordFor(startPath)) != sample);
+        QVERIFY(sampleSha256(recordFor(middlePath)) != sample);
+        QVERIFY(sampleSha256(recordFor(endPath)) != sample);
+        QVERIFY(sha256(recordFor(outsidePath)) != sha256(first));
+
+        std::atomic_bool cancelled = true;
+        QVERIFY(sampleSha256(first, &cancelled).isEmpty());
+
+        const QString shortPath = directory.filePath(QStringLiteral("short.bin"));
+        QVERIFY(writeFile(shortPath, QByteArray("short")));
+        QVERIFY(sampleSha256(recordFor(shortPath)).isEmpty());
+        QVERIFY(!sha256(recordFor(shortPath)).isEmpty());
+
+        const QString emptyPath = directory.filePath(QStringLiteral("empty.bin"));
+        QVERIFY(writeFile(emptyPath, {}));
+        QVERIFY(sampleSha256(recordFor(emptyPath)).isEmpty());
+        QVERIFY(!sha256(recordFor(emptyPath)).isEmpty());
+
+        const QString replacedPath = directory.filePath(QStringLiteral("replaced.bin"));
+        QVERIFY(writeFile(replacedPath, content));
+        const FileRecord replaced = recordFor(replacedPath);
+        QVERIFY(QFile::remove(replacedPath));
+        QVERIFY(writeFile(replacedPath, content));
+        QVERIFY(!fileSignatureMatches(replaced));
+        QVERIFY(sha256(replaced).isEmpty());
+        QVERIFY(sampleSha256(replaced).isEmpty());
+
+        FileRecord missing = first;
+        missing.path = directory.filePath(QStringLiteral("missing.bin"));
+        QVERIFY(sha256(missing).isEmpty());
+        QVERIFY(sampleSha256(missing).isEmpty());
+    }
 };

 QTEST_GUILESS_MAIN(SearchCoreTest)