3 Commits

Author SHA1 Message Date
forust 9f7908b275 chore: release v0.3.0
ci-release / publish-linux-amd64 (push) Has been skipped
ci-release / verify (push) Failing after 1m46s
2026-07-26 16:46:25 +02:00
forust 010bd0f374 feat(ui): rename app and add input history
Import legacy SearchMyFiles settings once so the FolderScope identity change does not discard user profiles and preferences.
2026-07-26 16:46:08 +02:00
forust c24d0dd84d feat(search): optimize duplicate detection
ci-release / publish-linux-amd64 (push) Failing after 1m55s
ci-release / verify (push) Successful in 2m32s
2026-07-26 16:27:33 +02:00
21 changed files with 1456 additions and 123 deletions
+33
View File
@@ -5,6 +5,37 @@ All notable changes to FolderScope are documented here.
The project uses [Semantic Versioning](https://semver.org/). Release tags are
formatted as `vMAJOR.MINOR.PATCH`.
## [0.3.0] - 2026-07-26
### Added
- Added persistent input history for folder and wildcard fields in Search
Options.
### Changed
- Renamed the executable, desktop entry, icon, and application identity from
SearchMyFiles to FolderScope.
- Preserved existing settings and search profiles when migrating to the new
application identity.
- Kept the base-folder browser button compact so the input field uses the
available width.
## [0.2.5] - 2026-07-26
### Added
- Added an optional strict byte-by-byte verification mode for duplicate files.
### Changed
- Accelerated duplicate detection with size grouping, sampled block hashes, and
full SHA-256 hashing only for remaining candidates.
- Normalized overlapping search roots and deduplicated file candidates so a
path cannot be compared with itself.
- Strengthened persistent cache invalidation with nanosecond timestamps and
filesystem identity.
## [0.2.4] - 2026-07-25
### Fixed
@@ -66,3 +97,5 @@ formatted as `vMAJOR.MINOR.PATCH`.
[0.2.2]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.2
[0.2.3]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.3
[0.2.4]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.4
[0.2.5]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.2.5
[0.3.0]: https://gitea.forust.xyz/forust/folderscope/releases/tag/v0.3.0
+44 -7
View File
@@ -22,7 +22,7 @@ target_link_libraries(folderscope_core PUBLIC Qt6::Core)
target_compile_options(folderscope_core PRIVATE -Wall -Wextra -Wpedantic)
target_include_directories(folderscope_core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src")
qt_add_executable(searchmyfiles
qt_add_executable(folderscope
src/long_long_spin_box.cpp
src/long_long_spin_box.h
src/main.cpp
@@ -38,14 +38,14 @@ qt_add_executable(searchmyfiles
src/options_dialog.cpp
)
target_link_libraries(searchmyfiles PRIVATE folderscope_core Qt6::Widgets Qt6::Concurrent Qt6::DBus)
target_compile_options(searchmyfiles PRIVATE -Wall -Wextra -Wpedantic)
target_compile_definitions(searchmyfiles PRIVATE FOLDERSCOPE_VERSION="${PROJECT_VERSION}")
target_link_libraries(folderscope PRIVATE folderscope_core Qt6::Widgets Qt6::Concurrent Qt6::DBus)
target_compile_options(folderscope PRIVATE -Wall -Wextra -Wpedantic)
target_compile_definitions(folderscope PRIVATE FOLDERSCOPE_VERSION="${PROJECT_VERSION}")
install(TARGETS searchmyfiles RUNTIME DESTINATION bin)
install(FILES packaging/searchmyfiles.desktop
install(TARGETS folderscope RUNTIME DESTINATION bin)
install(FILES packaging/folderscope.desktop
DESTINATION share/applications)
install(FILES packaging/searchmyfiles.svg
install(FILES packaging/folderscope.svg
DESTINATION share/icons/hicolor/scalable/apps)
include(CTest)
@@ -59,6 +59,43 @@ if (BUILD_TESTING)
target_compile_options(search_core_tests PRIVATE -Wall -Wextra -Wpedantic)
add_test(NAME search-core COMMAND search_core_tests)
qt_add_executable(search_cache_tests
src/search_cache.cpp
src/search_cache.h
tests/search_cache_test.cpp
)
target_link_libraries(search_cache_tests PRIVATE folderscope_core Qt6::Test)
target_compile_options(search_cache_tests PRIVATE -Wall -Wextra -Wpedantic)
target_include_directories(search_cache_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
add_test(NAME search-cache COMMAND search_cache_tests)
qt_add_executable(search_engine_tests
src/search_cache.cpp
src/search_cache.h
src/search_engine.cpp
src/search_engine.h
tests/search_engine_test.cpp
)
target_link_libraries(search_engine_tests PRIVATE
folderscope_core Qt6::Concurrent Qt6::Test)
target_compile_options(search_engine_tests PRIVATE -Wall -Wextra -Wpedantic)
target_include_directories(search_engine_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
add_test(NAME search-engine COMMAND search_engine_tests)
qt_add_executable(options_dialog_tests
src/long_long_spin_box.cpp
src/long_long_spin_box.h
src/options_dialog.cpp
src/options_dialog.h
tests/options_dialog_test.cpp
)
target_link_libraries(options_dialog_tests PRIVATE
folderscope_core Qt6::Widgets Qt6::Test)
target_compile_options(options_dialog_tests PRIVATE -Wall -Wextra -Wpedantic)
target_include_directories(options_dialog_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
add_test(NAME options-dialog COMMAND options_dialog_tests)
set_tests_properties(options-dialog PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen")
qt_add_executable(long_long_spin_box_tests
src/long_long_spin_box.cpp
src/long_long_spin_box.h
+2 -1
View File
@@ -12,6 +12,7 @@ single window.
- search across multiple root folders;
- filter by name patterns, size, timestamps, and file attributes;
- inspect duplicate files and files with matching names;
- optionally verify duplicate matches byte by byte in strict mode;
- search inside file contents, including text and hex;
- reuse a persistent, automatically invalidated cache for content matches and
duplicate hashes;
@@ -33,7 +34,7 @@ Requirements:
```bash
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
cmake --build build -j
./build/searchmyfiles
./build/folderscope
```
Run the automated tests:
+1 -1
View File
@@ -1 +1 @@
0.2.4
0.3.0
@@ -1,8 +1,8 @@
[Desktop Entry]
Type=Application
Name=SearchMyFiles
Name=FolderScope
Comment=Fast advanced file search
Exec=searchmyfiles
Icon=searchmyfiles
Exec=folderscope
Icon=folderscope
Terminal=false
Categories=Utility;FileTools;

Before

Width:  |  Height:  |  Size: 324 B

After

Width:  |  Height:  |  Size: 324 B

+25 -2
View File
@@ -1,15 +1,38 @@
#include <QApplication>
#include <QMetaType>
#include <QSettings>
#include <QTimer>
#include "main_window.h"
#include "search_core.h"
namespace {
void migrateLegacySettings()
{
QSettings settings;
const QString migrationKey =
QStringLiteral("migration/searchMyFilesSettingsImported");
if (settings.value(migrationKey, false).toBool())
return;
QSettings legacy(QStringLiteral("SearchMyFilesLinux"),
QStringLiteral("SearchMyFiles"));
for (const QString &key : legacy.allKeys()) {
if (!settings.contains(key))
settings.setValue(key, legacy.value(key));
}
settings.setValue(migrationKey, true);
}
}
int main(int argc, char **argv)
{
QApplication app(argc, argv);
QCoreApplication::setOrganizationName(QStringLiteral("SearchMyFilesLinux"));
QCoreApplication::setApplicationName(QStringLiteral("SearchMyFiles"));
QCoreApplication::setOrganizationName(QStringLiteral("FolderScope"));
QCoreApplication::setApplicationName(QStringLiteral("FolderScope"));
migrateLegacySettings();
QApplication::setStyle(QStringLiteral("Fusion"));
qRegisterMetaType<SearchOptions>();
qRegisterMetaType<QVector<FileRecord>>();
+6 -2
View File
@@ -23,7 +23,7 @@
MainWindow::MainWindow()
{
loadOptions();
setWindowTitle(tr("SearchMyFiles for Linux"));
setWindowTitle(tr("FolderScope"));
resize(1280, 760);
model_ = new ResultsModel(this);
model_->setDisplayOptions(sizeUnit_, showGmt_, markDuplicates_, duplicateColorSet_);
@@ -507,7 +507,7 @@ void MainWindow::startCurrentSearch()
progress_->show();
progress_->setRange(0, 0);
progress_->setFormat(tr("Scanning…"));
setWindowTitle(tr("%1 — %2 — SearchMyFiles for Linux")
setWindowTitle(tr("%1 — %2 — FolderScope")
.arg(options_.roots, options_.mode));
if (focusOnSearchStart_)
table_->setFocus();
@@ -977,6 +977,8 @@ void MainWindow::saveOptions()
settings.setValue(QStringLiteral("mode"), options_.mode);
settings.setValue(QStringLiteral("duplicateNameMode"), options_.duplicateNameMode);
settings.setValue(QStringLiteral("duplicateNameWithoutExtension"), options_.duplicateNameWithoutExtension);
settings.setValue(QStringLiteral("strictDuplicateComparison"),
options_.strictDuplicateComparison);
settings.setValue(QStringLiteral("showDuplicateCopiesOnly"), options_.showDuplicateCopiesOnly);
settings.setValue(QStringLiteral("includeSubfoldersInSummary"), options_.includeSubfoldersInSummary);
settings.setValue(QStringLiteral("hidden"), options_.hidden);
@@ -1023,6 +1025,8 @@ void MainWindow::loadOptions()
options_.mode = s.value(QStringLiteral("mode"), options_.mode).toString();
options_.duplicateNameMode = s.value(QStringLiteral("duplicateNameMode"), options_.duplicateNameMode).toString();
options_.duplicateNameWithoutExtension = s.value(QStringLiteral("duplicateNameWithoutExtension"), false).toBool();
options_.strictDuplicateComparison =
s.value(QStringLiteral("strictDuplicateComparison"), false).toBool();
options_.showDuplicateCopiesOnly =
s.value(QStringLiteral("showDuplicateCopiesOnly"),
s.value(QStringLiteral("showOnlyDuplicateFiles"), true)).toBool();
+85 -30
View File
@@ -8,6 +8,7 @@
#include <QMessageBox>
#include <QMimeData>
#include <QSettings>
#include <QSizePolicy>
#include "long_long_spin_box.h"
@@ -87,22 +88,29 @@ void OptionsDialog::setupUi(const SearchOptions &o)
auto *filesPage = new QWidget;
auto *filesForm = new QFormLayout(filesPage);
roots_ = new QLineEdit(o.roots);
roots_->setAcceptDrops(true);
roots_ = historyCombo(QStringLiteral("roots"), o.roots);
roots_->lineEdit()->setAcceptDrops(true);
roots_->setToolTip(tr("Drop folders here from your file manager"));
roots_->installEventFilter(this);
roots_->lineEdit()->installEventFilter(this);
auto *rootRow = new QWidget;
auto *rootLayout = new QHBoxLayout(rootRow);
rootLayout->setContentsMargins(0, 0, 0, 0);
rootLayout->addWidget(roots_);
auto *browse = new QPushButton(tr("Browse…"));
browse->setSizePolicy(QSizePolicy::Fixed, QSizePolicy::Fixed);
rootLayout->addWidget(browse);
rootLayout->setStretch(0, 1);
filesForm->addRow(tr("Base folders (separate with ;):"), rootRow);
fileMasks_ = addLine(filesForm, tr("Files wildcard:"), o.fileWildcards);
subfolderMasks_ = addLine(filesForm, tr("Subfolders wildcard:"), o.subfolderWildcards);
excludeFiles_ = addLine(filesForm, tr("Exclude files:"), o.excludeFiles);
excludeFolders_ = addLine(filesForm, tr("Exclude folders:"), o.excludeFolders);
includeFolders_ = addLine(filesForm, tr("Include only folders:"), o.includeFolders);
fileMasks_ = addHistoryCombo(filesForm, tr("Files wildcard:"),
QStringLiteral("fileWildcards"), o.fileWildcards);
subfolderMasks_ = addHistoryCombo(filesForm, tr("Subfolders wildcard:"),
QStringLiteral("subfolderWildcards"), o.subfolderWildcards);
excludeFiles_ = addHistoryCombo(filesForm, tr("Exclude files:"),
QStringLiteral("excludeFiles"), o.excludeFiles);
excludeFolders_ = addHistoryCombo(filesForm, tr("Exclude folders:"),
QStringLiteral("excludeFolders"), o.excludeFolders);
includeFolders_ = addHistoryCombo(filesForm, tr("Include only folders:"),
QStringLiteral("includeFolders"), o.includeFolders);
recursive_ = new QCheckBox(tr("Search subfolders")); recursive_->setChecked(o.recursive);
findFiles_ = new QCheckBox(tr("Find files")); findFiles_->setChecked(o.findFiles);
findFolders_ = new QCheckBox(tr("Find folders")); findFolders_->setChecked(o.findFolders);
@@ -112,6 +120,11 @@ void OptionsDialog::setupUi(const SearchOptions &o)
accurateProgress_->setChecked(o.accurateProgress);
useCache_ = new QCheckBox(tr("Use persistent cache for hashes and content searches"));
useCache_->setChecked(o.useCache);
strictDuplicateComparison_ =
new QCheckBox(tr("Strict duplicate comparison (byte by byte)"));
strictDuplicateComparison_->setObjectName(
QStringLiteral("strictDuplicateComparison"));
strictDuplicateComparison_->setChecked(o.strictDuplicateComparison);
maxDepth_ = new QSpinBox; maxDepth_->setRange(0, 1024); maxDepth_->setValue(o.maxDepth);
maxDepth_->setSpecialValueText(tr("Unlimited"));
maxResults_ = new QSpinBox; maxResults_->setRange(0, 100'000'000); maxResults_->setValue(o.maxResults);
@@ -124,6 +137,7 @@ void OptionsDialog::setupUi(const SearchOptions &o)
filesForm->addRow(retrieveOwner_);
filesForm->addRow(accurateProgress_);
filesForm->addRow(useCache_);
filesForm->addRow(strictDuplicateComparison_);
filesForm->addRow(tr("Stop after finding:"), maxResults_);
tabs->addTab(filesPage, tr("Files & folders"));
@@ -177,7 +191,7 @@ void OptionsDialog::setupUi(const SearchOptions &o)
connect(buttons, &QDialogButtonBox::accepted, this, &OptionsDialog::accept);
connect(buttons, &QDialogButtonBox::rejected, this, &QDialog::reject);
connect(browse, &QPushButton::clicked, this, [this] {
const QStringList existingFolders = patterns(roots_->text());
const QStringList existingFolders = patterns(roots_->currentText());
const QString startFolder = existingFolders.isEmpty()
? QDir::homePath() : existingFolders.constLast();
const QString folder = QFileDialog::getExistingDirectory(
@@ -189,7 +203,7 @@ void OptionsDialog::setupUi(const SearchOptions &o)
const QString cleanPath = QDir::cleanPath(folder);
if (!folders.contains(cleanPath))
folders.push_back(cleanPath);
roots_->setText(folders.join(u';'));
roots_->setEditText(folders.join(u';'));
});
layout->addWidget(buttons);
}
@@ -197,12 +211,12 @@ void OptionsDialog::setupUi(const SearchOptions &o)
SearchOptions OptionsDialog::options() const
{
SearchOptions o;
o.roots = roots_->text();
o.fileWildcards = fileMasks_->text();
o.subfolderWildcards = subfolderMasks_->text();
o.excludeFiles = excludeFiles_->text();
o.excludeFolders = excludeFolders_->text();
o.includeFolders = includeFolders_->text();
o.roots = roots_->currentText();
o.fileWildcards = fileMasks_->currentText();
o.subfolderWildcards = subfolderMasks_->currentText();
o.excludeFiles = excludeFiles_->currentText();
o.excludeFolders = excludeFolders_->currentText();
o.includeFolders = includeFolders_->currentText();
o.contains = contains_->toPlainText();
o.binary = binary_->isChecked();
o.multipleValues = multiple_->isChecked();
@@ -221,6 +235,7 @@ SearchOptions OptionsDialog::options() const
o.retrieveOwner = retrieveOwner_->isChecked();
o.accurateProgress = accurateProgress_->isChecked();
o.useCache = useCache_->isChecked();
o.strictDuplicateComparison = strictDuplicateComparison_->isChecked();
o.maxDepth = maxDepth_->value();
o.maxResults = maxResults_->value();
o.mode = mode_->currentText();
@@ -237,14 +252,35 @@ void OptionsDialog::accept()
if (!validateDate(minTime_, tr("From time"))
|| !validateDate(maxTime_, tr("To time")))
return;
rememberHistory(QStringLiteral("roots"), roots_->currentText());
rememberHistory(QStringLiteral("fileWildcards"), fileMasks_->currentText());
rememberHistory(QStringLiteral("subfolderWildcards"), subfolderMasks_->currentText());
rememberHistory(QStringLiteral("excludeFiles"), excludeFiles_->currentText());
rememberHistory(QStringLiteral("excludeFolders"), excludeFolders_->currentText());
rememberHistory(QStringLiteral("includeFolders"), includeFolders_->currentText());
QDialog::accept();
}
QLineEdit *OptionsDialog::addLine(QFormLayout *layout, const QString &label, const QString &value)
QComboBox *OptionsDialog::historyCombo(const QString &key, const QString &value)
{
auto *edit = new QLineEdit(value);
layout->addRow(label, edit);
return edit;
auto *combo = new QComboBox;
combo->setEditable(true);
combo->setInsertPolicy(QComboBox::NoInsert);
combo->setObjectName(QStringLiteral("history%1").arg(key));
QSettings settings;
const QStringList history = settings.value(QStringLiteral("inputHistory/%1").arg(key)).toStringList();
combo->addItems(history);
combo->setEditText(value);
return combo;
}
QComboBox *OptionsDialog::addHistoryCombo(QFormLayout *layout, const QString &label,
const QString &key, const QString &value)
{
auto *combo = historyCombo(key, value);
layout->addRow(label, combo);
return combo;
}
LongLongSpinBox *OptionsDialog::sizeSpin(qint64 value)
@@ -277,6 +313,22 @@ bool OptionsDialog::validateDate(QLineEdit *edit, const QString &label)
return false;
}
void OptionsDialog::rememberHistory(const QString &key, const QString &value)
{
if (value.isEmpty())
return;
QSettings settings;
const QString settingsKey = QStringLiteral("inputHistory/%1").arg(key);
QStringList history = settings.value(settingsKey).toStringList();
history.removeAll(value);
history.prepend(value);
constexpr qsizetype maxHistoryEntries = 20;
if (history.size() > maxHistoryEntries)
history.erase(history.begin() + maxHistoryEntries, history.end());
settings.setValue(settingsKey, history);
}
QComboBox *OptionsDialog::attrCombo(const QString &value)
{
auto *combo = new QComboBox;
@@ -287,7 +339,7 @@ QComboBox *OptionsDialog::attrCombo(const QString &value)
bool OptionsDialog::eventFilter(QObject *obj, QEvent *event)
{
if (obj == roots_ && event->type() == QEvent::DragEnter) {
if (obj == roots_->lineEdit() && event->type() == QEvent::DragEnter) {
auto *dragEnter = static_cast<QDragEnterEvent *>(event);
if (dragEnter->mimeData()->hasUrls()) {
dragEnter->acceptProposedAction();
@@ -295,11 +347,11 @@ bool OptionsDialog::eventFilter(QObject *obj, QEvent *event)
}
return false;
}
if (obj == roots_ && event->type() == QEvent::Drop) {
if (obj == roots_->lineEdit() && event->type() == QEvent::Drop) {
auto *drop = static_cast<QDropEvent *>(event);
if (!drop->mimeData()->hasUrls())
return false;
QStringList folders = patterns(roots_->text());
QStringList folders = patterns(roots_->currentText());
for (const QUrl &url : drop->mimeData()->urls()) {
if (!url.isLocalFile())
continue;
@@ -307,7 +359,7 @@ bool OptionsDialog::eventFilter(QObject *obj, QEvent *event)
if (QFileInfo::exists(cleanPath) && !folders.contains(cleanPath))
folders.append(cleanPath);
}
roots_->setText(folders.join(u';'));
roots_->setEditText(folders.join(u';'));
drop->acceptProposedAction();
return true;
}
@@ -360,6 +412,7 @@ void OptionsDialog::saveProfile()
saveBool("accurateProgress", opt.accurateProgress);
saveBool("useCache", opt.useCache);
saveBool("duplicateNameWithoutExtension", opt.duplicateNameWithoutExtension);
saveBool("strictDuplicateComparison", opt.strictDuplicateComparison);
saveInt("lastMinutes", opt.lastMinutes);
saveInt("maxDepth", opt.maxDepth);
@@ -393,12 +446,12 @@ void OptionsDialog::loadProfile()
auto loadInt = [&](const char *key, int def) { return settings.value(QString::fromLatin1(key), def).toInt(); };
auto loadLong = [&](const char *key, qint64 def) { return settings.value(QString::fromLatin1(key), def).toLongLong(); };
roots_->setText(load("roots"));
fileMasks_->setText(load("fileWildcards"));
subfolderMasks_->setText(load("subfolderWildcards"));
excludeFiles_->setText(load("excludeFiles"));
excludeFolders_->setText(load("excludeFolders"));
includeFolders_->setText(load("includeFolders"));
roots_->setEditText(load("roots"));
fileMasks_->setEditText(load("fileWildcards"));
subfolderMasks_->setEditText(load("subfolderWildcards"));
excludeFiles_->setEditText(load("excludeFiles"));
excludeFolders_->setEditText(load("excludeFolders"));
includeFolders_->setEditText(load("includeFolders"));
contains_->setText(load("contains"));
binary_->setChecked(loadBool("binary", false));
multiple_->setChecked(loadBool("multipleValues", false));
@@ -419,6 +472,8 @@ void OptionsDialog::loadProfile()
retrieveOwner_->setChecked(loadBool("retrieveOwner", false));
accurateProgress_->setChecked(loadBool("accurateProgress", true));
useCache_->setChecked(loadBool("useCache", true));
strictDuplicateComparison_->setChecked(
loadBool("strictDuplicateComparison", false));
maxDepth_->setValue(loadInt("maxDepth", 0));
maxResults_->setValue(loadInt("maxResults", 0));
mode_->setCurrentText(load("mode"));
+11 -7
View File
@@ -29,10 +29,13 @@ protected:
private:
bool eventFilter(QObject *obj, QEvent *event) override;
static QLineEdit *addLine(QFormLayout *layout, const QString &label, const QString &value);
static QComboBox *historyCombo(const QString &key, const QString &value);
static QComboBox *addHistoryCombo(QFormLayout *layout, const QString &label,
const QString &key, const QString &value);
static LongLongSpinBox *sizeSpin(qint64 value);
static qint64 dateValue(const QLineEdit *edit);
bool validateDate(QLineEdit *edit, const QString &label);
void rememberHistory(const QString &key, const QString &value);
static QComboBox *attrCombo(const QString &value);
void setupUi(const SearchOptions &o);
@@ -42,12 +45,12 @@ private:
QComboBox *mode_{};
QComboBox *duplicateMode_{};
QCheckBox *duplicateWithoutExtension_{};
QLineEdit *roots_{};
QLineEdit *fileMasks_{};
QLineEdit *subfolderMasks_{};
QLineEdit *excludeFiles_{};
QLineEdit *excludeFolders_{};
QLineEdit *includeFolders_{};
QComboBox *roots_{};
QComboBox *fileMasks_{};
QComboBox *subfolderMasks_{};
QComboBox *excludeFiles_{};
QComboBox *excludeFolders_{};
QComboBox *includeFolders_{};
QTextEdit *contains_{};
QCheckBox *binary_{};
QCheckBox *multiple_{};
@@ -66,6 +69,7 @@ private:
QCheckBox *retrieveOwner_{};
QCheckBox *accurateProgress_{};
QCheckBox *useCache_{};
QCheckBox *strictDuplicateComparison_{};
QSpinBox *maxDepth_{};
QSpinBox *maxResults_{};
QComboBox *hidden_{};
+1 -1
View File
@@ -23,7 +23,7 @@ QVariant ResultsModel::headerData(int section, Qt::Orientation orientation, int
if (section == DuplicateNumber)
return tr("Identifies one set of identical files");
if (section == DuplicateGroup)
return tr("Order based on the base-folder list; 1 = preferred copy to keep");
return tr("Most-specific base folder wins; ties use newer files, then path. 1 = preferred copy to keep");
return {};
}
if (role != Qt::DisplayRole)
+41 -7
View File
@@ -10,14 +10,41 @@
#include <algorithm>
#include <utility>
SearchCache::SearchCache()
SearchCache::SearchCache(QString path)
{
if (path.isEmpty()) {
const QString cacheRoot = QStandardPaths::writableLocation(QStandardPaths::CacheLocation);
QDir().mkpath(cacheRoot);
path_ = QDir(cacheRoot).filePath(QStringLiteral("search-cache-v1.dat"));
} else {
path_ = std::move(path);
}
load();
}
bool SearchCache::findSampleHash(const FileRecord &record, QByteArray &digest)
{
if (!enabled_)
return false;
QMutexLocker lock(&mutex_);
auto it = entries_.find(record.path);
if (it == entries_.end() || !signatureMatches(*it, record)
|| it->sampleSha256.isEmpty())
return false;
it->lastUsed = QDateTime::currentSecsSinceEpoch();
digest = it->sampleSha256;
return true;
}
void SearchCache::storeSampleHash(const FileRecord &record, const QByteArray &digest)
{
if (!enabled_ || digest.isEmpty())
return;
QMutexLocker lock(&mutex_);
SearchCacheEntry &entry = currentEntry(record);
entry.sampleSha256 = digest;
}
bool SearchCache::findHash(const FileRecord &record, QByteArray &digest)
{
if (!enabled_)
@@ -75,9 +102,10 @@ void SearchCache::save()
return;
QDataStream stream(&file);
stream.setVersion(QDataStream::Qt_6_0);
stream << quint32(0x46534348) << quint32(1) << quint32(entries_.size());
stream << quint32(0x46534348) << quint32(2) << quint32(entries_.size());
for (auto it = entries_.cbegin(); it != entries_.cend(); ++it) {
stream << it.key() << it->size << it->modified << it->lastUsed << it->sha256;
stream << it.key() << it->size << it->modifiedNs << it->device << it->inode
<< it->lastUsed << it->sampleSha256 << it->sha256;
stream << quint32(it->contentMatches.size());
for (auto match = it->contentMatches.cbegin(); match != it->contentMatches.cend(); ++match)
stream << match.key() << match.value();
@@ -95,7 +123,10 @@ void SearchCache::clear()
bool SearchCache::signatureMatches(const SearchCacheEntry &entry, const FileRecord &record)
{
return entry.size == record.size && entry.modified == record.modified;
return entry.size == record.size
&& entry.modifiedNs == record.modifiedNs
&& entry.device == record.device
&& entry.inode == record.inode;
}
SearchCacheEntry &SearchCache::currentEntry(const FileRecord &record)
@@ -104,7 +135,9 @@ SearchCacheEntry &SearchCache::currentEntry(const FileRecord &record)
if (!signatureMatches(entry, record)) {
entry = {};
entry.size = record.size;
entry.modified = record.modified;
entry.modifiedNs = record.modifiedNs;
entry.device = record.device;
entry.inode = record.inode;
}
entry.lastUsed = QDateTime::currentSecsSinceEpoch();
return entry;
@@ -121,13 +154,14 @@ void SearchCache::load()
quint32 version = 0;
quint32 count = 0;
stream >> magic >> version >> count;
if (magic != 0x46534348 || version != 1 || count > 200'000)
if (magic != 0x46534348 || version != 2 || count > 200'000)
return;
for (quint32 index = 0; index < count && stream.status() == QDataStream::Ok; ++index) {
QString path;
SearchCacheEntry entry;
quint32 contentCount = 0;
stream >> path >> entry.size >> entry.modified >> entry.lastUsed >> entry.sha256;
stream >> path >> entry.size >> entry.modifiedNs >> entry.device >> entry.inode
>> entry.lastUsed >> entry.sampleSha256 >> entry.sha256;
stream >> contentCount;
if (contentCount > 256)
return;
+7 -2
View File
@@ -9,19 +9,24 @@
struct SearchCacheEntry {
qint64 size = -1;
qint64 modified = -1;
qint64 modifiedNs = -1;
quint64 device = 0;
quint64 inode = 0;
qint64 lastUsed = 0;
QByteArray sampleSha256;
QByteArray sha256;
QHash<QByteArray, bool> contentMatches;
};
class SearchCache {
public:
SearchCache();
explicit SearchCache(QString path = {});
void setEnabled(bool enabled) { enabled_ = enabled; }
bool enabled() const { return enabled_; }
bool findSampleHash(const FileRecord &record, QByteArray &digest);
void storeSampleHash(const FileRecord &record, const QByteArray &digest);
bool findHash(const FileRecord &record, QByteArray &digest);
void storeHash(const FileRecord &record, const QByteArray &digest);
std::optional<bool> findContent(const FileRecord &record, const QByteArray &queryKey);
+188
View File
@@ -2,11 +2,16 @@
#include <QCryptographicHash>
#include <QDateTime>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QRegularExpression>
#include <QSet>
#include <algorithm>
#include <array>
#include <pwd.h>
#include <sys/stat.h>
QStringList patterns(const QString &text)
{
@@ -54,6 +59,129 @@ bool wildcardMatch(const QString &value, const QStringList &items, Qt::CaseSensi
return false;
}
namespace {
struct RootCandidate {
QString path;
};
bool isDescendantPath(const QString &parent, const QString &child,
QString *relativePath = nullptr)
{
const QString relative = QDir(parent).relativeFilePath(child);
const bool descendant = relative != QStringLiteral(".")
&& relative != QStringLiteral("..")
&& !relative.startsWith(QStringLiteral("../"))
&& !QDir::isAbsolutePath(relative);
if (descendant && relativePath)
*relativePath = relative;
return descendant;
}
bool canCollapseInto(const RootCandidate &parent, const RootCandidate &child,
const SearchOptions &options, const QStringList &excludedFolders,
Qt::CaseSensitivity cs)
{
if (!QFileInfo(parent.path).isDir())
return false;
QString relative;
if (!isDescendantPath(parent.path, child.path, &relative))
return false;
QString current = parent.path;
const QStringList segments = relative.split(u'/', Qt::SkipEmptyParts);
for (const QString &segment : segments) {
current = QDir(current).filePath(segment);
const QFileInfo info(current);
if ((!options.followLinks && info.isSymLink())
|| (!excludedFolders.isEmpty()
&& (wildcardMatch(info.fileName(), excludedFolders, cs)
|| wildcardMatch(QDir::cleanPath(current), excludedFolders, cs)))) {
return false;
}
}
return true;
}
}
EffectiveRoots effectiveSearchRoots(const SearchOptions &options)
{
QVector<RootCandidate> unique;
QSet<QString> identities;
int skipped = 0;
for (const QString &rootText : patterns(options.roots)) {
const QFileInfo info(rootText);
const QString path = QDir::cleanPath(info.absoluteFilePath());
const QString canonical = info.canonicalFilePath();
const QString identity = canonical.isEmpty()
? path : QDir::cleanPath(canonical);
if (identities.contains(identity)) {
++skipped;
continue;
}
identities.insert(identity);
unique.push_back({path});
}
const QStringList subfolderMasks = patterns(options.subfolderWildcards).isEmpty()
? QStringList{QStringLiteral("*")} : patterns(options.subfolderWildcards);
const bool universalSubfolderMask =
subfolderMasks.contains(QStringLiteral("*"));
if (!options.recursive || options.maxDepth > 0 || options.maxResults > 0
|| !universalSubfolderMask) {
EffectiveRoots result;
result.skipped = skipped;
for (const RootCandidate &root : unique) {
result.paths.push_back(root.path);
result.priorityPaths.push_back(root.path);
}
return result;
}
const Qt::CaseSensitivity cs =
options.caseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive;
const QStringList excludedFolders = patterns(options.excludeFolders);
EffectiveRoots result;
result.skipped = skipped;
for (const RootCandidate &root : unique)
result.priorityPaths.push_back(root.path);
for (qsizetype childIndex = 0; childIndex < unique.size(); ++childIndex) {
bool redundant = false;
for (qsizetype parentIndex = 0; parentIndex < unique.size(); ++parentIndex) {
if (parentIndex == childIndex)
continue;
if (canCollapseInto(unique[parentIndex], unique[childIndex], options,
excludedFolders, cs)) {
redundant = true;
break;
}
}
if (redundant) {
++result.skipped;
} else {
result.paths.push_back(unique[childIndex].path);
}
}
return result;
}
int preferredRootIndex(const QString &path, const QStringList &roots)
{
const QString normalizedPath = QDir::cleanPath(QFileInfo(path).absoluteFilePath());
int bestIndex = -1;
qsizetype bestLength = 0;
for (qsizetype index = 0; index < roots.size(); ++index) {
const QString root = QDir::cleanPath(QFileInfo(roots[index]).absoluteFilePath());
if (normalizedPath != root && !isDescendantPath(root, normalizedPath))
continue;
if (root.size() > bestLength) {
bestIndex = int(index);
bestLength = root.size();
}
}
return bestIndex;
}
bool shouldShowDuplicateResult(bool copiesOnly, int keeperPriority)
{
if (keeperPriority < 1)
@@ -99,6 +227,26 @@ bool isCancelled(const std::atomic_bool *cancelled)
{
return cancelled && cancelled->load(std::memory_order_relaxed);
}
qint64 modifiedNanoseconds(const struct stat &st)
{
return qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec;
}
bool signatureMatches(const FileRecord &record, const struct stat &st)
{
return record.size == st.st_size
&& record.modifiedNs == modifiedNanoseconds(st)
&& record.device == quint64(st.st_dev)
&& record.inode == quint64(st.st_ino);
}
bool statMatches(const FileRecord &record)
{
struct stat st {};
const QByteArray nativePath = QFile::encodeName(record.path);
return ::stat(nativePath.constData(), &st) == 0 && signatureMatches(record, st);
}
}
bool fileContains(const QString &path, const SearchOptions &options,
@@ -187,6 +335,46 @@ QByteArray sha256(const QString &path, const std::atomic_bool *cancelled)
return isCancelled(cancelled) ? QByteArray{} : hash.result();
}
bool fileSignatureMatches(const FileRecord &record)
{
return statMatches(record);
}
QByteArray sha256(const FileRecord &record, const std::atomic_bool *cancelled)
{
if (isCancelled(cancelled) || !statMatches(record))
return {};
const QByteArray digest = sha256(record.path, cancelled);
return !digest.isEmpty() && statMatches(record) ? digest : QByteArray{};
}
QByteArray sampleSha256(const FileRecord &record, const std::atomic_bool *cancelled)
{
if (isCancelled(cancelled) || record.size <= duplicateSampleThreshold
|| !statMatches(record))
return {};
QFile file(record.path);
if (!file.open(QIODevice::ReadOnly))
return {};
const qint64 middleOffset = (record.size - duplicateSampleBlockSize) / 2;
const std::array<qint64, 3> offsets{
0, middleOffset, record.size - duplicateSampleBlockSize
};
QCryptographicHash hash(QCryptographicHash::Sha256);
for (const qint64 offset : offsets) {
if (isCancelled(cancelled) || !file.seek(offset))
return {};
const QByteArray block = file.read(duplicateSampleBlockSize);
if (block.size() != duplicateSampleBlockSize)
return {};
hash.addData(block);
}
const QByteArray digest = hash.result();
return !isCancelled(cancelled) && statMatches(record) ? digest : QByteArray{};
}
bool parseOptionalIsoDate(const QString &text, qint64 &seconds)
{
const QString trimmed = text.trimmed();
+19
View File
@@ -11,6 +11,9 @@
#include <optional>
#include <sys/types.h>
inline constexpr qint64 duplicateSampleBlockSize = 64LL * 1024;
inline constexpr qint64 duplicateSampleThreshold = 4LL * 1024 * 1024;
struct SearchOptions {
QString roots = QDir::homePath();
QString fileWildcards = QStringLiteral("*");
@@ -41,6 +44,7 @@ struct SearchOptions {
QString mode = QStringLiteral("Standard search");
QString duplicateNameMode = QStringLiteral("All files and folders");
bool duplicateNameWithoutExtension = false;
bool strictDuplicateComparison = false;
bool showDuplicateCopiesOnly = true;
bool includeSubfoldersInSummary = false;
QString hidden = QStringLiteral("Any");
@@ -60,6 +64,9 @@ struct FileRecord {
qint64 created = 0;
qint64 accessed = 0;
qint64 changed = 0;
qint64 modifiedNs = 0;
quint64 device = 0;
quint64 inode = 0;
QString type;
QString owner;
QString attributes;
@@ -67,6 +74,12 @@ struct FileRecord {
int duplicateCopy = 0;
};
struct EffectiveRoots {
QStringList paths;
QStringList priorityPaths;
int skipped = 0;
};
Q_DECLARE_METATYPE(SearchOptions)
Q_DECLARE_METATYPE(FileRecord)
Q_DECLARE_METATYPE(QVector<FileRecord>)
@@ -74,6 +87,8 @@ Q_DECLARE_METATYPE(QVector<FileRecord>)
QStringList patterns(const QString &text);
QStringList exclusionPatterns(const QString &text);
bool wildcardMatch(const QString &value, const QStringList &items, Qt::CaseSensitivity cs);
EffectiveRoots effectiveSearchRoots(const SearchOptions &options);
int preferredRootIndex(const QString &path, const QStringList &roots);
QString humanSize(qint64 bytes);
QString ownerName(uid_t uid);
std::optional<QByteArray> hexNeedle(QString text);
@@ -82,5 +97,9 @@ bool fileContains(const QString &path, const SearchOptions &options,
bool filesEqual(const QString &leftPath, const QString &rightPath,
const std::atomic_bool *cancelled = nullptr);
QByteArray sha256(const QString &path, const std::atomic_bool *cancelled = nullptr);
QByteArray sha256(const FileRecord &record, const std::atomic_bool *cancelled = nullptr);
QByteArray sampleSha256(const FileRecord &record,
const std::atomic_bool *cancelled = nullptr);
bool fileSignatureMatches(const FileRecord &record);
bool parseOptionalIsoDate(const QString &text, qint64 &seconds);
bool shouldShowDuplicateResult(bool copiesOnly, int keeperPriority);
+190 -46
View File
@@ -6,6 +6,7 @@
#include <QDir>
#include <QFileInfo>
#include <QThread>
#include <QThreadPool>
#include <QtConcurrent>
#include <algorithm>
@@ -18,6 +19,27 @@
namespace fs = std::filesystem;
namespace {
struct HashedRecord {
QByteArray digest;
FileRecord record;
};
bool keepFirst(const FileRecord &left, const FileRecord &right,
const QStringList &priorityRoots)
{
const int leftRoot = preferredRootIndex(left.path, priorityRoots);
const int rightRoot = preferredRootIndex(right.path, priorityRoots);
const int leftPriority = leftRoot < 0 ? priorityRoots.size() : leftRoot;
const int rightPriority = rightRoot < 0 ? priorityRoots.size() : rightRoot;
if (leftPriority != rightPriority)
return leftPriority < rightPriority;
if (left.modified != right.modified)
return left.modified > right.modified;
return left.path < right.path;
}
}
void SearchEngine::search(const SearchOptions &o)
{
cancelled_.store(false, std::memory_order_relaxed);
@@ -33,6 +55,8 @@ void SearchEngine::search(const SearchOptions &o)
cacheMisses_.store(0, std::memory_order_relaxed);
const auto started = std::chrono::steady_clock::now();
QVector<FileRecord> candidates;
QSet<QString> candidatePaths;
const EffectiveRoots roots = effectiveSearchRoots(o);
const Qt::CaseSensitivity cs = o.caseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive;
const QStringList fileMasks = patterns(o.fileWildcards).isEmpty()
? QStringList{QStringLiteral("*")} : patterns(o.fileWildcards);
@@ -46,7 +70,7 @@ void SearchEngine::search(const SearchOptions &o)
if (o.accurateProgress) {
emit phaseProgress(tr("Counting files"), 0, 0);
for (const QString &rootText : patterns(o.roots)) {
for (const QString &rootText : roots.paths) {
if (cancelled_.load(std::memory_order_relaxed))
break;
std::error_code ec;
@@ -84,7 +108,7 @@ void SearchEngine::search(const SearchOptions &o)
emit phaseProgress(tr("Scanning files"), 0, totalEntries);
}
for (const QString &rootText : patterns(o.roots)) {
for (const QString &rootText : roots.paths) {
if (cancelled_.load(std::memory_order_relaxed))
break;
std::error_code ec;
@@ -96,7 +120,8 @@ void SearchEngine::search(const SearchOptions &o)
auto process = [&](const fs::directory_entry &entry, const fs::path &baseRoot) {
if (cancelled_.load(std::memory_order_relaxed))
return;
const QString path = QString::fromStdString(entry.path().string());
const QString path = QDir::cleanPath(QFileInfo(
QString::fromStdString(entry.path().string())).absoluteFilePath());
const QString name = QString::fromStdString(entry.path().filename().string());
const bool isDir = entry.is_directory(ec);
if ((isDir && !o.findFolders) || (!isDir && !o.findFiles))
@@ -147,6 +172,9 @@ void SearchEngine::search(const SearchOptions &o)
|| !attrMatches(o.readonly, readOnly)
|| !attrMatches(o.executable, executable))
return;
if (candidatePaths.contains(path))
return;
candidatePaths.insert(path);
QString attrs;
attrs += isDir ? u'd' : u'-';
@@ -160,6 +188,8 @@ void SearchEngine::search(const SearchOptions &o)
st.st_mtime,
QFileInfo(path).birthTime().isValid() ? QFileInfo(path).birthTime().toSecsSinceEpoch() : 0,
st.st_atime, st.st_ctime,
qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec,
quint64(st.st_dev), quint64(st.st_ino),
isDir ? QStringLiteral("Folder") : QStringLiteral("File"),
o.retrieveOwner ? ownerName(st.st_uid) : QString{}, attrs, 0, 0
});
@@ -237,55 +267,103 @@ void SearchEngine::search(const SearchOptions &o)
candidates.resize(o.maxResults);
}
QThreadPool hashPool;
const int idealThreads = QThread::idealThreadCount();
hashPool.setMaxThreadCount(std::clamp(
idealThreads > 0 ? idealThreads / 2 : 1, 1, 4));
QVector<FileRecord> results;
if (o.mode == QStringLiteral("Duplicates search")
|| o.mode == QStringLiteral("Non-Duplicates search")) {
std::unordered_map<qint64, QVector<FileRecord>> bySize;
for (const auto &record : candidates)
if (record.type == QStringLiteral("File"))
QSet<QString> duplicateInputPaths;
for (const auto &record : candidates) {
if (record.type == QStringLiteral("File")
&& !duplicateInputPaths.contains(record.path)) {
duplicateInputPaths.insert(record.path);
bySize[record.size].push_back(record);
quint64 hashTotal = 0;
for (const auto &[size, records] : bySize) {
Q_UNUSED(size);
if (records.size() > 1)
hashTotal += records.size();
}
std::atomic<quint64> hashed = 0;
if (hashTotal)
emit phaseProgress(tr("Checking duplicates"), 0, hashTotal);
int group = 1;
QSet<QString> duplicatedPaths;
for (auto &[size, sameSize] : bySize) {
if (cancelled_.load(std::memory_order_relaxed))
break;
Q_UNUSED(size);
if (sameSize.size() < 2)
}
QVector<FileRecord> sampleCandidates;
QVector<FileRecord> fullHashCandidates;
for (const auto &[size, records] : bySize) {
if (records.size() < 2)
continue;
auto hashes = QtConcurrent::blockingMapped(sameSize, [this, &hashed, hashTotal](const FileRecord &r) {
auto result = std::pair<QByteArray, FileRecord>{cachedSha256(r), r};
const quint64 done = hashed.fetch_add(1, std::memory_order_relaxed) + 1;
if (done == hashTotal || done % 16 == 0)
emit phaseProgress(tr("Checking duplicates"), done, hashTotal);
if (size > duplicateSampleThreshold)
sampleCandidates += records;
else
fullHashCandidates += records;
}
std::atomic<qsizetype> sampled = 0;
const qsizetype sampleTotal = sampleCandidates.size();
if (sampleTotal)
emit phaseProgress(tr("Sampling duplicate candidates"), 0, sampleTotal);
const auto samples = QtConcurrent::blockingMapped(
&hashPool, sampleCandidates,
[this, &sampled, sampleTotal](const FileRecord &record) {
HashedRecord result{cachedSampleSha256(record), record};
const qsizetype done = sampled.fetch_add(1, std::memory_order_relaxed) + 1;
if (done == sampleTotal || done % 16 == 0)
emit phaseProgress(tr("Sampling duplicate candidates"), done, sampleTotal);
return result;
});
if (cancelled_.load(std::memory_order_relaxed))
break;
QHash<QByteArray, QVector<FileRecord>> sameHash;
for (auto &[hash, record] : hashes)
if (!hash.isEmpty())
sameHash[hash].push_back(record);
for (const auto &hashCandidates : sameHash) {
if (finishIfCancelled())
return;
QHash<qint64, QHash<QByteArray, QVector<FileRecord>>> bySample;
for (const auto &sample : samples)
if (!sample.digest.isEmpty())
bySample[sample.record.size][sample.digest].push_back(sample.record);
for (const auto &sameSize : bySample)
for (const auto &sameSample : sameSize)
if (sameSample.size() > 1)
fullHashCandidates += sameSample;
std::atomic<qsizetype> hashed = 0;
const qsizetype hashTotal = fullHashCandidates.size();
if (hashTotal)
emit phaseProgress(tr("Hashing duplicate candidates"), 0, hashTotal);
const auto hashes = QtConcurrent::blockingMapped(
&hashPool, fullHashCandidates,
[this, &hashed, hashTotal](const FileRecord &record) {
HashedRecord result{cachedSha256(record), record};
const qsizetype done = hashed.fetch_add(1, std::memory_order_relaxed) + 1;
if (done == hashTotal || done % 16 == 0)
emit phaseProgress(tr("Hashing duplicate candidates"), done, hashTotal);
return result;
});
if (finishIfCancelled())
return;
QHash<qint64, QHash<QByteArray, QVector<FileRecord>>> byHash;
for (const auto &hash : hashes)
if (!hash.digest.isEmpty())
byHash[hash.record.size][hash.digest].push_back(hash.record);
int group = 1;
QSet<QString> duplicatedPaths;
for (const auto &sameSize : byHash) {
for (const auto &hashCandidates : sameSize) {
if (cancelled_.load(std::memory_order_relaxed))
break;
if (hashCandidates.size() < 2)
continue;
QVector<QVector<FileRecord>> exactGroups;
if (!o.strictDuplicateComparison) {
exactGroups.push_back(hashCandidates);
} else {
for (const auto &record : hashCandidates) {
if (cancelled_.load(std::memory_order_relaxed))
break;
bool placed = false;
for (auto &exact : exactGroups) {
if (filesEqual(exact.front().path, record.path, &cancelled_)) {
if (fileSignatureMatches(exact.front())
&& fileSignatureMatches(record)
&& filesEqual(exact.front().path, record.path, &cancelled_)
&& fileSignatureMatches(exact.front())
&& fileSignatureMatches(record)) {
exact.push_back(record);
placed = true;
break;
@@ -294,11 +372,17 @@ void SearchEngine::search(const SearchOptions &o)
if (!placed)
exactGroups.push_back({record});
}
}
for (auto &exact : exactGroups) {
if (cancelled_.load(std::memory_order_relaxed))
break;
if (exact.size() < 2)
continue;
std::stable_sort(exact.begin(), exact.end(),
[&roots](const FileRecord &left,
const FileRecord &right) {
return keepFirst(left, right, roots.priorityPaths);
});
int duplicateCopy = 1;
for (auto &record : exact) {
record.group = group;
@@ -343,13 +427,50 @@ void SearchEngine::search(const SearchOptions &o)
[](const FileRecord &r) { return r.type != QStringLiteral("File"); }), sameName.end());
if (sameName.size() < 2)
continue;
std::stable_sort(sameName.begin(), sameName.end(),
[&roots](const FileRecord &left,
const FileRecord &right) {
return keepFirst(left, right, roots.priorityPaths);
});
if (o.duplicateNameMode.contains(QStringLiteral("identical"), Qt::CaseInsensitive)) {
const QByteArray firstHash = cachedSha256(sameName.front());
bool identical = !firstHash.isEmpty();
for (qsizetype i = 1; identical && i < sameName.size(); ++i) {
identical = cachedSha256(sameName[i]) == firstHash
&& filesEqual(
sameName.front().path, sameName[i].path, &cancelled_);
bool identical = std::all_of(
sameName.cbegin(), sameName.cend(),
[&sameName](const FileRecord &record) {
return record.size == sameName.front().size;
});
if (identical && sameName.front().size > duplicateSampleThreshold) {
const auto samples = QtConcurrent::blockingMapped(
&hashPool, sameName, [this](const FileRecord &record) {
return cachedSampleSha256(record);
});
identical = !samples.isEmpty() && !samples.front().isEmpty()
&& std::all_of(samples.cbegin(), samples.cend(),
[&samples](const QByteArray &digest) {
return digest == samples.front();
});
}
QVector<QByteArray> hashes;
if (identical) {
hashes = QtConcurrent::blockingMapped(
&hashPool, sameName, [this](const FileRecord &record) {
return cachedSha256(record);
});
identical = !hashes.isEmpty() && !hashes.front().isEmpty()
&& std::all_of(hashes.cbegin(), hashes.cend(),
[&hashes](const QByteArray &digest) {
return digest == hashes.front();
});
}
if (identical && o.strictDuplicateComparison) {
const FileRecord &first = sameName.front();
for (qsizetype index = 1; identical && index < sameName.size(); ++index) {
const FileRecord &record = sameName[index];
identical = fileSignatureMatches(first)
&& fileSignatureMatches(record)
&& filesEqual(first.path, record.path, &cancelled_)
&& fileSignatureMatches(first)
&& fileSignatureMatches(record);
}
}
const bool wantsNonIdentical = o.duplicateNameMode.startsWith(QStringLiteral("Only non"));
if (identical == wantsNonIdentical)
@@ -376,7 +497,7 @@ void SearchEngine::search(const SearchOptions &o)
while (parent.cdUp()) {
const QString ancestor = parent.absolutePath();
bool belongsToRoot = false;
for (const QString &root : patterns(o.roots)) {
for (const QString &root : roots.paths) {
if (ancestor == QDir(root).absolutePath()) {
belongsToRoot = true;
break;
@@ -401,8 +522,8 @@ void SearchEngine::search(const SearchOptions &o)
}
const QFileInfo info(it.key());
results.push_back({it.key(), info.fileName(), info.absolutePath(), total, totalOnDisk,
newest, 0, 0, 0, tr("%1 files").arg(it.value().size()),
{}, {}, 0, 0});
newest, 0, 0, 0, 0, 0, 0,
tr("%1 files").arg(it.value().size()), {}, {}, 0, 0});
}
if (finishIfCancelled())
return;
@@ -415,21 +536,42 @@ void SearchEngine::search(const SearchOptions &o)
const double seconds = std::chrono::duration<double>(
std::chrono::steady_clock::now() - started).count();
cache_.save();
emit finished(results, tr("%1 results, %2 entries scanned in %3 seconds; cache: %4 hits")
QString message = tr("%1 results, %2 entries scanned in %3 seconds; cache: %4 hits")
.arg(results.size()).arg(scanned).arg(seconds, 0, 'f', 2)
.arg(cacheHits_.load(std::memory_order_relaxed)));
.arg(cacheHits_.load(std::memory_order_relaxed));
if (roots.skipped > 0)
message += tr("; %1 redundant roots skipped").arg(roots.skipped);
emit finished(results, message);
}
QByteArray SearchEngine::cachedSampleSha256(const FileRecord &record)
{
QByteArray digest;
if (cache_.findSampleHash(record, digest)) {
if (!fileSignatureMatches(record))
return {};
cacheHits_.fetch_add(1, std::memory_order_relaxed);
return digest;
}
cacheMisses_.fetch_add(1, std::memory_order_relaxed);
digest = sampleSha256(record, &cancelled_);
if (!digest.isEmpty() && !cancelled_.load(std::memory_order_relaxed))
cache_.storeSampleHash(record, digest);
return digest;
}
QByteArray SearchEngine::cachedSha256(const FileRecord &record)
{
QByteArray digest;
if (cache_.findHash(record, digest)) {
if (!fileSignatureMatches(record))
return {};
cacheHits_.fetch_add(1, std::memory_order_relaxed);
return digest;
}
cacheMisses_.fetch_add(1, std::memory_order_relaxed);
digest = sha256(record.path, &cancelled_);
if (!cancelled_.load(std::memory_order_relaxed))
digest = sha256(record, &cancelled_);
if (!digest.isEmpty() && !cancelled_.load(std::memory_order_relaxed))
cache_.storeHash(record, digest);
return digest;
}
@@ -441,6 +583,8 @@ bool SearchEngine::cachedFileContains(const FileRecord &record, const SearchOpti
stream << o.contains << o.binary << o.multipleValues << o.multipleAnd << o.caseSensitive;
const QByteArray queryKey = QCryptographicHash::hash(queryData, QCryptographicHash::Sha256);
if (const auto cached = cache_.findContent(record, queryKey)) {
if (!fileSignatureMatches(record))
return false;
cacheHits_.fetch_add(1, std::memory_order_relaxed);
return *cached;
}
+4 -1
View File
@@ -6,6 +6,7 @@
#include <atomic>
#include <chrono>
#include <utility>
#include "search_core.h"
#include "search_cache.h"
@@ -13,7 +14,8 @@
class SearchEngine final : public QObject {
Q_OBJECT
public:
explicit SearchEngine(QObject *parent = nullptr) : QObject(parent) {}
explicit SearchEngine(QObject *parent = nullptr, QString cachePath = {})
: QObject(parent), cache_(std::move(cachePath)) {}
public slots:
void stop() { cancelled_.store(true, std::memory_order_relaxed); }
@@ -32,6 +34,7 @@ signals:
void finished(const QVector<FileRecord> &results, const QString &message);
private:
QByteArray cachedSampleSha256(const FileRecord &record);
QByteArray cachedSha256(const FileRecord &record);
bool cachedFileContains(const FileRecord &record, const SearchOptions &o);
+61
View File
@@ -0,0 +1,61 @@
#include "options_dialog.h"
#include <QCheckBox>
#include <QComboBox>
#include <QSettings>
#include <QtTest>
class OptionsDialogTest final : public QObject {
Q_OBJECT
private slots:
void exposesStrictDuplicateComparison()
{
SearchOptions options;
QVERIFY(!options.strictDuplicateComparison);
options.strictDuplicateComparison = true;
OptionsDialog dialog(options);
auto *strict = dialog.findChild<QCheckBox *>(
QStringLiteral("strictDuplicateComparison"));
QVERIFY(strict);
QVERIFY(strict->isChecked());
QVERIFY(dialog.options().strictDuplicateComparison);
strict->setChecked(false);
QVERIFY(!dialog.options().strictDuplicateComparison);
}
void remembersInputHistory()
{
QSettings settings;
settings.remove(QStringLiteral("inputHistory"));
SearchOptions options;
OptionsDialog first(options);
auto *roots = first.findChild<QComboBox *>(QStringLiteral("historyroots"));
QVERIFY(roots);
roots->setEditText(QStringLiteral("/tmp/first;/tmp/second"));
QVERIFY(QMetaObject::invokeMethod(&first, "accept"));
QCOMPARE(settings.value(QStringLiteral("inputHistory/roots")).toStringList(),
QStringList{QStringLiteral("/tmp/first;/tmp/second")});
OptionsDialog second(options);
auto *history = second.findChild<QComboBox *>(QStringLiteral("historyroots"));
QVERIFY(history);
QCOMPARE(history->itemText(0), QStringLiteral("/tmp/first;/tmp/second"));
history->setEditText(QStringLiteral("/tmp/new"));
QVERIFY(QMetaObject::invokeMethod(&second, "accept"));
QCOMPARE(settings.value(QStringLiteral("inputHistory/roots")).toStringList(),
(QStringList{QStringLiteral("/tmp/new"),
QStringLiteral("/tmp/first;/tmp/second")}));
settings.remove(QStringLiteral("inputHistory"));
}
};
QTEST_MAIN(OptionsDialogTest)
#include "options_dialog_test.moc"
+112
View File
@@ -0,0 +1,112 @@
#include "search_cache.h"
#include <QDataStream>
#include <QFile>
#include <QTemporaryDir>
#include <QtTest>
class SearchCacheTest final : public QObject {
Q_OBJECT
private:
static FileRecord record()
{
FileRecord value;
value.path = QStringLiteral("/tmp/cache-test.bin");
value.size = 1234;
value.modifiedNs = 5678;
value.device = 9;
value.inode = 10;
return value;
}
private slots:
void storesAndReloadsBothHashes()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
const QString path = directory.filePath(QStringLiteral("cache.dat"));
const FileRecord value = record();
const QByteArray sample("sample");
const QByteArray full("full");
const QByteArray query("query");
{
SearchCache cache(path);
cache.storeSampleHash(value, sample);
cache.storeHash(value, full);
cache.storeContent(value, query, true);
cache.save();
}
SearchCache cache(path);
QByteArray digest;
QVERIFY(cache.findSampleHash(value, digest));
QCOMPARE(digest, sample);
QVERIFY(cache.findHash(value, digest));
QCOMPARE(digest, full);
QCOMPARE(cache.findContent(value, query), std::optional<bool>(true));
}
void invalidatesEverySignatureField()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
SearchCache cache(directory.filePath(QStringLiteral("cache.dat")));
const FileRecord value = record();
cache.storeSampleHash(value, QByteArray("sample"));
cache.storeHash(value, QByteArray("full"));
const auto misses = [&cache](FileRecord changed) {
QByteArray digest;
return !cache.findSampleHash(changed, digest)
&& !cache.findHash(changed, digest);
};
FileRecord changed = value;
++changed.size;
QVERIFY(misses(changed));
changed = value;
++changed.modifiedNs;
QVERIFY(misses(changed));
changed = value;
++changed.device;
QVERIFY(misses(changed));
changed = value;
++changed.inode;
QVERIFY(misses(changed));
}
void ignoresVersionOneData()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
const QString path = directory.filePath(QStringLiteral("cache.dat"));
QFile file(path);
QVERIFY(file.open(QIODevice::WriteOnly));
QDataStream stream(&file);
stream.setVersion(QDataStream::Qt_6_0);
stream << quint32(0x46534348) << quint32(1) << quint32(0);
file.close();
SearchCache cache(path);
QByteArray digest;
QVERIFY(!cache.findHash(record(), digest));
cache.storeHash(record(), QByteArray("full"));
cache.save();
QFile saved(path);
QVERIFY(saved.open(QIODevice::ReadOnly));
QDataStream savedStream(&saved);
savedStream.setVersion(QDataStream::Qt_6_0);
quint32 magic = 0;
quint32 version = 0;
savedStream >> magic >> version;
QCOMPARE(magic, quint32(0x46534348));
QCOMPARE(version, quint32(2));
}
};
QTEST_GUILESS_MAIN(SearchCacheTest)
#include "search_cache_test.moc"
+202
View File
@@ -5,6 +5,35 @@
#include <QTemporaryDir>
#include <QtTest>
#include <sys/stat.h>
namespace {
FileRecord recordFor(const QString &path)
{
struct stat st {};
const QByteArray nativePath = QFile::encodeName(path);
if (::stat(nativePath.constData(), &st) != 0)
return {};
FileRecord record;
record.path = path;
record.size = st.st_size;
record.modified = st.st_mtime;
record.modifiedNs =
qint64(st.st_mtim.tv_sec) * 1'000'000'000LL + st.st_mtim.tv_nsec;
record.device = st.st_dev;
record.inode = st.st_ino;
record.type = QStringLiteral("File");
return record;
}
bool writeFile(const QString &path, const QByteArray &data)
{
QFile file(path);
return file.open(QIODevice::WriteOnly)
&& file.write(data) == data.size();
}
}
class SearchCoreTest final : public QObject {
Q_OBJECT
@@ -23,6 +52,113 @@ private slots:
Qt::CaseSensitive));
}
void collapsesRedundantSearchRoots()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
QDir root(directory.path());
QVERIFY(root.mkpath(QStringLiteral("child")));
QVERIFY(root.mkpath(QStringLiteral("misc")));
SearchOptions options;
const QString child = directory.filePath(QStringLiteral("child"));
const QString misc = directory.filePath(QStringLiteral("misc"));
options.roots = QStringLiteral("%1;%2/;%1;%3")
.arg(child, directory.path(), misc);
options.excludeFolders = QStringLiteral("unrelated");
const EffectiveRoots effective = effectiveSearchRoots(options);
QCOMPARE(effective.paths, QStringList({directory.path()}));
QCOMPARE(effective.skipped, 3);
}
void preservesNestedRootsThatExpandCoverage()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
QDir root(directory.path());
QVERIFY(root.mkpath(QStringLiteral("child")));
const QString child = directory.filePath(QStringLiteral("child"));
SearchOptions options;
options.roots = directory.path() + u';' + child;
options.recursive = false;
QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
options.recursive = true;
options.maxDepth = 1;
QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
options.maxDepth = 0;
options.maxResults = 1;
QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
options.maxResults = 0;
options.subfolderWildcards = QStringLiteral("matched-*");
QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
options.subfolderWildcards = QStringLiteral("*");
options.excludeFolders = QStringLiteral("child");
QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
options.excludeFolders = QStringLiteral("unrelated");
QCOMPARE(effectiveSearchRoots(options).paths,
QStringList({directory.path()}));
}
void respectsSymlinksWhenCollapsingRoots()
{
QTemporaryDir directory;
QTemporaryDir external;
QVERIFY(directory.isValid());
QVERIFY(external.isValid());
QDir externalRoot(external.path());
QVERIFY(externalRoot.mkpath(QStringLiteral("child")));
const QString link = directory.filePath(QStringLiteral("link"));
QVERIFY(QFile::link(external.path(), link));
const QString linkedChild = QDir(link).filePath(QStringLiteral("child"));
SearchOptions options;
options.roots = directory.path() + u';' + linkedChild;
options.followLinks = false;
QCOMPARE(effectiveSearchRoots(options).paths.size(), 2);
options.followLinks = true;
QCOMPARE(effectiveSearchRoots(options).paths,
QStringList({directory.path()}));
options.roots = external.path() + u';' + link;
const EffectiveRoots aliases = effectiveSearchRoots(options);
QCOMPARE(aliases.paths.size(), 1);
QCOMPARE(aliases.skipped, 1);
}
void prefersTheMostSpecificConfiguredRoot()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
QDir root(directory.path());
QVERIFY(root.mkpath(QStringLiteral("Temp")));
QVERIFY(root.mkpath(QStringLiteral("misc")));
const QString temp = directory.filePath(QStringLiteral("Temp"));
const QString misc = directory.filePath(QStringLiteral("misc"));
const QStringList roots{directory.path(), temp, misc};
QCOMPARE(preferredRootIndex(
directory.filePath(QStringLiteral("base.mp4")), roots), 0);
QCOMPARE(preferredRootIndex(
QDir(temp).filePath(QStringLiteral("copy.mp4")), roots), 1);
QCOMPARE(preferredRootIndex(
QDir(misc).filePath(QStringLiteral("copy.mp4")), roots), 2);
QCOMPARE(preferredRootIndex(
directory.filePath(QStringLiteral("unrelated/file.mp4")), roots), 0);
const QStringList similarRoots{directory.filePath(QStringLiteral("foo"))};
QCOMPARE(preferredRootIndex(
directory.filePath(QStringLiteral("foobar/file.mp4")), similarRoots), -1);
}
void parsesOptionalIsoDates()
{
qint64 seconds = -1;
@@ -108,6 +244,72 @@ private slots:
QVERIFY(!filesEqual(path, path, &cancelled));
QVERIFY(sha256(path, &cancelled).isEmpty());
}
void samplesLargeFilesAndValidatesSignatures()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
const QByteArray content(duplicateSampleThreshold + 1024 * 1024, 'x');
const QString firstPath = directory.filePath(QStringLiteral("first.bin"));
const QString samePath = directory.filePath(QStringLiteral("same.bin"));
const QString outsidePath = directory.filePath(QStringLiteral("outside.bin"));
const QString startPath = directory.filePath(QStringLiteral("start.bin"));
const QString middlePath = directory.filePath(QStringLiteral("middle.bin"));
const QString endPath = directory.filePath(QStringLiteral("end.bin"));
QVERIFY(writeFile(firstPath, content));
QVERIFY(writeFile(samePath, content));
QByteArray outside = content;
outside[1024 * 1024] = 'y';
QVERIFY(writeFile(outsidePath, outside));
QByteArray start = content;
start[0] = 'y';
QVERIFY(writeFile(startPath, start));
QByteArray middle = content;
middle[(content.size() - duplicateSampleBlockSize) / 2] = 'y';
QVERIFY(writeFile(middlePath, middle));
QByteArray end = content;
end[end.size() - 1] = 'y';
QVERIFY(writeFile(endPath, end));
const FileRecord first = recordFor(firstPath);
const QByteArray sample = sampleSha256(first);
QVERIFY(!sample.isEmpty());
QCOMPARE(sampleSha256(recordFor(samePath)), sample);
QCOMPARE(sampleSha256(recordFor(outsidePath)), sample);
QVERIFY(sampleSha256(recordFor(startPath)) != sample);
QVERIFY(sampleSha256(recordFor(middlePath)) != sample);
QVERIFY(sampleSha256(recordFor(endPath)) != sample);
QVERIFY(sha256(recordFor(outsidePath)) != sha256(first));
std::atomic_bool cancelled = true;
QVERIFY(sampleSha256(first, &cancelled).isEmpty());
const QString shortPath = directory.filePath(QStringLiteral("short.bin"));
QVERIFY(writeFile(shortPath, QByteArray("short")));
QVERIFY(sampleSha256(recordFor(shortPath)).isEmpty());
QVERIFY(!sha256(recordFor(shortPath)).isEmpty());
const QString emptyPath = directory.filePath(QStringLiteral("empty.bin"));
QVERIFY(writeFile(emptyPath, {}));
QVERIFY(sampleSha256(recordFor(emptyPath)).isEmpty());
QVERIFY(!sha256(recordFor(emptyPath)).isEmpty());
const QString replacedPath = directory.filePath(QStringLiteral("replaced.bin"));
QVERIFY(writeFile(replacedPath, content));
const FileRecord replaced = recordFor(replacedPath);
QVERIFY(QFile::remove(replacedPath));
QVERIFY(writeFile(replacedPath, content));
QVERIFY(!fileSignatureMatches(replaced));
QVERIFY(sha256(replaced).isEmpty());
QVERIFY(sampleSha256(replaced).isEmpty());
FileRecord missing = first;
missing.path = directory.filePath(QStringLiteral("missing.bin"));
QVERIFY(sha256(missing).isEmpty());
QVERIFY(sampleSha256(missing).isEmpty());
}
};
QTEST_GUILESS_MAIN(SearchCoreTest)
+408
View File
@@ -0,0 +1,408 @@
#include "search_engine.h"
#include <QDateTime>
#include <QDir>
#include <QElapsedTimer>
#include <QFile>
#include <QSignalSpy>
#include <QTemporaryDir>
#include <QtTest>
#include <algorithm>
#include <unistd.h>
namespace {
bool writeFile(const QString &path, const QByteArray &data)
{
QFile file(path);
return file.open(QIODevice::WriteOnly)
&& file.write(data) == data.size();
}
QVector<FileRecord> runSearch(SearchEngine &engine, SearchOptions options)
{
QSignalSpy finished(&engine, &SearchEngine::finished);
engine.search(options);
if (finished.size() != 1)
return {};
return finished.takeFirst().at(0).value<QVector<FileRecord>>();
}
QSet<QString> namesOf(const QVector<FileRecord> &records)
{
QSet<QString> names;
for (const auto &record : records)
names.insert(record.name);
return names;
}
SearchOptions optionsFor(const QString &root, const QString &mode)
{
SearchOptions options;
options.roots = root;
options.mode = mode;
options.recursive = true;
options.accurateProgress = false;
options.showDuplicateCopiesOnly = false;
options.useCache = false;
return options;
}
}
class SearchEngineTest final : public QObject {
Q_OBJECT
private slots:
void filtersBySampleThenFullHashAndSupportsStrictMode()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
QVERIFY(writeFile(directory.filePath(QStringLiteral("duplicate-a.bin")),
QByteArray("same content")));
QVERIFY(writeFile(directory.filePath(QStringLiteral("duplicate-b.bin")),
QByteArray("same content")));
QVERIFY(writeFile(directory.filePath(QStringLiteral("unique-size.bin")),
QByteArray("unique")));
const QByteArray large(duplicateSampleThreshold + 1024 * 1024, 'x');
QByteArray different = large;
different[1024 * 1024] = 'y';
QVERIFY(writeFile(directory.filePath(QStringLiteral("sample-a.bin")), large));
QVERIFY(writeFile(directory.filePath(QStringLiteral("sample-b.bin")), different));
SearchEngine engine(nullptr, directory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
directory.path(), QStringLiteral("Duplicates search"));
QVector<FileRecord> results = runSearch(engine, options);
QCOMPARE(namesOf(results),
QSet<QString>({QStringLiteral("duplicate-a.bin"),
QStringLiteral("duplicate-b.bin")}));
options.strictDuplicateComparison = true;
results = runSearch(engine, options);
QCOMPARE(namesOf(results),
QSet<QString>({QStringLiteral("duplicate-a.bin"),
QStringLiteral("duplicate-b.bin")}));
options.mode = QStringLiteral("Non-Duplicates search");
results = runSearch(engine, options);
QCOMPARE(namesOf(results),
QSet<QString>({QStringLiteral("unique-size.bin"),
QStringLiteral("sample-a.bin"),
QStringLiteral("sample-b.bin")}));
}
void appliesContentPipelineToDuplicateNames()
{
QTemporaryDir directory;
QVERIFY(directory.isValid());
QDir root(directory.path());
QVERIFY(root.mkpath(QStringLiteral("one")));
QVERIFY(root.mkpath(QStringLiteral("two")));
const QString first =
directory.filePath(QStringLiteral("one/report.bin"));
const QString second =
directory.filePath(QStringLiteral("two/report.bin"));
QVERIFY(writeFile(first, QByteArray("same")));
QVERIFY(writeFile(second, QByteArray("same")));
SearchEngine engine(nullptr, directory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
directory.path(), QStringLiteral("Duplicate names search"));
options.duplicateNameMode = QStringLiteral("Only identical content");
QVector<FileRecord> results = runSearch(engine, options);
QCOMPARE(results.size(), 2);
QVERIFY(writeFile(second, QByteArray("diff")));
results = runSearch(engine, options);
QVERIFY(results.isEmpty());
options.duplicateNameMode = QStringLiteral("Only non-identical content");
results = runSearch(engine, options);
QCOMPARE(results.size(), 2);
options.strictDuplicateComparison = true;
results = runSearch(engine, options);
QCOMPARE(results.size(), 2);
}
void assignsKeeperPriorityFromConfiguredRoots()
{
QTemporaryDir directory;
QTemporaryDir cacheDirectory;
QVERIFY(directory.isValid());
QVERIFY(cacheDirectory.isValid());
QDir root(directory.path());
QVERIFY(root.mkpath(QStringLiteral("Temp")));
QVERIFY(root.mkpath(QStringLiteral("misc")));
const QString temp = directory.filePath(QStringLiteral("Temp"));
const QString misc = directory.filePath(QStringLiteral("misc"));
const QByteArray content("same");
const QString basePath = directory.filePath(QStringLiteral("base.mp4"));
const QString tempPath = QDir(temp).filePath(QStringLiteral("temp.mp4"));
const QString miscPath = QDir(misc).filePath(QStringLiteral("misc.mp4"));
QVERIFY(writeFile(basePath, content));
QVERIFY(writeFile(tempPath, content));
QVERIFY(writeFile(miscPath, content));
SearchEngine engine(nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
QStringList({directory.path(), temp, misc}).join(u';'),
QStringLiteral("Duplicates search"));
QVector<FileRecord> results = runSearch(engine, options);
QCOMPARE(results.size(), 3);
QHash<QString, int> priorities;
for (const FileRecord &record : results)
priorities.insert(record.path, record.duplicateCopy);
QCOMPARE(priorities.value(basePath), 1);
QCOMPARE(priorities.value(tempPath), 2);
QCOMPARE(priorities.value(miscPath), 3);
}
void prefersNewerFilesWhenRootPriorityTies()
{
QTemporaryDir directory;
QTemporaryDir cacheDirectory;
QVERIFY(directory.isValid());
QVERIFY(cacheDirectory.isValid());
QDir root(directory.path());
QVERIFY(root.mkpath(QStringLiteral("child")));
const QString older = QDir(directory.filePath(QStringLiteral("child")))
.filePath(QStringLiteral("older.mp4"));
const QString newer = directory.filePath(QStringLiteral("newer.mp4"));
QVERIFY(writeFile(older, QByteArray("same")));
QVERIFY(writeFile(newer, QByteArray("same")));
QFile olderFile(older);
QVERIFY(olderFile.open(QIODevice::ReadOnly));
QVERIFY(olderFile.setFileTime(QDateTime::fromSecsSinceEpoch(1),
QFileDevice::FileModificationTime));
olderFile.close();
SearchEngine engine(nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
directory.path(), QStringLiteral("Duplicates search"));
const QVector<FileRecord> results = runSearch(engine, options);
QCOMPARE(results.size(), 2);
QHash<QString, int> priorities;
for (const FileRecord &record : results)
priorities.insert(record.path, record.duplicateCopy);
QCOMPARE(priorities.value(newer), 1);
QCOMPARE(priorities.value(older), 2);
}
void reusesSampleAndFullHashesOnWarmSearch()
{
QTemporaryDir directory;
QTemporaryDir cacheDirectory;
QVERIFY(directory.isValid());
QVERIFY(cacheDirectory.isValid());
const QByteArray content(duplicateSampleThreshold + 1024 * 1024, 'x');
const QString first = directory.filePath(QStringLiteral("first.bin"));
const QString second = directory.filePath(QStringLiteral("second.bin"));
QVERIFY(writeFile(first, content));
QVERIFY(writeFile(second, content));
const QString cachePath =
cacheDirectory.filePath(QStringLiteral("duplicate-cache.dat"));
SearchEngine engine(nullptr, cachePath);
SearchOptions options = optionsFor(
directory.path(), QStringLiteral("Duplicates search"));
options.useCache = true;
QSignalSpy finished(&engine, &SearchEngine::finished);
QElapsedTimer timer;
timer.start();
engine.search(options);
const qint64 coldMilliseconds = timer.elapsed();
QCOMPARE(finished.size(), 1);
finished.clear();
if (::geteuid() != 0) {
QVERIFY(QFile::setPermissions(first, {}));
QVERIFY(QFile::setPermissions(second, {}));
}
timer.restart();
engine.search(options);
const qint64 warmMilliseconds = timer.elapsed();
QCOMPARE(finished.size(), 1);
const QList<QVariant> arguments = finished.takeFirst();
QCOMPARE(arguments.at(0).value<QVector<FileRecord>>().size(), 2);
QVERIFY(arguments.at(1).toString().contains(QStringLiteral("cache: 4 hits")));
qInfo().noquote()
<< QStringLiteral("duplicate benchmark: cold %1 ms, warm %2 ms")
.arg(coldMilliseconds)
.arg(warmMilliseconds);
if (::geteuid() != 0) {
options.strictDuplicateComparison = true;
finished.clear();
engine.search(options);
QCOMPARE(finished.size(), 1);
QVERIFY(finished.takeFirst().at(0).value<QVector<FileRecord>>().isEmpty());
}
QFile::remove(cachePath);
}
void doesNotFullyHashDifferentSamplesOrSizes()
{
QTemporaryDir directory;
QTemporaryDir cacheDirectory;
QVERIFY(directory.isValid());
QVERIFY(cacheDirectory.isValid());
const QByteArray firstContent(
duplicateSampleThreshold + 1024 * 1024, 'x');
QByteArray secondContent = firstContent;
secondContent[0] = 'y';
QVERIFY(writeFile(directory.filePath(QStringLiteral("large-a.bin")),
firstContent));
QVERIFY(writeFile(directory.filePath(QStringLiteral("large-b.bin")),
secondContent));
QVERIFY(writeFile(directory.filePath(QStringLiteral("small-a.bin")),
QByteArray("one")));
QVERIFY(writeFile(directory.filePath(QStringLiteral("small-b.bin")),
QByteArray("different size")));
SearchEngine engine(
nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
directory.path(), QStringLiteral("Duplicates search"));
options.useCache = true;
QSignalSpy finished(&engine, &SearchEngine::finished);
engine.search(options);
QCOMPARE(finished.size(), 1);
finished.clear();
engine.search(options);
QCOMPARE(finished.size(), 1);
const QList<QVariant> arguments = finished.takeFirst();
QVERIFY(arguments.at(0).value<QVector<FileRecord>>().isEmpty());
QVERIFY(arguments.at(1).toString().contains(QStringLiteral("cache: 2 hits")));
}
void collapsesCurrentOverlappingRootLayoutBeforeHashing()
{
QTemporaryDir directory;
QTemporaryDir cacheDirectory;
QVERIFY(directory.isValid());
QVERIFY(cacheDirectory.isValid());
QDir root(directory.path());
const QStringList children{
QStringLiteral(".Prt"), QStringLiteral("misc"), QStringLiteral("Music"),
QStringLiteral("Буфер"), QStringLiteral("Temp")
};
QStringList roots{directory.path()};
for (qsizetype index = 0; index < children.size(); ++index) {
QVERIFY(root.mkpath(children[index]));
const QString child = directory.filePath(children[index]);
roots.push_back(child);
QVERIFY(writeFile(
QDir(child).filePath(QStringLiteral("unique.bin")),
QByteArray(duplicateSampleThreshold + index + 1,
char('a' + index))));
}
SearchEngine engine(
nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
roots.join(u';'), QStringLiteral("Duplicates search"));
options.accurateProgress = true;
options.excludeFolders =
directory.filePath(QStringLiteral("Meta"));
QSignalSpy phases(&engine, &SearchEngine::phaseProgress);
QSignalSpy finished(&engine, &SearchEngine::finished);
QElapsedTimer timer;
timer.start();
engine.search(options);
const qint64 elapsedMilliseconds = timer.elapsed();
QCOMPARE(finished.size(), 1);
const QList<QVariant> arguments = finished.takeFirst();
QVERIFY(arguments.at(0).value<QVector<FileRecord>>().isEmpty());
QVERIFY(arguments.at(1).toString().contains(
QStringLiteral("5 redundant roots skipped")));
quint64 scanningCompleted = 0;
quint64 scanningTotal = 0;
for (const QList<QVariant> &phase : phases) {
const QString name = phase.at(0).toString();
QVERIFY(name != QStringLiteral("Sampling duplicate candidates"));
QVERIFY(name != QStringLiteral("Hashing duplicate candidates"));
if (name == QStringLiteral("Scanning files")) {
scanningCompleted =
std::max(scanningCompleted, phase.at(1).toULongLong());
scanningTotal = phase.at(2).toULongLong();
}
}
QVERIFY(scanningTotal > 0);
QCOMPARE(scanningCompleted, scanningTotal);
qInfo().noquote()
<< QStringLiteral("overlapping-roots benchmark: %1 ms, 5 large files, 0 hash phases")
.arg(elapsedMilliseconds);
}
void deduplicatesCandidatesWhenNestedRootsCannotBeCollapsed()
{
QTemporaryDir directory;
QTemporaryDir cacheDirectory;
QVERIFY(directory.isValid());
QVERIFY(cacheDirectory.isValid());
QDir root(directory.path());
QVERIFY(root.mkpath(QStringLiteral("child")));
const QString child = directory.filePath(QStringLiteral("child"));
QVERIFY(writeFile(QDir(child).filePath(QStringLiteral("only.bin")),
QByteArray(duplicateSampleThreshold + 1, 'x')));
SearchEngine engine(
nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
directory.path() + u';' + child, QStringLiteral("Standard search"));
options.maxDepth = 2;
QVector<FileRecord> results = runSearch(engine, options);
QCOMPARE(results.size(), 1);
QCOMPARE(results.front().name, QStringLiteral("only.bin"));
options.mode = QStringLiteral("Duplicates search");
QSignalSpy phases(&engine, &SearchEngine::phaseProgress);
QSignalSpy finished(&engine, &SearchEngine::finished);
engine.search(options);
QCOMPARE(finished.size(), 1);
const QList<QVariant> arguments = finished.takeFirst();
QVERIFY(arguments.at(0).value<QVector<FileRecord>>().isEmpty());
QVERIFY(!arguments.at(1).toString().contains(
QStringLiteral("redundant roots skipped")));
for (const QList<QVariant> &phase : phases) {
const QString name = phase.at(0).toString();
QVERIFY(name != QStringLiteral("Sampling duplicate candidates"));
QVERIFY(name != QStringLiteral("Hashing duplicate candidates"));
}
}
void leavesUnreadableCandidatesAsNonDuplicates()
{
if (::geteuid() == 0)
QSKIP("Root can read files regardless of their permission bits");
QTemporaryDir directory;
QTemporaryDir cacheDirectory;
QVERIFY(directory.isValid());
QVERIFY(cacheDirectory.isValid());
const QString first = directory.filePath(QStringLiteral("first.bin"));
const QString second = directory.filePath(QStringLiteral("second.bin"));
QVERIFY(writeFile(first, QByteArray("blocked")));
QVERIFY(writeFile(second, QByteArray("blocked")));
QVERIFY(QFile::setPermissions(first, {}));
QVERIFY(QFile::setPermissions(second, {}));
SearchEngine engine(
nullptr, cacheDirectory.filePath(QStringLiteral("cache.dat")));
SearchOptions options = optionsFor(
directory.path(), QStringLiteral("Duplicates search"));
QVERIFY(runSearch(engine, options).isEmpty());
options.mode = QStringLiteral("Non-Duplicates search");
QCOMPARE(namesOf(runSearch(engine, options)),
QSet<QString>({QStringLiteral("first.bin"),
QStringLiteral("second.bin")}));
}
};
QTEST_GUILESS_MAIN(SearchEngineTest)
#include "search_engine_test.moc"