Added current fileId to DiffTaskResult; implemented folder blocklist feature; fixed indentations of some files; bumped version

This commit is contained in:
Jonathan Treffler 2025-07-02 14:41:59 +02:00
parent 85d63d5ae2
commit acf8990de1
6 changed files with 253 additions and 212 deletions

View file

@ -5,7 +5,7 @@
<name>Groupfolder Filesystem Snapshots</name> <name>Groupfolder Filesystem Snapshots</name>
<summary>Allows restoring a groupfolder to a previous snapshot in the filesystem</summary> <summary>Allows restoring a groupfolder to a previous snapshot in the filesystem</summary>
<description>App proving a PHP API for other apps, that allows (partially) restoring a groupfolder to a previous snapshot in the filesystem. Requires a filesystem with snapshot support (tested with and made for ZFS). It is made for other apps to integrate with, IT DOES NOT WORK STANDALONE</description> <description>App proving a PHP API for other apps, that allows (partially) restoring a groupfolder to a previous snapshot in the filesystem. Requires a filesystem with snapshot support (tested with and made for ZFS). It is made for other apps to integrate with, IT DOES NOT WORK STANDALONE</description>
<version>1.3.1</version> <version>1.4.0</version>
<licence>agpl</licence> <licence>agpl</licence>
<author homepage="https://verdigado.com/">verdigado eG</author> <author homepage="https://verdigado.com/">verdigado eG</author>
<author mail="jonathan.treffler@verdigado.com">Jonathan Treffler</author> <author mail="jonathan.treffler@verdigado.com">Jonathan Treffler</author>

View file

@ -16,7 +16,9 @@ class DiffTaskResult extends Entity implements JsonSerializable {
protected $beforeSize; protected $beforeSize;
protected $currentFileExists; protected $currentFileExists;
protected $currentPath; protected $currentFileId;
protected $currentPath;
protected $currentSize; protected $currentSize;
protected $reverted; protected $reverted;
@ -26,6 +28,7 @@ class DiffTaskResult extends Entity implements JsonSerializable {
$this->addType('beforeFileExists','boolean'); $this->addType('beforeFileExists','boolean');
$this->addType('beforeSize','integer'); $this->addType('beforeSize','integer');
$this->addType('currentFileExists','boolean'); $this->addType('currentFileExists','boolean');
$this->addType('currentFileId','integer');
$this->addType('currentSize','integer'); $this->addType('currentSize','integer');
$this->addType('reverted','boolean'); $this->addType('reverted','boolean');
} }
@ -42,6 +45,7 @@ class DiffTaskResult extends Entity implements JsonSerializable {
], ],
'current' => [ 'current' => [
'fileExists' => $this->currentFileExists, 'fileExists' => $this->currentFileExists,
'fileId' => $this->currentFileId,
'path' => $this->currentPath, 'path' => $this->currentPath,
'size' => $this->currentSize, 'size' => $this->currentSize,
], ],

View file

@ -0,0 +1,33 @@
<?php
namespace OCA\GroupfolderFilesystemSnapshots\Migration;
use Closure;
use OCP\DB\ISchemaWrapper;
use OCP\DB\Types;
use OCP\Migration\SimpleMigrationStep;
use OCP\Migration\IOutput;
class Version140Date20250701164500 extends SimpleMigrationStep {
const RESULTS_TABLE = "groupfolder_snapshots_task_results";
/**
* @param IOutput $output
* @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
* @param array $options
* @return null|ISchemaWrapper
*/
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) {
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
$table = $schema->getTable(self::RESULTS_TABLE);
if(!$table->hasColumn('current_file_id')) {
$table->addColumn('current_file_id', Types::BIGINT, [
'notnull' => false,
]);
}
return $schema;
}
}

View file

@ -5,227 +5,223 @@ namespace OCA\GroupfolderFilesystemSnapshots;
use OCA\GroupfolderFilesystemSnapshots\Helpers\FileHelper; use OCA\GroupfolderFilesystemSnapshots\Helpers\FileHelper;
class RecursiveDiff { class RecursiveDiff {
private $scan1files = [];
private $scan1folders = [];
public string $dir1; private $scan2files = [];
public string $dir2; private $scan2folders = [];
private $prefix; private $subJobs = [];
private $newResultCallback; private $subJobProgress = [];
private $progressCallback; private $progress = 0;
private $scan1files = []; public function __construct(
private $scan1folders = []; public readonly string $dir1,
public readonly string $dir2,
private readonly string $prefix = "",
private readonly array $folderBlocklist = [],
private $newResultCallback,
private $progressCallback,
){}
private $scan2files = []; public function scan() {
private $scan2folders = []; $scan_num_files = 0;
private $subJobs = []; if(file_exists($this->dir1) && is_dir($this->dir1)) {
[$this->scan1files, $this->scan1folders] = FileHelper::getFilesAndFolders($this->dir1);
}
if(file_exists($this->dir2) && is_dir($this->dir2)) {
[$this->scan2files, $this->scan2folders] = FileHelper::getFilesAndFolders($this->dir2);
}
private $subJobProgress = []; $scan_num_files += sizeof($this->scan1files);
private $progress = 0; $scan_num_files += sizeof($this->scan2files);
$allSubfolders = array_unique(array_merge($this->scan1folders, $this->scan2folders));
public function __construct($dir1, $dir2, $prefix = "", $newResultCallback, $progressCallback){ foreach($allSubfolders as $key=>$folder) {
$this->dir1 = $dir1; $subdir1 = $this->dir1 . DIRECTORY_SEPARATOR . $folder;
$this->dir2 = $dir2; $subdir2 = $this->dir2 . DIRECTORY_SEPARATOR . $folder;
$this->prefix = $prefix; $subprefix = $this->prefix . DIRECTORY_SEPARATOR . $folder;
$subFolderBlocklist = $this->folderBlocklist[$folder] ?? [];
$this->newResultCallback = $newResultCallback; if($subFolderBlocklist === true) {
$this->progressCallback = $progressCallback; continue;
} }
$newJob = new RecursiveDiff($subdir1, $subdir2, $subprefix, $subFolderBlocklist, $this->newResultCallback, function($numDoneFiles) use ($key) {
$this->subJobProgress[$key] = $numDoneFiles;
public function scan() { $this->updateProgress();
$scan_num_files = 0; });
if(file_exists($this->dir1) && is_dir($this->dir1)) { $this->subJobs[] = $newJob;
[$this->scan1files, $this->scan1folders] = FileHelper::getFilesAndFolders($this->dir1);
}
if(file_exists($this->dir2) && is_dir($this->dir2)) {
[$this->scan2files, $this->scan2folders] = FileHelper::getFilesAndFolders($this->dir2);
}
$scan_num_files += sizeof($this->scan1files); $scan_num_files += $newJob->scan();
$scan_num_files += sizeof($this->scan2files); }
$allSubfolders = array_unique(array_merge($this->scan1folders, $this->scan2folders));
foreach($allSubfolders as $key=>$folder) { return $scan_num_files;
$subdir1 = $this->dir1 . DIRECTORY_SEPARATOR . $folder; }
$subdir2 = $this->dir2 . DIRECTORY_SEPARATOR . $folder;
$subprefix = $this->prefix . DIRECTORY_SEPARATOR . $folder;
$newJob = new RecursiveDiff($subdir1, $subdir2, $subprefix, $this->newResultCallback, function($numDoneFiles) use ($key) {
$this->subJobProgress[$key] = $numDoneFiles;
$this->updateProgress(); private function updateProgress() {
}); ($this->progressCallback)(array_sum($this->subJobProgress) + $this->progress);
}
$this->subJobs[] = $newJob; function diff() {
$diff = [];
$scan_num_files += $newJob->scan(); foreach($this->subJobs as $job) {
} $result = $job->diff();
array_push($diff, ...$result);
}
return $scan_num_files; $fileCreations = array_diff($this->scan2files, $this->scan1files);
} $fileCreationsFilesizes = FileHelper::getFilesizesOfFiles($this->dir2, $fileCreations);
private function updateProgress() { $fileDeletions = array_diff($this->scan1files, $this->scan2files);
($this->progressCallback)(array_sum($this->subJobProgress) + $this->progress); $fileDeletionsFilesizes = FileHelper::getFilesizesOfFiles($this->dir1, $fileDeletions);
}
function diff() { $filePossibleEdits = array_intersect($this->scan1files, $this->scan2files);
$diff = [];
foreach($this->subJobs as $job) { /*$diff[] = [
$result = $job->diff(); "type" => "DEBUG",
array_push($diff, ...$result); "prefix" => $this->prefix,
} "fileCreations" => $fileCreations,
"fileCreationsFilesizes" => $fileCreationsFilesizes,
"fileDeletions" => $fileDeletions,
"fileDeletionsFilesizes" => $fileDeletionsFilesizes,
//"folderCreations" => $folderCreations,
//"folderDeletions" => $folderDeletions,
"allSubfolders" => $allSubfolders,
];*/
$fileCreations = array_diff($this->scan2files, $this->scan1files); // search for creations and deletions, that are actually renames
$fileCreationsFilesizes = FileHelper::getFilesizesOfFiles($this->dir2, $fileCreations); foreach($fileCreations as $creationIndex=>$creation) {
$creationPath = $this->dir2 . DIRECTORY_SEPARATOR . $creation;
$creationSize = $fileCreationsFilesizes[$creationIndex];
$renameContenders = array_keys($fileDeletionsFilesizes, $creationSize);
$fileDeletions = array_diff($this->scan1files, $this->scan2files); if(sizeof($renameContenders) != 0) {
$fileDeletionsFilesizes = FileHelper::getFilesizesOfFiles($this->dir1, $fileDeletions); /*$diff[] = [
"type" => "DEBUG",
"comparing" => [
"creation" => $creationIndex,
"deletions" => $renameContenders,
],
];*/
$filePossibleEdits = array_intersect($this->scan1files, $this->scan2files); $creationSHA = sha1_file($creationPath);
foreach($renameContenders as $contender) {
$deletion = $fileDeletions[$contender];
$deletionPath = $this->dir1 . DIRECTORY_SEPARATOR . $deletion;
$deletionSHA = sha1_file($deletionPath);
/*$diff[] = [ if($deletionSHA == $creationSHA) {
"type" => "DEBUG", ($this->newResultCallback)(
"prefix" => $this->prefix, type: "RENAME",
"fileCreations" => $fileCreations, beforeFileExists: True,
"fileCreationsFilesizes" => $fileCreationsFilesizes, beforePath: $this->prefix . DIRECTORY_SEPARATOR . $deletion,
"fileDeletions" => $fileDeletions, beforeSize: $creationSize,
"fileDeletionsFilesizes" => $fileDeletionsFilesizes, currentFileExists: True,
//"folderCreations" => $folderCreations, currentPath: $this->prefix . DIRECTORY_SEPARATOR . $creation,
//"folderDeletions" => $folderDeletions, currentSize: $creationSize,
"allSubfolders" => $allSubfolders, );
];*/
// search for creations and deletions, that are actually renames unset($fileCreations[$creationIndex]);
foreach($fileCreations as $creationIndex=>$creation) { unset($fileDeletions[$contender]);
$creationPath = $this->dir2 . DIRECTORY_SEPARATOR . $creation;
$creationSize = $fileCreationsFilesizes[$creationIndex];
$renameContenders = array_keys($fileDeletionsFilesizes, $creationSize);
if(sizeof($renameContenders) != 0) { $this->progress += 2;
/*$diff[] = [ $this->updateProgress();
"type" => "DEBUG",
"comparing" => [
"creation" => $creationIndex,
"deletions" => $renameContenders,
],
];*/
$creationSHA = sha1_file($creationPath); break;
foreach($renameContenders as $contender) { }
$deletion = $fileDeletions[$contender]; }
$deletionPath = $this->dir1 . DIRECTORY_SEPARATOR . $deletion; }
$deletionSHA = sha1_file($deletionPath); }
if($deletionSHA == $creationSHA) { foreach($fileCreations as $index=>$creation) {
($this->newResultCallback)( ($this->newResultCallback)(
type: "RENAME", type: "CREATION",
beforeFileExists: True, beforeFileExists: False,
beforePath: $this->prefix . DIRECTORY_SEPARATOR . $deletion, beforePath: NULL,
beforeSize: $creationSize, beforeSize: NULL,
currentFileExists: True, currentFileExists: True,
currentPath: $this->prefix . DIRECTORY_SEPARATOR . $creation, currentPath: $this->prefix . DIRECTORY_SEPARATOR . $creation,
currentSize: $creationSize, currentSize: $fileCreationsFilesizes[$index],
); );
unset($fileCreations[$creationIndex]); $this->progress++;
unset($fileDeletions[$contender]); $this->updateProgress();
}
$this->progress += 2; foreach($fileDeletions as $index=>$deletion) {
$this->updateProgress(); ($this->newResultCallback)(
type: "DELETION",
beforeFileExists: True,
beforePath: $this->prefix . DIRECTORY_SEPARATOR . $deletion,
beforeSize: $fileDeletionsFilesizes[$index],
currentFileExists: False,
currentPath: NULL,
currentSize: NULL,
);
break; $this->progress++;
} $this->updateProgress();
} }
}
}
foreach($fileCreations as $index=>$creation) { foreach($filePossibleEdits as $possibleEdit) {
($this->newResultCallback)( $file1 = $this->dir1 . DIRECTORY_SEPARATOR . $possibleEdit;
type: "CREATION", $file2 = $this->dir2 . DIRECTORY_SEPARATOR . $possibleEdit;
beforeFileExists: False, $file1Size = filesize($file1);
beforePath: NULL, $file2Size = filesize($file2);
beforeSize: NULL,
currentFileExists: True,
currentPath: $this->prefix . DIRECTORY_SEPARATOR . $creation,
currentSize: $fileCreationsFilesizes[$index],
);
$this->progress++; $this->progress += 2;
$this->updateProgress(); $this->updateProgress();
}
foreach($fileDeletions as $index=>$deletion) { if(filemtime($file1) == filemtime($file2)) {
($this->newResultCallback)( //not different because same mtime
type: "DELETION", continue;
beforeFileExists: True, } else {
beforePath: $this->prefix . DIRECTORY_SEPARATOR . $deletion, // mtime different, but could just have gotten touched without modifications
beforeSize: $fileDeletionsFilesizes[$index], if($file1Size == $file2Size) {
currentFileExists: False, // if filesize is the same check for binary differences
currentPath: NULL, $handle1 = fopen($file1, 'rb');
currentSize: NULL, $handle2 = fopen($file2, 'rb');
);
$this->progress++; $filesdifferent = false;
$this->updateProgress();
}
foreach($filePossibleEdits as $possibleEdit) { while(!feof($handle1)) {
$file1 = $this->dir1 . DIRECTORY_SEPARATOR . $possibleEdit; if(fread($handle1, 8192) != fread($handle2, 8192)) {
$file2 = $this->dir2 . DIRECTORY_SEPARATOR . $possibleEdit; // files are different
$file1Size = filesize($file1); $filesdifferent = true;
$file2Size = filesize($file2); break;
}
}
$this->progress += 2; fclose($handle1);
$this->updateProgress(); fclose($handle2);
if(filemtime($file1) == filemtime($file2)) { if(!$filesdifferent) {
//not different because same mtime continue;
continue; }
} else { }
// mtime different, but could just have gotten touched without modifications }
if($file1Size == $file2Size) {
// if filesize is the same check for binary differences
$handle1 = fopen($file1, 'rb');
$handle2 = fopen($file2, 'rb');
$filesdifferent = false;
($this->newResultCallback)(
type: "EDIT",
beforeFileExists: True,
beforePath: $this->prefix . DIRECTORY_SEPARATOR . $possibleEdit,
beforeSize: $file1Size,
currentFileExists: True,
currentPath: $this->prefix . DIRECTORY_SEPARATOR . $possibleEdit,
currentSize: $file2Size,
);
}
while(!feof($handle1)) { return $diff;
if(fread($handle1, 8192) != fread($handle2, 8192)) { }
// files are different
$filesdifferent = true;
break;
}
}
fclose($handle1);
fclose($handle2);
if(!$filesdifferent) {
continue;
}
}
}
($this->newResultCallback)(
type: "EDIT",
beforeFileExists: True,
beforePath: $this->prefix . DIRECTORY_SEPARATOR . $possibleEdit,
beforeSize: $file1Size,
currentFileExists: True,
currentPath: $this->prefix . DIRECTORY_SEPARATOR . $possibleEdit,
currentSize: $file2Size,
);
}
return $diff;
}
} }

View file

@ -6,27 +6,27 @@ use OCP\IURLGenerator;
use OCP\Settings\IIconSection; use OCP\Settings\IIconSection;
class SnapshotsSection implements IIconSection { class SnapshotsSection implements IIconSection {
private IL10N $l; private IL10N $l;
private IURLGenerator $urlGenerator; private IURLGenerator $urlGenerator;
public function __construct(IL10N $l, IURLGenerator $urlGenerator) { public function __construct(IL10N $l, IURLGenerator $urlGenerator) {
$this->l = $l; $this->l = $l;
$this->urlGenerator = $urlGenerator; $this->urlGenerator = $urlGenerator;
} }
public function getIcon(): string { public function getIcon(): string {
return $this->urlGenerator->imagePath('core', 'actions/settings-dark.svg'); return $this->urlGenerator->imagePath('core', 'actions/settings-dark.svg');
} }
public function getID(): string { public function getID(): string {
return 'groupfolder_filesystem_snapshots'; return 'groupfolder_filesystem_snapshots';
} }
public function getName(): string { public function getName(): string {
return $this->l->t('Groupfolder Filesystem Snapshots'); return $this->l->t('Groupfolder Filesystem Snapshots');
} }
public function getPriority(): int { public function getPriority(): int {
return 98; return 98;
} }
} }

View file

@ -58,7 +58,8 @@ class DiffTaskService {
} }
} }
function create(string $relativePathInGroupfolder, int $groupfolderId, string $snapshotId, string $userId, Callable $progressCallback = null): ?DiffTask { function create(string $relativePathInGroupfolder, int $groupfolderId, string $snapshotId, string $userId, array $folderBlocklist, Callable $progressCallback = null): ?DiffTask {
$parentNode = $this->pathManager->getGroupfolderMountById($groupfolderId)->get($relativePathInGroupfolder);
$snapshotPath = $this->pathManager->getGroupFolderSnapshotDirectory($groupfolderId, $relativePathInGroupfolder, $snapshotId); $snapshotPath = $this->pathManager->getGroupFolderSnapshotDirectory($groupfolderId, $relativePathInGroupfolder, $snapshotId);
$groupfolderPath = $this->pathManager->getGroupFolderDirectory($groupfolderId, $relativePathInGroupfolder); $groupfolderPath = $this->pathManager->getGroupFolderDirectory($groupfolderId, $relativePathInGroupfolder);
@ -80,16 +81,25 @@ class DiffTaskService {
$snapshotPath, $snapshotPath,
$groupfolderPath, $groupfolderPath,
"", "",
function(string $type, bool $beforeFileExists, ?string $beforePath, ?int $beforeSize, bool $currentFileExists, ?string $currentPath, ?int $currentSize) use ($task) { $folderBlocklist,
function(string $type, bool $beforeFileExists, ?string $beforePath, ?int $beforeSize, bool $currentFileExists, ?string $currentPath, ?int $currentSize) use ($task, $parentNode) {
$newResult = new DiffTaskResult(); $newResult = new DiffTaskResult();
$newResult->setTaskId($task->getId()); $newResult->setTaskId($task->getId());
$newResult->setType($type); $newResult->setType($type);
$newResult->setBeforeFileExists($beforeFileExists); $newResult->setBeforeFileExists($beforeFileExists);
$newResult->setBeforePath($beforePath); if($beforeFileExists) {
$newResult->setBeforeSize($beforeSize); $newResult->setBeforePath($beforePath);
$newResult->setBeforeSize($beforeSize);
}
$newResult->setCurrentFileExists($currentFileExists); $newResult->setCurrentFileExists($currentFileExists);
$newResult->setCurrentPath($currentPath); if($currentFileExists) {
$newResult->setCurrentSize($currentSize); $newResult->setCurrentFileId($parentNode->get($currentPath)?->getId());
$newResult->setCurrentPath($currentPath);
$newResult->setCurrentSize($currentSize);
}
$newResult = $this->diffTaskResultMapper->insert($newResult); $newResult = $this->diffTaskResultMapper->insert($newResult);
}, },
function($numDoneFiles) use ($progressCallback, &$numFiles) { function($numDoneFiles) use ($progressCallback, &$numFiles) {
@ -97,8 +107,7 @@ class DiffTaskService {
($progressCallback)([ ($progressCallback)([
"overallFiles" => $numFiles, "overallFiles" => $numFiles,
"doneFiles" => $numDoneFiles, "doneFiles" => $numDoneFiles,
"progress" => number_format(($numDoneFiles / $numFiles),2), "progress" => round(($numDoneFiles / $numFiles),2),
"progressPercent" => (number_format(($numDoneFiles / $numFiles),2) * 100) . "%",
]); ]);
} }
}, },
@ -113,7 +122,6 @@ class DiffTaskService {
"overallFiles" => $numFiles, "overallFiles" => $numFiles,
"doneFiles" => $numFiles, "doneFiles" => $numFiles,
"progress" => 1.0, "progress" => 1.0,
"progressPercent" => "100.00%",
"result" => $task, "result" => $task,
]); ]);
} }