From 95938d32d957d8e1977e9949a18e5f14a0ce23fc Mon Sep 17 00:00:00 2001 From: Louis Chemineau Date: Wed, 15 Nov 2023 18:30:40 +0100 Subject: [PATCH] Add migration from the old to new metadata tables Signed-off-by: Louis Chemineau --- core/BackgroundJobs/MigrateMetadataJob.php | 107 ++++++++++++++++++ .../Version28000Date20231004103301.php | 1 + lib/composer/composer/autoload_classmap.php | 2 + lib/composer/composer/autoload_static.php | 2 + lib/private/Repair.php | 2 + .../Repair/AddMetadataMigrationJob.php | 43 +++++++ 6 files changed, 157 insertions(+) create mode 100644 core/BackgroundJobs/MigrateMetadataJob.php create mode 100644 lib/private/Repair/AddMetadataMigrationJob.php diff --git a/core/BackgroundJobs/MigrateMetadataJob.php b/core/BackgroundJobs/MigrateMetadataJob.php new file mode 100644 index 00000000000..4f1856c2335 --- /dev/null +++ b/core/BackgroundJobs/MigrateMetadataJob.php @@ -0,0 +1,107 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\BackgroundJobs; + +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; +use OCP\BackgroundJob\TimedJob; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IConfig; +use OCP\IDBConnection; + +class MigrateMetadataJob extends TimedJob { + public function __construct( + ITimeFactory $time, + private IConfig $config, + private IFilesMetadataManager $filesMetadataManager, + private IDBConnection $connection, + private IJobList $jobList, + ) { + parent::__construct($time); + + $this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE); + $this->setInterval(24 * 3600); + } + + protected function run(mixed $argument): void { + if (!$this->connection->tableExists('file_metadata')) { + return; + } + + $startTime = time(); + + $selectQuery = $this->connection->getQueryBuilder() + ->select('*') + ->from('file_metadata') + ->setMaxResults(200); + + $deleteQuery = $this->connection->getQueryBuilder(); + $deleteQuery->delete('file_metadata') + ->where($deleteQuery->expr()->eq('id', $deleteQuery->createParameter('id'))) + ->where($deleteQuery->expr()->eq('group_name', $deleteQuery->createParameter('group_name'))) + ->where($deleteQuery->expr()->eq('value', $deleteQuery->createParameter('value'))); + + do { + $this->connection->beginTransaction(); + + $results = $selectQuery->executeQuery(); + + while ($row = $results->fetch()) { + $metadata = $this->filesMetadataManager->getMetadata($row['id'], true); + + switch ($row['group_name']) { + case 'size': + $metadata->setArray('photos-size', json_decode($row['value'], true)); + break; + case 'gps': + $metadata->setArray('photos-gps', json_decode($row['value'], true)); + break; + case 'photos_place': + $metadata->setString('photos-place', $row['value'], true); + break; + } + + $this->filesMetadataManager->saveMetadata($metadata); + $deleteQuery->setParameter('id', $row['id']); + $deleteQuery->setParameter('group_name', $row['group_name']); + $deleteQuery->setParameter('value', $row['value']); + $deleteQuery->executeStatement(); + } + + $results->closeCursor(); + + $this->connection->commit(); + + // Stop if execution time is more than one hour. + if (time() - $startTime > 60 * 60) { + return; + } + } while ($results->rowCount() !== 0); + + $this->connection->dropTable('file_metadata'); + $this->jobList->remove(MigrateMetadataJob::class); + } +} diff --git a/core/Migrations/Version28000Date20231004103301.php b/core/Migrations/Version28000Date20231004103301.php index 8888bf51910..f1ce479d0fa 100644 --- a/core/Migrations/Version28000Date20231004103301.php +++ b/core/Migrations/Version28000Date20231004103301.php @@ -31,6 +31,7 @@ use OCP\DB\Types; use OCP\Migration\IOutput; use OCP\Migration\SimpleMigrationStep; +// Create new tables for the Metadata API (files_metadata and files_metadata_index). class Version28000Date20231004103301 extends SimpleMigrationStep { public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { /** @var ISchemaWrapper $schema */ diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 85fbe3d9151..39085325067 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1003,6 +1003,7 @@ return array( 'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => $baseDir . '/core/BackgroundJobs/CheckForUserCertificates.php', 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MigrateMetadataJob' => $baseDir . '/core/BackgroundJobs/MigrateMetadataJob.php', 'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php', @@ -1602,6 +1603,7 @@ return array( 'OC\\RepairException' => $baseDir . '/lib/private/RepairException.php', 'OC\\Repair\\AddBruteForceCleanupJob' => $baseDir . '/lib/private/Repair/AddBruteForceCleanupJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', + 'OC\\Repair\\AddMetadataMigrationJob' => $baseDir . '/lib/private/Repair/AddMetadataMigrationJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 1881fd88eb3..498f4d8c0b1 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1036,6 +1036,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CheckForUserCertificates.php', 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MigrateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MigrateMetadataJob.php', 'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php', @@ -1635,6 +1636,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\RepairException' => __DIR__ . '/../../..' . '/lib/private/RepairException.php', 'OC\\Repair\\AddBruteForceCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddBruteForceCleanupJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', + 'OC\\Repair\\AddMetadataMigrationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataMigrationJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 967d079d6db..e70c3f7e39d 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -46,6 +46,7 @@ use OC\DB\Connection; use OC\DB\ConnectionAdapter; use OC\Repair\AddBruteForceCleanupJob; use OC\Repair\AddCleanupUpdaterBackupsJob; +use OC\Repair\AddMetadataMigrationJob; use OC\Repair\CleanTags; use OC\Repair\ClearFrontendCaches; use OC\Repair\ClearGeneratedAvatarCache; @@ -211,6 +212,7 @@ class Repair implements IOutput { \OCP\Server::get(CleanUpAbandonedApps::class), \OCP\Server::get(AddMissingSecretJob::class), \OCP\Server::get(AddRemoveOldTasksBackgroundJob::class), + \OCP\Server::get(AddMetadataMigrationJob::class), ]; } diff --git a/lib/private/Repair/AddMetadataMigrationJob.php b/lib/private/Repair/AddMetadataMigrationJob.php new file mode 100644 index 00000000000..f73768721dc --- /dev/null +++ b/lib/private/Repair/AddMetadataMigrationJob.php @@ -0,0 +1,43 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OC\Repair; + +use OC\Core\BackgroundJobs\MigrateMetadataJob; +use OCP\BackgroundJob\IJobList; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class AddMetadataMigrationJob implements IRepairStep { + public function __construct( + private IJobList $jobList, + ) { + } + + public function getName() { + return 'Queue a job to migrate legacy metadata tables'; + } + + public function run(IOutput $output) { + $this->jobList->add(MigrateMetadataJob::class); + } +}