Background job to cleanup leftover chunked uploads

Signed-off-by: Roeland Jago Douma <roeland@famdouma.nl>
pull/12227/head
Roeland Jago Douma 6 years ago
parent a1f9ed1d7d
commit cb742e7045
No known key found for this signature in database
GPG Key ID: F941078878347C0C

@ -16,6 +16,7 @@ return array(
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => $baseDir . '/../lib/BackgroundJob/RefreshWebcalJob.php',
'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => $baseDir . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\UploadCleanup' => $baseDir . '/../lib/BackgroundJob/UploadCleanup.php',
'OCA\\DAV\\CalDAV\\Activity\\Backend' => $baseDir . '/../lib/CalDAV/Activity/Backend.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => $baseDir . '/../lib/CalDAV/Activity/Filter/Calendar.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => $baseDir . '/../lib/CalDAV/Activity/Filter/Todo.php',
@ -181,6 +182,7 @@ return array(
'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => $baseDir . '/../lib/SystemTag/SystemTagsRelationsCollection.php',
'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',
'OCA\\DAV\\Upload\\UploadFolder' => $baseDir . '/../lib/Upload/UploadFolder.php',

@ -31,6 +31,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/RefreshWebcalJob.php',
'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\UploadCleanup' => __DIR__ . '/..' . '/../lib/BackgroundJob/UploadCleanup.php',
'OCA\\DAV\\CalDAV\\Activity\\Backend' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Backend.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Calendar.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Todo.php',
@ -196,6 +197,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagsRelationsCollection.php',
'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',
'OCA\\DAV\\Upload\\UploadFolder' => __DIR__ . '/..' . '/../lib/Upload/UploadFolder.php',

@ -0,0 +1,87 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\BackgroundJob;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\BackgroundJob\TimedJob;
use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
class UploadCleanup extends TimedJob {
/** @var IRootFolder */
private $rootFolder;
/** @var IJobList */
private $jobList;
public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) {
parent::__construct($time);
$this->rootFolder = $rootFolder;
$this->jobList = $jobList;
// Run once a day
$this->setInterval(60*60*24);
}
protected function run($argument) {
$uid = $argument['uid'];
$folder = $argument['folder'];
$userFolder = $this->rootFolder->getUserFolder($uid);
$userRoot = $userFolder->getParent();
try {
/** @var Folder $uploads */
$uploads = $userRoot->get('uploads');
/** @var Folder $uploadFolder */
$uploadFolder = $uploads->get($folder);
} catch (NotFoundException $e) {
$this->jobList->remove(self::class, $argument);
return;
}
$files = $uploadFolder->getDirectoryListing();
// Remove if all files have an mtime of more than a day
$time = $this->time->getTime() - 60 * 60 * 24;
// The folder has to be more than a day old
$initial = $uploadFolder->getMTime() < $time;
$expire = array_reduce($files, function(bool $carry, File $file) use ($time) {
return $carry && $file->getMTime() < $time;
}, $initial);
if ($expire) {
$uploadFolder->delete();
$this->jobList->remove(self::class, $argument);
}
}
}

@ -35,6 +35,7 @@ use OCA\DAV\Connector\Sabre\Principal;
use OCA\DAV\DAV\GroupPrincipalBackend;
use OCA\DAV\DAV\SystemPrincipalBackend;
use OCA\DAV\CalDAV\Principal\Collection;
use OCA\DAV\Upload\CleanupService;
use Sabre\DAV\SimpleCollection;
class RootCollection extends SimpleCollection {
@ -120,7 +121,10 @@ class RootCollection extends SimpleCollection {
$systemAddressBookRoot = new AddressBookRoot(new SystemPrincipalBackend(), $systemCardDavBackend, 'principals/system');
$systemAddressBookRoot->disableListing = $disableListing;
$uploadCollection = new Upload\RootCollection($userPrincipalBackend, 'principals/users');
$uploadCollection = new Upload\RootCollection(
$userPrincipalBackend,
'principals/users',
\OC::$server->query(CleanupService::class));
$uploadCollection->disableListing = $disableListing;
$avatarCollection = new Avatars\RootCollection($userPrincipalBackend, 'principals/users');

@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Upload;
use OCA\DAV\BackgroundJob\UploadCleanup;
use OCP\BackgroundJob\IJobList;
use OCP\IUserSession;
class CleanupService {
/** @var IUserSession */
private $userSession;
/** @var IJobList */
private $jobList;
public function __construct(IUserSession $userSession, IJobList $jobList) {
$this->userSession = $userSession;
$this->jobList = $jobList;
}
public function addJob(string $folder) {
$this->jobList->add(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]);
}
public function removeJob(string $folder) {
$this->jobList->remove(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]);
}
}

@ -1,4 +1,5 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
@ -23,20 +24,31 @@
namespace OCA\DAV\Upload;
use Sabre\DAVACL\AbstractPrincipalCollection;
use Sabre\DAVACL\PrincipalBackend;
class RootCollection extends AbstractPrincipalCollection {
/** @var CleanupService */
private $cleanupService;
public function __construct(PrincipalBackend\BackendInterface $principalBackend,
string $principalPrefix,
CleanupService $cleanupService) {
parent::__construct($principalBackend, $principalPrefix);
$this->cleanupService = $cleanupService;
}
/**
* @inheritdoc
*/
function getChildForPrincipal(array $principalInfo) {
return new UploadHome($principalInfo);
public function getChildForPrincipal(array $principalInfo): UploadHome {
return new UploadHome($principalInfo, $this->cleanupService);
}
/**
* @inheritdoc
*/
function getName() {
public function getName(): string {
return 'uploads';
}

@ -22,16 +22,22 @@
*/
namespace OCA\DAV\Upload;
use OCA\DAV\BackgroundJob\UploadCleanup;
use OCA\DAV\Connector\Sabre\Directory;
use OCP\BackgroundJob\IJobList;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\ICollection;
class UploadFolder implements ICollection {
/** @var Directory */
private $node;
/** @var CleanupService */
private $cleanupService;
function __construct(Directory $node) {
function __construct(Directory $node, CleanupService $cleanupService) {
$this->node = $node;
$this->cleanupService = $cleanupService;
}
function createFile($name, $data = null) {
@ -65,6 +71,9 @@ class UploadFolder implements ICollection {
function delete() {
$this->node->delete();
// Background cleanup job is not needed anymore
$this->cleanupService->removeJob($this->getName());
}
function getName() {

@ -30,51 +30,56 @@ use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\ICollection;
class UploadHome implements ICollection {
/**
* UploadHome constructor.
*
* @param array $principalInfo
*/
public function __construct($principalInfo) {
/** @var array */
private $principalInfo;
/** @var CleanupService */
private $cleanupService;
public function __construct(array $principalInfo, CleanupService $cleanupService) {
$this->principalInfo = $principalInfo;
$this->cleanupService = $cleanupService;
}
function createFile($name, $data = null) {
public function createFile($name, $data = null) {
throw new Forbidden('Permission denied to create file (filename ' . $name . ')');
}
function createDirectory($name) {
public function createDirectory($name) {
$this->impl()->createDirectory($name);
// Add a cleanup job
$this->cleanupService->addJob($name);
}
function getChild($name) {
return new UploadFolder($this->impl()->getChild($name));
public function getChild($name): UploadFolder {
return new UploadFolder($this->impl()->getChild($name), $this->cleanupService);
}
function getChildren() {
public function getChildren(): array {
return array_map(function($node) {
return new UploadFolder($node);
return new UploadFolder($node, $this->cleanupService);
}, $this->impl()->getChildren());
}
function childExists($name) {
public function childExists($name): bool {
return !is_null($this->getChild($name));
}
function delete() {
public function delete() {
$this->impl()->delete();
}
function getName() {
public function getName() {
list(,$name) = \Sabre\Uri\split($this->principalInfo['uri']);
return $name;
}
function setName($name) {
public function setName($name) {
throw new Forbidden('Permission denied to rename this folder');
}
function getLastModified() {
public function getLastModified() {
return $this->impl()->getLastModified();
}

Loading…
Cancel
Save