mirror of https://github.com/nextcloud/server.git
Revert "fix(performance): Do not set up filesystem on every call"
Joas Schilling
1 year ago
committed by
GitHub
parent
93e703bbfc
commit
98ed72b3ed
@ -1,62 +0,0 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* @copyright 2023 Anna Larch <anna.larch@gmx.net>
|
||||
*
|
||||
* @author Anna Larch <anna.larch@gmx.net>
|
||||
*
|
||||
* @license GNU AGPL version 3 or any later version
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
namespace OCA\Files\BackgroundJob;
|
||||
|
||||
use OC\Cache\File;
|
||||
use OCP\AppFramework\Utility\ITimeFactory;
|
||||
use OCP\BackgroundJob\Job;
|
||||
use OCP\BackgroundJob\TimedJob;
|
||||
use OCP\Files\IRootFolder;
|
||||
use OCP\IUser;
|
||||
use OCP\IUserManager;
|
||||
use Psr\Log\LoggerInterface;
|
||||
|
||||
class FileChunkCleanupJob extends TimedJob {
|
||||
private IUserManager $userManager;
|
||||
private IRootFolder $rootFolder;
|
||||
private LoggerInterface $logger;
|
||||
|
||||
public function __construct(IUserManager $userManager, IRootFolder $rootFolder, LoggerInterface $logger, ITimeFactory $timeFactory) {
|
||||
parent::__construct($timeFactory);
|
||||
$this->setInterval(3600*24);
|
||||
$this->setTimeSensitivity(Job::TIME_INSENSITIVE);
|
||||
$this->userManager = $userManager;
|
||||
$this->rootFolder = $rootFolder;
|
||||
$this->logger = $logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* This job cleans up all backups except the latest 3 from the updaters backup directory
|
||||
*/
|
||||
public function run($argument): void {
|
||||
$this->userManager->callForSeenUsers(function (IUser $user): void {
|
||||
$this->logger->debug('Running chunk cleanup job for user '. $user->getUID());
|
||||
$fileCache = new File();
|
||||
$fileCache->setUpStorage($user->getUID());
|
||||
$fileCache->gc();
|
||||
$this->logger->debug('Finished running chunk cleanup job for user '. $user->getUID());
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,184 @@
|
||||
<?php
|
||||
/**
|
||||
* @copyright Copyright (c) 2016, ownCloud, Inc.
|
||||
*
|
||||
* @author Bart Visscher <bartv@thisnet.nl>
|
||||
* @author Christoph Wurst <christoph@winzerhof-wurst.at>
|
||||
* @author Felix Moeller <mail@felixmoeller.de>
|
||||
* @author Jörn Friedrich Dreyer <jfd@butonic.de>
|
||||
* @author Morris Jobke <hey@morrisjobke.de>
|
||||
* @author Robin Appelman <robin@icewind.nl>
|
||||
* @author Roeland Jago Douma <roeland@famdouma.nl>
|
||||
* @author Thomas Müller <thomas.mueller@tmit.eu>
|
||||
* @author Thomas Tanghus <thomas@tanghus.net>
|
||||
* @author Vincent Petry <vincent@nextcloud.com>
|
||||
*
|
||||
* @license AGPL-3.0
|
||||
*
|
||||
* This code is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3,
|
||||
* as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License, version 3,
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
*
|
||||
*/
|
||||
class OC_FileChunking {
|
||||
protected $info;
|
||||
protected $cache;
|
||||
|
||||
/**
|
||||
* TTL of chunks
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
protected $ttl;
|
||||
|
||||
public static function decodeName($name) {
|
||||
preg_match('/(?P<name>.*)-chunking-(?P<transferid>\d+)-(?P<chunkcount>\d+)-(?P<index>\d+)/', $name, $matches);
|
||||
return $matches;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string[] $info
|
||||
*/
|
||||
public function __construct($info) {
|
||||
$this->info = $info;
|
||||
$this->ttl = \OC::$server->getConfig()->getSystemValue('cache_chunk_gc_ttl', 86400);
|
||||
}
|
||||
|
||||
public function getPrefix() {
|
||||
$name = $this->info['name'];
|
||||
$transferid = $this->info['transferid'];
|
||||
|
||||
return $name.'-chunking-'.$transferid.'-';
|
||||
}
|
||||
|
||||
protected function getCache() {
|
||||
if (!isset($this->cache)) {
|
||||
$this->cache = new \OC\Cache\File();
|
||||
}
|
||||
return $this->cache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores the given $data under the given $key - the number of stored bytes is returned
|
||||
*
|
||||
* @param string $index
|
||||
* @param resource $data
|
||||
* @return int
|
||||
*/
|
||||
public function store($index, $data) {
|
||||
$cache = $this->getCache();
|
||||
$name = $this->getPrefix().$index;
|
||||
$cache->set($name, $data, $this->ttl);
|
||||
|
||||
return $cache->size($name);
|
||||
}
|
||||
|
||||
public function isComplete() {
|
||||
$prefix = $this->getPrefix();
|
||||
$cache = $this->getCache();
|
||||
$chunkcount = (int)$this->info['chunkcount'];
|
||||
|
||||
for ($i = ($chunkcount - 1); $i >= 0; $i--) {
|
||||
if (!$cache->hasKey($prefix.$i)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles the chunks into the file specified by the path.
|
||||
* Chunks are deleted afterwards.
|
||||
*
|
||||
* @param resource $f target path
|
||||
*
|
||||
* @return integer assembled file size
|
||||
*
|
||||
* @throws \OC\InsufficientStorageException when file could not be fully
|
||||
* assembled due to lack of free space
|
||||
*/
|
||||
public function assemble($f) {
|
||||
$cache = $this->getCache();
|
||||
$prefix = $this->getPrefix();
|
||||
$count = 0;
|
||||
for ($i = 0; $i < $this->info['chunkcount']; $i++) {
|
||||
$chunk = $cache->get($prefix.$i);
|
||||
// remove after reading to directly save space
|
||||
$cache->remove($prefix.$i);
|
||||
$count += fwrite($f, $chunk);
|
||||
// let php release the memory to work around memory exhausted error with php 5.6
|
||||
$chunk = null;
|
||||
}
|
||||
|
||||
return $count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the size of the chunks already present
|
||||
* @return integer size in bytes
|
||||
*/
|
||||
public function getCurrentSize() {
|
||||
$cache = $this->getCache();
|
||||
$prefix = $this->getPrefix();
|
||||
$total = 0;
|
||||
for ($i = 0; $i < $this->info['chunkcount']; $i++) {
|
||||
$total += $cache->size($prefix.$i);
|
||||
}
|
||||
return $total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all chunks which belong to this transmission
|
||||
*/
|
||||
public function cleanup() {
|
||||
$cache = $this->getCache();
|
||||
$prefix = $this->getPrefix();
|
||||
for ($i = 0; $i < $this->info['chunkcount']; $i++) {
|
||||
$cache->remove($prefix.$i);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes one specific chunk
|
||||
* @param string $index
|
||||
*/
|
||||
public function remove($index) {
|
||||
$cache = $this->getCache();
|
||||
$prefix = $this->getPrefix();
|
||||
$cache->remove($prefix.$index);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles the chunks into the file specified by the path.
|
||||
* Also triggers the relevant hooks and proxies.
|
||||
*
|
||||
* @param \OC\Files\Storage\Storage $storage storage
|
||||
* @param string $path target path relative to the storage
|
||||
* @return bool true on success or false if file could not be created
|
||||
*
|
||||
* @throws \OC\ServerNotAvailableException
|
||||
*/
|
||||
public function file_assemble($storage, $path) {
|
||||
// use file_put_contents as method because that best matches what this function does
|
||||
if (\OC\Files\Filesystem::isValidPath($path)) {
|
||||
$target = $storage->fopen($path, 'w');
|
||||
if ($target) {
|
||||
$count = $this->assemble($target);
|
||||
fclose($target);
|
||||
return $count > 0;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,72 @@
|
||||
<?php
|
||||
/**
|
||||
* @author Roeland Jago Douma <rullzer@owncloud.com>
|
||||
*
|
||||
* @copyright Copyright (c) 2016, ownCloud, Inc.
|
||||
* @license AGPL-3.0
|
||||
*
|
||||
* This code is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3,
|
||||
* as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License, version 3,
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||
*
|
||||
*/
|
||||
|
||||
namespace Test;
|
||||
|
||||
use OCP\ICache;
|
||||
|
||||
class FileChunkingTest extends \Test\TestCase {
|
||||
public function dataIsComplete() {
|
||||
return [
|
||||
[1, [], false],
|
||||
[1, [0], true],
|
||||
[2, [], false],
|
||||
[2, [0], false],
|
||||
[2, [1], false],
|
||||
[2, [0,1], true],
|
||||
[10, [], false],
|
||||
[10, [0,1,2,3,4,5,6,7,8], false],
|
||||
[10, [1,2,3,4,5,6,7,8,9], false],
|
||||
[10, [0,1,2,3,5,6,7,8,9], false],
|
||||
[10, [0,1,2,3,4,5,6,7,8,9], true],
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @dataProvider dataIsComplete
|
||||
* @param $total
|
||||
* @param array $present
|
||||
* @param $expected
|
||||
*/
|
||||
public function testIsComplete($total, array $present, $expected) {
|
||||
$fileChunking = $this->getMockBuilder(\OC_FileChunking::class)
|
||||
->setMethods(['getCache'])
|
||||
->setConstructorArgs([[
|
||||
'name' => 'file',
|
||||
'transferid' => '42',
|
||||
'chunkcount' => $total,
|
||||
]])
|
||||
->getMock();
|
||||
|
||||
$cache = $this->createMock(ICache::class);
|
||||
|
||||
$cache->expects($this->atLeastOnce())
|
||||
->method('hasKey')
|
||||
->willReturnCallback(function ($key) use ($present) {
|
||||
$data = explode('-', $key);
|
||||
return in_array($data[3], $present);
|
||||
});
|
||||
|
||||
$fileChunking->method('getCache')->willReturn($cache);
|
||||
|
||||
$this->assertEquals($expected, $fileChunking->isComplete());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue