Skip to content

perf(files/cache): SQL aggregation for folder size calculation #59586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
solracsf wants to merge 1 commit into
base: master
Choose a base branch
from
Open

perf(files/cache): SQL aggregation for folder size calculation #59586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 30 additions & 28 deletions lib/private/Files/Cache/Cache.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1015,7 +1015,6 @@ public function calculateFolderSize($path, $entry = null) {
/**
* inner function because we can't add new params to the public function without breaking any child classes
*
* @param string $path
* @param array|null|ICacheEntry $entry (optional) meta data of the folder
* @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown
* @return int|float
Expand All @@ -1029,7 +1028,19 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
$id = $entry['fileid'];

$query = $this->getQueryBuilder();
$query->select('size', 'unencrypted_size')
$sizeCol = $query->getColumnName('size');
$unencryptedSizeCol = $query->getColumnName('unencrypted_size');
$query->selectAlias($query->func()->sum('size'), 'size_sum')
->selectAlias($query->func()->min('size'), 'size_min')
->selectAlias($query->func()->max('unencrypted_size'), 'unencrypted_max')
->selectAlias(
$query->createFunction("SUM(CASE WHEN $unencryptedSizeCol > 0 THEN $unencryptedSizeCol ELSE $sizeCol END)"),
'unencrypted_sum'
)
->selectAlias(
$query->createFunction("MIN(CASE WHEN $unencryptedSizeCol > 0 THEN $unencryptedSizeCol ELSE $sizeCol END)"),
'unencrypted_min'
)
->from('filecache')
->whereStorageId($this->getNumericStorageId())
->whereParent($id);
Expand All @@ -1038,34 +1049,24 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
}

$result = $query->executeQuery();
$rows = $result->fetchAll();
$agg = $result->fetch();
$result->closeCursor();

if ($rows) {
$sizes = array_map(function (array $row) {
return Util::numericToNumber($row['size']);
}, $rows);
$unencryptedOnlySizes = array_map(function (array $row) {
return Util::numericToNumber($row['unencrypted_size']);
}, $rows);
$unencryptedSizes = array_map(function (array $row) {
return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']);
}, $rows);

$sum = array_sum($sizes);
$min = min($sizes);

$unencryptedSum = array_sum($unencryptedSizes);
$unencryptedMin = min($unencryptedSizes);
$unencryptedMax = max($unencryptedOnlySizes);
// SUM() returns NULL on empty set
if ($agg && $agg['size_sum'] !== null) {
$sum = Util::numericToNumber($agg['size_sum']);
$min = Util::numericToNumber($agg['size_min']);
$unencryptedMax = Util::numericToNumber($agg['unencrypted_max'] ?? 0);
$unencryptedSum = Util::numericToNumber($agg['unencrypted_sum'] ?? 0);
$unencryptedMin = Util::numericToNumber($agg['unencrypted_min'] ?? 0);

$sum = 0 + $sum;
$min = 0 + $min;
if ($min === -1) {
$totalSize = $min;
} else {
$totalSize = $sum;
}
$unencryptedSum = 0 + $unencryptedSum;
$unencryptedMin = 0 + $unencryptedMin;

$totalSize = ($min === -1) ? $min : $sum;

if ($unencryptedMin === -1 || $min === -1) {
$unencryptedTotal = $unencryptedMin;
} else {
Expand All @@ -1077,15 +1078,16 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
$unencryptedMax = 0;
}

// only set unencrypted size for a folder if any child entries have it set, or the folder is empty
// only set unencrypted size for a folder if any child entries have it set
// or if the folder is empty
$shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0;
if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) {
if ($shouldWriteUnEncryptedSize) {
// if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes
// if all children have an unencrypted size of 0
// just set the folder unencrypted size to 0 instead of summing the sizes
if ($unencryptedMax === 0) {
$unencryptedTotal = 0;
}

$this->update($id, [
'size' => $totalSize,
'unencrypted_size' => $unencryptedTotal,
Expand Down
22 changes: 14 additions & 8 deletions lib/private/Files/Cache/Propagator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,29 +218,32 @@ public function commitBatch(): void {
$query = $this->connection->getQueryBuilder();
$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->set('etag', $query->createParameter('etag'))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('fileid', $query->createParameter('fileid')));

$queryWithSize = $this->connection->getQueryBuilder();
$queryWithSize->update('filecache')
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
->set('etag', $queryWithSize->expr()->literal(uniqid()))
->set('etag', $queryWithSize->createParameter('etag'))
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($queryWithSize->expr()->eq('fileid', $queryWithSize->createParameter('fileid')));

while ($row = $result->fetchAssociative()) {
$item = $this->batch[$row['path']];
$newEtag = uniqid();
if ($item['size'] && $row['size'] > -1) {
$queryWithSize->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT)
->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->executeStatement();
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
$queryWithSize->executeStatement();
} else {
$query->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->executeStatement();
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
$query->executeStatement();
}
}
}
Expand All @@ -249,28 +252,31 @@ public function commitBatch(): void {
$query = $this->connection->getQueryBuilder();
$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->set('etag', $query->createParameter('etag'))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));

$queryWithSize = $this->connection->getQueryBuilder();
$queryWithSize->update('filecache')
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
->set('etag', $queryWithSize->expr()->literal(uniqid()))
->set('etag', $queryWithSize->createParameter('etag'))
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($queryWithSize->expr()->eq('path_hash', $queryWithSize->createParameter('hash')));

foreach ($this->batch as $item) {
$newEtag = uniqid();
if ($item['size']) {
$queryWithSize->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT)
->executeStatement();
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
$queryWithSize->executeStatement();
} else {
$query->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->executeStatement();
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
$query->executeStatement();
}
}
}
Expand Down
Loading