diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php index 59ae152..a0456a2 100644 --- a/lib/Controller/QueueController.php +++ b/lib/Controller/QueueController.php @@ -17,7 +17,6 @@ use OCA\ContextChat\Db\QueueFile; use OCA\ContextChat\Db\QueueMapper; use OCA\ContextChat\Service\ProviderConfigService; -use OCA\ContextChat\Service\QueueService; use OCA\ContextChat\Service\StorageService; use OCA\ContextChat\Type\Source; use OCP\AppFramework\Http; @@ -39,15 +38,11 @@ use Psr\Log\LoggerInterface; class QueueController extends OCSController { - private const INDEX_COMPLETION_THRESHOLD = 0.02; // 2% - public function __construct( string $appName, IRequest $request, private LoggerInterface $logger, private IAppConfig $appConfig, - private QueueService $queueService, - private StorageService $storageService, private IJobList $jobList, private ITimeFactory $timeFactory, private QueueMapper $queueMapper, @@ -182,7 +177,7 @@ public function deleteDocumentsQueueItems(IDBConnection $db, QueueMapper $queueM try { $this->setInitialIndexCompletion(); - } catch (\Exception $e) { + } catch (\Throwable $e) { $this->logger->warning('Could not check for initial index completion', ['exception' => $e]); } @@ -356,46 +351,43 @@ private function setInitialIndexCompletion(): void { try { $lastEnqueuedDbId = $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true); - if ($lastEnqueuedDbId !== -1) { - $initiallyQueuedFilesExist = $this->queueMapper->existsQueueItemsUpToDbId($lastEnqueuedDbId); - if ($initiallyQueuedFilesExist) { - $this->logger->debug('Initially queued files still in the queue, intial indexing has not completed.'); + if ($lastEnqueuedDbId === -1) { + // Instances that completed their crawl before this value was introduced never got + // it set, and the migration only backfills it while the referenced file is still + // queued. Seed it from the queue instead of falling through to counting every file + // in the file cache on every single request. + $maxQueuedDbId = $this->queueMapper->getMaxId(); + if ($maxQueuedDbId === null) { + // no crawl jobs left and nothing queued, so everything has been indexed once + $this->logger->info('Initial index completion detected, setting last indexed time'); + $this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true); return; } - $this->logger->info('Initial index completion detected, setting last indexed time'); - $this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true); + $this->appConfig->setAppValueInt('last_enqueued_db_id', $maxQueuedDbId, lazy: true); + $lastEnqueuedDbId = $maxQueuedDbId; + } + + $initiallyQueuedFilesExist = $this->queueMapper->existsQueueItemsUpToDbId($lastEnqueuedDbId); + if ($initiallyQueuedFilesExist) { + $this->logger->debug('Initially queued files still in the queue, intial indexing has not completed.'); return; } + $this->logger->info('Initial index completion detected, setting last indexed time'); + $this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true); + return; } catch (\Exception $e) { $this->logger->warning('Could not get last enqueued file\'s DB id', ['exception' => $e]); } - // last enqueued file's ID could not be retrieved, falling back to file counting method + // last enqueued file's ID could not be retrieved, fall back to checking whether the queue is + // empty. With no crawl jobs left, nothing more will be queued for the initial index. try { - $queuedNewFilesCount = $this->queueService->countNewFiles(); - $eligibleFilesCount = $this->storageService->countFiles(); - // if the new files in the queue are less than 2% of the total eligible files, we consider the - // initial indexing complete this allows for some margin of error in case some files were - // added while we were indexing but still ensures that we have indexed the vast majority of - // files at least once - if (self::withinThreshold($queuedNewFilesCount, $eligibleFilesCount)) { + if ($this->queueMapper->getMaxId() === null) { $this->logger->info('Initial index completion detected, setting last indexed time'); $this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true); - return; } - } catch (\OCP\DB\Exception $e) { - $this->logger->warning('Could not count queued new files or total eligible files', ['exception' => $e]); - return; + } catch (\Exception $e) { + $this->logger->warning('Could not check whether the queue is empty', ['exception' => $e]); } - - // we are still indexing files that were never indexed before. - $this->logger->debug('Initial indexing not completed yet', [ - 'queuedNewFilesCount' => $queuedNewFilesCount, - 'eligibleFilesCount' => $eligibleFilesCount, - ]); - } - - private static function withinThreshold(int $current, int $total, float $threshold = self::INDEX_COMPLETION_THRESHOLD): bool { - return ((float)($total - $current) / (float)$total) < $threshold; } } diff --git a/lib/Db/QueueMapper.php b/lib/Db/QueueMapper.php index 2af16ce..c444e82 100644 --- a/lib/Db/QueueMapper.php +++ b/lib/Db/QueueMapper.php @@ -72,6 +72,24 @@ public function removeFromQueue(array $ids): void { } } + /** + * @return int|null The highest queue item id, or null if the queue is empty + * @throws \OCP\DB\Exception + */ + public function getMaxId(): ?int { + $qb = $this->db->getQueryBuilder(); + $qb->select('id') + ->from($this->getTableName()) + ->orderBy('id', 'DESC') + ->setMaxResults(1); + + $result = $qb->executeQuery(); + $id = $result->fetchOne(); + $result->closeCursor(); + + return ($id === false || $id === null) ? null : (int)$id; + } + /** * @param int $dbId * @return bool @@ -87,7 +105,7 @@ public function existsQueueItemsUpToDbId(int $dbId) : bool { try { $this->findEntity($qb); return true; - } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception) { + } catch (DoesNotExistException|MultipleObjectsReturnedException) { return false; } } diff --git a/lib/Service/StatisticsService.php b/lib/Service/StatisticsService.php index 4f066b2..7d214ef 100644 --- a/lib/Service/StatisticsService.php +++ b/lib/Service/StatisticsService.php @@ -13,9 +13,12 @@ use OCA\ContextChat\Db\QueueContentItemMapper; use OCA\ContextChat\Logger; use OCP\AppFramework\Services\IAppConfig; +use OCP\AppFramework\Utility\ITimeFactory; use OCP\DB\Exception; class StatisticsService { + private const ELIGIBLE_FILES_COUNT_TTL = 60 * 60; // 1 hour + public function __construct( private IAppConfig $appConfig, private ActionScheduler $actionService, @@ -25,6 +28,7 @@ public function __construct( private QueueContentItemMapper $contentQueue, private Logger $logger, private FsEventMapper $fsEventMapper, + private ITimeFactory $timeFactory, ) { } @@ -39,12 +43,7 @@ public function getStatistics(): array { $stats['intial_indexing_completed_at'] = $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true); } - try { - $stats['eligible_files_count'] = $this->storageService->countFiles(); - } catch (Exception $e) { - $this->logger->error($e->getMessage(), ['exception' => $e]); - $stats['eligible_files_count'] = 0; - } + $stats['eligible_files_count'] = $this->getEligibleFilesCount(); try { $stats['queued_actions_count'] = $this->actionService->count(); } catch (Exception $e) { @@ -109,4 +108,25 @@ public function getStatistics(): array { return $stats; } + + /** + * countFiles() walks every mount of the instance, so its result is cached for an hour + */ + private function getEligibleFilesCount(): int { + $now = $this->timeFactory->getTime(); + $countedAt = $this->appConfig->getAppValueInt('eligible_files_count_time', 0, lazy: true); + if ($now - $countedAt < self::ELIGIBLE_FILES_COUNT_TTL) { + return $this->appConfig->getAppValueInt('eligible_files_count', 0, lazy: true); + } + + try { + $count = $this->storageService->countFiles(); + } catch (Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + return 0; + } + $this->appConfig->setAppValueInt('eligible_files_count', $count, lazy: true); + $this->appConfig->setAppValueInt('eligible_files_count_time', $now, lazy: true); + return $count; + } } diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index b937259..dfcbd50 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -55,7 +55,11 @@ public function __construct( public function countFiles(): int { $totalCount = 0; foreach ($this->getMounts() as $mount) { - $totalCount += $this->countFilesInMount($mount['storage_id'], $mount['root_id']); + // use the overridden root so home mounts are counted from their `files/` folder, the + // same root the crawl in getFilesInMount() uses; counting from the storage root would + // also include `uploads/`, `cache/`, `files_encryption/` and friends, which are never + // queued for indexing + $totalCount += $this->countFilesInMount($mount['storage_id'], $mount['overridden_root']); } return $totalCount; } @@ -66,50 +70,33 @@ public function countFiles(): int { * @return int */ public function countFilesInMount(int $storageId, int $rootId): int { - $qb = $this->getCacheQueryBuilder(); - try { - $qb->selectFileCache(); - $qb->andWhere($qb->expr()->eq('filecache.fileid', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))); - $result = $qb->executeQuery(); - /** @var array{path:string}|false $root */ - $root = $result->fetch(); - $result->closeCursor(); - } catch (DBException $e) { - $this->logger->error('Could not fetch storage root', ['exception' => $e]); - return 0; - } - - if ($root === false) { + $rootPath = $this->getPathOfFileId($rootId); + if ($rootPath === null) { $this->logger->error('Could not fetch storage root'); return 0; } $mimeTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Application::MIMETYPES); + $path = $rootPath === '' ? '' : $rootPath . '/'; + // `_` and `%` are LIKE wildcards. Group folder roots start with two underscores + // (`__groupfolders/28/`), so an unescaped pattern has no literal prefix at all and + // fs_storage_path_prefix degenerates into a scan of the entire storage. + $pathPattern = $this->db->escapeLikeParameter($path) . '%'; - $qb = $this->getCacheQueryBuilder(); + $qb = $this->db->getQueryBuilder(); try { - $path = $root['path'] === '' ? '' : $root['path'] . '/'; - $qb->select($qb->func()->count('*')) - ->from('filecache', 'filecache'); - - // End to end encrypted files are descendants of a folder with encrypted=1 - // Use a subquery to check the `encrypted` status of the parent folder - $subQuery = $this->getCacheQueryBuilder()->select('p.encrypted') - ->from('filecache', 'p') - ->andWhere($qb->expr()->eq('p.fileid', 'filecache.parent')) - ->getSQL(); - - $qb->andWhere( - $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) - ); - $qb->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))); - $qb - ->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($path . '%'))) - ->andWhere($qb->expr()->notLike('filecache.path', $qb->createNamedParameter('files_versions/%'))) - ->andWhere($qb->expr()->notLike('filecache.path', $qb->createNamedParameter('files_trashbin/%'))) - ->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId))) + ->from('filecache', 'filecache') + // End to end encrypted files are descendants of a folder with encrypted=1. + // `fileid` is the primary key, so joining the parent row matches at most one row + // and is equivalent to the correlated subquery this replaces, except that the + // planner can satisfy it with a primary key lookup instead of re-running a + // subquery for every candidate row. + ->innerJoin('filecache', 'filecache', 'p', $qb->expr()->eq('p.fileid', 'filecache.parent')) + ->where($qb->expr()->eq('p.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($pathPattern))) ->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY))) ->andWhere($qb->expr()->lte('filecache.size', $qb->createNamedParameter(Application::CC_MAX_SIZE, IQueryBuilder::PARAM_INT))) ->andWhere($qb->expr()->gt('filecache.size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))); @@ -125,7 +112,28 @@ public function countFilesInMount(int $storageId, int $rootId): int { $this->logger->warning('Could not count files in mount: storage=' . $storageId . ' root=' . $rootId); return 0; } - return $countInMount; + return (int)$countInMount; + } + + /** + * @param int $fileId + * @return string|null The file cache path, or null if the row does not exist or is unreadable + */ + private function getPathOfFileId(int $fileId): ?string { + $qb = $this->db->getQueryBuilder(); + try { + $qb->select('path') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + $path = $result->fetchOne(); + $result->closeCursor(); + } catch (DBException $e) { + $this->logger->error('Could not fetch path of file ' . $fileId, ['exception' => $e]); + return null; + } + + return $path === false ? null : (string)$path; } private function isFileAccessAvailable(): bool { @@ -269,7 +277,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) ); $qb - ->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($path . '%'))) + ->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($this->db->escapeLikeParameter($path) . '%'))) ->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId))) ->andWhere($qb->expr()->gt('filecache.fileid', $qb->createNamedParameter($lastFileId))) ->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)));