fix: initial indexing completion fixes and perf improvements

lib/Controller/QueueController.php

@@ -17,7 +17,6 @@
 use OCA\ContextChat\Db\QueueFile;
 use OCA\ContextChat\Db\QueueMapper;
 use OCA\ContextChat\Service\ProviderConfigService;
-use OCA\ContextChat\Service\QueueService;
 use OCA\ContextChat\Service\StorageService;
 use OCA\ContextChat\Type\Source;
 use OCP\AppFramework\Http;
@@ -39,15 +38,11 @@
 use Psr\Log\LoggerInterface;
 
 class QueueController extends OCSController {
-	private const INDEX_COMPLETION_THRESHOLD = 0.02; // 2%
-
 	public function __construct(
 		string $appName,
 		IRequest $request,
 		private LoggerInterface $logger,
 		private IAppConfig $appConfig,
-		private QueueService $queueService,
-		private StorageService $storageService,
 		private IJobList $jobList,
 		private ITimeFactory $timeFactory,
 		private QueueMapper $queueMapper,
@@ -182,7 +177,7 @@ public function deleteDocumentsQueueItems(IDBConnection $db, QueueMapper $queueM
 
 		try {
 			$this->setInitialIndexCompletion();
-		} catch (\Exception $e) {
+		} catch (\Throwable $e) {
 			$this->logger->warning('Could not check for initial index completion', ['exception' => $e]);
 		}
 
@@ -356,46 +351,43 @@ private function setInitialIndexCompletion(): void {
 
 		try {
 			$lastEnqueuedDbId = $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true);
-			if ($lastEnqueuedDbId !== -1) {
-				$initiallyQueuedFilesExist = $this->queueMapper->existsQueueItemsUpToDbId($lastEnqueuedDbId);
-				if ($initiallyQueuedFilesExist) {
-					$this->logger->debug('Initially queued files still in the queue, intial indexing has not completed.');
+			if ($lastEnqueuedDbId === -1) {
+				// Instances that completed their crawl before this value was introduced never got
+				// it set, and the migration only backfills it while the referenced file is still
+				// queued. Seed it from the queue instead of falling through to counting every file
+				// in the file cache on every single request.
+				$maxQueuedDbId = $this->queueMapper->getMaxId();
+				if ($maxQueuedDbId === null) {
+					// no crawl jobs left and nothing queued, so everything has been indexed once
+					$this->logger->info('Initial index completion detected, setting last indexed time');
+					$this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true);
 					return;
 				}
-				$this->logger->info('Initial index completion detected, setting last indexed time');
-				$this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true);
+				$this->appConfig->setAppValueInt('last_enqueued_db_id', $maxQueuedDbId, lazy: true);
+				$lastEnqueuedDbId = $maxQueuedDbId;
+			}
+
+			$initiallyQueuedFilesExist = $this->queueMapper->existsQueueItemsUpToDbId($lastEnqueuedDbId);
+			if ($initiallyQueuedFilesExist) {
+				$this->logger->debug('Initially queued files still in the queue, intial indexing has not completed.');
 				return;
 			}
+			$this->logger->info('Initial index completion detected, setting last indexed time');
+			$this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true);
+			return;
 		} catch (\Exception $e) {
 			$this->logger->warning('Could not get last enqueued file\'s DB id', ['exception' => $e]);
 		}
 
-		// last enqueued file's ID could not be retrieved, falling back to file counting method
+		// last enqueued file's ID could not be retrieved, fall back to checking whether the queue is
+		// empty. With no crawl jobs left, nothing more will be queued for the initial index.
 		try {
-			$queuedNewFilesCount = $this->queueService->countNewFiles();
-			$eligibleFilesCount = $this->storageService->countFiles();
-			// if the new files in the queue are less than 2% of the total eligible files, we consider the
-			// initial indexing complete this allows for some margin of error in case some files were
-			// added while we were indexing but still ensures that we have indexed the vast majority of
-			// files at least once
-			if (self::withinThreshold($queuedNewFilesCount, $eligibleFilesCount)) {
+			if ($this->queueMapper->getMaxId() === null) {
 				$this->logger->info('Initial index completion detected, setting last indexed time');
 				$this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true);
-				return;
 			}
-		} catch (\OCP\DB\Exception $e) {
-			$this->logger->warning('Could not count queued new files or total eligible files', ['exception' => $e]);
-			return;
+		} catch (\Exception $e) {
+			$this->logger->warning('Could not check whether the queue is empty', ['exception' => $e]);
 		}
-
-		// we are still indexing files that were never indexed before.
-		$this->logger->debug('Initial indexing not completed yet', [
-			'queuedNewFilesCount' => $queuedNewFilesCount,
-			'eligibleFilesCount' => $eligibleFilesCount,
-		]);
-	}
-
-	private static function withinThreshold(int $current, int $total, float $threshold = self::INDEX_COMPLETION_THRESHOLD): bool {
-		return ((float)($total - $current) / (float)$total) < $threshold;
 	}
 }

lib/Db/QueueMapper.php

@@ -72,6 +72,24 @@ public function removeFromQueue(array $ids): void {
 		}
 	}
 
+	/**
+	 * @return int|null The highest queue item id, or null if the queue is empty
+	 * @throws \OCP\DB\Exception
+	 */
+	public function getMaxId(): ?int {
+		$qb = $this->db->getQueryBuilder();
+		$qb->select('id')
+			->from($this->getTableName())
+			->orderBy('id', 'DESC')
+			->setMaxResults(1);
+
+		$result = $qb->executeQuery();
+		$id = $result->fetchOne();
+		$result->closeCursor();
+
+		return ($id === false || $id === null) ? null : (int)$id;
+	}
+
 	/**
 	 * @param int $dbId
 	 * @return bool
@@ -87,7 +105,7 @@ public function existsQueueItemsUpToDbId(int $dbId) : bool {
 		try {
 			$this->findEntity($qb);
 			return true;
-		} catch (DoesNotExistException|MultipleObjectsReturnedException|Exception) {
+		} catch (DoesNotExistException|MultipleObjectsReturnedException) {
 			return false;
 		}
 	}

lib/Service/StatisticsService.php

@@ -13,9 +13,12 @@
 use OCA\ContextChat\Db\QueueContentItemMapper;
 use OCA\ContextChat\Logger;
 use OCP\AppFramework\Services\IAppConfig;
+use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\DB\Exception;
 
 class StatisticsService {
+	private const ELIGIBLE_FILES_COUNT_TTL = 60 * 60; // 1 hour
+
 	public function __construct(
 		private IAppConfig $appConfig,
 		private ActionScheduler $actionService,
@@ -25,6 +28,7 @@ public function __construct(
 		private QueueContentItemMapper $contentQueue,
 		private Logger $logger,
 		private FsEventMapper $fsEventMapper,
+		private ITimeFactory $timeFactory,
 	) {
 	}
 
@@ -39,12 +43,7 @@ public function getStatistics(): array {
 			$stats['intial_indexing_completed_at'] = $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true);
 		}
 
-		try {
-			$stats['eligible_files_count'] = $this->storageService->countFiles();
-		} catch (Exception $e) {
-			$this->logger->error($e->getMessage(), ['exception' => $e]);
-			$stats['eligible_files_count'] = 0;
-		}
+		$stats['eligible_files_count'] = $this->getEligibleFilesCount();
 		try {
 			$stats['queued_actions_count'] = $this->actionService->count();
 		} catch (Exception $e) {
@@ -109,4 +108,25 @@ public function getStatistics(): array {
 
 		return $stats;
 	}
+
+	/**
+	 * countFiles() walks every mount of the instance, so its result is cached for an hour
+	 */
+	private function getEligibleFilesCount(): int {
+		$now = $this->timeFactory->getTime();
+		$countedAt = $this->appConfig->getAppValueInt('eligible_files_count_time', 0, lazy: true);
+		if ($now - $countedAt < self::ELIGIBLE_FILES_COUNT_TTL) {
+			return $this->appConfig->getAppValueInt('eligible_files_count', 0, lazy: true);
+		}
+
+		try {
+			$count = $this->storageService->countFiles();
+		} catch (Exception $e) {
+			$this->logger->error($e->getMessage(), ['exception' => $e]);
+			return 0;
+		}
+		$this->appConfig->setAppValueInt('eligible_files_count', $count, lazy: true);
+		$this->appConfig->setAppValueInt('eligible_files_count_time', $now, lazy: true);
+		return $count;
+	}
 }

lib/Service/StorageService.php

@@ -55,7 +55,11 @@ public function __construct(
 	public function countFiles(): int {
 		$totalCount = 0;
 		foreach ($this->getMounts() as $mount) {
-			$totalCount += $this->countFilesInMount($mount['storage_id'], $mount['root_id']);
+			// use the overridden root so home mounts are counted from their `files/` folder, the
+			// same root the crawl in getFilesInMount() uses; counting from the storage root would
+			// also include `uploads/`, `cache/`, `files_encryption/` and friends, which are never
+			// queued for indexing
+			$totalCount += $this->countFilesInMount($mount['storage_id'], $mount['overridden_root']);
 		}
 		return $totalCount;
 	}
@@ -66,50 +70,33 @@ public function countFiles(): int {
 	 * @return int
 	 */
 	public function countFilesInMount(int $storageId, int $rootId): int {
-		$qb = $this->getCacheQueryBuilder();
-		try {
-			$qb->selectFileCache();
-			$qb->andWhere($qb->expr()->eq('filecache.fileid', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT)));
-			$result = $qb->executeQuery();
-			/** @var array{path:string}|false $root */
-			$root = $result->fetch();
-			$result->closeCursor();
-		} catch (DBException $e) {
-			$this->logger->error('Could not fetch storage root', ['exception' => $e]);
-			return 0;
-		}
-
-		if ($root === false) {
+		$rootPath = $this->getPathOfFileId($rootId);
+		if ($rootPath === null) {
 			$this->logger->error('Could not fetch storage root');
 			return 0;
 		}
 
 		$mimeTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Application::MIMETYPES);
+		$path = $rootPath === '' ? '' : $rootPath . '/';
+		// `_` and `%` are LIKE wildcards. Group folder roots start with two underscores
+		// (`__groupfolders/28/`), so an unescaped pattern has no literal prefix at all and
+		// fs_storage_path_prefix degenerates into a scan of the entire storage.
+		$pathPattern = $this->db->escapeLikeParameter($path) . '%';
 
-		$qb = $this->getCacheQueryBuilder();
+		$qb = $this->db->getQueryBuilder();
 
 		try {
-			$path = $root['path'] === '' ? '' : $root['path'] . '/';
-
 			$qb->select($qb->func()->count('*'))
-				->from('filecache', 'filecache');
-
-			// End to end encrypted files are descendants of a folder with encrypted=1
-			// Use a subquery to check the `encrypted` status of the parent folder
-			$subQuery = $this->getCacheQueryBuilder()->select('p.encrypted')
-				->from('filecache', 'p')
-				->andWhere($qb->expr()->eq('p.fileid', 'filecache.parent'))
-				->getSQL();
-
-			$qb->andWhere(
-				$qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))
-			);
-			$qb->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)));
-			$qb
-				->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($path . '%')))
-				->andWhere($qb->expr()->notLike('filecache.path', $qb->createNamedParameter('files_versions/%')))
-				->andWhere($qb->expr()->notLike('filecache.path', $qb->createNamedParameter('files_trashbin/%')))
-				->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId)))
+				->from('filecache', 'filecache')
+				// End to end encrypted files are descendants of a folder with encrypted=1.
+				// `fileid` is the primary key, so joining the parent row matches at most one row
+				// and is equivalent to the correlated subquery this replaces, except that the
+				// planner can satisfy it with a primary key lookup instead of re-running a
+				// subquery for every candidate row.
+				->innerJoin('filecache', 'filecache', 'p', $qb->expr()->eq('p.fileid', 'filecache.parent'))
+				->where($qb->expr()->eq('p.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
+				->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
+				->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($pathPattern)))
 				->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)))
 				->andWhere($qb->expr()->lte('filecache.size', $qb->createNamedParameter(Application::CC_MAX_SIZE, IQueryBuilder::PARAM_INT)))
 				->andWhere($qb->expr()->gt('filecache.size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
@@ -125,7 +112,28 @@ public function countFilesInMount(int $storageId, int $rootId): int {
 			$this->logger->warning('Could not count files in mount: storage=' . $storageId . ' root=' . $rootId);
 			return 0;
 		}
-		return $countInMount;
+		return (int)$countInMount;
+	}
+
+	/**
+	 * @param int $fileId
+	 * @return string|null The file cache path, or null if the row does not exist or is unreadable
+	 */
+	private function getPathOfFileId(int $fileId): ?string {
+		$qb = $this->db->getQueryBuilder();
+		try {
+			$qb->select('path')
+				->from('filecache')
+				->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
+			$result = $qb->executeQuery();
+			$path = $result->fetchOne();
+			$result->closeCursor();
+		} catch (DBException $e) {
+			$this->logger->error('Could not fetch path of file ' . $fileId, ['exception' => $e]);
+			return null;
+		}
+
+		return $path === false ? null : (string)$path;
 	}
 
 	private function isFileAccessAvailable(): bool {
@@ -269,7 +277,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId
 				$qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))
 			);
 			$qb
-				->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($path . '%')))
+				->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($this->db->escapeLikeParameter($path) . '%')))
 				->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId)))
 				->andWhere($qb->expr()->gt('filecache.fileid', $qb->createNamedParameter($lastFileId)))
 				->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)));