Feat/link faces to contacts

README.md

@@ -126,6 +126,12 @@ Go to "Apps" in your nextcloud, search for "recognize" and click install.
 
 Any configuration is done in Settings/Recognize of your Nextcloud instance.
 
+#### Linking people to contacts
+
+Recognized people can be linked to the contacts of the Contacts app. Once a day, recognize compares the photos of all contacts that are not linked yet with the recognized people and links a person when exactly one person matches the photo; linked people that have no name yet take the name of the contact. The linked contact is shown on the page of the person in the Photos app, where the link can also be made, changed or removed by hand, and a photo of the person can be used as the photo of the contact. In the Contacts app, the photo of a contact can be picked from the recognized faces.
+
+Automatic linking can be turned off in the admin settings and runs on demand with `occ recognize:link-contacts`. It needs the built-in face recognition; with the TaskProcessing backend, people can only be linked by hand.
+
 #### Ignoring directories
 
 If you want path/to/your/folder/* to be excluded from image recognition, add a file `path/to/your/folder/.noimage`. If you want to exclude it from music genre recognition, add a file `path/to/your/folder/.nomusic`. If you want to exclude it from video recognition, add a file `path/to/your/folder/.novideo`. If you want to exclude it from all recognition, add a file `path/to/your/folder/.nomedia`.

appinfo/info.xml

@@ -96,6 +96,7 @@ The app does not send any sensitive data to cloud providers or similar services.
 	</dependencies>
 	<background-jobs>
 		<job>OCA\Recognize\BackgroundJobs\MaintenanceJob</job>
+		<job>OCA\Recognize\BackgroundJobs\ScheduleLinkContactsJob</job>
 	</background-jobs>
 
 	<repair-steps>
@@ -121,6 +122,7 @@ The app does not send any sensitive data to cloud providers or similar services.
 		<command>OCA\Recognize\Command\DownloadModels</command>
 		<command>OCA\Recognize\Command\Classify</command>
 		<command>OCA\Recognize\Command\ClusterFaces</command>
+		<command>OCA\Recognize\Command\LinkContacts</command>
 	</commands>
 
 	<settings>

appinfo/routes.php

@@ -42,4 +42,15 @@
 		['name' => 'admin#get_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'GET'],
 		['name' => 'admin#set_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'PUT'],
 	],
+	'ocs' => [
+		// Links between people (face clusters) and contacts, used by the Photos and Contacts apps
+		['name' => 'contact_link#searchContacts', 'url' => '/api/v1/contacts', 'verb' => 'GET'],
+		['name' => 'contact_link#getContact', 'url' => '/api/v1/contacts/{addressBook}/{uid}', 'verb' => 'GET'],
+		['name' => 'contact_link#listFaces', 'url' => '/api/v1/faces', 'verb' => 'GET'],
+		['name' => 'contact_link#listDetections', 'url' => '/api/v1/faces/{clusterId}/detections', 'verb' => 'GET', 'requirements' => ['clusterId' => '\d+']],
+		['name' => 'contact_link#linkFace', 'url' => '/api/v1/faces/{clusterId}/contact', 'verb' => 'PUT', 'requirements' => ['clusterId' => '\d+']],
+		['name' => 'contact_link#unlinkFace', 'url' => '/api/v1/faces/{clusterId}/contact', 'verb' => 'DELETE', 'requirements' => ['clusterId' => '\d+']],
+		['name' => 'contact_link#getDetectionPreview', 'url' => '/api/v1/faces/detections/{detectionId}/preview', 'verb' => 'GET', 'requirements' => ['detectionId' => '\d+']],
+		['name' => 'contact_link#setContactPhoto', 'url' => '/api/v1/faces/detections/{detectionId}/contact-photo', 'verb' => 'POST', 'requirements' => ['detectionId' => '\d+']],
+	],
 ];

lib/AppInfo/Application.php

@@ -9,6 +9,7 @@
 
 use OCA\DAV\Connector\Sabre\Principal;
 use OCA\DAV\Events\SabrePluginAddEvent;
+use OCA\Recognize\Capabilities;
 use OCA\Recognize\Dav\Faces\PropFindPlugin;
 use OCA\Recognize\Hooks\FileListener;
 use OCA\Recognize\TaskProcessing\AudioClassificationTaskType;
@@ -64,6 +65,8 @@ public function register(IRegistrationContext $context): void {
 		/** Register $principalBackend for the DAV collection */
 		$context->registerServiceAlias('principalBackend', Principal::class);
 
+		$context->registerCapability(Capabilities::class);
+
 		$context->registerTaskProcessingTaskType(ImageClassificationTaskType::class);
 		$context->registerTaskProcessingTaskType(VideoClassificationTaskType::class);
 		$context->registerTaskProcessingTaskType(AudioClassificationTaskType::class);

lib/BackgroundJobs/ClusterFacesJob.php

@@ -7,6 +7,7 @@
 declare(strict_types=1);
 namespace OCA\Recognize\BackgroundJobs;
 
+use OCA\Recognize\Service\ContactLinkService;
 use OCA\Recognize\Service\FaceClusterAnalyzer;
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\SettingsService;
@@ -20,13 +21,15 @@ final class ClusterFacesJob extends QueuedJob {
 	private IJobList $jobList;
 	private LoggerInterface $logger;
 	private SettingsService $settingsService;
+	private ContactLinkService $contactLinkService;
 
-	public function __construct(ITimeFactory $time, Logger $logger, IJobList $jobList, FaceClusterAnalyzer $clusterAnalyzer, SettingsService $settingsService) {
+	public function __construct(ITimeFactory $time, Logger $logger, IJobList $jobList, FaceClusterAnalyzer $clusterAnalyzer, SettingsService $settingsService, ContactLinkService $contactLinkService) {
 		parent::__construct($time);
 		$this->logger = $logger;
 		$this->jobList = $jobList;
 		$this->clusterAnalyzer = $clusterAnalyzer;
 		$this->settingsService = $settingsService;
+		$this->contactLinkService = $contactLinkService;
 	}
 
 	/**
@@ -47,6 +50,9 @@ protected function run($argument): void {
 				}
 			}
 			$this->clusterAnalyzer->calculateClusters($userId, $batchSize);
+			if ($this->contactLinkService->isAutoLinkingAvailable()) {
+				$this->jobList->add(LinkContactsJob::class, ['userId' => $userId]);
+			}
 		} catch (\Throwable $e) {
 			$this->settingsService->setSetting('clusterFaces.status', 'false');
 			$this->logger->error('Failed to calculate face clusters', ['exception' => $e]);

lib/BackgroundJobs/LinkContactsJob.php

@@ -0,0 +1,46 @@
+<?php
+
+/*
+ * Copyright (c) 2026 The Recognize contributors.
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+declare(strict_types=1);
+namespace OCA\Recognize\BackgroundJobs;
+
+use OCA\Recognize\Service\ContactLinkService;
+use OCA\Recognize\Service\Logger;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\QueuedJob;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Links the face clusters of one user to their contacts.
+ *
+ * Queued by {@see ScheduleLinkContactsJob} for every user, and by
+ * {@see ClusterFacesJob} when the clusters of a user changed.
+ */
+final class LinkContactsJob extends QueuedJob {
+	private LoggerInterface $logger;
+	private ContactLinkService $contactLinkService;
+
+	public function __construct(ITimeFactory $time, Logger $logger, ContactLinkService $contactLinkService) {
+		parent::__construct($time);
+		$this->logger = $logger;
+		$this->contactLinkService = $contactLinkService;
+	}
+
+	/**
+	 * @param array{userId: string} $argument
+	 */
+	protected function run($argument): void {
+		$userId = $argument['userId'];
+		if (!$this->contactLinkService->isAutoLinkingAvailable()) {
+			return;
+		}
+		try {
+			$this->contactLinkService->linkContactsForUser($userId);
+		} catch (\Throwable $e) {
+			$this->logger->error('Failed to link face clusters of user ' . $userId . ' to contacts', ['exception' => $e]);
+		}
+	}
+}

lib/BackgroundJobs/ScheduleLinkContactsJob.php

@@ -0,0 +1,57 @@
+<?php
+
+/*
+ * Copyright (c) 2026 The Recognize contributors.
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+declare(strict_types=1);
+namespace OCA\Recognize\BackgroundJobs;
+
+use OCA\Recognize\Db\FaceClusterMapper;
+use OCA\Recognize\Service\ContactLinkService;
+use OCA\Recognize\Service\Logger;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJobList;
+use OCP\BackgroundJob\TimedJob;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Queues a {@see LinkContactsJob} for every user with face clusters, so
+ * contacts that got a photo since the last run are linked as well.
+ */
+final class ScheduleLinkContactsJob extends TimedJob {
+	public const INTERVAL = 24 * 60 * 60;
+
+	private LoggerInterface $logger;
+	private IJobList $jobList;
+	private FaceClusterMapper $clusterMapper;
+	private ContactLinkService $contactLinkService;
+
+	public function __construct(ITimeFactory $time, Logger $logger, IJobList $jobList, FaceClusterMapper $clusterMapper, ContactLinkService $contactLinkService) {
+		parent::__construct($time);
+		$this->logger = $logger;
+		$this->jobList = $jobList;
+		$this->clusterMapper = $clusterMapper;
+		$this->contactLinkService = $contactLinkService;
+		$this->setInterval(self::INTERVAL);
+		$this->setTimeSensitivity(self::TIME_INSENSITIVE);
+	}
+
+	/**
+	 * @param array $argument
+	 */
+	protected function run($argument): void {
+		if (!$this->contactLinkService->isAutoLinkingAvailable()) {
+			return;
+		}
+		try {
+			$userIds = $this->clusterMapper->findUserIds();
+		} catch (\OCP\DB\Exception $e) {
+			$this->logger->error('Failed to list users with face clusters', ['exception' => $e]);
+			return;
+		}
+		foreach ($userIds as $userId) {
+			$this->jobList->add(LinkContactsJob::class, ['userId' => $userId]);
+		}
+	}
+}

lib/Capabilities.php

@@ -0,0 +1,30 @@
+<?php
+
+/*
+ * Copyright (c) 2026 The Recognize contributors.
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+declare(strict_types=1);
+namespace OCA\Recognize;
+
+use OCA\Recognize\AppInfo\Application;
+use OCP\Capabilities\ICapability;
+
+/**
+ * Tells other apps which integrations this version of recognize offers.
+ */
+final class Capabilities implements ICapability {
+	/**
+	 * @return array{recognize: array{faces: array{contactLinking: bool}}}
+	 */
+	public function getCapabilities(): array {
+		return [
+			Application::APP_ID => [
+				'faces' => [
+					// People can be linked to contacts, and faces can be used as contact photos
+					'contactLinking' => true,
+				],
+			],
+		];
+	}
+}

lib/Classifiers/Classifier.php

@@ -158,44 +158,10 @@ public function classifyFiles(string $model, array $queueFiles, int $timeout): \
 
 		$this->logger->debug('Classifying '.var_export($paths, true));
 
-		$command = [
-			$this->config->getAppValueString('node_binary', lazy: true),
-			dirname(__DIR__, 2) . '/src/classifier_'.$model.'.js',
-			'-'
-		];
-
-		if (trim($this->config->getAppValueString('nice_binary', '', lazy: true)) !== '') {
-			$command = [
-				$this->config->getAppValueString('nice_binary', lazy: true),
-				"-" . $this->config->getAppValueString('nice_value', '0', lazy: true),
-				...$command,
-			];
-		}
-
-		$this->logger->debug('Running '.var_export($command, true));
-
-		$proc = new Process($command, __DIR__);
-		$env = [];
-		if ($this->config->getAppValueString('tensorflow.gpu', 'false') === 'true') {
-			$env['RECOGNIZE_GPU'] = 'true';
-		}
-		if ($this->config->getAppValueString('tensorflow.purejs', 'false', lazy: true) === 'true') {
-			$env['RECOGNIZE_PUREJS'] = 'true';
-		}
-		// Set cores
-		$cores = $this->config->getAppValueString('tensorflow.cores', '0');
-		if ($cores !== '0') {
-			$env['RECOGNIZE_CORES'] = $cores;
-		}
-		$proc->setEnv($env);
-		$proc->setTimeout(count($paths) * $timeout);
-		$proc->setInput(implode("\n", $paths));
+		$proc = $this->createClassifierProcess($model, $paths, $timeout);
 		try {
 			$proc->start();
-
-			if ((int)$cores !== 0) {
-				@exec('taskset -cp ' . implode(',', range(0, (int)$cores, 1)) . ' ' . ((string)$proc->getPid()));
-			}
+			$this->pinToConfiguredCores($proc);
 
 			$i = 0;
 			$errOut = '';
@@ -261,6 +227,102 @@ public function classifyFiles(string $model, array $queueFiles, int $timeout): \
 		}
 	}
 
+	/**
+	 * Run the classifier of a model on local image files that are not part of the
+	 * queue, such as images that are not stored in Nextcloud at all.
+	 *
+	 * @param string $model
+	 * @param list<string> $paths Absolute paths of the images
+	 * @param int $timeout Seconds granted per image
+	 * @return list<mixed> The decoded result for each path, in the order of the paths
+	 * @throws \ErrorException|\RuntimeException
+	 */
+	public function classifyLocalFiles(string $model, array $paths, int $timeout): array {
+		if (count($paths) === 0) {
+			return [];
+		}
+		$this->logger->debug('Classifying local files '.var_export($paths, true));
+		$proc = $this->createClassifierProcess($model, $paths, $timeout);
+		try {
+			$proc->start();
+			$this->pinToConfiguredCores($proc);
+			$proc->wait();
+		} catch (ProcessTimedOutException $e) {
+			$this->logger->warning($proc->getErrorOutput());
+			throw new \RuntimeException('Classifier process timeout');
+		} catch (RuntimeException $e) {
+			$this->logger->warning($proc->getErrorOutput());
+			throw new \ErrorException('Classifier process could not be started');
+		}
+		$this->logger->debug('Classifier process output: ' . $proc->getErrorOutput());
+
+		$results = [];
+		foreach (explode("\n", $proc->getOutput()) as $line) {
+			if (trim($line) === '') {
+				continue;
+			}
+			try {
+				/** @var mixed $decoded */
+				$decoded = json_decode($line, true, 512, JSON_OBJECT_AS_ARRAY | JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_IGNORE);
+				/** @psalm-suppress MixedAssignment */
+				$results[] = $decoded;
+			} catch (\JsonException $e) {
+				$this->logger->warning('Classifier process emitted invalid JSON: ' . $line, ['exception' => $e]);
+			}
+		}
+		if (count($results) !== count($paths)) {
+			$this->logger->warning('Classifier process output: '.$proc->getErrorOutput());
+			throw new \ErrorException('Classifier process error');
+		}
+		return $results;
+	}
+
+	/**
+	 * @param list<string> $paths
+	 */
+	private function createClassifierProcess(string $model, array $paths, int $timeout): Process {
+		$command = [
+			$this->config->getAppValueString('node_binary', lazy: true),
+			dirname(__DIR__, 2) . '/src/classifier_'.$model.'.js',
+			'-'
+		];
+
+		if (trim($this->config->getAppValueString('nice_binary', '', lazy: true)) !== '') {
+			$command = [
+				$this->config->getAppValueString('nice_binary', lazy: true),
+				"-" . $this->config->getAppValueString('nice_value', '0', lazy: true),
+				...$command,
+			];
+		}
+
+		$this->logger->debug('Running '.var_export($command, true));
+
+		$proc = new Process($command, __DIR__);
+		$env = [];
+		if ($this->config->getAppValueString('tensorflow.gpu', 'false') === 'true') {
+			$env['RECOGNIZE_GPU'] = 'true';
+		}
+		if ($this->config->getAppValueString('tensorflow.purejs', 'false', lazy: true) === 'true') {
+			$env['RECOGNIZE_PUREJS'] = 'true';
+		}
+		// Set cores
+		$cores = $this->config->getAppValueString('tensorflow.cores', '0');
+		if ($cores !== '0') {
+			$env['RECOGNIZE_CORES'] = $cores;
+		}
+		$proc->setEnv($env);
+		$proc->setTimeout(count($paths) * $timeout);
+		$proc->setInput(implode("\n", $paths));
+		return $proc;
+	}
+
+	private function pinToConfiguredCores(Process $proc): void {
+		$cores = (int)$this->config->getAppValueString('tensorflow.cores', '0');
+		if ($cores !== 0) {
+			@exec('taskset -cp ' . implode(',', range(0, $cores, 1)) . ' ' . ((string)$proc->getPid()));
+		}
+	}
+
 	/**
 	 * Get path of file to process.
 	 * If the file is an image and not JPEG, it will be converted using ImageMagick.