diff --git a/README.md b/README.md
index e7898879e..f11af4a0b 100644
--- a/README.md
+++ b/README.md
@@ -126,6 +126,12 @@ Go to "Apps" in your nextcloud, search for "recognize" and click install.
Any configuration is done in Settings/Recognize of your Nextcloud instance.
+#### Linking people to contacts
+
+Recognized people can be linked to the contacts of the Contacts app. Once a day, recognize compares the photos of all contacts that are not linked yet with the recognized people and links a person when exactly one person matches the photo; linked people that have no name yet take the name of the contact. The linked contact is shown on the page of the person in the Photos app, where the link can also be made, changed or removed by hand, and a photo of the person can be used as the photo of the contact. In the Contacts app, the photo of a contact can be picked from the recognized faces.
+
+Automatic linking can be turned off in the admin settings and runs on demand with `occ recognize:link-contacts`. It needs the built-in face recognition; with the TaskProcessing backend, people can only be linked by hand.
+
#### Ignoring directories
If you want path/to/your/folder/* to be excluded from image recognition, add a file `path/to/your/folder/.noimage`. If you want to exclude it from music genre recognition, add a file `path/to/your/folder/.nomusic`. If you want to exclude it from video recognition, add a file `path/to/your/folder/.novideo`. If you want to exclude it from all recognition, add a file `path/to/your/folder/.nomedia`.
diff --git a/appinfo/info.xml b/appinfo/info.xml
index e7e201b50..d1ef3826f 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -96,6 +96,7 @@ The app does not send any sensitive data to cloud providers or similar services.
OCA\Recognize\BackgroundJobs\MaintenanceJob
+ OCA\Recognize\BackgroundJobs\ScheduleLinkContactsJob
@@ -121,6 +122,7 @@ The app does not send any sensitive data to cloud providers or similar services.
OCA\Recognize\Command\DownloadModels
OCA\Recognize\Command\Classify
OCA\Recognize\Command\ClusterFaces
+ OCA\Recognize\Command\LinkContacts
diff --git a/appinfo/routes.php b/appinfo/routes.php
index 7e0983022..448e8ad4d 100644
--- a/appinfo/routes.php
+++ b/appinfo/routes.php
@@ -42,4 +42,15 @@
['name' => 'admin#get_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'GET'],
['name' => 'admin#set_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'PUT'],
],
+ 'ocs' => [
+ // Links between people (face clusters) and contacts, used by the Photos and Contacts apps
+ ['name' => 'contact_link#searchContacts', 'url' => '/api/v1/contacts', 'verb' => 'GET'],
+ ['name' => 'contact_link#getContact', 'url' => '/api/v1/contacts/{addressBook}/{uid}', 'verb' => 'GET'],
+ ['name' => 'contact_link#listFaces', 'url' => '/api/v1/faces', 'verb' => 'GET'],
+ ['name' => 'contact_link#listDetections', 'url' => '/api/v1/faces/{clusterId}/detections', 'verb' => 'GET', 'requirements' => ['clusterId' => '\d+']],
+ ['name' => 'contact_link#linkFace', 'url' => '/api/v1/faces/{clusterId}/contact', 'verb' => 'PUT', 'requirements' => ['clusterId' => '\d+']],
+ ['name' => 'contact_link#unlinkFace', 'url' => '/api/v1/faces/{clusterId}/contact', 'verb' => 'DELETE', 'requirements' => ['clusterId' => '\d+']],
+ ['name' => 'contact_link#getDetectionPreview', 'url' => '/api/v1/faces/detections/{detectionId}/preview', 'verb' => 'GET', 'requirements' => ['detectionId' => '\d+']],
+ ['name' => 'contact_link#setContactPhoto', 'url' => '/api/v1/faces/detections/{detectionId}/contact-photo', 'verb' => 'POST', 'requirements' => ['detectionId' => '\d+']],
+ ],
];
diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php
index e4be8f757..c7aab4c0d 100644
--- a/lib/AppInfo/Application.php
+++ b/lib/AppInfo/Application.php
@@ -9,6 +9,7 @@
use OCA\DAV\Connector\Sabre\Principal;
use OCA\DAV\Events\SabrePluginAddEvent;
+use OCA\Recognize\Capabilities;
use OCA\Recognize\Dav\Faces\PropFindPlugin;
use OCA\Recognize\Hooks\FileListener;
use OCA\Recognize\TaskProcessing\AudioClassificationTaskType;
@@ -64,6 +65,8 @@ public function register(IRegistrationContext $context): void {
/** Register $principalBackend for the DAV collection */
$context->registerServiceAlias('principalBackend', Principal::class);
+ $context->registerCapability(Capabilities::class);
+
$context->registerTaskProcessingTaskType(ImageClassificationTaskType::class);
$context->registerTaskProcessingTaskType(VideoClassificationTaskType::class);
$context->registerTaskProcessingTaskType(AudioClassificationTaskType::class);
diff --git a/lib/BackgroundJobs/ClusterFacesJob.php b/lib/BackgroundJobs/ClusterFacesJob.php
index 81908657d..058cd9852 100644
--- a/lib/BackgroundJobs/ClusterFacesJob.php
+++ b/lib/BackgroundJobs/ClusterFacesJob.php
@@ -7,6 +7,7 @@
declare(strict_types=1);
namespace OCA\Recognize\BackgroundJobs;
+use OCA\Recognize\Service\ContactLinkService;
use OCA\Recognize\Service\FaceClusterAnalyzer;
use OCA\Recognize\Service\Logger;
use OCA\Recognize\Service\SettingsService;
@@ -20,13 +21,15 @@ final class ClusterFacesJob extends QueuedJob {
private IJobList $jobList;
private LoggerInterface $logger;
private SettingsService $settingsService;
+ private ContactLinkService $contactLinkService;
- public function __construct(ITimeFactory $time, Logger $logger, IJobList $jobList, FaceClusterAnalyzer $clusterAnalyzer, SettingsService $settingsService) {
+ public function __construct(ITimeFactory $time, Logger $logger, IJobList $jobList, FaceClusterAnalyzer $clusterAnalyzer, SettingsService $settingsService, ContactLinkService $contactLinkService) {
parent::__construct($time);
$this->logger = $logger;
$this->jobList = $jobList;
$this->clusterAnalyzer = $clusterAnalyzer;
$this->settingsService = $settingsService;
+ $this->contactLinkService = $contactLinkService;
}
/**
@@ -47,6 +50,9 @@ protected function run($argument): void {
}
}
$this->clusterAnalyzer->calculateClusters($userId, $batchSize);
+ if ($this->contactLinkService->isAutoLinkingAvailable()) {
+ $this->jobList->add(LinkContactsJob::class, ['userId' => $userId]);
+ }
} catch (\Throwable $e) {
$this->settingsService->setSetting('clusterFaces.status', 'false');
$this->logger->error('Failed to calculate face clusters', ['exception' => $e]);
diff --git a/lib/BackgroundJobs/LinkContactsJob.php b/lib/BackgroundJobs/LinkContactsJob.php
new file mode 100644
index 000000000..f33633487
--- /dev/null
+++ b/lib/BackgroundJobs/LinkContactsJob.php
@@ -0,0 +1,46 @@
+logger = $logger;
+ $this->contactLinkService = $contactLinkService;
+ }
+
+ /**
+ * @param array{userId: string} $argument
+ */
+ protected function run($argument): void {
+ $userId = $argument['userId'];
+ if (!$this->contactLinkService->isAutoLinkingAvailable()) {
+ return;
+ }
+ try {
+ $this->contactLinkService->linkContactsForUser($userId);
+ } catch (\Throwable $e) {
+ $this->logger->error('Failed to link face clusters of user ' . $userId . ' to contacts', ['exception' => $e]);
+ }
+ }
+}
diff --git a/lib/BackgroundJobs/ScheduleLinkContactsJob.php b/lib/BackgroundJobs/ScheduleLinkContactsJob.php
new file mode 100644
index 000000000..6984316e9
--- /dev/null
+++ b/lib/BackgroundJobs/ScheduleLinkContactsJob.php
@@ -0,0 +1,57 @@
+logger = $logger;
+ $this->jobList = $jobList;
+ $this->clusterMapper = $clusterMapper;
+ $this->contactLinkService = $contactLinkService;
+ $this->setInterval(self::INTERVAL);
+ $this->setTimeSensitivity(self::TIME_INSENSITIVE);
+ }
+
+ /**
+ * @param array $argument
+ */
+ protected function run($argument): void {
+ if (!$this->contactLinkService->isAutoLinkingAvailable()) {
+ return;
+ }
+ try {
+ $userIds = $this->clusterMapper->findUserIds();
+ } catch (\OCP\DB\Exception $e) {
+ $this->logger->error('Failed to list users with face clusters', ['exception' => $e]);
+ return;
+ }
+ foreach ($userIds as $userId) {
+ $this->jobList->add(LinkContactsJob::class, ['userId' => $userId]);
+ }
+ }
+}
diff --git a/lib/Capabilities.php b/lib/Capabilities.php
new file mode 100644
index 000000000..35072c55a
--- /dev/null
+++ b/lib/Capabilities.php
@@ -0,0 +1,30 @@
+ [
+ 'faces' => [
+ // People can be linked to contacts, and faces can be used as contact photos
+ 'contactLinking' => true,
+ ],
+ ],
+ ];
+ }
+}
diff --git a/lib/Classifiers/Classifier.php b/lib/Classifiers/Classifier.php
index a1c88994c..c1b00f994 100644
--- a/lib/Classifiers/Classifier.php
+++ b/lib/Classifiers/Classifier.php
@@ -158,44 +158,10 @@ public function classifyFiles(string $model, array $queueFiles, int $timeout): \
$this->logger->debug('Classifying '.var_export($paths, true));
- $command = [
- $this->config->getAppValueString('node_binary', lazy: true),
- dirname(__DIR__, 2) . '/src/classifier_'.$model.'.js',
- '-'
- ];
-
- if (trim($this->config->getAppValueString('nice_binary', '', lazy: true)) !== '') {
- $command = [
- $this->config->getAppValueString('nice_binary', lazy: true),
- "-" . $this->config->getAppValueString('nice_value', '0', lazy: true),
- ...$command,
- ];
- }
-
- $this->logger->debug('Running '.var_export($command, true));
-
- $proc = new Process($command, __DIR__);
- $env = [];
- if ($this->config->getAppValueString('tensorflow.gpu', 'false') === 'true') {
- $env['RECOGNIZE_GPU'] = 'true';
- }
- if ($this->config->getAppValueString('tensorflow.purejs', 'false', lazy: true) === 'true') {
- $env['RECOGNIZE_PUREJS'] = 'true';
- }
- // Set cores
- $cores = $this->config->getAppValueString('tensorflow.cores', '0');
- if ($cores !== '0') {
- $env['RECOGNIZE_CORES'] = $cores;
- }
- $proc->setEnv($env);
- $proc->setTimeout(count($paths) * $timeout);
- $proc->setInput(implode("\n", $paths));
+ $proc = $this->createClassifierProcess($model, $paths, $timeout);
try {
$proc->start();
-
- if ((int)$cores !== 0) {
- @exec('taskset -cp ' . implode(',', range(0, (int)$cores, 1)) . ' ' . ((string)$proc->getPid()));
- }
+ $this->pinToConfiguredCores($proc);
$i = 0;
$errOut = '';
@@ -261,6 +227,102 @@ public function classifyFiles(string $model, array $queueFiles, int $timeout): \
}
}
+ /**
+ * Run the classifier of a model on local image files that are not part of the
+ * queue, such as images that are not stored in Nextcloud at all.
+ *
+ * @param string $model
+ * @param list $paths Absolute paths of the images
+ * @param int $timeout Seconds granted per image
+ * @return list The decoded result for each path, in the order of the paths
+ * @throws \ErrorException|\RuntimeException
+ */
+ public function classifyLocalFiles(string $model, array $paths, int $timeout): array {
+ if (count($paths) === 0) {
+ return [];
+ }
+ $this->logger->debug('Classifying local files '.var_export($paths, true));
+ $proc = $this->createClassifierProcess($model, $paths, $timeout);
+ try {
+ $proc->start();
+ $this->pinToConfiguredCores($proc);
+ $proc->wait();
+ } catch (ProcessTimedOutException $e) {
+ $this->logger->warning($proc->getErrorOutput());
+ throw new \RuntimeException('Classifier process timeout');
+ } catch (RuntimeException $e) {
+ $this->logger->warning($proc->getErrorOutput());
+ throw new \ErrorException('Classifier process could not be started');
+ }
+ $this->logger->debug('Classifier process output: ' . $proc->getErrorOutput());
+
+ $results = [];
+ foreach (explode("\n", $proc->getOutput()) as $line) {
+ if (trim($line) === '') {
+ continue;
+ }
+ try {
+ /** @var mixed $decoded */
+ $decoded = json_decode($line, true, 512, JSON_OBJECT_AS_ARRAY | JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_IGNORE);
+ /** @psalm-suppress MixedAssignment */
+ $results[] = $decoded;
+ } catch (\JsonException $e) {
+ $this->logger->warning('Classifier process emitted invalid JSON: ' . $line, ['exception' => $e]);
+ }
+ }
+ if (count($results) !== count($paths)) {
+ $this->logger->warning('Classifier process output: '.$proc->getErrorOutput());
+ throw new \ErrorException('Classifier process error');
+ }
+ return $results;
+ }
+
+ /**
+ * @param list $paths
+ */
+ private function createClassifierProcess(string $model, array $paths, int $timeout): Process {
+ $command = [
+ $this->config->getAppValueString('node_binary', lazy: true),
+ dirname(__DIR__, 2) . '/src/classifier_'.$model.'.js',
+ '-'
+ ];
+
+ if (trim($this->config->getAppValueString('nice_binary', '', lazy: true)) !== '') {
+ $command = [
+ $this->config->getAppValueString('nice_binary', lazy: true),
+ "-" . $this->config->getAppValueString('nice_value', '0', lazy: true),
+ ...$command,
+ ];
+ }
+
+ $this->logger->debug('Running '.var_export($command, true));
+
+ $proc = new Process($command, __DIR__);
+ $env = [];
+ if ($this->config->getAppValueString('tensorflow.gpu', 'false') === 'true') {
+ $env['RECOGNIZE_GPU'] = 'true';
+ }
+ if ($this->config->getAppValueString('tensorflow.purejs', 'false', lazy: true) === 'true') {
+ $env['RECOGNIZE_PUREJS'] = 'true';
+ }
+ // Set cores
+ $cores = $this->config->getAppValueString('tensorflow.cores', '0');
+ if ($cores !== '0') {
+ $env['RECOGNIZE_CORES'] = $cores;
+ }
+ $proc->setEnv($env);
+ $proc->setTimeout(count($paths) * $timeout);
+ $proc->setInput(implode("\n", $paths));
+ return $proc;
+ }
+
+ private function pinToConfiguredCores(Process $proc): void {
+ $cores = (int)$this->config->getAppValueString('tensorflow.cores', '0');
+ if ($cores !== 0) {
+ @exec('taskset -cp ' . implode(',', range(0, $cores, 1)) . ' ' . ((string)$proc->getPid()));
+ }
+ }
+
/**
* Get path of file to process.
* If the file is an image and not JPEG, it will be converted using ImageMagick.
diff --git a/lib/Command/LinkContacts.php b/lib/Command/LinkContacts.php
new file mode 100644
index 000000000..54b4ac911
--- /dev/null
+++ b/lib/Command/LinkContacts.php
@@ -0,0 +1,81 @@
+setName('recognize:link-contacts')
+ ->setDescription('Link face clusters to contacts by matching the photos of the contacts against the clusters')
+ ->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Only link the contacts of this user');
+ }
+
+ /**
+ * Execute the command
+ *
+ * @param InputInterface $input
+ * @param OutputInterface $output
+ *
+ * @return int
+ */
+ protected function execute(InputInterface $input, OutputInterface $output): int {
+ $this->logger->setCliOutput($output);
+
+ if (!$this->analyzer->isAvailable()) {
+ $output->writeln('Contact photos cannot be analyzed: ' . $this->analyzer->getUnavailableReason() . '');
+ return 1;
+ }
+
+ /** @var string|null $user */
+ $user = $input->getOption('user');
+ if (is_string($user) && $user !== '') {
+ $userIds = [$user];
+ } else {
+ try {
+ $userIds = $this->clusterMapper->findUserIds();
+ } catch (Exception $e) {
+ $this->logger->error($e->getMessage(), ['exception' => $e]);
+ return 1;
+ }
+ }
+
+ foreach ($userIds as $userId) {
+ $this->logger->info('Linking face clusters of user ' . $userId . ' to contacts');
+ try {
+ $report = $this->contactLinkService->linkContactsForUser($userId);
+ $output->writeln($userId . ': ' . $report['linked'] . ' linked, ' . $report['analyzed'] . ' contact photos analyzed, ' . $report['ambiguous'] . ' ambiguous');
+ } catch (\Throwable $e) {
+ $this->logger->error($e->getMessage(), ['exception' => $e]);
+ }
+ }
+ return 0;
+ }
+}
diff --git a/lib/Command/ResetFaces.php b/lib/Command/ResetFaces.php
index 188c540e4..0f5ce8be7 100644
--- a/lib/Command/ResetFaces.php
+++ b/lib/Command/ResetFaces.php
@@ -7,6 +7,7 @@
declare(strict_types=1);
namespace OCA\Recognize\Command;
+use OCA\Recognize\Db\ContactFaceMapper;
use OCA\Recognize\Db\FaceClusterMapper;
use OCA\Recognize\Db\FaceDetectionMapper;
use Symfony\Component\Console\Command\Command;
@@ -16,11 +17,13 @@
final class ResetFaces extends Command {
private FaceDetectionMapper $faceDetectionMapper;
private FaceClusterMapper $clusterMapper;
+ private ContactFaceMapper $contactFaceMapper;
- public function __construct(FaceDetectionMapper $faceDetectionMapper, FaceClusterMapper $clusterMapper) {
+ public function __construct(FaceDetectionMapper $faceDetectionMapper, FaceClusterMapper $clusterMapper, ContactFaceMapper $contactFaceMapper) {
parent::__construct();
$this->faceDetectionMapper = $faceDetectionMapper;
$this->clusterMapper = $clusterMapper;
+ $this->contactFaceMapper = $contactFaceMapper;
}
/**
@@ -45,6 +48,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
try {
$this->clusterMapper->deleteAll();
$this->faceDetectionMapper->deleteAll();
+ $this->contactFaceMapper->deleteAll();
} catch (\Exception $ex) {
$output->writeln('Failed to reset faces');
$output->writeln($ex->getMessage());
diff --git a/lib/Contacts/ContactCard.php b/lib/Contacts/ContactCard.php
new file mode 100644
index 000000000..0f211860a
--- /dev/null
+++ b/lib/Contacts/ContactCard.php
@@ -0,0 +1,163 @@
+uid = $uid !== '' ? $uid : null;
+ }
+
+ public function getAddressBookId(): int {
+ return $this->addressBookId;
+ }
+
+ public function getAddressBookUri(): string {
+ return $this->addressBookUri;
+ }
+
+ public function getAddressBookName(): string {
+ return $this->addressBookName;
+ }
+
+ public function isAddressBookReadOnly(): bool {
+ return $this->addressBookReadOnly;
+ }
+
+ public function getUri(): string {
+ return $this->uri;
+ }
+
+ public function getUid(): string {
+ if ($this->uid === null) {
+ $vCard = $this->getVCard();
+ $this->uid = isset($vCard->UID) ? (string)$vCard->UID : '';
+ }
+ return $this->uid;
+ }
+
+ public function getVCard(): VCard {
+ if ($this->vCard === null) {
+ /** @var VCard $vCard */
+ $vCard = Reader::read($this->cardData);
+ $this->vCard = $vCard;
+ }
+ return $this->vCard;
+ }
+
+ public function getDisplayName(): string {
+ $vCard = $this->getVCard();
+ $fn = isset($vCard->FN) ? trim((string)$vCard->FN) : '';
+ if ($fn !== '') {
+ return $fn;
+ }
+ if (isset($vCard->N)) {
+ // N holds family name, given name, additional names, prefix and suffix
+ $parts = array_map(static fn ($part): string => trim((string)$part), $vCard->N->getParts());
+ $ordered = array_filter([$parts[3] ?? '', $parts[1] ?? '', $parts[2] ?? '', $parts[0] ?? '', $parts[4] ?? '']);
+ if (count($ordered) > 0) {
+ return implode(' ', $ordered);
+ }
+ }
+ return $this->getUid();
+ }
+
+ public function hasPhoto(): bool {
+ return $this->getPhoto() !== null;
+ }
+
+ /**
+ * The embedded photo of the contact. Photos referenced by URL are not
+ * fetched, as the server never downloads remote content on behalf of a user.
+ *
+ * @return array{mimeType: string, data: string}|null
+ */
+ public function getPhoto(): ?array {
+ $vCard = $this->getVCard();
+ if (!isset($vCard->PHOTO)) {
+ return null;
+ }
+ $photo = $vCard->PHOTO;
+ $value = $photo->getValue();
+ if ($value === '') {
+ return null;
+ }
+
+ if ($photo->getValueType() === 'URI') {
+ if (!str_starts_with($value, 'data:')) {
+ return null;
+ }
+ $comma = strpos($value, ',');
+ if ($comma === false) {
+ return null;
+ }
+ $meta = substr($value, strlen('data:'), $comma - strlen('data:'));
+ $payload = substr($value, $comma + 1);
+ $data = str_contains($meta, ';base64') ? base64_decode($payload, true) : rawurldecode($payload);
+ if ($data === false || $data === '') {
+ return null;
+ }
+ $mimeType = strtolower(explode(';', $meta)[0]);
+ return ['mimeType' => $mimeType !== '' ? $mimeType : 'application/octet-stream', 'data' => $data];
+ }
+
+ $type = '';
+ foreach (['TYPE', 'MEDIATYPE'] as $parameterName) {
+ if (!isset($photo[$parameterName])) {
+ continue;
+ }
+ $parameter = $photo[$parameterName];
+ if ($parameter instanceof Parameter) {
+ $type = strtolower((string)$parameter->getValue());
+ break;
+ }
+ }
+ if ($type !== '' && !str_starts_with($type, 'image/')) {
+ $type = 'image/' . $type;
+ }
+ return ['mimeType' => $type !== '' ? $type : 'application/octet-stream', 'data' => $value];
+ }
+
+ /**
+ * @return array{uid: string, addressBook: string, addressBookName: string, displayName: string, hasPhoto: bool, readOnly: bool}
+ */
+ public function toArray(): array {
+ return [
+ 'uid' => $this->getUid(),
+ 'addressBook' => $this->addressBookUri,
+ 'addressBookName' => $this->addressBookName,
+ 'displayName' => $this->getDisplayName(),
+ 'hasPhoto' => $this->hasPhoto(),
+ 'readOnly' => $this->addressBookReadOnly,
+ ];
+ }
+}
diff --git a/lib/Controller/AdminController.php b/lib/Controller/AdminController.php
index 0b1a149e8..781ba2ef3 100644
--- a/lib/Controller/AdminController.php
+++ b/lib/Controller/AdminController.php
@@ -17,6 +17,7 @@
use OCA\Recognize\BackgroundJobs\ClusterFacesJob;
use OCA\Recognize\BackgroundJobs\SchedulerJob;
use OCA\Recognize\BackgroundJobs\StorageCrawlJob;
+use OCA\Recognize\Db\ContactFaceMapper;
use OCA\Recognize\Db\FaceClusterMapper;
use OCA\Recognize\Db\FaceDetectionMapper;
use OCA\Recognize\Service\QueueService;
@@ -43,9 +44,11 @@ final class AdminController extends Controller {
private IAppConfig $config;
private FaceDetectionMapper $faceDetections;
private IBinaryFinder $binaryFinder;
+ private ContactFaceMapper $contactFaceMapper;
- public function __construct(string $appName, IRequest $request, TagManager $tagManager, IJobList $jobList, SettingsService $settingsService, QueueService $queue, FaceClusterMapper $clusterMapper, FaceDetectionMapper $detectionMapper, IAppConfig $config, FaceDetectionMapper $faceDetections, IBinaryFinder $binaryFinder) {
+ public function __construct(string $appName, IRequest $request, TagManager $tagManager, IJobList $jobList, SettingsService $settingsService, QueueService $queue, FaceClusterMapper $clusterMapper, FaceDetectionMapper $detectionMapper, IAppConfig $config, FaceDetectionMapper $faceDetections, IBinaryFinder $binaryFinder, ContactFaceMapper $contactFaceMapper) {
parent::__construct($appName, $request);
+ $this->contactFaceMapper = $contactFaceMapper;
$this->tagManager = $tagManager;
$this->jobList = $jobList;
$this->settingsService = $settingsService;
@@ -89,6 +92,7 @@ public function resetFaces(): JSONResponse {
try {
$this->clusterMapper->deleteAll();
$this->detectionMapper->deleteAll();
+ $this->contactFaceMapper->deleteAll();
} catch (Exception $e) {
return new JSONResponse([], Http::STATUS_INTERNAL_SERVER_ERROR);
}
diff --git a/lib/Controller/ContactLinkController.php b/lib/Controller/ContactLinkController.php
new file mode 100644
index 000000000..e7048c424
--- /dev/null
+++ b/lib/Controller/ContactLinkController.php
@@ -0,0 +1,325 @@
+, array{}>
+ */
+ #[NoAdminRequired]
+ public function searchContacts(string $search = '', int $limit = 50): DataResponse {
+ $userId = $this->requireUser();
+ $limit = max(1, min($limit, self::MAX_SEARCH_RESULTS));
+ $contacts = $this->contactsService->searchContacts($userId, trim($search), $limit);
+ return new DataResponse(array_map(fn (ContactCard $contact): array => $this->contactsService->serializeContact($userId, $contact), $contacts));
+ }
+
+ /**
+ * Get a contact of the current user along with the face clusters linked to it.
+ *
+ * @param string $addressBook The URI of the address book
+ * @param string $uid The UID of the contact
+ * @return DataResponse}, array{}>
+ * @throws OCSNotFoundException The contact does not exist
+ */
+ #[NoAdminRequired]
+ public function getContact(string $addressBook, string $uid): DataResponse {
+ $userId = $this->requireUser();
+ $contact = $this->requireContact($userId, $addressBook, $uid);
+ $clusters = $this->clusterMapper->findByContact($userId, $contact->getAddressBookUri(), $contact->getUid());
+ return new DataResponse([
+ 'contact' => $this->contactsService->serializeContact($userId, $contact),
+ 'faces' => array_map(fn (FaceCluster $cluster): array => $this->serializeCluster($cluster), $clusters),
+ ]);
+ }
+
+ /**
+ * List the face clusters of the current user with the contact they are linked to.
+ *
+ * @return DataResponse, array{}>
+ */
+ #[NoAdminRequired]
+ public function listFaces(): DataResponse {
+ $userId = $this->requireUser();
+ $clusters = $this->clusterMapper->findByUserId($userId);
+ return new DataResponse(array_map(fn (FaceCluster $cluster): array => $this->serializeCluster($cluster), $clusters));
+ }
+
+ /**
+ * List the faces that make up a cluster, the largest ones first.
+ *
+ * @param int $clusterId The id of the cluster
+ * @param int $limit Maximum number of faces
+ * @return DataResponse, array{}>
+ * @throws OCSNotFoundException The cluster does not exist
+ */
+ #[NoAdminRequired]
+ public function listDetections(int $clusterId, int $limit = 100): DataResponse {
+ $userId = $this->requireUser();
+ $cluster = $this->requireCluster($userId, $clusterId);
+ $limit = max(1, min($limit, self::MAX_DETECTIONS));
+ $detections = $this->detectionMapper->findByClusterId($cluster->getId());
+ usort($detections, static fn (FaceDetection $a, FaceDetection $b): int => ($b->getWidth() * $b->getHeight()) <=> ($a->getWidth() * $a->getHeight()));
+ $detections = array_slice($detections, 0, $limit);
+ return new DataResponse(array_map(static fn (FaceDetection $detection): array => self::serializeDetection($detection), $detections));
+ }
+
+ /**
+ * Link a face cluster to a contact.
+ *
+ * @param int $clusterId The id of the cluster
+ * @param string $addressBook The URI of the address book of the contact
+ * @param string $uid The UID of the contact
+ * @return DataResponse
+ * @throws OCSNotFoundException The cluster or the contact does not exist
+ */
+ #[NoAdminRequired]
+ public function linkFace(int $clusterId, string $addressBook, string $uid): DataResponse {
+ $userId = $this->requireUser();
+ $cluster = $this->requireCluster($userId, $clusterId);
+ $contact = $this->requireContact($userId, $addressBook, $uid);
+ try {
+ $cluster = $this->contactLinkService->linkCluster($cluster, $contact, FaceCluster::LINK_SOURCE_MANUAL);
+ } catch (\OCP\DB\Exception $e) {
+ $this->logger->error('Could not link face cluster ' . $clusterId . ' to a contact', ['exception' => $e]);
+ throw new OCSException('Could not store the link', Http::STATUS_INTERNAL_SERVER_ERROR);
+ }
+ return new DataResponse($this->serializeCluster($cluster));
+ }
+
+ /**
+ * Remove the link between a face cluster and its contact. The cluster will not be linked automatically again.
+ *
+ * @param int $clusterId The id of the cluster
+ * @return DataResponse
+ * @throws OCSNotFoundException The cluster does not exist
+ */
+ #[NoAdminRequired]
+ public function unlinkFace(int $clusterId): DataResponse {
+ $userId = $this->requireUser();
+ $cluster = $this->requireCluster($userId, $clusterId);
+ try {
+ $cluster = $this->contactLinkService->unlinkCluster($cluster);
+ } catch (\OCP\DB\Exception $e) {
+ $this->logger->error('Could not unlink face cluster ' . $clusterId, ['exception' => $e]);
+ throw new OCSException('Could not remove the link', Http::STATUS_INTERNAL_SERVER_ERROR);
+ }
+ return new DataResponse($this->serializeCluster($cluster));
+ }
+
+ /**
+ * Get the face of a detection cut out of its photo.
+ *
+ * @param int $detectionId The id of the face detection
+ * @param int $size Width and height of the image in pixels
+ * @return DataDisplayResponse
+ * @throws OCSNotFoundException The detection or its photo does not exist
+ */
+ #[NoAdminRequired]
+ #[NoCSRFRequired]
+ public function getDetectionPreview(int $detectionId, int $size = 256): DataDisplayResponse {
+ $userId = $this->requireUser();
+ $detection = $this->requireDetection($userId, $detectionId);
+ $size = max(self::MIN_PREVIEW_SIZE, min($size, self::MAX_PREVIEW_SIZE));
+ try {
+ $face = $this->contactLinkService->cropFace($userId, $detection, $size);
+ } catch (NotFoundException $e) {
+ throw new OCSNotFoundException('Photo not found');
+ } catch (Exception $e) {
+ $this->logger->warning('Could not crop face ' . $detectionId, ['exception' => $e]);
+ throw new OCSException('Could not crop the face', Http::STATUS_INTERNAL_SERVER_ERROR);
+ }
+ $response = new DataDisplayResponse($face['data'], Http::STATUS_OK, ['Content-Type' => $face['mimeType']]);
+ $response->cacheFor(60 * 60 * 24, false, true);
+ return $response;
+ }
+
+ /**
+ * Use the face of a detection as the photo of a contact.
+ *
+ * Without a contact, the photo goes to the contact the cluster of the detection is linked to.
+ *
+ * @param int $detectionId The id of the face detection
+ * @param string|null $addressBook The URI of the address book of the contact
+ * @param string|null $uid The UID of the contact
+ * @return DataResponse
+ * @throws OCSNotFoundException The detection, its photo or the contact does not exist
+ * @throws OCSBadRequestException No contact was given and the cluster is not linked to one
+ * @throws OCSForbiddenException The address book of the contact is read-only
+ */
+ #[NoAdminRequired]
+ public function setContactPhoto(int $detectionId, ?string $addressBook = null, ?string $uid = null): DataResponse {
+ $userId = $this->requireUser();
+ $detection = $this->requireDetection($userId, $detectionId);
+
+ if (($addressBook === null) !== ($uid === null)) {
+ throw new OCSBadRequestException('Both the address book and the UID of the contact are required');
+ }
+ if ($addressBook !== null && $uid !== null) {
+ $contact = $this->requireContact($userId, $addressBook, $uid);
+ } else {
+ $clusterId = $detection->getClusterId();
+ $cluster = $clusterId !== null && $clusterId > 0 ? $this->requireCluster($userId, $clusterId) : null;
+ $contact = $cluster !== null ? $this->contactLinkService->getLinkedContact($cluster) : null;
+ if ($contact === null) {
+ throw new OCSBadRequestException('This person is not linked to a contact');
+ }
+ }
+ if ($contact->isAddressBookReadOnly()) {
+ throw new OCSForbiddenException('The address book of this contact is read-only');
+ }
+
+ try {
+ $this->contactLinkService->setContactPhotoFromDetection($userId, $detection, $contact);
+ } catch (NotFoundException $e) {
+ throw new OCSNotFoundException('Photo not found');
+ } catch (Exception $e) {
+ $this->logger->warning('Could not set the photo of a contact from face ' . $detectionId, ['exception' => $e]);
+ throw new OCSException('Could not set the contact photo', Http::STATUS_INTERNAL_SERVER_ERROR);
+ }
+
+ $updated = $this->contactsService->findContact($userId, $contact->getAddressBookUri(), $contact->getUid()) ?? $contact;
+ return new DataResponse($this->contactsService->serializeContact($userId, $updated));
+ }
+
+ /**
+ * @throws OCSForbiddenException
+ */
+ private function requireUser(): string {
+ if ($this->userId === null) {
+ throw new OCSForbiddenException('Not logged in');
+ }
+ return $this->userId;
+ }
+
+ /**
+ * @throws OCSNotFoundException
+ */
+ private function requireCluster(string $userId, int $clusterId): FaceCluster {
+ try {
+ $cluster = $this->clusterMapper->find($clusterId);
+ } catch (DoesNotExistException|MultipleObjectsReturnedException|\OCP\DB\Exception $e) {
+ throw new OCSNotFoundException('Person not found');
+ }
+ if ($cluster->getUserId() !== $userId) {
+ throw new OCSNotFoundException('Person not found');
+ }
+ return $cluster;
+ }
+
+ /**
+ * @throws OCSNotFoundException
+ */
+ private function requireDetection(string $userId, int $detectionId): FaceDetection {
+ try {
+ $detection = $this->detectionMapper->find($detectionId);
+ } catch (DoesNotExistException|MultipleObjectsReturnedException|\OCP\DB\Exception $e) {
+ throw new OCSNotFoundException('Face not found');
+ }
+ if ($detection->getUserId() !== $userId) {
+ throw new OCSNotFoundException('Face not found');
+ }
+ return $detection;
+ }
+
+ /**
+ * @throws OCSNotFoundException
+ */
+ private function requireContact(string $userId, string $addressBook, string $uid): ContactCard {
+ $contact = $this->contactsService->findContact($userId, $addressBook, $uid);
+ if ($contact === null) {
+ throw new OCSNotFoundException('Contact not found');
+ }
+ return $contact;
+ }
+
+ /**
+ * @return array{id: int, title: string, name: string, linkSource: string|null, contact: array|null, preview: array{fileId: int, detection: array}|null}
+ */
+ private function serializeCluster(FaceCluster $cluster): array {
+ $contact = $this->contactLinkService->getLinkedContact($cluster);
+ try {
+ $previewDetection = $this->detectionMapper->findDetectionForPreviewImageByClusterId($cluster->getId());
+ $preview = ['fileId' => $previewDetection->getFileId(), 'detection' => self::serializeDetection($previewDetection)];
+ } catch (DoesNotExistException|MultipleObjectsReturnedException|\OCP\DB\Exception $e) {
+ $preview = null;
+ }
+ return [
+ 'id' => $cluster->getId(),
+ 'title' => $cluster->getTitle(),
+ // The name of the person on the recognize WebDAV endpoint, which the Photos app uses to address it
+ 'name' => $cluster->getTitle() !== '' ? $cluster->getTitle() : (string)$cluster->getId(),
+ 'linkSource' => $cluster->getContactLinkSource(),
+ 'contact' => $contact !== null ? $this->contactsService->serializeContact($cluster->getUserId(), $contact) : null,
+ 'preview' => $preview,
+ ];
+ }
+
+ /**
+ * @return array{id: int, fileId: int, x: float, y: float, width: float, height: float}
+ */
+ private static function serializeDetection(FaceDetection $detection): array {
+ return [
+ 'id' => $detection->getId(),
+ 'fileId' => $detection->getFileId(),
+ 'x' => $detection->getX(),
+ 'y' => $detection->getY(),
+ 'width' => $detection->getWidth(),
+ 'height' => $detection->getHeight(),
+ ];
+ }
+}
diff --git a/lib/Dav/Faces/FaceRoot.php b/lib/Dav/Faces/FaceRoot.php
index 97b0d8625..73391fd9a 100644
--- a/lib/Dav/Faces/FaceRoot.php
+++ b/lib/Dav/Faces/FaceRoot.php
@@ -50,6 +50,10 @@ public function __construct(FaceClusterMapper $clusterMapper, FaceCluster $clust
/**
* @inheritDoc
*/
+ public function getCluster(): FaceCluster {
+ return $this->cluster;
+ }
+
public function getName(): string {
return $this->cluster->getTitle() !== '' ? $this->cluster->getTitle() : ''.$this->cluster->getId();
}
diff --git a/lib/Dav/Faces/PropFindPlugin.php b/lib/Dav/Faces/PropFindPlugin.php
index 41721e2a1..7335f9b8f 100644
--- a/lib/Dav/Faces/PropFindPlugin.php
+++ b/lib/Dav/Faces/PropFindPlugin.php
@@ -13,6 +13,8 @@
use OCA\Recognize\Db\FaceClusterMapper;
use OCA\Recognize\Db\FaceDetectionMapper;
use OCA\Recognize\Db\FaceDetectionWithTitle;
+use OCA\Recognize\Service\ContactLinkService;
+use OCA\Recognize\Service\ContactsService;
use OCP\AppFramework\Db\DoesNotExistException;
use OCP\AppFramework\Db\MultipleObjectsReturnedException;
use OCP\AppFramework\Services\IAppConfig;
@@ -36,6 +38,8 @@ final class PropFindPlugin extends ServerPlugin {
public const REALPATH_PROPERTYNAME = '{http://nextcloud.org/ns}realpath';
public const NBITEMS_PROPERTYNAME = '{http://nextcloud.org/ns}nbItems';
public const FACE_PREVIEW_IMAGE_PROPERTYNAME = '{http://nextcloud.org/ns}face-preview-image';
+ public const FACE_CLUSTER_ID_PROPERTYNAME = '{http://nextcloud.org/ns}face-cluster-id';
+ public const FACE_CONTACT_PROPERTYNAME = '{http://nextcloud.org/ns}face-contact';
public const API_KEY_TIMEOUT = 60 * 60 * 24;
@@ -49,6 +53,8 @@ public function __construct(
private LoggerInterface $logger,
private ITimeFactory $timeFactory,
private IAppConfig $appConfig,
+ private ContactLinkService $contactLinkService,
+ private ContactsService $contactsService,
) {
}
@@ -95,6 +101,15 @@ public function propFind(PropFind $propFind, INode $node): void {
if ($node instanceof FaceRoot) {
$propFind->handle(self::FACE_PREVIEW_IMAGE_PROPERTYNAME, fn () => $node->getPreviewImage());
+ $propFind->handle(self::FACE_CLUSTER_ID_PROPERTYNAME, fn () => $node->getCluster()->getId());
+ $propFind->handle(self::FACE_CONTACT_PROPERTYNAME, function () use ($node): string {
+ $cluster = $node->getCluster();
+ $contact = $this->contactLinkService->getLinkedContact($cluster);
+ return json_encode([
+ 'linkSource' => $cluster->getContactLinkSource(),
+ 'contact' => $contact !== null ? $this->contactsService->serializeContact($cluster->getUserId(), $contact) : null,
+ ], \JSON_THROW_ON_ERROR | \JSON_INVALID_UTF8_SUBSTITUTE);
+ });
}
if ($node instanceof File) {
diff --git a/lib/Db/ContactFace.php b/lib/Db/ContactFace.php
new file mode 100644
index 000000000..8cb732c0a
--- /dev/null
+++ b/lib/Db/ContactFace.php
@@ -0,0 +1,66 @@
+addType('userId', 'string');
+ $this->addType('addressBook', 'string');
+ $this->addType('contactUid', 'string');
+ $this->addType('photoHash', 'string');
+ $this->addType('faceVector', 'json');
+ $this->addType('faceCount', 'integer');
+ }
+
+ /**
+ * @return list|null The embedding, or null when the photo holds no usable face
+ */
+ public function getVector(): ?array {
+ /** @var list|null $vector */
+ $vector = $this->getter('faceVector');
+ return $vector;
+ }
+
+ /**
+ * @param list|null $vector
+ */
+ public function setVector(?array $vector): void {
+ $this->setter('faceVector', [$vector]);
+ }
+}
diff --git a/lib/Db/ContactFaceMapper.php b/lib/Db/ContactFaceMapper.php
new file mode 100644
index 000000000..c4dd5b4f1
--- /dev/null
+++ b/lib/Db/ContactFaceMapper.php
@@ -0,0 +1,47 @@
+
+ */
+final class ContactFaceMapper extends QBMapper {
+ public function __construct(IDBConnection $db) {
+ parent::__construct($db, 'recognize_contact_faces', ContactFace::class);
+ }
+
+ /**
+ * @throws DoesNotExistException
+ * @throws MultipleObjectsReturnedException
+ * @throws Exception
+ */
+ public function findByContact(string $userId, string $addressBook, string $contactUid): ContactFace {
+ $qb = $this->db->getQueryBuilder();
+ $qb->select(ContactFace::$columns)
+ ->from($this->getTableName())
+ ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId)))
+ ->andWhere($qb->expr()->eq('address_book', $qb->createPositionalParameter($addressBook)))
+ ->andWhere($qb->expr()->eq('contact_uid', $qb->createPositionalParameter($contactUid)));
+ return $this->findEntity($qb);
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function deleteAll(): void {
+ $qb = $this->db->getQueryBuilder();
+ $qb->delete($this->getTableName())
+ ->executeStatement();
+ }
+}
diff --git a/lib/Db/FaceCluster.php b/lib/Db/FaceCluster.php
index 7f89afa06..5a53d886b 100644
--- a/lib/Db/FaceCluster.php
+++ b/lib/Db/FaceCluster.php
@@ -17,25 +17,47 @@
* @method setTitle(string $title)
* @method string getUserId()
* @method setUserId(string $userId)
+ * @method string|null getContactUid()
+ * @method setContactUid(string|null $contactUid)
+ * @method string|null getContactAddressBook()
+ * @method setContactAddressBook(string|null $contactAddressBook)
+ * @method string|null getContactLinkSource()
+ * @method setContactLinkSource(string|null $contactLinkSource)
*/
final class FaceCluster extends Entity {
+ /** The link was established by the scheduled matching of contact photos. */
+ public const LINK_SOURCE_AUTO = 'auto';
+ /** The link, or the absence of one, was decided by the user and is never changed automatically. */
+ public const LINK_SOURCE_MANUAL = 'manual';
+
protected $title;
protected $userId;
+ protected ?string $contactUid = null;
+ protected ?string $contactAddressBook = null;
+ protected ?string $contactLinkSource = null;
/**
* @var string[]
*/
- public static array $columns = ['id', 'title', 'user_id'];
+ public static array $columns = ['id', 'title', 'user_id', 'contact_uid', 'contact_address_book', 'contact_link_source'];
/**
* @var string[]
*/
- public static array $fields = ['id', 'title', 'userId'];
+ public static array $fields = ['id', 'title', 'userId', 'contactUid', 'contactAddressBook', 'contactLinkSource'];
public function __construct() {
// add types in constructor
$this->addType('title', 'string');
$this->addType('userId', 'string');
+ $this->addType('contactUid', 'string');
+ $this->addType('contactAddressBook', 'string');
+ $this->addType('contactLinkSource', 'string');
+ }
+
+ public function isLinked(): bool {
+ return $this->getContactUid() !== null && $this->getContactUid() !== ''
+ && $this->getContactAddressBook() !== null && $this->getContactAddressBook() !== '';
}
public function toArray(): array {
diff --git a/lib/Db/FaceClusterMapper.php b/lib/Db/FaceClusterMapper.php
index 0212a12a3..7cfabbf5b 100644
--- a/lib/Db/FaceClusterMapper.php
+++ b/lib/Db/FaceClusterMapper.php
@@ -89,6 +89,70 @@ public function findByDetectionId(int $detectionId): array {
return $this->findEntities($qb);
}
+ /**
+ * Clusters of a user that are linked to a contact.
+ *
+ * @throws \OCP\DB\Exception
+ * @return list<\OCA\Recognize\Db\FaceCluster>
+ */
+ public function findLinkedByUserId(string $userId): array {
+ $qb = $this->db->getQueryBuilder();
+ $qb->select(FaceCluster::$columns)
+ ->from('recognize_face_clusters')
+ ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId)))
+ ->andWhere($qb->expr()->isNotNull('contact_uid'));
+ return $this->findEntities($qb);
+ }
+
+ /**
+ * Clusters of a user that are linked to the given contact.
+ *
+ * @throws \OCP\DB\Exception
+ * @return list<\OCA\Recognize\Db\FaceCluster>
+ */
+ public function findByContact(string $userId, string $addressBook, string $contactUid): array {
+ $qb = $this->db->getQueryBuilder();
+ $qb->select(FaceCluster::$columns)
+ ->from('recognize_face_clusters')
+ ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId)))
+ ->andWhere($qb->expr()->eq('contact_address_book', $qb->createPositionalParameter($addressBook)))
+ ->andWhere($qb->expr()->eq('contact_uid', $qb->createPositionalParameter($contactUid)));
+ return $this->findEntities($qb);
+ }
+
+ /**
+ * Clusters of a user that may still be linked to a contact automatically: they
+ * hold at least one detection, are not linked, and the user never decided on a
+ * link for them.
+ *
+ * @throws \OCP\DB\Exception
+ * @return list<\OCA\Recognize\Db\FaceCluster>
+ */
+ public function findAutoLinkCandidatesByUserId(string $userId): array {
+ $qb = $this->db->getQueryBuilder();
+ $columns = array_map(fn ($c) => 'fc.'.$c, FaceCluster::$columns);
+ $qb->selectDistinct($columns)
+ ->from('recognize_face_clusters', 'fc')
+ ->innerJoin('fc', 'recognize_face_detections', 'fd', 'fc.id = fd.cluster_id')
+ ->where($qb->expr()->eq('fc.user_id', $qb->createPositionalParameter($userId)))
+ ->andWhere($qb->expr()->isNull('fc.contact_uid'))
+ ->andWhere($qb->expr()->isNull('fc.contact_link_source'));
+ return $this->findEntities($qb);
+ }
+
+ /**
+ * @return list
+ * @throws \OCP\DB\Exception
+ */
+ public function findUserIds(): array {
+ $qb = $this->db->getQueryBuilder();
+ $qb->selectDistinct('user_id')
+ ->from('recognize_face_clusters');
+ /** @var list */
+ $list = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ return $list;
+ }
+
public function delete(Entity $entity): Entity {
/** @var FaceCluster $entity */
$qb = $this->db->getQueryBuilder();
diff --git a/lib/Migration/Version013002000Date20261020094821.php b/lib/Migration/Version013002000Date20261020094821.php
new file mode 100644
index 000000000..6c2728102
--- /dev/null
+++ b/lib/Migration/Version013002000Date20261020094821.php
@@ -0,0 +1,90 @@
+hasTable('recognize_face_clusters')) {
+ $table = $schema->getTable('recognize_face_clusters');
+ if (!$table->hasColumn('contact_uid')) {
+ $table->addColumn('contact_uid', Types::STRING, [
+ 'notnull' => false,
+ 'length' => 255,
+ ]);
+ $table->addColumn('contact_address_book', Types::STRING, [
+ 'notnull' => false,
+ 'length' => 255,
+ ]);
+ $table->addColumn('contact_link_source', Types::STRING, [
+ 'notnull' => false,
+ 'length' => 16,
+ ]);
+ $table->addIndex(['user_id', 'contact_address_book', 'contact_uid'], 'recognize_faceclust_contact');
+ $changed = true;
+ }
+ }
+
+ if (!$schema->hasTable('recognize_contact_faces')) {
+ $table = $schema->createTable('recognize_contact_faces');
+ $table->addColumn('id', Types::BIGINT, [
+ 'autoincrement' => true,
+ 'notnull' => true,
+ 'length' => 20,
+ ]);
+ $table->addColumn('user_id', Types::STRING, [
+ 'notnull' => true,
+ 'length' => 64,
+ ]);
+ $table->addColumn('address_book', Types::STRING, [
+ 'notnull' => true,
+ 'length' => 255,
+ ]);
+ $table->addColumn('contact_uid', Types::STRING, [
+ 'notnull' => true,
+ 'length' => 255,
+ ]);
+ $table->addColumn('photo_hash', Types::STRING, [
+ 'notnull' => true,
+ 'length' => 40,
+ ]);
+ $table->addColumn('face_vector', Types::TEXT, [
+ 'notnull' => false,
+ ]);
+ $table->addColumn('face_count', Types::INTEGER, [
+ 'notnull' => true,
+ 'default' => 0,
+ ]);
+
+ $table->setPrimaryKey(['id'], 'recognize_contactface_id');
+ $table->addUniqueIndex(['user_id', 'address_book', 'contact_uid'], 'recognize_contactface_contact');
+ $changed = true;
+ }
+
+ return $changed ? $schema : null;
+ }
+}
diff --git a/lib/Service/ContactLinkService.php b/lib/Service/ContactLinkService.php
new file mode 100644
index 000000000..f0c8438aa
--- /dev/null
+++ b/lib/Service/ContactLinkService.php
@@ -0,0 +1,364 @@
+settingsService->getSetting('faces.contactLinking') === 'true' && $this->analyzer->isAvailable();
+ }
+
+ /**
+ * Link the unlinked contacts with a photo of a user to their face clusters.
+ *
+ * A contact is linked to a cluster when its photo matches that cluster and no
+ * other one, and no other contact matches that cluster. Everything else is left
+ * for the user to decide.
+ *
+ * @return array{linked: int, analyzed: int, ambiguous: int}
+ * @throws \OCP\DB\Exception
+ * @throws Exception When the contact photos cannot be analyzed
+ */
+ public function linkContactsForUser(string $userId): array {
+ $report = ['linked' => 0, 'analyzed' => 0, 'ambiguous' => 0];
+
+ if (!$this->analyzer->isAvailable()) {
+ $this->logger->debug('Not linking contacts of user ' . $userId . ': ' . $this->analyzer->getUnavailableReason());
+ return $report;
+ }
+
+ $candidates = [];
+ foreach ($this->clusterMapper->findAutoLinkCandidatesByUserId($userId) as $cluster) {
+ $candidates[$cluster->getId()] = $cluster;
+ }
+ if (count($candidates) === 0) {
+ $this->logger->debug('No face clusters of user ' . $userId . ' left to link to a contact');
+ return $report;
+ }
+
+ $dimensions = $this->analyzer->getDimensions();
+ $centroids = [];
+ foreach ($candidates as $clusterId => $cluster) {
+ $sample = $this->detectionMapper->findClusterSample($clusterId, self::CLUSTER_SAMPLE_SIZE);
+ if (count($sample) === 0 || count(reset($sample)->getVector()) !== $dimensions) {
+ continue;
+ }
+ $centroids[$clusterId] = FaceClusterAnalyzer::calculateCentroidOfDetections($sample);
+ }
+ if (count($centroids) === 0) {
+ return $report;
+ }
+
+ $linkedContacts = [];
+ foreach ($this->clusterMapper->findLinkedByUserId($userId) as $cluster) {
+ $linkedContacts[self::contactKey((string)$cluster->getContactAddressBook(), (string)$cluster->getContactUid())] = true;
+ }
+
+ /** @var array $contacts */
+ $contacts = [];
+ /** @var array> $vectors */
+ $vectors = [];
+ /** @var array $toAnalyze */
+ $toAnalyze = [];
+ foreach ($this->contactsService->getAddressBooks($userId) as $book) {
+ foreach ($this->contactsService->getContacts($userId, $book['uri']) as $contact) {
+ $key = self::contactKey($contact->getAddressBookUri(), $contact->getUid());
+ if (isset($linkedContacts[$key]) || isset($contacts[$key])) {
+ continue;
+ }
+ $photo = $contact->getPhoto();
+ if ($photo === null) {
+ continue;
+ }
+ $contacts[$key] = $contact;
+ $hash = sha1($photo['data']);
+ $cached = $this->findCachedFace($userId, $contact);
+ if ($cached !== null && $cached->getPhotoHash() === $hash) {
+ $vector = $cached->getVector();
+ if ($vector === null) {
+ continue;
+ }
+ if (count($vector) === $dimensions) {
+ $vectors[$key] = $vector;
+ continue;
+ }
+ }
+ if (count($toAnalyze) < self::MAX_PHOTOS_PER_RUN) {
+ $toAnalyze[$key] = ['contact' => $contact, 'photo' => $photo, 'hash' => $hash, 'cached' => $cached];
+ }
+ }
+ }
+
+ if (count($toAnalyze) > 0) {
+ $analyzed = $this->analyzer->analyze(array_map(static fn (array $entry): string => $entry['photo']['data'], $toAnalyze));
+ $report['analyzed'] = count($analyzed);
+ foreach ($analyzed as $key => $result) {
+ $entry = $toAnalyze[$key];
+ $this->storeCachedFace($userId, $entry['contact'], $entry['hash'], $result['vector'], $result['faceCount'], $entry['cached']);
+ if ($result['vector'] !== null && count($result['vector']) === $dimensions) {
+ $vectors[$key] = $result['vector'];
+ }
+ }
+ }
+
+ $threshold = $this->analyzer->getMatchThreshold();
+ /** @var array> $clustersByContact */
+ $clustersByContact = [];
+ /** @var array> $contactsByCluster */
+ $contactsByCluster = [];
+ foreach ($vectors as $key => $vector) {
+ foreach ($centroids as $clusterId => $centroid) {
+ if (self::distance($vector, $centroid) <= $threshold) {
+ $clustersByContact[$key][] = $clusterId;
+ $contactsByCluster[$clusterId][] = $key;
+ }
+ }
+ }
+
+ foreach ($clustersByContact as $key => $clusterIds) {
+ if (count($clusterIds) !== 1 || count($contactsByCluster[$clusterIds[0]]) !== 1) {
+ $report['ambiguous']++;
+ $this->logger->debug('Contact ' . $contacts[$key]->getDisplayName() . ' of user ' . $userId . ' matches several face clusters or shares a cluster with another contact, leaving the decision to the user');
+ continue;
+ }
+ $this->linkCluster($candidates[$clusterIds[0]], $contacts[$key], FaceCluster::LINK_SOURCE_AUTO);
+ $report['linked']++;
+ }
+
+ $this->logger->info('Linked ' . $report['linked'] . ' face clusters of user ' . $userId . ' to contacts (' . $report['analyzed'] . ' contact photos analyzed, ' . $report['ambiguous'] . ' ambiguous)');
+ return $report;
+ }
+
+ /**
+ * Link a cluster to a contact.
+ *
+ * A cluster without a name takes the name of the contact when linked
+ * automatically, unless the user already has a person of that name.
+ *
+ * @param string $source One of the FaceCluster::LINK_SOURCE_* constants
+ * @throws \OCP\DB\Exception
+ */
+ public function linkCluster(FaceCluster $cluster, ContactCard $contact, string $source): FaceCluster {
+ $cluster->setContactUid($contact->getUid());
+ $cluster->setContactAddressBook($contact->getAddressBookUri());
+ $cluster->setContactLinkSource($source);
+ if ($source === FaceCluster::LINK_SOURCE_AUTO && $cluster->getTitle() === '') {
+ $title = $contact->getDisplayName();
+ if ($title !== '' && preg_match('/^[0-9]+$/', $title) !== 1 && !$this->titleExists($cluster->getUserId(), $title)) {
+ $cluster->setTitle($title);
+ }
+ }
+ /** @var FaceCluster $cluster */
+ $cluster = $this->clusterMapper->update($cluster);
+ return $cluster;
+ }
+
+ /**
+ * Remove the link of a cluster. The cluster is not linked again automatically.
+ *
+ * @throws \OCP\DB\Exception
+ */
+ public function unlinkCluster(FaceCluster $cluster): FaceCluster {
+ $cluster->setContactUid(null);
+ $cluster->setContactAddressBook(null);
+ $cluster->setContactLinkSource(FaceCluster::LINK_SOURCE_MANUAL);
+ /** @var FaceCluster $cluster */
+ $cluster = $this->clusterMapper->update($cluster);
+ return $cluster;
+ }
+
+ /**
+ * @return ContactCard|null The linked contact, or null when the cluster is not linked or the contact is gone
+ */
+ public function getLinkedContact(FaceCluster $cluster): ?ContactCard {
+ if (!$cluster->isLinked()) {
+ return null;
+ }
+ return $this->contactsService->findContact($cluster->getUserId(), (string)$cluster->getContactAddressBook(), (string)$cluster->getContactUid());
+ }
+
+ /**
+ * Cut the face of a detection out of its photo, as a square with some room around the face.
+ *
+ * @param int $size Width and height of the returned image
+ * @return array{mimeType: string, data: string}
+ * @throws NotFoundException When the photo is not available to the user
+ * @throws Exception When the photo cannot be processed
+ */
+ public function cropFace(string $userId, FaceDetection $detection, int $size): array {
+ $file = $this->rootFolder->getUserFolder($userId)->getFirstNodeById($detection->getFileId());
+ if (!$file instanceof File) {
+ throw new NotFoundException('Photo ' . $detection->getFileId() . ' not found for user');
+ }
+ $previewSize = $size > 256 ? self::PREVIEW_SIZE_LARGE : self::PREVIEW_SIZE_SMALL;
+ try {
+ $preview = $this->previewManager->getPreview($file, $previewSize, $previewSize, false, IPreview::MODE_FILL);
+ $image = @imagecreatefromstring($preview->getContent());
+ } catch (\Throwable $e) {
+ throw new Exception('Could not generate a preview of photo ' . $file->getId(), 0, $e);
+ }
+ if ($image === false) {
+ throw new Exception('Could not decode the preview of photo ' . $file->getId());
+ }
+
+ $width = imagesx($image);
+ $height = imagesy($image);
+ $faceWidth = $detection->getWidth() * $width;
+ $faceHeight = $detection->getHeight() * $height;
+ $centerX = ($detection->getX() + $detection->getWidth() / 2) * $width;
+ $centerY = ($detection->getY() + $detection->getHeight() / 2) * $height;
+
+ $side = (int)round(max($faceWidth, $faceHeight) * (1 + 2 * self::CROP_MARGIN));
+ $side = max(1, min($side, $width, $height));
+ $left = (int)round(min(max($centerX - $side / 2, 0), $width - $side));
+ $top = (int)round(min(max($centerY - $side / 2, 0), $height - $side));
+
+ $cropped = imagecrop($image, ['x' => $left, 'y' => $top, 'width' => $side, 'height' => $side]);
+ imagedestroy($image);
+ if ($cropped === false) {
+ throw new Exception('Could not crop the face out of photo ' . $file->getId());
+ }
+ if ($side !== $size) {
+ $scaled = imagescale($cropped, $size, $size);
+ imagedestroy($cropped);
+ if ($scaled === false) {
+ throw new Exception('Could not scale the face of photo ' . $file->getId());
+ }
+ $cropped = $scaled;
+ }
+
+ ob_start();
+ imageinterlace($cropped, true);
+ $written = imagejpeg($cropped, null, 90);
+ $data = (string)ob_get_clean();
+ imagedestroy($cropped);
+ if (!$written) {
+ throw new Exception('Could not encode the face of photo ' . $file->getId());
+ }
+ return ['mimeType' => 'image/jpeg', 'data' => $data];
+ }
+
+ /**
+ * Use the face of a detection as the photo of a contact.
+ *
+ * @throws NotFoundException When the photo is not available to the user
+ * @throws Exception When the photo cannot be processed or the contact cannot be written
+ */
+ public function setContactPhotoFromDetection(string $userId, FaceDetection $detection, ContactCard $contact): void {
+ $face = $this->cropFace($userId, $detection, self::CONTACT_PHOTO_SIZE);
+ $this->contactsService->setPhoto($contact, $face['mimeType'], $face['data']);
+ }
+
+ private function findCachedFace(string $userId, ContactCard $contact): ?ContactFace {
+ try {
+ return $this->contactFaceMapper->findByContact($userId, $contact->getAddressBookUri(), $contact->getUid());
+ } catch (DoesNotExistException|MultipleObjectsReturnedException|\OCP\DB\Exception $e) {
+ return null;
+ }
+ }
+
+ /**
+ * @param list|null $vector
+ */
+ private function storeCachedFace(string $userId, ContactCard $contact, string $hash, ?array $vector, int $faceCount, ?ContactFace $existing): void {
+ $face = $existing ?? new ContactFace();
+ $face->setUserId($userId);
+ $face->setAddressBook($contact->getAddressBookUri());
+ $face->setContactUid($contact->getUid());
+ $face->setPhotoHash($hash);
+ $face->setVector($vector);
+ $face->setFaceCount($faceCount);
+ try {
+ if ($existing === null) {
+ $this->contactFaceMapper->insert($face);
+ } else {
+ $this->contactFaceMapper->update($face);
+ }
+ } catch (\OCP\DB\Exception $e) {
+ $this->logger->warning('Could not cache the face of contact ' . $contact->getUid(), ['exception' => $e]);
+ }
+ }
+
+ private function titleExists(string $userId, string $title): bool {
+ try {
+ $this->clusterMapper->findByUserAndTitle($userId, $title);
+ return true;
+ } catch (DoesNotExistException $e) {
+ return false;
+ } catch (MultipleObjectsReturnedException|\OCP\DB\Exception $e) {
+ return true;
+ }
+ }
+
+ private static function contactKey(string $addressBook, string $uid): string {
+ return $addressBook . "\0" . $uid;
+ }
+
+ /**
+ * @param list $a
+ * @param list $b
+ */
+ private static function distance(array $a, array $b): float {
+ $sum = 0.0;
+ foreach ($a as $i => $value) {
+ $diff = $value - ($b[$i] ?? 0.0);
+ $sum += $diff * $diff;
+ }
+ return sqrt($sum);
+ }
+}
diff --git a/lib/Service/ContactPhotoAnalyzer.php b/lib/Service/ContactPhotoAnalyzer.php
new file mode 100644
index 000000000..294de5e4e
--- /dev/null
+++ b/lib/Service/ContactPhotoAnalyzer.php
@@ -0,0 +1,201 @@
+settingsService->getSetting('faces.enabled') === 'true'
+ && $this->settingsService->getSetting('taskprocessing.enabled') !== 'true'
+ && trim($this->settingsService->getSetting('node_binary')) !== '';
+ }
+
+ /**
+ * Why {@see isAvailable()} is false, for logs and the admin settings.
+ */
+ public function getUnavailableReason(): string {
+ if ($this->settingsService->getSetting('faces.enabled') !== 'true') {
+ return 'Face recognition is disabled';
+ }
+ if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') {
+ return 'Contact photos cannot be analyzed by the TaskProcessing backend';
+ }
+ return 'No Node.js binary is configured';
+ }
+
+ /**
+ * The number of dimensions of the embeddings this analyzer produces.
+ */
+ public function getDimensions(): int {
+ return FaceClusterAnalyzer::DIMENSIONS;
+ }
+
+ /**
+ * The largest distance between a contact photo embedding and the centre of a
+ * cluster at which the two are considered to show the same person.
+ */
+ public function getMatchThreshold(): float {
+ return FaceClusterAnalyzer::MAX_CLUSTER_EDGE_LENGTH;
+ }
+
+ /**
+ * Analyze several photos in one classifier run.
+ *
+ * @param array $imagesByKey The raw image bytes, keyed by an identifier of the caller's choice
+ * @return array|null, faceCount: int}> One result per key; the vector is
+ * null when the photo holds no usable face
+ * @throws Exception When the classifier cannot be run
+ */
+ public function analyze(array $imagesByKey): array {
+ if (count($imagesByKey) === 0) {
+ return [];
+ }
+ if (!$this->isAvailable()) {
+ throw new Exception($this->getUnavailableReason());
+ }
+
+ $paths = [];
+ $results = [];
+ foreach ($imagesByKey as $key => $imageData) {
+ $path = $this->writeTemporaryJpeg($imageData);
+ if ($path === null) {
+ $results[$key] = ['vector' => null, 'faceCount' => 0];
+ continue;
+ }
+ $paths[$key] = $path;
+ }
+
+ if (count($paths) > 0) {
+ $timeout = $this->settingsService->getSetting('tensorflow.purejs') === 'true'
+ ? ClusteringFaceClassifier::IMAGE_PUREJS_TIMEOUT
+ : ClusteringFaceClassifier::IMAGE_TIMEOUT;
+ try {
+ $faces = $this->faceClassifier->classifyLocalFiles(ClusteringFaceClassifier::MODEL_NAME, array_values($paths), $timeout);
+ } catch (\ErrorException|\RuntimeException $e) {
+ throw new Exception('Could not analyze contact photos: ' . $e->getMessage(), 0, $e);
+ } finally {
+ foreach ($paths as $path) {
+ @unlink($path);
+ }
+ }
+ foreach (array_keys($paths) as $i => $key) {
+ $results[$key] = $this->pickFace(is_array($faces[$i]) ? $faces[$i] : []);
+ }
+ }
+
+ return $results;
+ }
+
+ /**
+ * Store the image as a JPEG the classifier can read, scaled down when it is too large.
+ *
+ * @return string|null The path, or null when the data is not a readable image
+ */
+ private function writeTemporaryJpeg(string $imageData): ?string {
+ $image = @imagecreatefromstring($imageData);
+ if ($image === false) {
+ $this->logger->debug('Contact photo could not be decoded as an image');
+ return null;
+ }
+ $width = imagesx($image);
+ $height = imagesy($image);
+ if ($width > self::MAX_IMAGE_DIMENSION || $height > self::MAX_IMAGE_DIMENSION) {
+ $scale = self::MAX_IMAGE_DIMENSION / max($width, $height);
+ $scaled = imagescale($image, (int)round($width * $scale), (int)round($height * $scale));
+ if ($scaled !== false) {
+ imagedestroy($image);
+ $image = $scaled;
+ }
+ }
+ $path = $this->tempManager->getTemporaryFile('.jpg');
+ if ($path === false) {
+ imagedestroy($image);
+ throw new Exception('Could not create a temporary file for a contact photo');
+ }
+ $written = imagejpeg($image, $path, 90);
+ imagedestroy($image);
+ if (!$written) {
+ @unlink($path);
+ return null;
+ }
+ return $path;
+ }
+
+ /**
+ * Pick the face that stands for the contact out of the faces found in the photo.
+ *
+ * @param array $faces The faces the classifier found
+ * @return array{vector: list|null, faceCount: int}
+ */
+ private function pickFace(array $faces): array {
+ $usable = [];
+ foreach ($faces as $face) {
+ if (!is_array($face) || !isset($face['vector']) || !is_array($face['vector'])) {
+ continue;
+ }
+ if ((float)($face['score'] ?? 0) < ClusteringFaceClassifier::MIN_FACE_RECOGNITION_SCORE) {
+ continue;
+ }
+ $roll = abs((float)($face['angle']['roll'] ?? 0));
+ $yaw = abs((float)($face['angle']['yaw'] ?? 0));
+ if ($roll > ClusteringFaceClassifier::MAX_FACE_ROLL || $yaw > ClusteringFaceClassifier::MAX_FACE_YAW) {
+ continue;
+ }
+ $usable[] = $face;
+ }
+
+ if (count($usable) === 0) {
+ return ['vector' => null, 'faceCount' => 0];
+ }
+
+ usort($usable, static function (array $a, array $b): int {
+ return self::area($b) <=> self::area($a);
+ });
+
+ if (count($usable) > 1 && self::area($usable[0]) < self::MIN_FACE_DOMINANCE * self::area($usable[1])) {
+ $this->logger->debug('Contact photo shows several faces of similar size, not using it');
+ return ['vector' => null, 'faceCount' => count($usable)];
+ }
+
+ /** @var list $vector */
+ $vector = array_values(array_map('floatval', $usable[0]['vector']));
+ return ['vector' => $vector, 'faceCount' => count($usable)];
+ }
+
+ private static function area(array $face): float {
+ return (float)($face['width'] ?? 0) * (float)($face['height'] ?? 0);
+ }
+}
diff --git a/lib/Service/ContactsService.php b/lib/Service/ContactsService.php
new file mode 100644
index 000000000..9354d948d
--- /dev/null
+++ b/lib/Service/ContactsService.php
@@ -0,0 +1,236 @@
+> */
+ private array $addressBooksByUser = [];
+
+ public function __construct(
+ private CardDavBackend $cardDavBackend,
+ private IURLGenerator $urlGenerator,
+ private IAppManager $appManager,
+ ) {
+ }
+
+ /**
+ * The address books a user can read, own ones and ones shared with them.
+ *
+ * @return list
+ */
+ public function getAddressBooks(string $userId): array {
+ if (!isset($this->addressBooksByUser[$userId])) {
+ $books = [];
+ /** @var array $row */
+ foreach ($this->cardDavBackend->getAddressBooksForUser('principals/users/' . $userId) as $row) {
+ $books[] = [
+ 'id' => (int)$row['id'],
+ 'uri' => (string)$row['uri'],
+ 'displayName' => (string)($row['{DAV:}displayname'] ?? $row['uri']),
+ 'readOnly' => (bool)($row[self::READ_ONLY_PROPERTY] ?? false),
+ ];
+ }
+ $this->addressBooksByUser[$userId] = $books;
+ }
+ return $this->addressBooksByUser[$userId];
+ }
+
+ /**
+ * The Contacts app identifies an address book by the last segment of its DAV
+ * href, so a percent-encoded URI is accepted as well.
+ *
+ * @return array{id: int, uri: string, displayName: string, readOnly: bool}|null
+ */
+ public function findAddressBook(string $userId, string $addressBookUri): ?array {
+ $decodedUri = rawurldecode($addressBookUri);
+ foreach ($this->getAddressBooks($userId) as $book) {
+ if ($book['uri'] === $addressBookUri || $book['uri'] === $decodedUri) {
+ return $book;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * All contacts of one address book of the user.
+ *
+ * @return list
+ */
+ public function getContacts(string $userId, string $addressBookUri): array {
+ $book = $this->findAddressBook($userId, $addressBookUri);
+ if ($book === null) {
+ return [];
+ }
+ $cards = [];
+ /** @var array $row */
+ foreach ($this->cardDavBackend->getCards($book['id']) as $row) {
+ $card = $this->rowToCard($book, $row);
+ if ($card !== null) {
+ $cards[] = $card;
+ }
+ }
+ return $cards;
+ }
+
+ /**
+ * @return ContactCard|null The contact, or null when the user has no such contact
+ */
+ public function findContact(string $userId, string $addressBookUri, string $uid): ?ContactCard {
+ $book = $this->findAddressBook($userId, $addressBookUri);
+ if ($book === null) {
+ return null;
+ }
+ $row = $this->cardDavBackend->getCardByUid($book['id'], $uid);
+ if ($row === null) {
+ return null;
+ }
+ return $this->rowToCard($book, $row);
+ }
+
+ /**
+ * Contacts of the user whose name or email matches the pattern, across all
+ * their address books. An empty pattern lists the first contacts of each book.
+ *
+ * @return list
+ */
+ public function searchContacts(string $userId, string $pattern, int $limit): array {
+ $books = [];
+ foreach ($this->getAddressBooks($userId) as $book) {
+ $books[$book['id']] = $book;
+ }
+ if (count($books) === 0) {
+ return [];
+ }
+
+ $rows = $this->cardDavBackend->searchPrincipalUri(
+ 'principals/users/' . $userId,
+ $pattern,
+ ['FN', 'N', 'EMAIL', 'NICKNAME'],
+ ['limit' => $limit, 'escape_like_param' => true],
+ );
+
+ $cards = [];
+ /** @var array $row */
+ foreach ($rows as $row) {
+ $book = $books[(int)$row['addressbookid']] ?? null;
+ if ($book === null) {
+ continue;
+ }
+ $card = $this->rowToCard($book, $row);
+ if ($card !== null) {
+ $cards[] = $card;
+ }
+ }
+ usort($cards, static fn (ContactCard $a, ContactCard $b): int => strcasecmp($a->getDisplayName(), $b->getDisplayName()));
+ return $cards;
+ }
+
+ /**
+ * Replace the photo of a contact.
+ *
+ * @param string $mimeType image/jpeg or image/png
+ * @param string $data The raw image bytes
+ * @throws Exception When the address book cannot be written to
+ */
+ public function setPhoto(ContactCard $contact, string $mimeType, string $data): void {
+ if ($contact->isAddressBookReadOnly()) {
+ throw new Exception('The address book of this contact is read-only');
+ }
+
+ $vCard = $contact->getVCard();
+ $vCard->remove('PHOTO');
+ if ($vCard->getDocumentType() === VCard::VCARD40) {
+ // https://tools.ietf.org/html/rfc6350#section-6.2.4
+ $vCard->add('PHOTO', 'data:' . $mimeType . ';base64,' . base64_encode($data), ['VALUE' => 'URI']);
+ } else {
+ // https://tools.ietf.org/html/rfc2426#section-3.1.4
+ $vCard->add('PHOTO', $data, ['ENCODING' => 'b', 'TYPE' => strtoupper(substr($mimeType, strlen('image/')))]);
+ }
+
+ $this->cardDavBackend->updateCard($contact->getAddressBookId(), $contact->getUri(), $vCard->serialize());
+ }
+
+ /**
+ * The page of the contact in the Contacts app, or null when that app is not available to the user.
+ */
+ public function getContactUrl(string $userId, ContactCard $contact): ?string {
+ if (!$this->appManager->isEnabledForUser('contacts')) {
+ return null;
+ }
+ // The Contacts app knows the address book by the last segment of its DAV href
+ return $this->urlGenerator->linkToRouteAbsolute('contacts.contacts.direct', [
+ 'contact' => $contact->getUid() . '~' . \Sabre\HTTP\encodePath($contact->getAddressBookUri()),
+ ]);
+ }
+
+ /**
+ * Where the photo of the contact can be fetched, which is the same endpoint the Contacts app uses.
+ */
+ public function getPhotoUrl(string $userId, ContactCard $contact): string {
+ return $this->urlGenerator->getAbsoluteURL(
+ 'remote.php/dav/addressbooks/users/' . rawurlencode($userId) . '/' . rawurlencode($contact->getAddressBookUri()) . '/' . rawurlencode($contact->getUri()) . '?photo'
+ );
+ }
+
+ /**
+ * @return array{uid: string, addressBook: string, addressBookName: string, displayName: string, hasPhoto: bool, readOnly: bool, url: string|null, photoUrl: string|null}
+ */
+ public function serializeContact(string $userId, ContactCard $contact): array {
+ return array_merge($contact->toArray(), [
+ 'url' => $this->getContactUrl($userId, $contact),
+ 'photoUrl' => $contact->hasPhoto() ? $this->getPhotoUrl($userId, $contact) : null,
+ ]);
+ }
+
+ /**
+ * @param array{id: int, uri: string, displayName: string, readOnly: bool} $book
+ * @param array $row A row of the cards table
+ */
+ private function rowToCard(array $book, array $row): ?ContactCard {
+ $cardData = $row['carddata'] ?? null;
+ if (is_resource($cardData)) {
+ $cardData = stream_get_contents($cardData);
+ }
+ if (!is_string($cardData) || $cardData === '') {
+ return null;
+ }
+ $card = new ContactCard(
+ $book['id'],
+ $book['uri'],
+ $book['displayName'],
+ $book['readOnly'],
+ (string)$row['uri'],
+ isset($row['uid']) ? (string)$row['uid'] : null,
+ $cardData,
+ );
+ try {
+ if ($card->getUid() === '') {
+ return null;
+ }
+ } catch (\Throwable $e) {
+ // Unparsable card data
+ return null;
+ }
+ return $card;
+ }
+}
diff --git a/lib/Service/SettingsService.php b/lib/Service/SettingsService.php
index 8ea9d217b..6569c0e76 100644
--- a/lib/Service/SettingsService.php
+++ b/lib/Service/SettingsService.php
@@ -36,6 +36,7 @@ final class SettingsService {
'imagenet.enabled' => 'false',
'landmarks.enabled' => 'false',
'faces.enabled' => 'false',
+ 'faces.contactLinking' => 'true',
'musicnn.enabled' => 'false',
'movinet.enabled' => 'false',
'taskprocessing.enabled' => '',
diff --git a/psalm.xml b/psalm.xml
index 2aebeb64a..012d05cb2 100644
--- a/psalm.xml
+++ b/psalm.xml
@@ -16,6 +16,7 @@
+
diff --git a/src/components/ViewAdmin.vue b/src/components/ViewAdmin.vue
index 21e1d894d..bc4008805 100644
--- a/src/components/ViewAdmin.vue
+++ b/src/components/ViewAdmin.vue
@@ -88,6 +88,17 @@
:title="t('recognize', 'The number of files to process per job run (A job will be scheduled every 5 minutes; For normal operation ~500 or more, in WASM mode ~50 is recommended)')"
@update:model-value="onChange" />
+
+
+ {{ t('recognize', 'Link recognized people to contacts automatically (compares the photos of contacts with the recognized faces once a day; links can be changed in the Photos and Contacts apps)') }}
+
+
+ {{ t('recognize', 'Automatic linking is not available with the TaskProcessing backend, as it cannot analyze contact photos. People can still be linked to contacts by hand.') }}
+
+
@@ -445,9 +456,9 @@ const TASK_PROCESSING_TASK_TYPES = {
musicnn: 'recognize:audio:classification',
}
-const SETTINGS = ['tensorflow.cores', 'tensorflow.gpu', 'tensorflow.purejs', 'imagenet.enabled', 'landmarks.enabled', 'faces.enabled', 'musicnn.enabled', 'movinet.enabled', 'node_binary', 'ffmpeg_binary', 'faces.status', 'imagenet.status', 'landmarks.status', 'movinet.status', 'musicnn.status', 'faces.lastFile', 'imagenet.lastFile', 'landmarks.lastFile', 'movinet.lastFile', 'musicnn.lastFile', 'faces.batchSize', 'imagenet.batchSize', 'landmarks.batchSize', 'movinet.batchSize', 'musicnn.batchSize', 'clusterFaces.status', 'clusterFaces.lastRun', 'nice_binary', 'nice_value', 'concurrency.enabled', 'taskprocessing.enabled']
+const SETTINGS = ['tensorflow.cores', 'tensorflow.gpu', 'tensorflow.purejs', 'imagenet.enabled', 'landmarks.enabled', 'faces.enabled', 'faces.contactLinking', 'musicnn.enabled', 'movinet.enabled', 'node_binary', 'ffmpeg_binary', 'faces.status', 'imagenet.status', 'landmarks.status', 'movinet.status', 'musicnn.status', 'faces.lastFile', 'imagenet.lastFile', 'landmarks.lastFile', 'movinet.lastFile', 'musicnn.lastFile', 'faces.batchSize', 'imagenet.batchSize', 'landmarks.batchSize', 'movinet.batchSize', 'musicnn.batchSize', 'clusterFaces.status', 'clusterFaces.lastRun', 'nice_binary', 'nice_value', 'concurrency.enabled', 'taskprocessing.enabled']
-const BOOLEAN_SETTINGS = ['tensorflow.gpu', 'tensorflow.purejs', 'imagenet.enabled', 'landmarks.enabled', 'faces.enabled', 'musicnn.enabled', 'movinet.enabled', 'faces.status', 'imagenet.status', 'landmarks.status', 'movinet.status', 'musicnn.status', 'faces.lastFile', 'imagenet.lastFile', 'landmarks.lastFile', 'movinet.lastFile', 'musicnn.lastFile', 'clusterFaces.status', 'concurrency.enabled', 'taskprocessing.enabled']
+const BOOLEAN_SETTINGS = ['tensorflow.gpu', 'tensorflow.purejs', 'imagenet.enabled', 'landmarks.enabled', 'faces.enabled', 'faces.contactLinking', 'musicnn.enabled', 'movinet.enabled', 'faces.status', 'imagenet.status', 'landmarks.status', 'movinet.status', 'musicnn.status', 'faces.lastFile', 'imagenet.lastFile', 'landmarks.lastFile', 'movinet.lastFile', 'musicnn.lastFile', 'clusterFaces.status', 'concurrency.enabled', 'taskprocessing.enabled']
const MAX_RELATIVE_DATE = 1000 * 60 * 60 * 24 * 7 // one week
diff --git a/tests/ContactCardTest.php b/tests/ContactCardTest.php
new file mode 100644
index 000000000..3c888c818
--- /dev/null
+++ b/tests/ContactCardTest.php
@@ -0,0 +1,66 @@
+card("BEGIN:VCARD\r\nVERSION:4.0\r\nUID:uid-1\r\nN:Doe;Jane;;;\r\nEND:VCARD\r\n");
+ self::assertSame('Jane Doe', $card->getDisplayName());
+
+ $card = $this->card("BEGIN:VCARD\r\nVERSION:4.0\r\nUID:uid-1\r\nN:Doe;John;Quincy;Dr.;Jr.\r\nEND:VCARD\r\n");
+ self::assertSame('Dr. John Quincy Doe Jr.', $card->getDisplayName());
+
+ $card = $this->card("BEGIN:VCARD\r\nVERSION:4.0\r\nUID:uid-1\r\nFN:Jane Q. Doe\r\nN:Doe;Jane;;;\r\nEND:VCARD\r\n");
+ self::assertSame('Jane Q. Doe', $card->getDisplayName());
+ }
+
+ public function testUidIsReadFromCardWhenNotGiven(): void {
+ $card = $this->card("BEGIN:VCARD\r\nVERSION:3.0\r\nUID:from-card\r\nFN:Jane\r\nEND:VCARD\r\n", null);
+ self::assertSame('from-card', $card->getUid());
+ }
+
+ public function testPhotoOfVCard4DataUri(): void {
+ $card = $this->card("BEGIN:VCARD\r\nVERSION:4.0\r\nUID:uid-1\r\nFN:Jane\r\nPHOTO:data:image/png;base64," . self::PNG_PIXEL . "\r\nEND:VCARD\r\n");
+ $photo = $card->getPhoto();
+ self::assertNotNull($photo);
+ self::assertSame('image/png', $photo['mimeType']);
+ self::assertSame(base64_decode(self::PNG_PIXEL), $photo['data']);
+ self::assertTrue($card->hasPhoto());
+ }
+
+ public function testPhotoOfVCard3BinaryProperty(): void {
+ $card = $this->card("BEGIN:VCARD\r\nVERSION:3.0\r\nUID:uid-1\r\nFN:Jane\r\nPHOTO;ENCODING=b;TYPE=PNG:" . self::PNG_PIXEL . "\r\nEND:VCARD\r\n");
+ $photo = $card->getPhoto();
+ self::assertNotNull($photo);
+ self::assertSame('image/png', $photo['mimeType']);
+ self::assertSame(base64_decode(self::PNG_PIXEL), $photo['data']);
+ }
+
+ public function testRemotePhotoIsNotFetched(): void {
+ $card = $this->card("BEGIN:VCARD\r\nVERSION:4.0\r\nUID:uid-1\r\nFN:Jane\r\nPHOTO;VALUE=URI:https://example.com/jane.png\r\nEND:VCARD\r\n");
+ self::assertNull($card->getPhoto());
+ self::assertFalse($card->hasPhoto());
+ }
+
+ public function testCardWithoutPhoto(): void {
+ $card = $this->card("BEGIN:VCARD\r\nVERSION:4.0\r\nUID:uid-1\r\nFN:Jane\r\nEND:VCARD\r\n");
+ self::assertNull($card->getPhoto());
+ $array = $card->toArray();
+ self::assertSame('uid-1', $array['uid']);
+ self::assertSame('contacts', $array['addressBook']);
+ self::assertFalse($array['hasPhoto']);
+ self::assertFalse($array['readOnly']);
+ }
+}
diff --git a/tests/ContactLinkTest.php b/tests/ContactLinkTest.php
new file mode 100644
index 000000000..a3d154202
--- /dev/null
+++ b/tests/ContactLinkTest.php
@@ -0,0 +1,130 @@
+createUser(self::TEST_USER1, self::TEST_USER1);
+ \OC::$server->get(\OCP\IUserManager::class)->registerBackend($backend);
+ }
+
+ public function setUp(): void {
+ parent::setUp();
+ $this->faceDetectionMapper = \OC::$server->get(FaceDetectionMapper::class);
+ $this->faceClusterMapper = \OC::$server->get(FaceClusterMapper::class);
+ $this->contactLinkService = \OC::$server->get(ContactLinkService::class);
+
+ foreach ($this->faceClusterMapper->findByUserId(self::TEST_USER1) as $cluster) {
+ $this->faceClusterMapper->delete($cluster);
+ }
+ foreach ($this->faceDetectionMapper->findByUserId(self::TEST_USER1) as $detection) {
+ $this->faceDetectionMapper->delete($detection);
+ }
+ }
+
+ private function createCluster(string $title, int $detections = 1): FaceCluster {
+ $cluster = new FaceCluster();
+ $cluster->setUserId(self::TEST_USER1);
+ $cluster->setTitle($title);
+ /** @var FaceCluster $cluster */
+ $cluster = $this->faceClusterMapper->insert($cluster);
+ for ($i = 0; $i < $detections; $i++) {
+ $detection = new FaceDetection();
+ $detection->setUserId(self::TEST_USER1);
+ $detection->setFileId(1000 + $cluster->getId() * 10 + $i);
+ $detection->setVector(array_fill(0, 128, 0.0));
+ $detection->setX(0.1);
+ $detection->setY(0.1);
+ $detection->setHeight(0.5);
+ $detection->setWidth(0.5);
+ $detection->setClusterId($cluster->getId());
+ $this->faceDetectionMapper->insert($detection);
+ }
+ return $cluster;
+ }
+
+ private function createContact(string $uid, string $name): ContactCard {
+ return new ContactCard(1, 'contacts', 'Contacts', false, $uid . '.vcf', $uid, "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:$uid\r\nFN:$name\r\nEND:VCARD\r\n");
+ }
+
+ public function testManualLinkAndUnlink(): void {
+ $cluster = $this->createCluster('Someone');
+ $contact = $this->createContact('uid-a', 'Alice');
+
+ $cluster = $this->contactLinkService->linkCluster($cluster, $contact, FaceCluster::LINK_SOURCE_MANUAL);
+ self::assertTrue($cluster->isLinked());
+ self::assertSame('uid-a', $cluster->getContactUid());
+ self::assertSame('contacts', $cluster->getContactAddressBook());
+ self::assertSame(FaceCluster::LINK_SOURCE_MANUAL, $cluster->getContactLinkSource());
+ // A manual link keeps the name the user gave the person
+ self::assertSame('Someone', $cluster->getTitle());
+
+ self::assertCount(1, $this->faceClusterMapper->findByContact(self::TEST_USER1, 'contacts', 'uid-a'));
+ self::assertCount(1, $this->faceClusterMapper->findLinkedByUserId(self::TEST_USER1));
+ self::assertCount(0, $this->faceClusterMapper->findAutoLinkCandidatesByUserId(self::TEST_USER1));
+
+ $cluster = $this->contactLinkService->unlinkCluster($cluster);
+ self::assertFalse($cluster->isLinked());
+ self::assertNull($cluster->getContactUid());
+ self::assertSame(FaceCluster::LINK_SOURCE_MANUAL, $cluster->getContactLinkSource());
+ self::assertCount(0, $this->faceClusterMapper->findLinkedByUserId(self::TEST_USER1));
+ // A person the user unlinked is not offered to the automatic linking again
+ self::assertCount(0, $this->faceClusterMapper->findAutoLinkCandidatesByUserId(self::TEST_USER1));
+ }
+
+ public function testAutoLinkNamesUnnamedClusterAfterContact(): void {
+ $unnamed = $this->createCluster('');
+ $named = $this->createCluster('Bob');
+ $this->createCluster('Empty', 0);
+
+ $candidates = $this->faceClusterMapper->findAutoLinkCandidatesByUserId(self::TEST_USER1);
+ // Only clusters with detections can be matched
+ self::assertCount(2, $candidates);
+
+ $unnamed = $this->contactLinkService->linkCluster($unnamed, $this->createContact('uid-a', 'Alice'), FaceCluster::LINK_SOURCE_AUTO);
+ self::assertSame('Alice', $unnamed->getTitle());
+ self::assertSame(FaceCluster::LINK_SOURCE_AUTO, $unnamed->getContactLinkSource());
+
+ // A name that is already taken is not used twice
+ $another = $this->createCluster('');
+ $another = $this->contactLinkService->linkCluster($another, $this->createContact('uid-a2', 'Alice'), FaceCluster::LINK_SOURCE_AUTO);
+ self::assertSame('', $another->getTitle());
+ self::assertTrue($another->isLinked());
+
+ // A cluster that already has a name keeps it
+ $named = $this->contactLinkService->linkCluster($named, $this->createContact('uid-b', 'Robert'), FaceCluster::LINK_SOURCE_AUTO);
+ self::assertSame('Bob', $named->getTitle());
+
+ self::assertCount(0, $this->faceClusterMapper->findAutoLinkCandidatesByUserId(self::TEST_USER1));
+ self::assertContains(self::TEST_USER1, $this->faceClusterMapper->findUserIds());
+ }
+
+ public function testDeletingClusterRemovesLink(): void {
+ $cluster = $this->createCluster('Carol');
+ $this->contactLinkService->linkCluster($cluster, $this->createContact('uid-c', 'Carol'), FaceCluster::LINK_SOURCE_MANUAL);
+ $this->faceClusterMapper->delete($cluster);
+ self::assertCount(0, $this->faceClusterMapper->findByContact(self::TEST_USER1, 'contacts', 'uid-c'));
+ }
+}
diff --git a/tests/stubs/oca_dav_carddav_carddavbackend.php b/tests/stubs/oca_dav_carddav_carddavbackend.php
new file mode 100644
index 000000000..c3ad2c9f3
--- /dev/null
+++ b/tests/stubs/oca_dav_carddav_carddavbackend.php
@@ -0,0 +1,50 @@
+