cache remote collections

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

lib/AP.php

@@ -31,6 +31,7 @@ declare(strict_types=1);
 
 namespace OCA\Social;
 
+use OCA\Social\Model\ActivityPub\OrderedCollection;
 use OCA\Social\Tools\Traits\TArrayTools;
 use OCA\Social\Exceptions\ItemUnknownException;
 use OCA\Social\Exceptions\RedundancyLimitException;
@@ -305,6 +306,10 @@ class AP {
 				$item = new Note();
 				break;
 
+			case OrderedCollection::TYPE:
+				$item = new OrderedCollection();
+				break;
+
 			case SocialAppNotification::TYPE:
 				$item = new SocialAppNotification();
 				break;

lib/Command/CacheRefresh.php

@@ -85,6 +85,9 @@ class CacheRefresh extends Base {
 		$result = $this->cacheActorService->manageCacheRemoteActors($input->getOption('force'));
 		$output->writeLn($result . ' remote accounts updated');
 
+		$result = $this->cacheActorService->manageDetailsRemoteActors($input->getOption('force'));
+		$output->writeLn($result . ' remote accounts details updated');
+
 		$result = $this->documentService->manageCacheDocuments();
 		$output->writeLn($result . ' documents cached');
 

lib/Cron/Cache.php

@@ -86,6 +86,11 @@ class Cache extends TimedJob {
 		} catch (Exception $e) {
 		}
 
+		try {
+			$this->cacheActorService->manageDetailsRemoteActors();
+		} catch (Exception $e) {
+		}
+
 		try {
 			$this->documentService->manageCacheDocuments();
 		} catch (Exception $e) {

lib/Db/CacheActorsRequest.php

@@ -30,6 +30,7 @@ declare(strict_types=1);
 
 namespace OCA\Social\Db;
 
+use DateInterval;
 use DateTime;
 use Exception;
 use OCA\Social\Exceptions\CacheActorDoesNotExistException;
@@ -41,6 +42,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
 
 class CacheActorsRequest extends CacheActorsRequestBuilder {
 	public const CACHE_TTL = 60 * 24 * 10; // 10d
+	public const DETAILS_TTL = 60 * 18; // 18h
 
 
 	/**
@@ -101,9 +103,6 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
 	}
 
 
-	/**
-	 * Insert cache about an Actor in database.
-	 */
 	public function update(Person $actor): int {
 		$qb = $this->getCacheActorsUpdateSql();
 		$qb->set('following', $qb->createNamedParameter($actor->getFollowing()))
@@ -150,6 +149,24 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
 	}
 
 
+	public function updateDetails(Person $actor): int {
+		$qb = $this->getCacheActorsUpdateSql();
+		$qb->set('details', $qb->createNamedParameter(json_encode($actor->getDetailsAll())));
+
+		try {
+			$qb->set(
+				'details_update',
+				$qb->createNamedParameter(new DateTime('now'), IQueryBuilder::PARAM_DATE)
+			);
+		} catch (Exception $e) {
+		}
+
+		$qb->limitToIdString($actor->getId());
+
+		return $qb->executeStatement();
+	}
+
+
 	/**
 	 * get Cached version of an Actor, based on the UriId
 	 *
@@ -212,7 +229,6 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
 	public function searchAccounts(string $search): array {
 		$qb = $this->getCacheActorsSelectSql();
 		$qb->searchInAccount($search);
-		/** @var SocialQueryBuilder $qb */
 		$qb->leftJoinCacheDocuments('icon_id');
 		$this->leftJoinDetails($qb);
 		$qb->limitResults(25);
@@ -236,6 +252,22 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
 	}
 
 
+	/**
+	 * @return Person[]
+	 * @throws Exception
+	 */
+	public function getRemoteActorsToUpdateDetails(bool $force = false): array {
+		$qb = $this->getCacheActorsSelectSql();
+		$qb->limitToLocal(false);
+		if (!$force) {
+			$date = new DateTime('now');
+			$date->sub(new DateInterval('PT' . self::DETAILS_TTL . 'M'));
+			$qb->limitToDBFieldDateTime('details_update', $date, true);
+		}
+
+		return $this->getCacheActorsFromRequest($qb);
+	}
+
 	/**
 	 * delete cached version of an Actor, based on the UriId
 	 *

lib/Db/CacheActorsRequestBuilder.php

@@ -80,7 +80,8 @@ class CacheActorsRequestBuilder extends CoreRequestBuilder {
 		$qb->select(
 			'ca.nid', 'ca.id', 'ca.account', 'ca.following', 'ca.followers', 'ca.inbox',
 			'ca.shared_inbox', 'ca.outbox', 'ca.featured', 'ca.url', 'ca.type', 'ca.preferred_username',
-			'ca.name', 'ca.summary', 'ca.public_key', 'ca.local', 'ca.details', 'ca.source', 'ca.creation'
+			'ca.name', 'ca.summary', 'ca.public_key', 'ca.local', 'ca.details', 'ca.source', 'ca.creation',
+			'ca.details_update'
 		)
 		   ->from(self::TABLE_CACHE_ACTORS, 'ca');
 

lib/Db/CoreRequestBuilder.php

@@ -115,6 +115,7 @@ class CoreRequestBuilder {
 			'public_key',
 			'source',
 			'details',
+			'details_update',
 			'creation'
 		],
 		self::TABLE_CACHE_DOCUMENTS => [

lib/Migration/Version1000Date20221118000001.php

@@ -896,6 +896,12 @@ class Version1000Date20221118000001 extends SimpleMigrationStep {
 				'notnull' => false,
 			]
 		);
+		$table->addColumn(
+			'details_update', Types::DATETIME,
+			[
+				'notnull' => false,
+			]
+		);
 
 		$table->setPrimaryKey(['nid']);
 		$table->addUniqueIndex(['id_prim']);

lib/Migration/Version1000Date20230407000001.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * Nextcloud - Social Support
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2023, Maxence Lange <maxence@artificial-owl.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Social\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version1000Date20230407000001 extends SimpleMigrationStep {
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		// fix nid as primary on social_cache_actor
+		if ($schema->hasTable('social_cache_actor')) {
+			$table = $schema->getTable('social_cache_actor');
+
+			if (!$table->hasColumn('details_update')) {
+				$table->addColumn(
+					'details_update', Types::DATETIME,
+					[
+						'notnull' => false
+					]
+				);
+			}
+		}
+
+		return $schema;
+	}
+}

lib/Model/ActivityPub/OrderedCollection.php

@@ -76,6 +76,9 @@ class OrderedCollection extends ACore implements JsonSerializable {
 
 	public function import(array $data): self {
 		parent::import($data);
+		$this->setFirst($this->validate(ACore::AS_USERNAME, 'first', $data, ''))
+			 ->setLast($this->validate(ACore::AS_USERNAME, 'last', $data, ''))
+			 ->setTotalItems($this->getInt('totalItems', $data));
 
 		return $this;
 	}

lib/Service/CacheActorService.php

@@ -45,6 +45,7 @@ use OCA\Social\Exceptions\RetrieveAccountFormatException;
 use OCA\Social\Exceptions\SocialAppConfigException;
 use OCA\Social\Exceptions\UnauthorizedFediverseException;
 use OCA\Social\Model\ActivityPub\Actor\Person;
+use OCA\Social\Model\ActivityPub\OrderedCollection;
 use OCA\Social\Model\Client\Options\ProbeOptions;
 use OCA\Social\Tools\Exceptions\MalformedArrayException;
 use OCA\Social\Tools\Exceptions\RequestContentException;
@@ -65,7 +66,7 @@ use Psr\Log\LoggerInterface;
 class CacheActorService {
 	use TArrayTools;
 
-	private \OCP\IURLGenerator $urlGenerator;
+	private IURLGenerator $urlGenerator;
 	private ActorsRequest $actorsRequest;
 	private CacheActorsRequest $cacheActorsRequest;
 	private CurlService $curlService;
@@ -305,6 +306,68 @@ class CacheActorService {
 	}
 
 
+	/**
+	 * @return int
+	 * @throws Exception
+	 */
+	public function manageDetailsRemoteActors(bool $force = false): int {
+		$update = $this->cacheActorsRequest->getRemoteActorsToUpdateDetails($force);
+
+		// WARNING: risk of race condition if something else update details on remote actor.
+		// Any details update on remote cache-actor must be managed from here.
+		foreach ($update as $item) {
+			try {
+				$this->addRemoteActorDetailCount($item);
+				$this->cacheActorsRequest->updateDetails($item);
+			} catch (Exception $e) {
+			}
+		}
+
+		return sizeof($update);
+	}
+
+
+	public function addRemoteActorDetailCount(Person $actor): void {
+		try {
+			$followers = $this->getCollectionFromId($actor->getFollowers());
+			$following = $this->getCollectionFromId($actor->getFollowing());
+			$outbox = $this->getCollectionFromId($actor->getOutbox());
+		} catch (InvalidResourceException $e) {
+			return;
+		}
+
+		$count = [
+			'followers' => $followers->getTotalItems(),
+			'following' => $following->getTotalItems(),
+			'post' => $outbox->getTotalItems()
+		];
+		$actor->setDetailArray('count', $count);
+	}
+
+
+	/**
+	 * @param string $id
+	 *
+	 * @return OrderedCollection
+	 * @throws InvalidResourceException
+	 */
+	private function getCollectionFromId(string $id): OrderedCollection {
+		try {
+			$object = $this->curlService->retrieveObject($id);
+			/** @var OrderedCollection $collection */
+			$collection = AP::$activityPub->getItemFromData($object);
+		} catch (Exception $e) {
+			throw new InvalidResourceException();
+		}
+
+		if ($collection->getType() !== OrderedCollection::TYPE) {
+			throw new InvalidResourceException();
+		}
+
+		return $collection;
+	}
+
+
 	/**
 	 * @param Person $actor
 	 *
@@ -342,7 +405,7 @@ class CacheActorService {
 	}
 
 
-	public function getFromNids(array $ids):array {
+	public function getFromNids(array $ids): array {
 		return $this->cacheActorsRequest->getFromNids($ids);
 	}
 }