kopia lustrzana https://github.com/nextcloud/social
cache remote collections
Signed-off-by: Maxence Lange <maxence@artificial-owl.com>pull/1721/head
rodzic
8d44442a33
commit
1c6f5dd361
|
@ -31,6 +31,7 @@ declare(strict_types=1);
|
||||||
|
|
||||||
namespace OCA\Social;
|
namespace OCA\Social;
|
||||||
|
|
||||||
|
use OCA\Social\Model\ActivityPub\OrderedCollection;
|
||||||
use OCA\Social\Tools\Traits\TArrayTools;
|
use OCA\Social\Tools\Traits\TArrayTools;
|
||||||
use OCA\Social\Exceptions\ItemUnknownException;
|
use OCA\Social\Exceptions\ItemUnknownException;
|
||||||
use OCA\Social\Exceptions\RedundancyLimitException;
|
use OCA\Social\Exceptions\RedundancyLimitException;
|
||||||
|
@ -305,6 +306,10 @@ class AP {
|
||||||
$item = new Note();
|
$item = new Note();
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case OrderedCollection::TYPE:
|
||||||
|
$item = new OrderedCollection();
|
||||||
|
break;
|
||||||
|
|
||||||
case SocialAppNotification::TYPE:
|
case SocialAppNotification::TYPE:
|
||||||
$item = new SocialAppNotification();
|
$item = new SocialAppNotification();
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -85,6 +85,9 @@ class CacheRefresh extends Base {
|
||||||
$result = $this->cacheActorService->manageCacheRemoteActors($input->getOption('force'));
|
$result = $this->cacheActorService->manageCacheRemoteActors($input->getOption('force'));
|
||||||
$output->writeLn($result . ' remote accounts updated');
|
$output->writeLn($result . ' remote accounts updated');
|
||||||
|
|
||||||
|
$result = $this->cacheActorService->manageDetailsRemoteActors($input->getOption('force'));
|
||||||
|
$output->writeLn($result . ' remote accounts details updated');
|
||||||
|
|
||||||
$result = $this->documentService->manageCacheDocuments();
|
$result = $this->documentService->manageCacheDocuments();
|
||||||
$output->writeLn($result . ' documents cached');
|
$output->writeLn($result . ' documents cached');
|
||||||
|
|
||||||
|
|
|
@ -86,6 +86,11 @@ class Cache extends TimedJob {
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
$this->cacheActorService->manageDetailsRemoteActors();
|
||||||
|
} catch (Exception $e) {
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$this->documentService->manageCacheDocuments();
|
$this->documentService->manageCacheDocuments();
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
|
|
@ -30,6 +30,7 @@ declare(strict_types=1);
|
||||||
|
|
||||||
namespace OCA\Social\Db;
|
namespace OCA\Social\Db;
|
||||||
|
|
||||||
|
use DateInterval;
|
||||||
use DateTime;
|
use DateTime;
|
||||||
use Exception;
|
use Exception;
|
||||||
use OCA\Social\Exceptions\CacheActorDoesNotExistException;
|
use OCA\Social\Exceptions\CacheActorDoesNotExistException;
|
||||||
|
@ -41,6 +42,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
|
||||||
|
|
||||||
class CacheActorsRequest extends CacheActorsRequestBuilder {
|
class CacheActorsRequest extends CacheActorsRequestBuilder {
|
||||||
public const CACHE_TTL = 60 * 24 * 10; // 10d
|
public const CACHE_TTL = 60 * 24 * 10; // 10d
|
||||||
|
public const DETAILS_TTL = 60 * 18; // 18h
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -101,9 +103,6 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Insert cache about an Actor in database.
|
|
||||||
*/
|
|
||||||
public function update(Person $actor): int {
|
public function update(Person $actor): int {
|
||||||
$qb = $this->getCacheActorsUpdateSql();
|
$qb = $this->getCacheActorsUpdateSql();
|
||||||
$qb->set('following', $qb->createNamedParameter($actor->getFollowing()))
|
$qb->set('following', $qb->createNamedParameter($actor->getFollowing()))
|
||||||
|
@ -150,6 +149,24 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public function updateDetails(Person $actor): int {
|
||||||
|
$qb = $this->getCacheActorsUpdateSql();
|
||||||
|
$qb->set('details', $qb->createNamedParameter(json_encode($actor->getDetailsAll())));
|
||||||
|
|
||||||
|
try {
|
||||||
|
$qb->set(
|
||||||
|
'details_update',
|
||||||
|
$qb->createNamedParameter(new DateTime('now'), IQueryBuilder::PARAM_DATE)
|
||||||
|
);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
}
|
||||||
|
|
||||||
|
$qb->limitToIdString($actor->getId());
|
||||||
|
|
||||||
|
return $qb->executeStatement();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* get Cached version of an Actor, based on the UriId
|
* get Cached version of an Actor, based on the UriId
|
||||||
*
|
*
|
||||||
|
@ -212,7 +229,6 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
|
||||||
public function searchAccounts(string $search): array {
|
public function searchAccounts(string $search): array {
|
||||||
$qb = $this->getCacheActorsSelectSql();
|
$qb = $this->getCacheActorsSelectSql();
|
||||||
$qb->searchInAccount($search);
|
$qb->searchInAccount($search);
|
||||||
/** @var SocialQueryBuilder $qb */
|
|
||||||
$qb->leftJoinCacheDocuments('icon_id');
|
$qb->leftJoinCacheDocuments('icon_id');
|
||||||
$this->leftJoinDetails($qb);
|
$this->leftJoinDetails($qb);
|
||||||
$qb->limitResults(25);
|
$qb->limitResults(25);
|
||||||
|
@ -236,6 +252,22 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return Person[]
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function getRemoteActorsToUpdateDetails(bool $force = false): array {
|
||||||
|
$qb = $this->getCacheActorsSelectSql();
|
||||||
|
$qb->limitToLocal(false);
|
||||||
|
if (!$force) {
|
||||||
|
$date = new DateTime('now');
|
||||||
|
$date->sub(new DateInterval('PT' . self::DETAILS_TTL . 'M'));
|
||||||
|
$qb->limitToDBFieldDateTime('details_update', $date, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->getCacheActorsFromRequest($qb);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* delete cached version of an Actor, based on the UriId
|
* delete cached version of an Actor, based on the UriId
|
||||||
*
|
*
|
||||||
|
|
|
@ -80,7 +80,8 @@ class CacheActorsRequestBuilder extends CoreRequestBuilder {
|
||||||
$qb->select(
|
$qb->select(
|
||||||
'ca.nid', 'ca.id', 'ca.account', 'ca.following', 'ca.followers', 'ca.inbox',
|
'ca.nid', 'ca.id', 'ca.account', 'ca.following', 'ca.followers', 'ca.inbox',
|
||||||
'ca.shared_inbox', 'ca.outbox', 'ca.featured', 'ca.url', 'ca.type', 'ca.preferred_username',
|
'ca.shared_inbox', 'ca.outbox', 'ca.featured', 'ca.url', 'ca.type', 'ca.preferred_username',
|
||||||
'ca.name', 'ca.summary', 'ca.public_key', 'ca.local', 'ca.details', 'ca.source', 'ca.creation'
|
'ca.name', 'ca.summary', 'ca.public_key', 'ca.local', 'ca.details', 'ca.source', 'ca.creation',
|
||||||
|
'ca.details_update'
|
||||||
)
|
)
|
||||||
->from(self::TABLE_CACHE_ACTORS, 'ca');
|
->from(self::TABLE_CACHE_ACTORS, 'ca');
|
||||||
|
|
||||||
|
|
|
@ -115,6 +115,7 @@ class CoreRequestBuilder {
|
||||||
'public_key',
|
'public_key',
|
||||||
'source',
|
'source',
|
||||||
'details',
|
'details',
|
||||||
|
'details_update',
|
||||||
'creation'
|
'creation'
|
||||||
],
|
],
|
||||||
self::TABLE_CACHE_DOCUMENTS => [
|
self::TABLE_CACHE_DOCUMENTS => [
|
||||||
|
|
|
@ -896,6 +896,12 @@ class Version1000Date20221118000001 extends SimpleMigrationStep {
|
||||||
'notnull' => false,
|
'notnull' => false,
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
$table->addColumn(
|
||||||
|
'details_update', Types::DATETIME,
|
||||||
|
[
|
||||||
|
'notnull' => false,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
$table->setPrimaryKey(['nid']);
|
$table->setPrimaryKey(['nid']);
|
||||||
$table->addUniqueIndex(['id_prim']);
|
$table->addUniqueIndex(['id_prim']);
|
||||||
|
|
|
@ -0,0 +1,59 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Nextcloud - Social Support
|
||||||
|
*
|
||||||
|
* This file is licensed under the Affero General Public License version 3 or
|
||||||
|
* later. See the COPYING file.
|
||||||
|
*
|
||||||
|
* @author Maxence Lange <maxence@artificial-owl.com>
|
||||||
|
* @copyright 2023, Maxence Lange <maxence@artificial-owl.com>
|
||||||
|
* @license GNU AGPL version 3 or any later version
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
namespace OCA\Social\Migration;
|
||||||
|
|
||||||
|
use Closure;
|
||||||
|
use OCP\DB\ISchemaWrapper;
|
||||||
|
use OCP\DB\Types;
|
||||||
|
use OCP\Migration\IOutput;
|
||||||
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
|
class Version1000Date20230407000001 extends SimpleMigrationStep {
|
||||||
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ISchemaWrapper {
|
||||||
|
/** @var ISchemaWrapper $schema */
|
||||||
|
$schema = $schemaClosure();
|
||||||
|
|
||||||
|
// fix nid as primary on social_cache_actor
|
||||||
|
if ($schema->hasTable('social_cache_actor')) {
|
||||||
|
$table = $schema->getTable('social_cache_actor');
|
||||||
|
|
||||||
|
if (!$table->hasColumn('details_update')) {
|
||||||
|
$table->addColumn(
|
||||||
|
'details_update', Types::DATETIME,
|
||||||
|
[
|
||||||
|
'notnull' => false
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $schema;
|
||||||
|
}
|
||||||
|
}
|
|
@ -76,6 +76,9 @@ class OrderedCollection extends ACore implements JsonSerializable {
|
||||||
|
|
||||||
public function import(array $data): self {
|
public function import(array $data): self {
|
||||||
parent::import($data);
|
parent::import($data);
|
||||||
|
$this->setFirst($this->validate(ACore::AS_USERNAME, 'first', $data, ''))
|
||||||
|
->setLast($this->validate(ACore::AS_USERNAME, 'last', $data, ''))
|
||||||
|
->setTotalItems($this->getInt('totalItems', $data));
|
||||||
|
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,6 +45,7 @@ use OCA\Social\Exceptions\RetrieveAccountFormatException;
|
||||||
use OCA\Social\Exceptions\SocialAppConfigException;
|
use OCA\Social\Exceptions\SocialAppConfigException;
|
||||||
use OCA\Social\Exceptions\UnauthorizedFediverseException;
|
use OCA\Social\Exceptions\UnauthorizedFediverseException;
|
||||||
use OCA\Social\Model\ActivityPub\Actor\Person;
|
use OCA\Social\Model\ActivityPub\Actor\Person;
|
||||||
|
use OCA\Social\Model\ActivityPub\OrderedCollection;
|
||||||
use OCA\Social\Model\Client\Options\ProbeOptions;
|
use OCA\Social\Model\Client\Options\ProbeOptions;
|
||||||
use OCA\Social\Tools\Exceptions\MalformedArrayException;
|
use OCA\Social\Tools\Exceptions\MalformedArrayException;
|
||||||
use OCA\Social\Tools\Exceptions\RequestContentException;
|
use OCA\Social\Tools\Exceptions\RequestContentException;
|
||||||
|
@ -65,7 +66,7 @@ use Psr\Log\LoggerInterface;
|
||||||
class CacheActorService {
|
class CacheActorService {
|
||||||
use TArrayTools;
|
use TArrayTools;
|
||||||
|
|
||||||
private \OCP\IURLGenerator $urlGenerator;
|
private IURLGenerator $urlGenerator;
|
||||||
private ActorsRequest $actorsRequest;
|
private ActorsRequest $actorsRequest;
|
||||||
private CacheActorsRequest $cacheActorsRequest;
|
private CacheActorsRequest $cacheActorsRequest;
|
||||||
private CurlService $curlService;
|
private CurlService $curlService;
|
||||||
|
@ -305,6 +306,68 @@ class CacheActorService {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return int
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public function manageDetailsRemoteActors(bool $force = false): int {
|
||||||
|
$update = $this->cacheActorsRequest->getRemoteActorsToUpdateDetails($force);
|
||||||
|
|
||||||
|
// WARNING: risk of race condition if something else update details on remote actor.
|
||||||
|
// Any details update on remote cache-actor must be managed from here.
|
||||||
|
foreach ($update as $item) {
|
||||||
|
try {
|
||||||
|
$this->addRemoteActorDetailCount($item);
|
||||||
|
$this->cacheActorsRequest->updateDetails($item);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sizeof($update);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public function addRemoteActorDetailCount(Person $actor): void {
|
||||||
|
try {
|
||||||
|
$followers = $this->getCollectionFromId($actor->getFollowers());
|
||||||
|
$following = $this->getCollectionFromId($actor->getFollowing());
|
||||||
|
$outbox = $this->getCollectionFromId($actor->getOutbox());
|
||||||
|
} catch (InvalidResourceException $e) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$count = [
|
||||||
|
'followers' => $followers->getTotalItems(),
|
||||||
|
'following' => $following->getTotalItems(),
|
||||||
|
'post' => $outbox->getTotalItems()
|
||||||
|
];
|
||||||
|
$actor->setDetailArray('count', $count);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $id
|
||||||
|
*
|
||||||
|
* @return OrderedCollection
|
||||||
|
* @throws InvalidResourceException
|
||||||
|
*/
|
||||||
|
private function getCollectionFromId(string $id): OrderedCollection {
|
||||||
|
try {
|
||||||
|
$object = $this->curlService->retrieveObject($id);
|
||||||
|
/** @var OrderedCollection $collection */
|
||||||
|
$collection = AP::$activityPub->getItemFromData($object);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
throw new InvalidResourceException();
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($collection->getType() !== OrderedCollection::TYPE) {
|
||||||
|
throw new InvalidResourceException();
|
||||||
|
}
|
||||||
|
|
||||||
|
return $collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param Person $actor
|
* @param Person $actor
|
||||||
*
|
*
|
||||||
|
@ -342,7 +405,7 @@ class CacheActorService {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public function getFromNids(array $ids):array {
|
public function getFromNids(array $ids): array {
|
||||||
return $this->cacheActorsRequest->getFromNids($ids);
|
return $this->cacheActorsRequest->getFromNids($ids);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Ładowanie…
Reference in New Issue