From 5196852ccaaec66fd17f822e12db9c7ac96c4b20 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Mon, 26 Nov 2018 11:33:48 -0100 Subject: [PATCH] managing Cache from Cron and Command line Signed-off-by: Maxence Lange --- appinfo/info.xml | 5 + lib/Command/CacheRefresh.php | 117 ++++++++++++++++++++ lib/Cron/Cache.php | 114 +++++++++++++++++++ lib/Db/ActorsRequest.php | 18 +++ lib/Db/CacheActorsRequest.php | 55 ++++----- lib/Db/CacheDocumentsRequest.php | 24 ++++ lib/Db/CoreRequestBuilder.php | 73 +++++++++++- lib/Model/ActivityPub/Document.php | 21 ++-- lib/Service/ActivityPub/DocumentService.php | 23 +++- lib/Service/ActivityPub/PersonService.php | 18 +++ lib/Service/ActorService.php | 30 +++-- 11 files changed, 447 insertions(+), 51 deletions(-) create mode 100644 lib/Command/CacheRefresh.php create mode 100644 lib/Cron/Cache.php diff --git a/appinfo/info.xml b/appinfo/info.xml index ba6f24f8..43b4e10d 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -26,7 +26,12 @@ + + OCA\Social\Cron\Cache + + + OCA\Social\Command\CacheRefresh OCA\Social\Command\NoteCreate diff --git a/lib/Command/CacheRefresh.php b/lib/Command/CacheRefresh.php new file mode 100644 index 00000000..fae76806 --- /dev/null +++ b/lib/Command/CacheRefresh.php @@ -0,0 +1,117 @@ + + * @copyright 2018, Maxence Lange + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\Social\Command; + + +use Exception; +use OC\Core\Command\Base; +use OCA\Social\Service\ActivityPub\DocumentService; +use OCA\Social\Service\ActivityPub\PersonService; +use OCA\Social\Service\ActorService; +use OCA\Social\Service\ConfigService; +use OCA\Social\Service\MiscService; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + + +class CacheRefresh extends Base { + + + /** @var ActorService */ + private $actorService; + + /** @var PersonService */ + private $personService; + + /** @var DocumentService */ + private $documentService; + + /** @var ConfigService */ + private $configService; + + /** @var MiscService */ + private $miscService; + + + /** + * CacheUpdate constructor. + * + * @param ActorService $actorService + * @param PersonService $personService + * @param DocumentService $documentService + * @param ConfigService $configService + * @param MiscService $miscService + */ + public function __construct( + ActorService $actorService, PersonService $personService, DocumentService $documentService, + ConfigService $configService, MiscService $miscService + ) { + parent::__construct(); + + $this->actorService = $actorService; + $this->personService = $personService; + $this->documentService = $documentService; + $this->configService = $configService; + $this->miscService = $miscService; + } + + + /** + * + */ + protected function configure() { + parent::configure(); + $this->setName('social:cache:refresh') + ->setDescription('Update the cache'); + } + + + /** + * @param InputInterface $input + * @param OutputInterface $output + * + * @throws Exception + */ + protected function execute(InputInterface $input, OutputInterface $output) { + + $result = $this->actorService->manageCacheLocalActors(); + $output->writeLn($result . ' local accounts regenerated'); + + $result = $this->personService->manageCacheRemoteActors(); + $output->writeLn($result . ' remote accounts updated'); + + $result = $this->documentService->manageCacheDocuments(); + $output->writeLn($result . ' documents cached'); + } + + +} + diff --git a/lib/Cron/Cache.php b/lib/Cron/Cache.php new file mode 100644 index 00000000..df9b741a --- /dev/null +++ b/lib/Cron/Cache.php @@ -0,0 +1,114 @@ + + * @copyright 2018, Maxence Lange + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\Social\Cron; + + +use Exception; +use OC\BackgroundJob\TimedJob; +use OCA\Social\AppInfo\Application; +use OCA\Social\Service\ActivityPub\DocumentService; +use OCA\Social\Service\ActivityPub\PersonService; +use OCA\Social\Service\ActorService; +use OCA\Social\Service\ConfigService; +use OCA\Social\Service\MiscService; +use OCP\AppFramework\QueryException; + + +/** + * Class Cache + * + * @package OCA\Social\Cron + */ +class Cache extends TimedJob { + + + /** @var ActorService */ + private $actorService; + + /** @var PersonService */ + private $personService; + + /** @var DocumentService */ + private $documentService; + + /** @var ConfigService */ + private $configService; + + /** @var MiscService */ + private $miscService; + + + /** + * Cache constructor. + */ + public function __construct() { + $this->setInterval(12 * 60); // 12 minutes + } + + + /** + * @param mixed $argument + * + * @throws QueryException + */ + protected function run($argument) { + $app = new Application(); + $c = $app->getContainer(); + + $this->actorService = $c->query(ActorService::class); + $this->personService = $c->query(PersonService::class); + $this->documentService = $c->query(DocumentService::class); + $this->configService = $c->query(ConfigService::class); + $this->miscService = $c->query(MiscService::class); + + $this->manageCache(); + } + + + private function manageCache() { + try { + $this->actorService->manageCacheLocalActors(); + } catch (Exception $e) { + } + + try { + $this->personService->manageCacheRemoteActors(); + } catch (Exception $e) { + } + + try { + $this->documentService->manageCacheDocuments(); + } catch (Exception $e) { + } + } + + +} diff --git a/lib/Db/ActorsRequest.php b/lib/Db/ActorsRequest.php index beec0ebc..09b37e4b 100644 --- a/lib/Db/ActorsRequest.php +++ b/lib/Db/ActorsRequest.php @@ -161,6 +161,24 @@ class ActorsRequest extends ActorsRequestBuilder { } + /** + * @return Person[] + * @throws SocialAppConfigException + */ + public function getAll(): array { + $qb = $this->getActorsSelectSql(); + + $accounts = []; + $cursor = $qb->execute(); + while ($data = $cursor->fetch()) { + $accounts[] = $this->parseActorsSelectSql($data); + } + $cursor->closeCursor(); + + return $accounts; + } + + /** * @param string $search * diff --git a/lib/Db/CacheActorsRequest.php b/lib/Db/CacheActorsRequest.php index 724abf2d..dd3d9489 100644 --- a/lib/Db/CacheActorsRequest.php +++ b/lib/Db/CacheActorsRequest.php @@ -30,15 +30,20 @@ declare(strict_types=1); namespace OCA\Social\Db; +use DateTime; +use Exception; use OCA\Social\Exceptions\CacheActorDoesNotExistException; use OCA\Social\Model\ActivityPub\Person; use OCA\Social\Service\ConfigService; use OCA\Social\Service\MiscService; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IDBConnection; class CacheActorsRequest extends CacheActorsRequestBuilder { + const CACHE_TTL = 60 * 24; // 1d + /** * CacheActorsRequest constructor. * @@ -78,7 +83,11 @@ class CacheActorsRequest extends CacheActorsRequestBuilder { ->setValue('name', $qb->createNamedParameter($actor->getName())) ->setValue('summary', $qb->createNamedParameter($actor->getSummary())) ->setValue('public_key', $qb->createNamedParameter($actor->getPublicKey())) - ->setValue('source', $qb->createNamedParameter($actor->getSource())); + ->setValue('source', $qb->createNamedParameter($actor->getSource())) + ->setValue( + 'creation', + $qb->createNamedParameter(new DateTime('now'), IQueryBuilder::PARAM_DATE) + ); if ($actor->gotIcon()) { $iconId = $actor->getIcon() @@ -92,30 +101,6 @@ class CacheActorsRequest extends CacheActorsRequestBuilder { } -// /** -// * get Cached value about an Actor, based on the account. -// * -// * @param string $account -// * -// * @return CacheActor -// * @throws CacheActorDoesNotExistException -// */ -// public function getFromAccount(string $account): CacheActor { -// $qb = $this->getCacheActorsSelectSql(); -// $this->limitToAccount($qb, $account); -// -// $cursor = $qb->execute(); -// $data = $cursor->fetch(); -// $cursor->closeCursor(); -// -// if ($data === false) { -// throw new CacheActorDoesNotExistException(); -// } -// -// return $this->parseCacheActorsSelectSql($data); -// } - - /** * get Cached version of an Actor, based on the UriId * @@ -187,6 +172,26 @@ class CacheActorsRequest extends CacheActorsRequestBuilder { } + /** + * @return Person[] + * @throws Exception + */ + public function getRemoteActorsToUpdate(): array { + $qb = $this->getCacheActorsSelectSql(); + $this->limitToLocal($qb, false); + $this->limitToCreation($qb, self::CACHE_TTL); + + $update = []; + $cursor = $qb->execute(); + while ($data = $cursor->fetch()) { + $update[] = $this->parseCacheActorsSelectSql($data); + } + $cursor->closeCursor(); + + return $update; + } + + /** * delete cached version of an Actor, based on the UriId * diff --git a/lib/Db/CacheDocumentsRequest.php b/lib/Db/CacheDocumentsRequest.php index 70484f39..7bba3f50 100644 --- a/lib/Db/CacheDocumentsRequest.php +++ b/lib/Db/CacheDocumentsRequest.php @@ -31,6 +31,7 @@ namespace OCA\Social\Db; use DateTime; +use Exception; use OCA\Social\Exceptions\CacheDocumentDoesNotExistException; use OCA\Social\Model\ActivityPub\Document; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -38,6 +39,9 @@ use OCP\DB\QueryBuilder\IQueryBuilder; class CacheDocumentsRequest extends CacheDocumentsRequestBuilder { + const CACHE_TTL = 15; // 15 min + + /** * insert cache about an Actor in database. * @@ -135,5 +139,25 @@ class CacheDocumentsRequest extends CacheDocumentsRequestBuilder { return $this->parseCacheDocumentsSelectSql($data); } + + /** + * @return Document[] + * @throws Exception + */ + public function getNotCachedDocuments() { + $qb = $this->getCacheDocumentsSelectSql(); + $this->limitToDBFieldEmpty($qb, 'local_copy'); + $this->limitToCaching($qb, self::CACHE_TTL); + + $documents = []; + $cursor = $qb->execute(); + while ($data = $cursor->fetch()) { + $documents[] = $this->parseCacheDocumentsSelectSql($data); + } + $cursor->closeCursor(); + + return $documents; + } + } diff --git a/lib/Db/CoreRequestBuilder.php b/lib/Db/CoreRequestBuilder.php index 2e598bc2..538d9751 100644 --- a/lib/Db/CoreRequestBuilder.php +++ b/lib/Db/CoreRequestBuilder.php @@ -31,7 +31,10 @@ declare(strict_types=1); namespace OCA\Social\Db; +use DateInterval; +use DateTime; use Doctrine\DBAL\Query\QueryBuilder; +use Exception; use OCA\Social\Exceptions\InvalidResourceException; use OCA\Social\Model\ActivityPub\Document; use OCA\Social\Model\ActivityPub\Image; @@ -212,6 +215,38 @@ class CoreRequestBuilder { } + /** + * Limit the request to the creation + * + * @param IQueryBuilder $qb + * @param int $delay + * + * @throws Exception + */ + protected function limitToCreation(IQueryBuilder &$qb, int $delay = 0) { + $date = new DateTime('now'); + $date->sub(new DateInterval('PT' . $delay . 'M')); + + $this->limitToDBFieldDateTime($qb, 'creation', $date); + } + + + /** + * Limit the request to the creation + * + * @param IQueryBuilder $qb + * @param int $delay + * + * @throws Exception + */ + protected function limitToCaching(IQueryBuilder &$qb, int $delay = 0) { + $date = new DateTime('now'); + $date->sub(new DateInterval('PT' . $delay . 'M')); + + $this->limitToDBFieldDateTime($qb, 'caching', $date); + } + + /** * Limit the request to the url * @@ -320,7 +355,7 @@ class CoreRequestBuilder { * @param bool $cs - case sensitive * @param string $alias */ - private function limitToDBField( + protected function limitToDBField( IQueryBuilder &$qb, string $field, string $value, bool $cs = true, string $alias = '' ) { $expr = $qb->expr(); @@ -347,7 +382,7 @@ class CoreRequestBuilder { * @param string $field * @param int $value */ - private function limitToDBFieldInt(IQueryBuilder &$qb, string $field, int $value) { + protected function limitToDBFieldInt(IQueryBuilder &$qb, string $field, int $value) { $expr = $qb->expr(); $pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : ''; $field = $pf . $field; @@ -356,12 +391,42 @@ class CoreRequestBuilder { } + /** + * @param IQueryBuilder $qb + * @param string $field + */ + protected function limitToDBFieldEmpty(IQueryBuilder &$qb, string $field) { + $expr = $qb->expr(); + $pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : ''; + $field = $pf . $field; + + $qb->andWhere($expr->eq($field, $qb->createNamedParameter(''))); + } + + + /** + * @param IQueryBuilder $qb + * @param string $field + * @param DateTime $date + */ + protected function limitToDBFieldDateTime(IQueryBuilder &$qb, string $field, DateTime $date) { + $expr = $qb->expr(); + $pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : ''; + $field = $pf . $field; + + $orX = $expr->orX(); + $orX->add($expr->lte($field, $qb->createNamedParameter($date, IQueryBuilder::PARAM_DATE))); + $orX->add($expr->isNull($field)); + $qb->andWhere($orX); + } + + /** * @param IQueryBuilder $qb * @param string $field * @param array $values */ - private function limitToDBFieldArray(IQueryBuilder &$qb, string $field, array $values) { + protected function limitToDBFieldArray(IQueryBuilder &$qb, string $field, array $values) { $expr = $qb->expr(); $pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : ''; $field = $pf . $field; @@ -384,7 +449,7 @@ class CoreRequestBuilder { * @param string $field * @param string $value */ - private function searchInDBField(IQueryBuilder &$qb, string $field, string $value) { + protected function searchInDBField(IQueryBuilder &$qb, string $field, string $value) { $expr = $qb->expr(); $pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : ''; diff --git a/lib/Model/ActivityPub/Document.php b/lib/Model/ActivityPub/Document.php index 040d68d5..c7da00f8 100644 --- a/lib/Model/ActivityPub/Document.php +++ b/lib/Model/ActivityPub/Document.php @@ -31,6 +31,7 @@ declare(strict_types=1); namespace OCA\Social\Model\ActivityPub; +use DateTime; use JsonSerializable; use OCA\Social\Exceptions\UrlCloudException; @@ -55,8 +56,8 @@ class Document extends ACore implements JsonSerializable { /** @var string */ private $localCopy = ''; - /** @var string */ - private $caching = ''; + /** @var int */ + private $caching = 0; /** @var bool */ private $public = false; @@ -173,18 +174,18 @@ class Document extends ACore implements JsonSerializable { /** - * @return string + * @return int */ - public function getCaching(): string { + public function getCaching(): int { return $this->caching; } /** - * @param string $caching + * @param int $caching * * @return Document */ - public function setCaching(string $caching): Document { + public function setCaching(int $caching): Document { $this->caching = $caching; return $this; @@ -218,7 +219,13 @@ class Document extends ACore implements JsonSerializable { $this->setLocalCopy($this->get('local_copy', $data, '')); $this->setMediaType($this->get('media_type', $data, '')); $this->setMimeType($this->get('mime_type', $data, '')); - $this->setCaching($this->get('caching', $data, '')); + + if ($this->get('caching', $data, '') === '') { + $this->setCaching(0); + } else { + $date = new DateTime($this->get('caching', $data, '')); + $this->setCaching($date->getTimestamp()); + } } /** diff --git a/lib/Service/ActivityPub/DocumentService.php b/lib/Service/ActivityPub/DocumentService.php index 94e4c085..8f4f76c4 100644 --- a/lib/Service/ActivityPub/DocumentService.php +++ b/lib/Service/ActivityPub/DocumentService.php @@ -31,6 +31,7 @@ declare(strict_types=1); namespace OCA\Social\Service\ActivityPub; +use Exception; use OCA\Social\Db\CacheDocumentsRequest; use OCA\Social\Exceptions\CacheContentException; use OCA\Social\Exceptions\CacheContentSizeException; @@ -87,9 +88,8 @@ class DocumentService implements ICoreService { return $document; } - // TODO - check the size of the attachment, also to stop download after a certain size of content. - // TODO - ignore this is getCaching is older than 15 minutes - if ($document->getCaching() !== '') { + // TODO - ignore this if getCaching is older than 15 minutes + if ($document->getCaching() > (time() - (CacheDocumentsRequest::CACHE_TTL * 60))) { return $document; } @@ -125,6 +125,23 @@ class DocumentService implements ICoreService { } + /** + * @return int + * @throws CacheDocumentDoesNotExistException + * @throws NotPermittedException + * @throws Exception + */ + public function manageCacheDocuments(): int { + $update = $this->cacheDocumentsRequest->getNotCachedDocuments(); + + foreach ($update as $item) { + $this->cacheRemoteDocument($item->getId()); + } + + return sizeof($update); + } + + /** * @param ACore $item */ diff --git a/lib/Service/ActivityPub/PersonService.php b/lib/Service/ActivityPub/PersonService.php index a657661a..432ab2a9 100644 --- a/lib/Service/ActivityPub/PersonService.php +++ b/lib/Service/ActivityPub/PersonService.php @@ -263,6 +263,24 @@ class PersonService implements ICoreService { } + /** + * @throws Exception + * @return int + */ + public function manageCacheRemoteActors(): int { + $update = $this->cacheActorsRequest->getRemoteActorsToUpdate(); + + foreach ($update as $item) { + try { + $this->getFromId($item->getId(), true); + } catch (Exception $e) { + } + } + + return sizeof($update); + } + + /** * @param ACore $item */ diff --git a/lib/Service/ActorService.php b/lib/Service/ActorService.php index 25758d89..348d1a1f 100644 --- a/lib/Service/ActorService.php +++ b/lib/Service/ActorService.php @@ -128,18 +128,6 @@ class ActorService { } - /** - * @param string $search - * - * @deprecated - used !? - * @return Person[] - * @throws SocialAppConfigException - */ - public function searchLocalAccounts(string $search): array { - return $this->actorsRequest->searchFromUsername($search); - } - - /** * Method should be called by the frontend and will generate a fresh Social account for * the user, using the userId and the username. @@ -234,4 +222,22 @@ class ActorService { } + /** + * @throws Exception + * @return int + */ + public function manageCacheLocalActors(): int { + $update = $this->actorsRequest->getAll(); + + foreach ($update as $item) { + try { + $this->cacheLocalActorByUsername($item->getPreferredUsername(), true); + } catch (Exception $e) { + } + } + + return sizeof($update); + } + + }