managing Cache from Cron and Command line

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

appinfo/info.xml

@@ -26,7 +26,12 @@
 		</navigation>
 	</navigations>
 
+	<background-jobs>
+		<job>OCA\Social\Cron\Cache</job>
+	</background-jobs>
+
 	<commands>
+		<command>OCA\Social\Command\CacheRefresh</command>
 		<command>OCA\Social\Command\NoteCreate</command>
 	</commands>
 

lib/Command/CacheRefresh.php

@@ -0,0 +1,117 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * Nextcloud - Social Support
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018, Maxence Lange <maxence@artificial-owl.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\Social\Command;
+
+
+use Exception;
+use OC\Core\Command\Base;
+use OCA\Social\Service\ActivityPub\DocumentService;
+use OCA\Social\Service\ActivityPub\PersonService;
+use OCA\Social\Service\ActorService;
+use OCA\Social\Service\ConfigService;
+use OCA\Social\Service\MiscService;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+
+class CacheRefresh extends Base {
+
+
+	/** @var ActorService */
+	private $actorService;
+
+	/** @var PersonService */
+	private $personService;
+
+	/** @var DocumentService */
+	private $documentService;
+
+	/** @var ConfigService */
+	private $configService;
+
+	/** @var MiscService */
+	private $miscService;
+
+
+	/**
+	 * CacheUpdate constructor.
+	 *
+	 * @param ActorService $actorService
+	 * @param PersonService $personService
+	 * @param DocumentService $documentService
+	 * @param ConfigService $configService
+	 * @param MiscService $miscService
+	 */
+	public function __construct(
+		ActorService $actorService, PersonService $personService, DocumentService $documentService,
+		ConfigService $configService, MiscService $miscService
+	) {
+		parent::__construct();
+
+		$this->actorService = $actorService;
+		$this->personService = $personService;
+		$this->documentService = $documentService;
+		$this->configService = $configService;
+		$this->miscService = $miscService;
+	}
+
+
+	/**
+	 *
+	 */
+	protected function configure() {
+		parent::configure();
+		$this->setName('social:cache:refresh')
+			 ->setDescription('Update the cache');
+	}
+
+
+	/**
+	 * @param InputInterface $input
+	 * @param OutputInterface $output
+	 *
+	 * @throws Exception
+	 */
+	protected function execute(InputInterface $input, OutputInterface $output) {
+
+		$result = $this->actorService->manageCacheLocalActors();
+		$output->writeLn($result . ' local accounts regenerated');
+
+		$result = $this->personService->manageCacheRemoteActors();
+		$output->writeLn($result . ' remote accounts updated');
+
+		$result = $this->documentService->manageCacheDocuments();
+		$output->writeLn($result . ' documents cached');
+	}
+
+
+}
+

lib/Cron/Cache.php

@@ -0,0 +1,114 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * Nextcloud - Social Support
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018, Maxence Lange <maxence@artificial-owl.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\Social\Cron;
+
+
+use Exception;
+use OC\BackgroundJob\TimedJob;
+use OCA\Social\AppInfo\Application;
+use OCA\Social\Service\ActivityPub\DocumentService;
+use OCA\Social\Service\ActivityPub\PersonService;
+use OCA\Social\Service\ActorService;
+use OCA\Social\Service\ConfigService;
+use OCA\Social\Service\MiscService;
+use OCP\AppFramework\QueryException;
+
+
+/**
+ * Class Cache
+ *
+ * @package OCA\Social\Cron
+ */
+class Cache extends TimedJob {
+
+
+	/** @var ActorService */
+	private $actorService;
+
+	/** @var PersonService */
+	private $personService;
+
+	/** @var DocumentService */
+	private $documentService;
+
+	/** @var ConfigService */
+	private $configService;
+
+	/** @var MiscService */
+	private $miscService;
+
+
+	/**
+	 * Cache constructor.
+	 */
+	public function __construct() {
+		$this->setInterval(12 * 60); // 12 minutes
+	}
+
+
+	/**
+	 * @param mixed $argument
+	 *
+	 * @throws QueryException
+	 */
+	protected function run($argument) {
+		$app = new Application();
+		$c = $app->getContainer();
+
+		$this->actorService = $c->query(ActorService::class);
+		$this->personService = $c->query(PersonService::class);
+		$this->documentService = $c->query(DocumentService::class);
+		$this->configService = $c->query(ConfigService::class);
+		$this->miscService = $c->query(MiscService::class);
+
+		$this->manageCache();
+	}
+
+
+	private function manageCache() {
+		try {
+			$this->actorService->manageCacheLocalActors();
+		} catch (Exception $e) {
+		}
+
+		try {
+			$this->personService->manageCacheRemoteActors();
+		} catch (Exception $e) {
+		}
+
+		try {
+			$this->documentService->manageCacheDocuments();
+		} catch (Exception $e) {
+		}
+	}
+
+
+}

lib/Db/ActorsRequest.php

@@ -161,6 +161,24 @@ class ActorsRequest extends ActorsRequestBuilder {
 	}
 
 
+	/**
+	 * @return Person[]
+	 * @throws SocialAppConfigException
+	 */
+	public function getAll(): array {
+		$qb = $this->getActorsSelectSql();
+
+		$accounts = [];
+		$cursor = $qb->execute();
+		while ($data = $cursor->fetch()) {
+			$accounts[] = $this->parseActorsSelectSql($data);
+		}
+		$cursor->closeCursor();
+
+		return $accounts;
+	}
+
+
 	/**
 	 * @param string $search
 	 *

lib/Db/CacheActorsRequest.php

@@ -30,15 +30,20 @@ declare(strict_types=1);
 namespace OCA\Social\Db;
 
 
+use DateTime;
+use Exception;
 use OCA\Social\Exceptions\CacheActorDoesNotExistException;
 use OCA\Social\Model\ActivityPub\Person;
 use OCA\Social\Service\ConfigService;
 use OCA\Social\Service\MiscService;
+use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\IDBConnection;
 
 class CacheActorsRequest extends CacheActorsRequestBuilder {
 
 
+	const CACHE_TTL = 60 * 24; // 1d
+
 	/**
 	 * CacheActorsRequest constructor.
 	 *
@@ -78,7 +83,11 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
 		   ->setValue('name', $qb->createNamedParameter($actor->getName()))
 		   ->setValue('summary', $qb->createNamedParameter($actor->getSummary()))
 		   ->setValue('public_key', $qb->createNamedParameter($actor->getPublicKey()))
-		   ->setValue('source', $qb->createNamedParameter($actor->getSource()));
+		   ->setValue('source', $qb->createNamedParameter($actor->getSource()))
+		   ->setValue(
+			   'creation',
+			   $qb->createNamedParameter(new DateTime('now'), IQueryBuilder::PARAM_DATE)
+		   );
 
 		if ($actor->gotIcon()) {
 			$iconId = $actor->getIcon()
@@ -92,30 +101,6 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
 	}
 
 
-//	/**
-//	 * get Cached value about an Actor, based on the account.
-//	 *
-//	 * @param string $account
-//	 *
-//	 * @return CacheActor
-//	 * @throws CacheActorDoesNotExistException
-//	 */
-//	public function getFromAccount(string $account): CacheActor {
-//		$qb = $this->getCacheActorsSelectSql();
-//		$this->limitToAccount($qb, $account);
-//
-//		$cursor = $qb->execute();
-//		$data = $cursor->fetch();
-//		$cursor->closeCursor();
-//
-//		if ($data === false) {
-//			throw new CacheActorDoesNotExistException();
-//		}
-//
-//		return $this->parseCacheActorsSelectSql($data);
-//	}
-
-
 	/**
 	 * get Cached version of an Actor, based on the UriId
 	 *
@@ -187,6 +172,26 @@ class CacheActorsRequest extends CacheActorsRequestBuilder {
 	}
 
 
+	/**
+	 * @return Person[]
+	 * @throws Exception
+	 */
+	public function getRemoteActorsToUpdate(): array {
+		$qb = $this->getCacheActorsSelectSql();
+		$this->limitToLocal($qb, false);
+		$this->limitToCreation($qb, self::CACHE_TTL);
+
+		$update = [];
+		$cursor = $qb->execute();
+		while ($data = $cursor->fetch()) {
+			$update[] = $this->parseCacheActorsSelectSql($data);
+		}
+		$cursor->closeCursor();
+
+		return $update;
+	}
+
+
 	/**
 	 * delete cached version of an Actor, based on the UriId
 	 *

lib/Db/CacheDocumentsRequest.php

@@ -31,6 +31,7 @@ namespace OCA\Social\Db;
 
 
 use DateTime;
+use Exception;
 use OCA\Social\Exceptions\CacheDocumentDoesNotExistException;
 use OCA\Social\Model\ActivityPub\Document;
 use OCP\DB\QueryBuilder\IQueryBuilder;
@@ -38,6 +39,9 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
 class CacheDocumentsRequest extends CacheDocumentsRequestBuilder {
 
 
+	const CACHE_TTL = 15; // 15 min
+
+
 	/**
 	 * insert cache about an Actor in database.
 	 *
@@ -135,5 +139,25 @@ class CacheDocumentsRequest extends CacheDocumentsRequestBuilder {
 		return $this->parseCacheDocumentsSelectSql($data);
 	}
 
+
+	/**
+	 * @return Document[]
+	 * @throws Exception
+	 */
+	public function getNotCachedDocuments() {
+		$qb = $this->getCacheDocumentsSelectSql();
+		$this->limitToDBFieldEmpty($qb, 'local_copy');
+		$this->limitToCaching($qb, self::CACHE_TTL);
+
+		$documents = [];
+		$cursor = $qb->execute();
+		while ($data = $cursor->fetch()) {
+			$documents[] = $this->parseCacheDocumentsSelectSql($data);
+		}
+		$cursor->closeCursor();
+
+		return $documents;
+	}
+
 }
 

lib/Db/CoreRequestBuilder.php

@@ -31,7 +31,10 @@ declare(strict_types=1);
 namespace OCA\Social\Db;
 
 
+use DateInterval;
+use DateTime;
 use Doctrine\DBAL\Query\QueryBuilder;
+use Exception;
 use OCA\Social\Exceptions\InvalidResourceException;
 use OCA\Social\Model\ActivityPub\Document;
 use OCA\Social\Model\ActivityPub\Image;
@@ -212,6 +215,38 @@ class CoreRequestBuilder {
 	}
 
 
+	/**
+	 * Limit the request to the creation
+	 *
+	 * @param IQueryBuilder $qb
+	 * @param int $delay
+	 *
+	 * @throws Exception
+	 */
+	protected function limitToCreation(IQueryBuilder &$qb, int $delay = 0) {
+		$date = new DateTime('now');
+		$date->sub(new DateInterval('PT' . $delay . 'M'));
+
+		$this->limitToDBFieldDateTime($qb, 'creation', $date);
+	}
+
+
+	/**
+	 * Limit the request to the creation
+	 *
+	 * @param IQueryBuilder $qb
+	 * @param int $delay
+	 *
+	 * @throws Exception
+	 */
+	protected function limitToCaching(IQueryBuilder &$qb, int $delay = 0) {
+		$date = new DateTime('now');
+		$date->sub(new DateInterval('PT' . $delay . 'M'));
+
+		$this->limitToDBFieldDateTime($qb, 'caching', $date);
+	}
+
+
 	/**
 	 * Limit the request to the url
 	 *
@@ -320,7 +355,7 @@ class CoreRequestBuilder {
 	 * @param bool $cs - case sensitive
 	 * @param string $alias
 	 */
-	private function limitToDBField(
+	protected function limitToDBField(
 		IQueryBuilder &$qb, string $field, string $value, bool $cs = true, string $alias = ''
 	) {
 		$expr = $qb->expr();
@@ -347,7 +382,7 @@ class CoreRequestBuilder {
 	 * @param string $field
 	 * @param int $value
 	 */
-	private function limitToDBFieldInt(IQueryBuilder &$qb, string $field, int $value) {
+	protected function limitToDBFieldInt(IQueryBuilder &$qb, string $field, int $value) {
 		$expr = $qb->expr();
 		$pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : '';
 		$field = $pf . $field;
@@ -356,12 +391,42 @@ class CoreRequestBuilder {
 	}
 
 
+	/**
+	 * @param IQueryBuilder $qb
+	 * @param string $field
+	 */
+	protected function limitToDBFieldEmpty(IQueryBuilder &$qb, string $field) {
+		$expr = $qb->expr();
+		$pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : '';
+		$field = $pf . $field;
+
+		$qb->andWhere($expr->eq($field, $qb->createNamedParameter('')));
+	}
+
+
+	/**
+	 * @param IQueryBuilder $qb
+	 * @param string $field
+	 * @param DateTime $date
+	 */
+	protected function limitToDBFieldDateTime(IQueryBuilder &$qb, string $field, DateTime $date) {
+		$expr = $qb->expr();
+		$pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : '';
+		$field = $pf . $field;
+
+		$orX = $expr->orX();
+		$orX->add($expr->lte($field, $qb->createNamedParameter($date, IQueryBuilder::PARAM_DATE)));
+		$orX->add($expr->isNull($field));
+		$qb->andWhere($orX);
+	}
+
+
 	/**
 	 * @param IQueryBuilder $qb
 	 * @param string $field
 	 * @param array $values
 	 */
-	private function limitToDBFieldArray(IQueryBuilder &$qb, string $field, array $values) {
+	protected function limitToDBFieldArray(IQueryBuilder &$qb, string $field, array $values) {
 		$expr = $qb->expr();
 		$pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : '';
 		$field = $pf . $field;
@@ -384,7 +449,7 @@ class CoreRequestBuilder {
 	 * @param string $field
 	 * @param string $value
 	 */
-	private function searchInDBField(IQueryBuilder &$qb, string $field, string $value) {
+	protected function searchInDBField(IQueryBuilder &$qb, string $field, string $value) {
 		$expr = $qb->expr();
 
 		$pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : '';

lib/Model/ActivityPub/Document.php

@@ -31,6 +31,7 @@ declare(strict_types=1);
 namespace OCA\Social\Model\ActivityPub;
 
 
+use DateTime;
 use JsonSerializable;
 use OCA\Social\Exceptions\UrlCloudException;
 
@@ -55,8 +56,8 @@ class Document extends ACore implements JsonSerializable {
 	/** @var string */
 	private $localCopy = '';
 
-	/** @var string */
-	private $caching = '';
+	/** @var int */
+	private $caching = 0;
 
 	/** @var bool */
 	private $public = false;
@@ -173,18 +174,18 @@ class Document extends ACore implements JsonSerializable {
 
 
 	/**
-	 * @return string
+	 * @return int
 	 */
-	public function getCaching(): string {
+	public function getCaching(): int {
 		return $this->caching;
 	}
 
 	/**
-	 * @param string $caching
+	 * @param int $caching
 	 *
 	 * @return Document
 	 */
-	public function setCaching(string $caching): Document {
+	public function setCaching(int $caching): Document {
 		$this->caching = $caching;
 
 		return $this;
@@ -218,7 +219,13 @@ class Document extends ACore implements JsonSerializable {
 		$this->setLocalCopy($this->get('local_copy', $data, ''));
 		$this->setMediaType($this->get('media_type', $data, ''));
 		$this->setMimeType($this->get('mime_type', $data, ''));
-		$this->setCaching($this->get('caching', $data, ''));
+
+		if ($this->get('caching', $data, '') === '') {
+			$this->setCaching(0);
+		} else {
+			$date = new DateTime($this->get('caching', $data, ''));
+			$this->setCaching($date->getTimestamp());
+		}
 	}
 
 	/**

lib/Service/ActivityPub/DocumentService.php

@@ -31,6 +31,7 @@ declare(strict_types=1);
 namespace OCA\Social\Service\ActivityPub;
 
 
+use Exception;
 use OCA\Social\Db\CacheDocumentsRequest;
 use OCA\Social\Exceptions\CacheContentException;
 use OCA\Social\Exceptions\CacheContentSizeException;
@@ -87,9 +88,8 @@ class DocumentService implements ICoreService {
 			return $document;
 		}
 
-		// TODO - check the size of the attachment, also to stop download after a certain size of content.
-		// TODO - ignore this is getCaching is older than 15 minutes
-		if ($document->getCaching() !== '') {
+		// TODO - ignore this if getCaching is older than 15 minutes
+		if ($document->getCaching() > (time() - (CacheDocumentsRequest::CACHE_TTL * 60))) {
 			return $document;
 		}
 
@@ -125,6 +125,23 @@ class DocumentService implements ICoreService {
 	}
 
 
+	/**
+	 * @return int
+	 * @throws CacheDocumentDoesNotExistException
+	 * @throws NotPermittedException
+	 * @throws Exception
+	 */
+	public function manageCacheDocuments(): int {
+		$update = $this->cacheDocumentsRequest->getNotCachedDocuments();
+
+		foreach ($update as $item) {
+			$this->cacheRemoteDocument($item->getId());
+		}
+
+		return sizeof($update);
+	}
+
+
 	/**
 	 * @param ACore $item
 	 */

lib/Service/ActivityPub/PersonService.php

@@ -263,6 +263,24 @@ class PersonService implements ICoreService {
 	}
 
 
+	/**
+	 * @throws Exception
+	 * @return int
+	 */
+	public function manageCacheRemoteActors(): int {
+		$update = $this->cacheActorsRequest->getRemoteActorsToUpdate();
+
+		foreach ($update as $item) {
+			try {
+				$this->getFromId($item->getId(), true);
+			} catch (Exception $e) {
+			}
+		}
+
+		return sizeof($update);
+	}
+
+
 	/**
 	 * @param ACore $item
 	 */

lib/Service/ActorService.php

@@ -128,18 +128,6 @@ class ActorService {
 	}
 
 
-	/**
-	 * @param string $search
-	 *
-	 * @deprecated - used !?
-	 * @return Person[]
-	 * @throws SocialAppConfigException
-	 */
-	public function searchLocalAccounts(string $search): array {
-		return $this->actorsRequest->searchFromUsername($search);
-	}
-
-
 	/**
 	 * Method should be called by the frontend and will generate a fresh Social account for
 	 * the user, using the userId and the username.
@@ -234,4 +222,22 @@ class ActorService {
 	}
 
 
+	/**
+	 * @throws Exception
+	 * @return int
+	 */
+	public function manageCacheLocalActors(): int {
+		$update = $this->actorsRequest->getAll();
+
+		foreach ($update as $item) {
+			try {
+				$this->cacheLocalActorByUsername($item->getPreferredUsername(), true);
+			} catch (Exception $e) {
+			}
+		}
+
+		return sizeof($update);
+	}
+
+
 }