fill the server_hashtags with hashtags and trends

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

lib/Command/CacheRefresh.php

@@ -37,6 +37,7 @@ use OCA\Social\Service\AccountService;
 use OCA\Social\Service\CacheActorService;
 use OCA\Social\Service\ConfigService;
 use OCA\Social\Service\DocumentService;
+use OCA\Social\Service\HashtagService;
 use OCA\Social\Service\MiscService;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -54,6 +55,9 @@ class CacheRefresh extends Base {
 	/** @var DocumentService */
 	private $documentService;
 
+	/** @var HashtagService */
+	private $hashtagService;
+
 	/** @var ConfigService */
 	private $configService;
 
@@ -67,18 +71,21 @@ class CacheRefresh extends Base {
 	 * @param AccountService $accountService
 	 * @param CacheActorService $cacheActorService
 	 * @param DocumentService $documentService
+	 * @param HashtagService $hashtagService
 	 * @param ConfigService $configService
 	 * @param MiscService $miscService
 	 */
 	public function __construct(
-		AccountService $accountService, CacheActorService $cacheActorService,
-		DocumentService $documentService, ConfigService $configService, MiscService $miscService
+		AccountService $actorService, CacheActorService $cacheActorService,
+		DocumentService $documentService, HashtagService $hashtagService,
+		ConfigService $configService, MiscService $miscService
 	) {
 		parent::__construct();
 
 		$this->accountService = $accountService;
 		$this->cacheActorService = $cacheActorService;
 		$this->documentService = $documentService;
+		$this->hashtagService = $hashtagService;
 		$this->configService = $configService;
 		$this->miscService = $miscService;
 	}
@@ -116,6 +123,9 @@ class CacheRefresh extends Base {
 
 		$result = $this->documentService->manageCacheDocuments();
 		$output->writeLn($result . ' documents cached');
+
+		$result = $this->hashtagService->manageHashtags();
+		$output->writeLn($result . ' hashtags updated');
 	}
 
 

lib/Cron/Cache.php

@@ -38,6 +38,7 @@ use OCA\Social\Service\AccountService;
 use OCA\Social\Service\CacheActorService;
 use OCA\Social\Service\ConfigService;
 use OCA\Social\Service\DocumentService;
+use OCA\Social\Service\HashtagService;
 use OCA\Social\Service\MiscService;
 use OCP\AppFramework\QueryException;
 
@@ -59,6 +60,9 @@ class Cache extends TimedJob {
 	/** @var DocumentService */
 	private $documentService;
 
+	/** @var HashtagService */
+	private $hashtagService;
+
 	/** @var ConfigService */
 	private $configService;
 
@@ -86,6 +90,7 @@ class Cache extends TimedJob {
 		$this->accountService = $c->query(AccountService::class);
 		$this->cacheActorService = $c->query(CacheActorService::class);
 		$this->documentService = $c->query(DocumentService::class);
+		$this->hashtagService = $c->query(HashtagService::class);
 		$this->configService = $c->query(ConfigService::class);
 		$this->miscService = $c->query(MiscService::class);
 
@@ -113,6 +118,11 @@ class Cache extends TimedJob {
 			$this->documentService->manageCacheDocuments();
 		} catch (Exception $e) {
 		}
+
+		try {
+			$this->hashtagService->manageHashtags();
+		} catch (Exception $e) {
+		}
 	}
 
 

lib/Db/CoreRequestBuilder.php

@@ -191,6 +191,17 @@ class CoreRequestBuilder {
 	}
 
 
+	/**
+	 * Limit the request to the ActorId
+	 *
+	 * @param IQueryBuilder $qb
+	 * @param string $hashtag
+	 */
+	protected function limitToHashtag(IQueryBuilder &$qb, string $hashtag) {
+		$this->limitToDBField($qb, 'hashtag', $hashtag);
+	}
+
+
 	/**
 	 * Limit the request to the ActorId
 	 *
@@ -484,6 +495,27 @@ class CoreRequestBuilder {
 	}
 
 
+	/**
+	 * @param IQueryBuilder $qb
+	 * @param int $timestamp
+	 * @param string $field
+	 */
+	protected function limitToSince(IQueryBuilder $qb, int $timestamp, string $field) {
+		$dTime = new \DateTime();
+		$dTime->setTimestamp($timestamp);
+
+		$expr = $qb->expr();
+		$pf = ($qb->getType() === QueryBuilder::SELECT) ? $this->defaultSelectAlias . '.' : '';
+		$field = $pf . $field;
+
+		$orX = $expr->orX();
+		$orX->add($expr->gte($field, $qb->createNamedParameter($dTime, IQueryBuilder::PARAM_DATE)));
+
+		$qb->andWhere($orX);
+
+	}
+
+
 	/**
 	 * @param IQueryBuilder $qb
 	 * @param string $field

lib/Db/HashtagsRequest.php

@@ -0,0 +1,100 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * Nextcloud - Social Support
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018, Maxence Lange <maxence@artificial-owl.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\Social\Db;
+
+
+use daita\MySmallPhpTools\Traits\TArrayTools;
+use DateTime;
+use OCA\Social\Exceptions\FollowDoesNotExistException;
+use OCA\Social\Model\ActivityPub\Activity\Follow;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+
+
+/**
+ * Class HashtagsRequest
+ *
+ * @package OCA\Social\Db
+ */
+class HashtagsRequest extends HashtagsRequestBuilder {
+
+
+	use TArrayTools;
+
+
+	/**
+	 * Insert a new Hashtag.
+	 *
+	 * @param string $hashtag
+	 * @param array $trend
+	 */
+	public function save(string $hashtag, array $trend) {
+		$qb = $this->getHashtagsInsertSql();
+		$qb->setValue('hashtag', $qb->createNamedParameter($hashtag))
+		   ->setValue('trend', $qb->createNamedParameter(json_encode($trend)));
+
+		$qb->execute();
+	}
+
+
+	/**
+	 * Insert a new Hashtag.
+	 *
+	 * @param string $hashtag
+	 * @param array $trend
+	 */
+	public function update(string $hashtag, array $trend) {
+		$qb = $this->getHashtagsUpdateSql();
+		$qb->set('trend', $qb->createNamedParameter(json_encode($trend)));
+		$this->limitToHashtag($qb, $hashtag);
+
+		$qb->execute();
+	}
+
+
+	/**
+	 * @return array
+	 */
+	public function getAll(): array {
+		$qb = $this->getHashtagsSelectSql();
+
+		$hashtags = [];
+		$cursor = $qb->execute();
+		while ($data = $cursor->fetch()) {
+			$hashtags[] = $this->parseHashtagsSelectSql($data);
+		}
+		$cursor->closeCursor();
+
+		return $hashtags;
+	}
+
+
+}
+

lib/Db/HashtagsRequestBuilder.php

@@ -0,0 +1,121 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * Nextcloud - Social Support
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018, Maxence Lange <maxence@artificial-owl.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\Social\Db;
+
+
+use daita\MySmallPhpTools\Traits\TArrayTools;
+use OCA\Social\Exceptions\InvalidResourceException;
+use OCA\Social\Model\ActivityPub\Activity\Follow;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+
+
+/**
+ * Class HashtagsRequestBuilder
+ *
+ * @package OCA\Social\Db
+ */
+class HashtagsRequestBuilder extends CoreRequestBuilder {
+
+
+	use TArrayTools;
+
+
+	/**
+	 * Base of the Sql Insert request
+	 *
+	 * @return IQueryBuilder
+	 */
+	protected function getHashtagsInsertSql(): IQueryBuilder {
+		$qb = $this->dbConnection->getQueryBuilder();
+		$qb->insert(self::TABLE_SERVER_HASHTAGS);
+
+		return $qb;
+	}
+
+
+	/**
+	 * Base of the Sql Update request
+	 *
+	 * @return IQueryBuilder
+	 */
+	protected function getHashtagsUpdateSql(): IQueryBuilder {
+		$qb = $this->dbConnection->getQueryBuilder();
+		$qb->update(self::TABLE_SERVER_HASHTAGS);
+
+		return $qb;
+	}
+
+
+	/**
+	 * Base of the Sql Select request for Shares
+	 *
+	 * @return IQueryBuilder
+	 */
+	protected function getHashtagsSelectSql(): IQueryBuilder {
+		$qb = $this->dbConnection->getQueryBuilder();
+
+		/** @noinspection PhpMethodParametersCountMismatchInspection */
+		$qb->select('h.hashtag', 'h.trend')
+		   ->from(self::TABLE_SERVER_HASHTAGS, 'h');
+
+		$this->defaultSelectAlias = 'h';
+
+		return $qb;
+	}
+
+
+	/**
+	 * Base of the Sql Delete request
+	 *
+	 * @return IQueryBuilder
+	 */
+	protected function getHashtagsDeleteSql(): IQueryBuilder {
+		$qb = $this->dbConnection->getQueryBuilder();
+		$qb->delete(self::TABLE_SERVER_HASHTAGS);
+
+		return $qb;
+	}
+
+
+	/**
+	 * @param array $data
+	 *
+	 * @return array
+	 */
+	protected function parseHashtagsSelectSql($data): array {
+		return [
+			'hashtag' => $this->get('hashtag', $data, ''),
+			'trend'   => $this->getArray('trend', $data, [])
+		];
+	}
+
+}
+

lib/Db/NotesRequest.php

@@ -340,6 +340,26 @@ class NotesRequest extends NotesRequestBuilder {
 	}
 
 
+	/**
+	 * @param int $since
+	 *
+	 * @return Note[]
+	 */
+	public function getNotesSince(int $since): array {
+		$qb = $this->getNotesSelectSql();
+		$this->limitToSince($qb, $since, 'published_time');
+
+		$notes = [];
+		$cursor = $qb->execute();
+		while ($data = $cursor->fetch()) {
+			$notes[] = $this->parseNotesSelectSql($data);
+		}
+		$cursor->closeCursor();
+
+		return $notes;
+	}
+
+
 	/**
 	 * @param string $id
 	 */

lib/Exceptions/HashtagDoesNotExistException.php

@@ -0,0 +1,8 @@
+<?php
+
+namespace OCA\Social\Exceptions;
+
+class HashtagDoesNotExistException extends \Exception {
+
+}
+

lib/Service/HashtagService.php

@@ -0,0 +1,208 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * Nextcloud - Social Support
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018, Maxence Lange <maxence@artificial-owl.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\Social\Service;
+
+
+use daita\MySmallPhpTools\Traits\TArrayTools;
+use OCA\Social\Db\HashtagsRequest;
+use OCA\Social\Db\NotesRequest;
+use OCA\Social\Exceptions\HashtagDoesNotExistException;
+
+
+class HashtagService {
+
+
+	const TREND_1H = 3600;
+	const TREND_12H = 43200;
+	const TREND_1D = 86400;
+	const TREND_3D = 259200;
+	const TREND_10D = 864000;
+
+
+	use TArrayTools;
+
+
+	/** @var HashtagsRequest */
+	private $hashtagsRequest;
+
+	/** @var NotesRequest */
+	private $notesRequest;
+
+	/** @var ConfigService */
+	private $configService;
+
+	/** @var MiscService */
+	private $miscService;
+
+
+	/**
+	 * ImportService constructor.
+	 *
+	 * @param HashtagsRequest $hashtagsRequest
+	 * @param NotesRequest $notesRequest
+	 * @param ConfigService $configService
+	 * @param MiscService $miscService
+	 */
+	public function __construct(
+		HashtagsRequest $hashtagsRequest, NotesRequest $notesRequest, ConfigService $configService,
+		MiscService $miscService
+	) {
+		$this->hashtagsRequest = $hashtagsRequest;
+		$this->notesRequest = $notesRequest;
+		$this->configService = $configService;
+		$this->miscService = $miscService;
+	}
+
+	/*
+	 * note: trend is:
+	 * [
+	 *   '1h' => x,
+	 *   '12h' => x,
+	 *   '1d' => x,
+	 *   '3d' => x,
+	 *   '10d' => x
+	 * ]
+	 */
+
+	/**
+	 *
+	 */
+	public function manageHashtags(): int {
+		$current = $this->hashtagsRequest->getAll();
+
+		$time = time();
+		$hashtags = [
+			'1h'  => $this->getTrendSince($time - self::TREND_1H),
+			'12h' => $this->getTrendSince($time - self::TREND_12H),
+			'1d'  => $this->getTrendSince($time - self::TREND_1D),
+			'3d'  => $this->getTrendSince($time - self::TREND_3D),
+			'10d' => $this->getTrendSince($time - self::TREND_10D)
+		];
+
+		$count = 0;
+		$formatted = $this->formatTrend($hashtags);
+		foreach ($formatted as $hashtag => $trend) {
+			$count++;
+			try {
+				$this->getFromList($current, $hashtag);
+				$this->hashtagsRequest->update($hashtag, $trend);
+			} catch (HashtagDoesNotExistException $e) {
+				$this->hashtagsRequest->save($hashtag, $trend);
+			}
+		}
+
+		return $count;
+	}
+
+
+	/**
+	 * @param int $timestamp
+	 *
+	 * @return array
+	 */
+	private function getTrendSince(int $timestamp): array {
+		$result = [];
+
+		$notes = $this->notesRequest->getNotesSince($timestamp);
+		foreach ($notes as $note) {
+
+			foreach ($note->getHashtags() as $hashtag) {
+				if (array_key_exists($hashtag, $result)) {
+					$result[$hashtag]++;
+				} else {
+					$result[$hashtag] = 1;
+				}
+			}
+		}
+
+		return $result;
+	}
+
+
+	/**
+	 * @param array $hashtags
+	 *
+	 * @return array
+	 */
+	private function formatTrend(array $hashtags): array {
+		$trends = [];
+		foreach (end($hashtags) as $hashtag => $count) {
+			$trends[$hashtag] = [];
+		}
+
+		$all = array_keys($trends);
+		$periods = array_keys($hashtags);
+		foreach ($all as $hashtag) {
+			foreach ($periods as $period) {
+				$count = $this->countFromList($hashtags[$period], $hashtag);
+				$trends[$hashtag][$period] = $count;
+			}
+		}
+
+		return $trends;
+	}
+
+
+	/**
+	 * @param array $list
+	 * @param string $hashtag
+	 *
+	 * @return int
+	 */
+	private function countFromList(array $list, string $hashtag): int {
+		foreach ($list as $key => $count) {
+			if ($key === $hashtag) {
+				return $count;
+			}
+		}
+
+		return 0;
+	}
+
+
+	/**
+	 * @param array $list
+	 * @param string $hashtag
+	 *
+	 * @return array
+	 * @throws HashtagDoesNotExistException
+	 */
+	private function getFromList(array $list, string $hashtag): array {
+		foreach ($list as $item) {
+			if ($this->get('hashtag', $item, '') === $hashtag) {
+				return $item;
+			}
+		}
+
+		throw new HashtagDoesNotExistException();
+	}
+}
+