From 8b068521c8ae6f81ef669add5bcf8655b9e733a0 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 10:15:25 -0100 Subject: [PATCH 01/26] return token on post creation Signed-off-by: Maxence Lange --- lib/Controller/LocalController.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/Controller/LocalController.php b/lib/Controller/LocalController.php index 997b12bb..b5216943 100644 --- a/lib/Controller/LocalController.php +++ b/lib/Controller/LocalController.php @@ -140,9 +140,14 @@ class LocalController extends Controller { $post->setType($this->get('type', $data, NoteService::TYPE_PUBLIC)); /** @var ACore $activity */ - $this->postService->createPost($post, $activity); + $token = $this->postService->createPost($post, $activity); - return $this->directSuccess($activity->getObject()); + return $this->success( + [ + 'post' => $activity->getObject(), + 'token' => $token + ] + ); } catch (Exception $e) { return $this->fail($e); } From 0f40a13a1f8977f639a7cedc05abe7d74ec31def Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 10:40:32 -0100 Subject: [PATCH 02/26] summary can be up to 3000 chars Signed-off-by: Maxence Lange --- appinfo/database.xml | 6 +++--- appinfo/info.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/appinfo/database.xml b/appinfo/database.xml index c3b073fa..63d91c75 100644 --- a/appinfo/database.xml +++ b/appinfo/database.xml @@ -41,7 +41,7 @@ summary text - 500 + 3000 true @@ -198,7 +198,7 @@ summary text - 255 + 3000 true @@ -362,7 +362,7 @@ summary text - 500 + 3000 true diff --git a/appinfo/info.xml b/appinfo/info.xml index 29f0a4a3..add173bd 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -5,7 +5,7 @@ Social 🎉 Nextcloud becomes part of the federated social networks! - 0.0.50 + 0.0.51 agpl Maxence Lange Julius Härtl From 191d67e4d0863257c89eddf98c0654ff0a2c614f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Wed, 28 Nov 2018 14:49:47 +0100 Subject: [PATCH 03/26] Log token to browser console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- src/store/timeline.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/store/timeline.js b/src/store/timeline.js index 411c1343..a77fcb4e 100644 --- a/src/store/timeline.js +++ b/src/store/timeline.js @@ -58,7 +58,8 @@ const actions = { }, post(context, post) { return axios.post(OC.generateUrl('apps/social/api/v1/post'), { data: post }).then((response) => { - context.commit('addPost', { data: response.data }) + // eslint-disable-next-line no-console + console.log('Post created with token ' + response.data.result.token) }).catch((error) => { OC.Notification.showTemporary('Failed to create a post') console.error('Failed to create a post', error) From a3a2ac6dcc8580da036ff55b6de9b5659c4fb9bb Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 14:17:23 -0100 Subject: [PATCH 04/26] error must be 0 Signed-off-by: Maxence Lange --- lib/Db/CacheDocumentsRequest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Db/CacheDocumentsRequest.php b/lib/Db/CacheDocumentsRequest.php index 6c7f7b68..d71a113f 100644 --- a/lib/Db/CacheDocumentsRequest.php +++ b/lib/Db/CacheDocumentsRequest.php @@ -150,7 +150,8 @@ class CacheDocumentsRequest extends CacheDocumentsRequestBuilder { $qb = $this->getCacheDocumentsSelectSql(); $this->limitToDBFieldEmpty($qb, 'local_copy'); $this->limitToCaching($qb, self::CACHE_TTL); - + $this->limitToDBFieldInt($qb, 'error', 0); + $documents = []; $cursor = $qb->execute(); while ($data = $cursor->fetch()) { From e8b80e8502bb09136e4595efb0f84243274bb707 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 14:21:56 -0100 Subject: [PATCH 05/26] exception on 410 Signed-off-by: Maxence Lange --- lib/Service/CurlService.php | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/Service/CurlService.php b/lib/Service/CurlService.php index 1aa94cc3..2218a52d 100644 --- a/lib/Service/CurlService.php +++ b/lib/Service/CurlService.php @@ -33,6 +33,8 @@ namespace OCA\Social\Service; use daita\MySmallPhpTools\Model\Request; use daita\MySmallPhpTools\Traits\TArrayTools; use daita\MySmallPhpTools\Traits\TPathTools; +use Exception; +use OCA\Social\Exceptions\Request410Exception; use OCA\Social\Exceptions\RequestException; use OCA\Social\Exceptions\SocialAppConfigException; @@ -71,6 +73,7 @@ class CurlService { * * @return array * @throws RequestException + * @throws Request410Exception */ public function request(Request $request): array { $curl = $this->initRequest($request); @@ -85,6 +88,7 @@ class CurlService { $this->parseRequestResultCode301($code); // $this->parseRequestResultCode401($code); $this->parseRequestResultCode404($code, $request); + $this->parseRequestResultCode410($code); // $this->parseRequestResultCode503($code); // $this->parseRequestResultCode500($code); // $this->parseRequestResult($result); @@ -130,7 +134,7 @@ class CurlService { $request->setAddress($host); try { $this->request($request); - } catch (RequestException $e) { + } catch (Exception $e) { } } @@ -234,11 +238,24 @@ class CurlService { * * @param Request $request * - * @throws RequestException + * @throws Request410Exception */ private function parseRequestResultCode404(int $code, Request $request) { if ($code === 404) { - throw new RequestException('404 Not Found - ' . json_encode($request)); + throw new Request410Exception('404 Not Found - ' . json_encode($request)); + } + } + + /** + * @param int $code + * + * @param Request $request + * + * @throws Request410Exception + */ + private function parseRequestResultCode410(int $code) { + if ($code === 410) { + throw new Request410Exception(); } } From 005a8c57d0ff5b2d012efcdad3d47b580b9cc9a6 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 14:26:28 -0100 Subject: [PATCH 06/26] 410 and gone signature/account Signed-off-by: Maxence Lange --- lib/Controller/ActivityPubController.php | 7 ++++++- lib/Exceptions/Request410Exception.php | 8 ++++++++ lib/Exceptions/SignatureIsGoneException.php | 8 ++++++++ lib/Service/ActivityPub/PersonService.php | 2 ++ lib/Service/ActivityService.php | 18 ++++++++++++++++-- lib/Service/InstanceService.php | 5 +++-- 6 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 lib/Exceptions/Request410Exception.php create mode 100644 lib/Exceptions/SignatureIsGoneException.php diff --git a/lib/Controller/ActivityPubController.php b/lib/Controller/ActivityPubController.php index 90a1369a..119076cc 100644 --- a/lib/Controller/ActivityPubController.php +++ b/lib/Controller/ActivityPubController.php @@ -32,9 +32,10 @@ namespace OCA\Social\Controller; use daita\MySmallPhpTools\Traits\Nextcloud\TNCDataResponse; use Exception; +use OC\AppFramework\Http; use OCA\Social\AppInfo\Application; use OCA\Social\Db\NotesRequest; -use OCA\Social\Exceptions\SignatureException; +use OCA\Social\Exceptions\SignatureIsGoneException; use OCA\Social\Exceptions\UnknownItemException; use OCA\Social\Service\ActivityPub\FollowService; use OCA\Social\Service\ActivityService; @@ -184,6 +185,8 @@ class ActivityPubController extends Controller { } return $this->success([]); + } catch (SignatureIsGoneException $e) { + return $this->fail($e, [], Http::STATUS_GONE); } catch (Exception $e) { return $this->fail($e); } @@ -220,6 +223,8 @@ class ActivityPubController extends Controller { } return $this->success([]); + } catch (SignatureIsGoneException $e) { + return $this->fail($e, [], Http::STATUS_GONE); } catch (Exception $e) { return $this->fail($e); } diff --git a/lib/Exceptions/Request410Exception.php b/lib/Exceptions/Request410Exception.php new file mode 100644 index 00000000..85bc8405 --- /dev/null +++ b/lib/Exceptions/Request410Exception.php @@ -0,0 +1,8 @@ +getHeader('date')); @@ -394,7 +399,12 @@ class ActivityService { throw new SignatureException('object is too old'); } - $this->checkSignature($request); + try { + $this->checkSignature($request); + } catch (Request410Exception $e) { + throw new SignatureIsGoneException(); + } + } @@ -429,9 +439,12 @@ class ActivityService { * @param IRequest $request * * @throws InvalidResourceException + * @throws MalformedArrayException + * @throws Request410Exception * @throws RequestException * @throws SignatureException - * @throws MalformedArrayException + * @throws SocialAppConfigException + * @throws UrlCloudException * @throws Exception */ private function checkSignature(IRequest $request) { @@ -508,6 +521,7 @@ class ActivityService { * @throws RequestException * @throws SocialAppConfigException * @throws UrlCloudException + * @throws Request410Exception */ private function retrieveKey($keyId): string { $actor = $this->personService->getFromId($keyId); diff --git a/lib/Service/InstanceService.php b/lib/Service/InstanceService.php index 1eafcc33..0d7eefef 100644 --- a/lib/Service/InstanceService.php +++ b/lib/Service/InstanceService.php @@ -35,6 +35,7 @@ use daita\MySmallPhpTools\Model\Request; use daita\MySmallPhpTools\Traits\TArrayTools; use daita\MySmallPhpTools\Traits\TPathTools; use OCA\Social\Exceptions\InvalidResourceException; +use OCA\Social\Exceptions\Request410Exception; use OCA\Social\Exceptions\RequestException; use OCA\Social\Model\ActivityPub\ACore; use OCA\Social\Model\Instance; @@ -83,8 +84,7 @@ class InstanceService { public function retrieveAccount(string $account) { $account = $this->withoutBeginAt($account); - if (strstr(substr($account, 0, -3), '@') === false) - { + if (strstr(substr($account, 0, -3), '@') === false) { throw new InvalidResourceException(); } list($username, $host) = explode('@', $account); @@ -113,6 +113,7 @@ class InstanceService { * * @return mixed * @throws RequestException + * @throws Request410Exception */ public function retrieveObject($id) { $url = parse_url($id); From 676ec5acce574e4b0e50cbdaeb3a6ab69f645b21 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 14:27:38 -0100 Subject: [PATCH 07/26] cleaning Signed-off-by: Maxence Lange --- lib/Controller/LocalController.php | 1 - lib/Db/CacheDocumentsRequest.php | 2 +- lib/Db/CoreRequestBuilder.php | 5 +++-- lib/Db/RequestQueueRequest.php | 8 +++++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/Controller/LocalController.php b/lib/Controller/LocalController.php index b5216943..f462319c 100644 --- a/lib/Controller/LocalController.php +++ b/lib/Controller/LocalController.php @@ -420,7 +420,6 @@ class LocalController extends Controller { $cached = []; foreach ($documents as $id) { try { - $document = $this->documentService->cacheRemoteDocument($id); $cached[] = $document; } catch (Exception $e) { diff --git a/lib/Db/CacheDocumentsRequest.php b/lib/Db/CacheDocumentsRequest.php index d71a113f..dcb153ad 100644 --- a/lib/Db/CacheDocumentsRequest.php +++ b/lib/Db/CacheDocumentsRequest.php @@ -151,7 +151,7 @@ class CacheDocumentsRequest extends CacheDocumentsRequestBuilder { $this->limitToDBFieldEmpty($qb, 'local_copy'); $this->limitToCaching($qb, self::CACHE_TTL); $this->limitToDBFieldInt($qb, 'error', 0); - + $documents = []; $cursor = $qb->execute(); while ($data = $cursor->fetch()) { diff --git a/lib/Db/CoreRequestBuilder.php b/lib/Db/CoreRequestBuilder.php index e62d72fe..efe8b813 100644 --- a/lib/Db/CoreRequestBuilder.php +++ b/lib/Db/CoreRequestBuilder.php @@ -363,9 +363,10 @@ class CoreRequestBuilder { /** * @param IQueryBuilder $qb + * @param string $order */ - protected function orderByPriority(IQueryBuilder &$qb) { - $qb->orderBy('priority', 'desc'); + protected function orderByPriority(IQueryBuilder &$qb, string $order = 'desc') { + $qb->orderBy('priority', $order); } diff --git a/lib/Db/RequestQueueRequest.php b/lib/Db/RequestQueueRequest.php index 33ca298c..045e9c16 100644 --- a/lib/Db/RequestQueueRequest.php +++ b/lib/Db/RequestQueueRequest.php @@ -180,9 +180,11 @@ class RequestQueueRequest extends RequestQueueRequestBuilder { */ public function setAsFailure(RequestQueue &$queue) { $qb = $this->getQueueUpdateSql(); - $qb->set('status', $qb->createNamedParameter(RequestQueue::STATUS_STANDBY)); - // TODO - increment tries++ -// ->set('tries', 'tries+1'); + $func = $qb->func(); + $expr = $qb->expr(); + + $qb->set('status', $qb->createNamedParameter(RequestQueue::STATUS_STANDBY)) + ->set('tries', $func->add('tries', $expr->literal(1))); $this->limitToId($qb, $queue->getId()); $this->limitToStatus($qb, RequestQueue::STATUS_RUNNING); From 13d49b6ebe27df5d6a69e9d5940f92946cc93817 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 14:39:25 -0100 Subject: [PATCH 08/26] exception handling Signed-off-by: Maxence Lange --- lib/Controller/ActivityPubController.php | 6 +++-- lib/Controller/NavigationController.php | 6 +++-- lib/Db/ActorsRequest.php | 30 ++++++++++-------------- lib/Service/ActorService.php | 2 +- lib/Service/InstanceService.php | 1 + 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/Controller/ActivityPubController.php b/lib/Controller/ActivityPubController.php index 119076cc..acd79fa3 100644 --- a/lib/Controller/ActivityPubController.php +++ b/lib/Controller/ActivityPubController.php @@ -125,7 +125,6 @@ class ActivityPubController extends Controller { * @param string $username * * @return Response - * @throws \OC\User\NoUserException */ public function actor(string $username): Response { if (!$this->checkSourceActivityStreams()) { @@ -210,10 +209,13 @@ class ActivityPubController extends Controller { public function inbox(string $username): Response { try { - $actor = $this->actorService->getActor($username); + $this->activityService->checkRequest($this->request); $body = file_get_contents('php://input'); + // TODO - check the recipient <-> username +// $actor = $this->actorService->getActor($username); + $this->miscService->log('Inbox: ' . $body); $activity = $this->importService->import($body); diff --git a/lib/Controller/NavigationController.php b/lib/Controller/NavigationController.php index 864b9eb8..37f7050f 100644 --- a/lib/Controller/NavigationController.php +++ b/lib/Controller/NavigationController.php @@ -125,7 +125,6 @@ class NavigationController extends Controller { * @NoSubAdminRequired * * @return TemplateResponse - * @throws NoUserException */ public function navigate($path = ''): TemplateResponse { $data = [ @@ -162,6 +161,10 @@ class NavigationController extends Controller { $data['serverData']['firstrun'] = true; } catch (AccountAlreadyExistsException $e) { // we do nothing + } catch (NoUserException $e) { + // well, should not happens + } catch (SocialAppConfigException $e) { + // neither. } return new TemplateResponse(Application::APP_NAME, 'main', $data); @@ -231,7 +234,6 @@ class NavigationController extends Controller { * @param $username * * @return RedirectResponse|PublicTemplateResponse - * @throws NoUserException */ public function public($username) { if (\OC::$server->getUserSession() diff --git a/lib/Db/ActorsRequest.php b/lib/Db/ActorsRequest.php index 09b37e4b..00572296 100644 --- a/lib/Db/ActorsRequest.php +++ b/lib/Db/ActorsRequest.php @@ -60,32 +60,28 @@ class ActorsRequest extends ActorsRequestBuilder { * @param Person $actor * * @return string - * @throws \Exception + * @throws SocialAppConfigException */ public function create(Person $actor): string { $id = $this->configService->getUrlSocial() . '@' . $actor->getPreferredUsername(); - try { - $qb = $this->getActorsInsertSql(); + $qb = $this->getActorsInsertSql(); - $qb->setValue('id', $qb->createNamedParameter($id)) + $qb->setValue('id', $qb->createNamedParameter($id)) // ->setValue('type', $qb->createNamedParameter($actor->getType())) - ->setValue('user_id', $qb->createNamedParameter($actor->getUserId())) - ->setValue('name', $qb->createNamedParameter($actor->getName())) - ->setValue('summary', $qb->createNamedParameter($actor->getSummary())) - ->setValue( - 'preferred_username', $qb->createNamedParameter($actor->getPreferredUsername()) - ) - ->setValue('public_key', $qb->createNamedParameter($actor->getPublicKey())) - ->setValue('private_key', $qb->createNamedParameter($actor->getPrivateKey())); + ->setValue('user_id', $qb->createNamedParameter($actor->getUserId())) + ->setValue('name', $qb->createNamedParameter($actor->getName())) + ->setValue('summary', $qb->createNamedParameter($actor->getSummary())) + ->setValue( + 'preferred_username', $qb->createNamedParameter($actor->getPreferredUsername()) + ) + ->setValue('public_key', $qb->createNamedParameter($actor->getPublicKey())) + ->setValue('private_key', $qb->createNamedParameter($actor->getPrivateKey())); - $qb->execute(); + $qb->execute(); - return $id; - } catch (\Exception $e) { - throw $e; - } + return $id; } diff --git a/lib/Service/ActorService.php b/lib/Service/ActorService.php index 348d1a1f..70fb824e 100644 --- a/lib/Service/ActorService.php +++ b/lib/Service/ActorService.php @@ -142,7 +142,7 @@ class ActorService { * * @throws AccountAlreadyExistsException * @throws NoUserException - * @throws Exception + * @throws SocialAppConfigException */ public function createActor(string $userId, string $username) { diff --git a/lib/Service/InstanceService.php b/lib/Service/InstanceService.php index 0d7eefef..911a86c9 100644 --- a/lib/Service/InstanceService.php +++ b/lib/Service/InstanceService.php @@ -80,6 +80,7 @@ class InstanceService { * @return mixed * @throws RequestException * @throws InvalidResourceException + * @throws Request410Exception */ public function retrieveAccount(string $account) { $account = $this->withoutBeginAt($account); From 9a042eea457a807976f89f10787bd505df7a423d Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 17:18:37 -0100 Subject: [PATCH 09/26] retry-on-fail by cron and cli Signed-off-by: Maxence Lange --- appinfo/info.xml | 1 + composer.lock | 8 +- lib/Command/QueueProcess.php | 126 ++++++++++++++++++++++++++++++++ lib/Cron/Queue.php | 109 +++++++++++++++++++++++++++ lib/Db/RequestQueueRequest.php | 46 ++++++++---- lib/Model/RequestQueue.php | 11 ++- lib/Service/ActivityService.php | 15 +++- lib/Service/QueueService.php | 22 ++++++ 8 files changed, 313 insertions(+), 25 deletions(-) create mode 100644 lib/Command/QueueProcess.php create mode 100644 lib/Cron/Queue.php diff --git a/appinfo/info.xml b/appinfo/info.xml index add173bd..8e1f2608 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -33,6 +33,7 @@ OCA\Social\Command\CacheRefresh OCA\Social\Command\QueueStatus + OCA\Social\Command\QueueProcess OCA\Social\Command\NoteCreate diff --git a/composer.lock b/composer.lock index b0c6dab8..1e26539f 100644 --- a/composer.lock +++ b/composer.lock @@ -12,12 +12,12 @@ "source": { "type": "git", "url": "https://github.com/daita/my-small-php-tools.git", - "reference": "12090dc3ae29d2eb49d5274ca3f6ebfb76ce5997" + "reference": "56cff24fdde14d21e3903428c5ee629c839866af" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/daita/my-small-php-tools/zipball/12090dc3ae29d2eb49d5274ca3f6ebfb76ce5997", - "reference": "12090dc3ae29d2eb49d5274ca3f6ebfb76ce5997", + "url": "https://api.github.com/repos/daita/my-small-php-tools/zipball/56cff24fdde14d21e3903428c5ee629c839866af", + "reference": "56cff24fdde14d21e3903428c5ee629c839866af", "shasum": "" }, "require": { @@ -40,7 +40,7 @@ } ], "description": "My small PHP Tools", - "time": "2018-11-28T10:47:43+00:00" + "time": "2018-11-28T13:07:27+00:00" } ], "packages-dev": [], diff --git a/lib/Command/QueueProcess.php b/lib/Command/QueueProcess.php new file mode 100644 index 00000000..d7aeb1a9 --- /dev/null +++ b/lib/Command/QueueProcess.php @@ -0,0 +1,126 @@ + + * @copyright 2018, Maxence Lange + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\Social\Command; + + +use Exception; +use OC\Core\Command\Base; +use OCA\Social\Exceptions\ActorDoesNotExistException; +use OCA\Social\Exceptions\RequestException; +use OCA\Social\Exceptions\SocialAppConfigException; +use OCA\Social\Service\ActivityService; +use OCA\Social\Service\ConfigService; +use OCA\Social\Service\MiscService; +use OCA\Social\Service\QueueService; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + + +class QueueProcess extends Base { + + + /** @var ActivityService */ + private $activityService; + + /** @var QueueService */ + private $queueService; + + /** @var ConfigService */ + private $configService; + + /** @var MiscService */ + private $miscService; + + + /** + * NoteCreate constructor. + * + * @param ActivityService $activityService + * @param QueueService $queueService + * @param ConfigService $configService + * @param MiscService $miscService + */ + public function __construct( + ActivityService $activityService, QueueService $queueService, ConfigService $configService, + MiscService $miscService + ) { + parent::__construct(); + + $this->activityService = $activityService; + $this->queueService = $queueService; + $this->configService = $configService; + $this->miscService = $miscService; + } + + + /** + * + */ + protected function configure() { + parent::configure(); + $this->setName('social:queue:process') + ->setDescription('Process the request queue'); + } + + + /** + * @param InputInterface $input + * @param OutputInterface $output + */ + protected function execute(InputInterface $input, OutputInterface $output) { + + $requests = $this->queueService->getRequestStandby($total = 0); + + $output->writeLn('found a total of ' . $total . ' requests in the queue'); + if ($total === 0) { + return; + } + + $output->writeLn(sizeof($requests) . ' are processable at this time'); + if (sizeof($requests) === 0) { + return; + } + + foreach ($requests as $request) { + $output->write('.'); + try { + $this->activityService->manageRequest($request); + } catch (ActorDoesNotExistException $e) { + } catch (RequestException $e) { + } catch (SocialAppConfigException $e) { + } + } + + $output->writeLn('done'); + } + +} + diff --git a/lib/Cron/Queue.php b/lib/Cron/Queue.php new file mode 100644 index 00000000..d725b113 --- /dev/null +++ b/lib/Cron/Queue.php @@ -0,0 +1,109 @@ + + * @copyright 2018, Maxence Lange + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\Social\Cron; + + +use Exception; +use OC\BackgroundJob\TimedJob; +use OCA\Social\AppInfo\Application; +use OCA\Social\Exceptions\ActorDoesNotExistException; +use OCA\Social\Exceptions\RequestException; +use OCA\Social\Exceptions\SocialAppConfigException; +use OCA\Social\Service\ActivityPub\DocumentService; +use OCA\Social\Service\ActivityPub\PersonService; +use OCA\Social\Service\ActivityService; +use OCA\Social\Service\ActorService; +use OCA\Social\Service\CacheService; +use OCA\Social\Service\ConfigService; +use OCA\Social\Service\MiscService; +use OCA\Social\Service\QueueService; +use OCP\AppFramework\QueryException; + + +/** + * Class Queue + * + * @package OCA\Social\Cron + */ +class Queue extends TimedJob { + + + /** @var ActivityService */ + private $activityService; + + /** @var QueueService */ + private $queueService; + + /** @var MiscService */ + private $miscService; + + + /** + * Cache constructor. + */ + public function __construct() { + $this->setInterval(12 * 60); // 12 minutes + } + + + /** + * @param mixed $argument + * + * @throws QueryException + */ + protected function run($argument) { + $app = new Application(); + $c = $app->getContainer(); + + $this->queueService = $c->query(QueueService::class); + $this->activityService = $c->query(ActivityService::class); + $this->miscService = $c->query(MiscService::class); + + $this->manageQueue(); + } + + + private function manageQueue() { + $requests = $this->queueService->getRequestStandby($total = 0); + + foreach ($requests as $request) { + try { + $this->activityService->manageRequest($request); + } catch (ActorDoesNotExistException $e) { + } catch (RequestException $e) { + } catch (SocialAppConfigException $e) { + } + } + + } + +} + diff --git a/lib/Db/RequestQueueRequest.php b/lib/Db/RequestQueueRequest.php index 045e9c16..d358051e 100644 --- a/lib/Db/RequestQueueRequest.php +++ b/lib/Db/RequestQueueRequest.php @@ -57,18 +57,6 @@ class RequestQueueRequest extends RequestQueueRequestBuilder { public function multiple(array $queues) { foreach ($queues as $queue) { $this->create($queue); -// $qb->values( -// [ -// 'source' => $qb->createNamedParameter($queue->getSource()), -// 'activity' => $qb->createNamedParameter($queue->getActivity()), -// 'instance' => $qb->createNamedParameter( -// json_encode($queue->getInstance(), JSON_UNESCAPED_SLASHES) -// ), -// 'status' => $qb->createNamedParameter($queue->getStatus()), -// 'tries' => $qb->createNamedParameter($queue->getTries()), -// 'last' => $qb->createNamedParameter($queue->getLast()) -// ] -// ); } } @@ -92,14 +80,34 @@ class RequestQueueRequest extends RequestQueueRequestBuilder { ) ->setValue('priority', $qb->createNamedParameter($queue->getPriority())) ->setValue('status', $qb->createNamedParameter($queue->getStatus())) - ->setValue('tries', $qb->createNamedParameter($queue->getTries())) - ->setValue('last', $qb->createNamedParameter($queue->getLast())); + ->setValue('tries', $qb->createNamedParameter($queue->getTries())); $qb->execute(); } /** - * return Actor from database based on the username + * return Queue from database based on the status != 9 + * + * @return RequestQueue[] + */ + public function getStandby(): array { + $qb = $this->getQueueSelectSql(); + $this->limitToStatus($qb, RequestQueue::STATUS_STANDBY); + $this->orderByPriority($qb, 'desc'); + + $requests = []; + $cursor = $qb->execute(); + while ($data = $cursor->fetch()) { + $requests[] = $this->parseQueueSelectSql($data); + } + $cursor->closeCursor(); + + return $requests; + } + + + /** + * return Queue from database based on the token * * @param string $token * @param int $status @@ -197,5 +205,13 @@ class RequestQueueRequest extends RequestQueueRequestBuilder { $queue->setStatus(RequestQueue::STATUS_SUCCESS); } + + public function delete(RequestQueue $queue) { + $qb = $this->getQueueDeleteSql(); + $this->limitToId($qb, $queue->getId()); + + $qb->execute(); + } + } diff --git a/lib/Model/RequestQueue.php b/lib/Model/RequestQueue.php index 62f98155..d96253f3 100644 --- a/lib/Model/RequestQueue.php +++ b/lib/Model/RequestQueue.php @@ -32,6 +32,7 @@ namespace OCA\Social\Model; use daita\MySmallPhpTools\Traits\TArrayTools; +use DateTime; use JsonSerializable; @@ -299,9 +300,15 @@ class RequestQueue implements JsonSerializable { $this->setActivity($this->get('activity', $data, '')); $this->setStatus($this->getInt('status', $data, 0)); $this->setTries($this->getInt('tries', $data, 0)); - $this->setLast($this->getInt('last', $data, 0)); - } + $last = $this->get('last', $data, ''); + if ($last === '') { + $this->setLast(0); + } else { + $dTime = new DateTime($last); + $this->setLast($dTime->getTimestamp()); + } + } /** * @return array diff --git a/lib/Service/ActivityService.php b/lib/Service/ActivityService.php index 85b1ea8d..fcc8399a 100644 --- a/lib/Service/ActivityService.php +++ b/lib/Service/ActivityService.php @@ -251,7 +251,6 @@ class ActivityService { /** * @param RequestQueue $queue * - * @throws ActorDoesNotExistException * @throws RequestException * @throws SocialAppConfigException */ @@ -263,9 +262,16 @@ class ActivityService { return; } - $result = $this->generateRequest( - $queue->getInstance(), $queue->getActivity(), $queue->getAuthor() - ); + + try { + $result = $this->generateRequest( + $queue->getInstance(), $queue->getActivity(), $queue->getAuthor() + ); + } catch (ActorDoesNotExistException $e) { + $this->queueService->deleteRequest($queue); + } catch (Request410Exception $e) { + $this->queueService->deleteRequest($queue); + } try { if ($this->getint('_code', $result, 500) === 202) { @@ -341,6 +347,7 @@ class ActivityService { * * @return Request[] * @throws ActorDoesNotExistException + * @throws Request410Exception * @throws RequestException * @throws SocialAppConfigException */ diff --git a/lib/Service/QueueService.php b/lib/Service/QueueService.php index 6fc3b8f3..36cc46dd 100644 --- a/lib/Service/QueueService.php +++ b/lib/Service/QueueService.php @@ -147,6 +147,22 @@ class QueueService { } + public function getRequestStandby(int &$total = 0): array { + $requests = $this->requestQueueRequest->getStandby(); + $total = sizeof($requests); + + $result = []; + foreach ($requests as $request) { + $delay = floor(pow($request->getTries(), 4) / 3); + if ($request->getLast() < (time() - $delay)) { + $result[] = $request; + } + } + + return $result; + } + + /** * @param string $token * @param int $status @@ -186,6 +202,12 @@ class QueueService { } } + /** + * @param RequestQueue $queue + */ + public function deleteRequest(RequestQueue $queue) { + $this->requestQueueRequest->delete($queue); + } } From df2e2715d08498a6f9324f2ed8f66b1e253d2222 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 17:19:13 -0100 Subject: [PATCH 10/26] Cron/Queue Signed-off-by: Maxence Lange --- appinfo/info.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index 8e1f2608..51b79c41 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -28,13 +28,14 @@ OCA\Social\Cron\Cache + OCA\Social\Cron\Queue OCA\Social\Command\CacheRefresh + OCA\Social\Command\NoteCreate OCA\Social\Command\QueueStatus OCA\Social\Command\QueueProcess - OCA\Social\Command\NoteCreate From 8b9217292d8c67d64d148a9a13df17de97a9ef24 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 28 Nov 2018 19:21:28 -0100 Subject: [PATCH 11/26] to avoid conflict, filters failing instance when processing queue Signed-off-by: Maxence Lange --- lib/Command/QueueProcess.php | 2 +- lib/Controller/QueueController.php | 2 +- lib/Cron/Queue.php | 2 +- lib/Db/CoreRequestBuilder.php | 9 --------- lib/Db/RequestQueueRequest.php | 6 +++--- lib/Service/ActivityService.php | 18 +++++++++++++++++- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/lib/Command/QueueProcess.php b/lib/Command/QueueProcess.php index d7aeb1a9..780a87da 100644 --- a/lib/Command/QueueProcess.php +++ b/lib/Command/QueueProcess.php @@ -109,11 +109,11 @@ class QueueProcess extends Base { return; } + $this->activityService->manageInit(); foreach ($requests as $request) { $output->write('.'); try { $this->activityService->manageRequest($request); - } catch (ActorDoesNotExistException $e) { } catch (RequestException $e) { } catch (SocialAppConfigException $e) { } diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php index f1594c85..26cdddfa 100644 --- a/lib/Controller/QueueController.php +++ b/lib/Controller/QueueController.php @@ -98,10 +98,10 @@ class QueueController extends Controller { $this->async(); $requests = $this->queueService->getRequestFromToken($token, RequestQueue::STATUS_STANDBY); + $this->activityService->manageInit(); foreach ($requests as $request) { try { $this->activityService->manageRequest($request); - } catch (ActorDoesNotExistException $e) { } catch (RequestException $e) { } catch (SocialAppConfigException $e) { } diff --git a/lib/Cron/Queue.php b/lib/Cron/Queue.php index d725b113..58e759a4 100644 --- a/lib/Cron/Queue.php +++ b/lib/Cron/Queue.php @@ -93,11 +93,11 @@ class Queue extends TimedJob { private function manageQueue() { $requests = $this->queueService->getRequestStandby($total = 0); + $this->activityService->manageInit(); foreach ($requests as $request) { try { $this->activityService->manageRequest($request); - } catch (ActorDoesNotExistException $e) { } catch (RequestException $e) { } catch (SocialAppConfigException $e) { } diff --git a/lib/Db/CoreRequestBuilder.php b/lib/Db/CoreRequestBuilder.php index efe8b813..a87672d6 100644 --- a/lib/Db/CoreRequestBuilder.php +++ b/lib/Db/CoreRequestBuilder.php @@ -361,15 +361,6 @@ class CoreRequestBuilder { } - /** - * @param IQueryBuilder $qb - * @param string $order - */ - protected function orderByPriority(IQueryBuilder &$qb, string $order = 'desc') { - $qb->orderBy('priority', $order); - } - - /** * @param IQueryBuilder $qb * @param string $field diff --git a/lib/Db/RequestQueueRequest.php b/lib/Db/RequestQueueRequest.php index d358051e..4e07ffba 100644 --- a/lib/Db/RequestQueueRequest.php +++ b/lib/Db/RequestQueueRequest.php @@ -86,14 +86,14 @@ class RequestQueueRequest extends RequestQueueRequestBuilder { /** - * return Queue from database based on the status != 9 + * return Queue from database based on the status=0 * * @return RequestQueue[] */ public function getStandby(): array { $qb = $this->getQueueSelectSql(); $this->limitToStatus($qb, RequestQueue::STATUS_STANDBY); - $this->orderByPriority($qb, 'desc'); + $qb->orderBy('id', 'asc'); $requests = []; $cursor = $qb->execute(); @@ -122,7 +122,7 @@ class RequestQueueRequest extends RequestQueueRequestBuilder { $this->limitToStatus($qb, $status); } - $this->orderByPriority($qb); + $qb->orderBy('priority', 'desc'); $requests = []; $cursor = $qb->execute(); diff --git a/lib/Service/ActivityService.php b/lib/Service/ActivityService.php index fcc8399a..37bf5606 100644 --- a/lib/Service/ActivityService.php +++ b/lib/Service/ActivityService.php @@ -107,6 +107,10 @@ class ActivityService { private $miscService; + /** @var array */ + private $failInstances; + + /** * ActivityService constructor. * @@ -233,10 +237,12 @@ class ActivityService { $author = $this->getAuthorFromItem($activity); $instancePaths = $this->generateInstancePaths($activity); $token = $this->queueService->generateRequestQueue($instancePaths, $activity, $author); + $this->manageInit(); try { $directRequest = $this->queueService->getPriorityRequest($token); $this->manageRequest($directRequest); + } catch (RequestException $e) { } catch (NoHighPriorityRequestException $e) { } catch (EmptyQueueException $e) { return ''; @@ -248,6 +254,11 @@ class ActivityService { } + public function manageInit() { + $this->failInstances = []; + } + + /** * @param RequestQueue $queue * @@ -255,6 +266,11 @@ class ActivityService { * @throws SocialAppConfigException */ public function manageRequest(RequestQueue $queue) { + $host = $queue->getInstance() + ->getAddress(); + if (in_array($host, $this->failInstances)) { + throw new RequestException(); + } try { $this->queueService->initRequest($queue); @@ -262,7 +278,6 @@ class ActivityService { return; } - try { $result = $this->generateRequest( $queue->getInstance(), $queue->getActivity(), $queue->getAuthor() @@ -278,6 +293,7 @@ class ActivityService { $this->queueService->endRequest($queue, true); } else { $this->queueService->endRequest($queue, false); + $this->failInstances[] = $host; } } catch (QueueStatusException $e) { } From 2aeed37229aa8ca58072f577943dd32d963cf99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 26 Oct 2018 13:08:12 +0200 Subject: [PATCH 12/26] Properly align content container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- src/App.vue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/App.vue b/src/App.vue index ef25ee57..9dedeaf5 100644 --- a/src/App.vue +++ b/src/App.vue @@ -30,6 +30,7 @@ .app-social { width: 100%; } + .setup { margin: auto; width: 700px; @@ -37,6 +38,9 @@ .setup input[type=url] { width: 300px; + + #app-content .social__wrapper { + margin: 15px calc(50% - 350px - 75px); } From 0044b5b8ba2c11497a20afab9f3fbe3dda7b0d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Fri, 26 Oct 2018 13:08:46 +0200 Subject: [PATCH 13/26] Add user list component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- src/components/ProfileInfo.vue | 6 +-- src/components/UserEntry.vue | 84 ++++++++++++++++++++++++++++++++++ src/router.js | 11 +++-- src/views/ProfileFollowers.vue | 33 ++++++++++--- 4 files changed, 120 insertions(+), 14 deletions(-) create mode 100644 src/components/UserEntry.vue diff --git a/src/components/ProfileInfo.vue b/src/components/ProfileInfo.vue index 058bc000..d1f2bd0c 100644 --- a/src/components/ProfileInfo.vue +++ b/src/components/ProfileInfo.vue @@ -33,13 +33,13 @@ diff --git a/src/components/UserEntry.vue b/src/components/UserEntry.vue new file mode 100644 index 00000000..0f260bf0 --- /dev/null +++ b/src/components/UserEntry.vue @@ -0,0 +1,84 @@ + + + + + + diff --git a/src/router.js b/src/router.js index 69833e83..88ed9b95 100644 --- a/src/router.js +++ b/src/router.js @@ -53,29 +53,32 @@ export default new Router({ }, { path: '/:index(index.php/)?apps/social/account/@:account', + alias: './timeline', components: { - default: Profile + default: Profile, + details: ProfileTimeline }, props: true, - name: 'profile', children: [ { path: '', + name: 'profile', components: { details: ProfileTimeline } }, { path: 'followers', + name: 'profile.followers', components: { - default: Profile, details: ProfileFollowers } }, { path: 'following', + name: 'profile.following', + components: { - default: Profile, details: ProfileFollowers } } diff --git a/src/views/ProfileFollowers.vue b/src/views/ProfileFollowers.vue index a84a63a1..345ff5df 100644 --- a/src/views/ProfileFollowers.vue +++ b/src/views/ProfileFollowers.vue @@ -21,31 +21,50 @@ --> diff --git a/src/components/UserEntry.vue b/src/components/UserEntry.vue index 63c86de2..6c0ec8ef 100644 --- a/src/components/UserEntry.vue +++ b/src/components/UserEntry.vue @@ -24,13 +24,13 @@
- +
- - {{ item.displayName }} + + {{ item.preferredUsername }} -

{{ item.description }}

+

{{ item.account }}

diff --git a/src/views/Timeline.vue b/src/views/Timeline.vue index 9de7ace6..382603d1 100644 --- a/src/views/Timeline.vue +++ b/src/views/Timeline.vue @@ -1,6 +1,5 @@