Merge pull request #9610 from annando/delayed-posts

Delayed posts from feeds

src/Core/Worker.php

@@ -1226,7 +1226,7 @@ class Worker
 			$priority = $run_parameter;
 		} elseif (is_array($run_parameter)) {
 			if (isset($run_parameter['delayed'])) {
-				$delayed = $run_parameter['execute'];
+				$delayed = $run_parameter['delayed'];
 			}
 			if (isset($run_parameter['priority'])) {
 				$priority = $run_parameter['priority'];

src/Protocol/Feed.php

@@ -29,6 +29,7 @@ use Friendica\Content\Text\HTML;
 use Friendica\Core\Cache\Duration;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
+use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
@@ -311,6 +312,8 @@ class Feed
 			$total_items = $max_items;
 		}
 
+		$postings = [];
+
 		// Importing older entries first
 		for ($i = $total_items; $i >= 0; --$i) {
 			$entry = $entries->item($i);
@@ -608,19 +611,32 @@ class Feed
 				$notify = PRIORITY_MEDIUM;
 			}
 
-			$id = Item::insert($item, $notify);
+			$postings[] = ['item' => $item, 'notify' => $notify,
+				'taglist' => $taglist, 'attachments' => $attachments];
+		}
 
-			Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id);
+		if (!empty($postings)) {
+			$total = count($postings);
+			if ($total > 1) {
+				// Posts shouldn't be delayed more than a day
+				$interval = min(1440, self::getPollInterval($contact));
+				$delay = round(($interval * 60) / $total);
+				Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]);
+			} else {
+				$delay = 0;
+			}
 
-			if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
+			$post_delay = 0;
-				$feeditem = Item::selectFirst(['uri-id'], ['id' => $id]);
+
-				foreach ($taglist as $tag) {
+			foreach ($postings as $posting) {
-					Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
+				if ($delay > 0) {
-				}
+					$publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' second');
-				foreach ($attachments as $attachment) {
+					Logger::notice('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]);
-					$attachment['uri-id'] = $feeditem['uri-id']; 
+					$post_delay += $delay;
-					Post\Media::insert($attachment);
 				}
+
+				Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $publish_at],
+					'DelayedPublish', $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']);
 			}
 		}
 
@@ -675,7 +691,7 @@ class Feed
 					$oldest = $day;
 					$oldest_date = $date;
 				}
-			
+
 				if ($newest < $day) {
 					$newest = $day;
 					$newest_date = $date;
@@ -748,6 +764,55 @@ class Feed
 		}
 	}
 
+	/**
+	 * Get the poll interval for the given contact array
+	 *
+	 * @param array $contact
+	 * @return int Poll interval in minutes
+	 */
+	public static function getPollInterval(array $contact)
+	{
+		if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) {
+			$ratings = [0, 3, 7, 8, 9, 10];
+			if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) {
+				$rating = $contact['rating'];
+			} elseif (array_key_exists($contact['priority'], $ratings)) {
+				$rating = $ratings[$contact['priority']];
+			} else {
+				$rating = -1;
+			}
+		} else {
+			// Check once a week per default for all other networks
+			$rating = 9;
+		}
+
+		// Friendica and OStatus are checked once a day
+		if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) {
+			$rating = 8;
+		}
+
+		// Check archived contacts or contacts with unsupported protocols once a month
+		if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) {
+			$rating = 10;
+		}
+
+		if ($rating < 0) {
+			return 0;
+		}
+		/*
+		 * Based on $contact['priority'], should we poll this site now? Or later?
+		 */
+
+		$min_poll_interval = max(1, DI::config()->get('system', 'min_poll_interval'));
+
+		$poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 720 ,1440, 10080, 43200];
+
+		//$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute',
+		//	'1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month'];
+
+		return $poll_intervals[$rating];
+	}
+
 	/**
 	 * Convert a tag array to a tag string
 	 *
@@ -762,7 +827,7 @@ class Feed
 			if ($tagstr != "") {
 				$tagstr .= ", ";
 			}
-	
+
 			$tagstr .= "#[url=" . DI::baseUrl() . "/search?tag=" . urlencode($tag) . "]" . $tag . "[/url]";
 		}
 

src/Worker/DelayedPublish.php

@@ -0,0 +1,58 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Worker;
+
+use Friendica\Core\Logger;
+use Friendica\Model\Item;
+use Friendica\Model\Post;
+use Friendica\Model\Tag;
+
+class DelayedPublish
+{
+	 /**
+	 * Publish a post, used for delayed postings
+	  *
+	  * @param array $item
+	  * @param integer $notify
+	  * @param array $taglist
+	  * @param array $attachments
+	  * @return void
+	  */
+	public static function execute(array $item, int $notify = 0, array $taglist = [], array $attachments = [])
+	{
+		$id = Item::insert($item, $notify);
+
+		Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]);
+
+		if (!empty($id) && (!empty($taglist) || !empty($attachments))) {
+			$feeditem = Item::selectFirst(['uri-id'], ['id' => $id]);
+			foreach ($taglist as $tag) {
+				Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag);
+			}
+			foreach ($attachments as $attachment) {
+				$attachment['uri-id'] = $feeditem['uri-id'];
+				Post\Media::insert($attachment);
+			}
+		}
+
+	}
+}

src/Worker/PollContacts.php

@@ -26,7 +26,7 @@ use Friendica\Core\Protocol;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
-use Friendica\Model\Contact;
+use Friendica\Protocol\Feed;
 use Friendica\Util\DateTimeFormat;
 
 /**
@@ -58,47 +58,16 @@ class PollContacts
 		}
 
 		while ($contact = DBA::fetch($contacts)) {
-			if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) {
+			$interval = Feed::getPollInterval($contact);
-				$ratings = [0, 3, 7, 8, 9, 10];
+			if ($interval == 0) {
-				if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) {
-					$rating = $contact['rating'];
-				} elseif (array_key_exists($contact['priority'], $ratings)) {
-					$rating = $ratings[$contact['priority']];
-				} else {
-					$rating = -1;
-				}
-			} else {
-				// Check once a week per default for all other networks
-				$rating = 9;
-			}
-
-			// Friendica and OStatus are checked once a day
-			if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) {
-				$rating = 8;
-			}
-
-			// Check archived contacts or contacts with unsupported protocols once a month
-			if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) {
-				$rating = 10;
-			}
-
-			if ($rating < 0) {
 				continue;
 			}
-			/*
-			 * Based on $contact['priority'], should we poll this site now? Or later?
-			 */
-
-			$min_poll_interval = DI::config()->get('system', 'min_poll_interval');
-
-			$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute',
-				'1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month'];
 
 			$now = DateTimeFormat::utcNow();
-			$next_update = DateTimeFormat::utc($contact['last-update'] . ' + ' . $poll_intervals[$rating]);
+			$next_update = DateTimeFormat::utc($contact['last-update'] . ' + ' . $interval . ' minute');
 
-			if (empty($poll_intervals[$rating]) || ($now < $next_update))  {
+			if ($now < $next_update)  {
-				Logger::debug('No update', ['cid' => $contact['id'], 'rating' => $rating, 'next' => $next_update, 'now' => $now]);
+				Logger::debug('No update', ['cid' => $contact['id'], 'interval' => $interval, 'next' => $next_update, 'now' => $now]);
 				continue;
 			}