From a6a0333170491db85349062a8eb68f58b41e9542 Mon Sep 17 00:00:00 2001 From: Happyfeet01 <3295104+Happyfeet01@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:05:11 +0200 Subject: [PATCH 001/253] Update Dockerfile.apache Update libwp6 to libwp7 --- contrib/docker/Dockerfile.apache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/Dockerfile.apache b/contrib/docker/Dockerfile.apache index 9c33aee17..a400f8797 100644 --- a/contrib/docker/Dockerfile.apache +++ b/contrib/docker/Dockerfile.apache @@ -33,7 +33,7 @@ RUN apt-get update \ # Required for GD libxpm4 \ libxpm-dev \ - libwebp6 \ + libwebp7 \ libwebp-dev \ ## Video Processing ffmpeg \ From 1ea65db70dc9fbc3392f6fc6ce48a4fbe2c3e4e1 Mon Sep 17 00:00:00 2001 From: Happyfeet01 <3295104+Happyfeet01@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:05:55 +0200 Subject: [PATCH 002/253] Update Dockerfile.fpm update libwp6 to libwp7 --- contrib/docker/Dockerfile.fpm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/Dockerfile.fpm b/contrib/docker/Dockerfile.fpm index 0b8e5c113..1bb0a15f7 100644 --- a/contrib/docker/Dockerfile.fpm +++ b/contrib/docker/Dockerfile.fpm @@ -33,7 +33,7 @@ RUN apt-get update \ # Required for GD libxpm4 \ libxpm-dev \ - libwebp6 \ + libwebp7 \ libwebp-dev \ ## Video Processing ffmpeg \ From a04ba18113ed7c0740a9bac0c2c00f0a3d6e0a6c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 22 Aug 2023 00:25:15 -0600 Subject: [PATCH 003/253] Add Groups migrations --- .../2021_08_04_095125_create_groups_table.php | 50 +++++++++++++++++++ ...8_04_095143_create_group_members_table.php | 40 +++++++++++++++ ..._08_04_095238_create_group_posts_table.php | 42 ++++++++++++++++ ..._072457_create_group_invitations_table.php | 38 ++++++++++++++ 4 files changed, 170 insertions(+) create mode 100644 database/migrations/2021_08_04_095125_create_groups_table.php create mode 100644 database/migrations/2021_08_04_095143_create_group_members_table.php create mode 100644 database/migrations/2021_08_04_095238_create_group_posts_table.php create mode 100644 database/migrations/2021_08_16_072457_create_group_invitations_table.php diff --git a/database/migrations/2021_08_04_095125_create_groups_table.php b/database/migrations/2021_08_04_095125_create_groups_table.php new file mode 100644 index 000000000..0af73391c --- /dev/null +++ b/database/migrations/2021_08_04_095125_create_groups_table.php @@ -0,0 +1,50 @@ +bigInteger('id')->unsigned()->primary(); + $table->bigInteger('profile_id')->unsigned()->nullable()->index(); + $table->string('status')->nullable()->index(); + $table->string('name')->nullable(); + $table->text('description')->nullable(); + $table->text('rules')->nullable(); + $table->boolean('local')->default(true)->index(); + $table->string('remote_url')->nullable(); + $table->string('inbox_url')->nullable(); + $table->boolean('is_private')->default(false); + $table->boolean('local_only')->default(false); + $table->json('metadata')->nullable(); + $table->timestamps(); + }); + + Schema::table('statuses', function (Blueprint $table) { + $table->bigInteger('group_id')->unsigned()->nullable()->index(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('groups'); + + Schema::table('statuses', function (Blueprint $table) { + $table->dropColumn('group_id'); + }); + } +} diff --git a/database/migrations/2021_08_04_095143_create_group_members_table.php b/database/migrations/2021_08_04_095143_create_group_members_table.php new file mode 100644 index 000000000..33df26229 --- /dev/null +++ b/database/migrations/2021_08_04_095143_create_group_members_table.php @@ -0,0 +1,40 @@ +id(); + $table->bigInteger('group_id')->unsigned()->index(); + $table->bigInteger('profile_id')->unsigned()->index(); + $table->string('role')->default('member')->index(); + $table->boolean('local_group')->default(false)->index(); + $table->boolean('local_profile')->default(false)->index(); + $table->boolean('join_request')->default(false)->index(); + $table->timestamp('approved_at')->nullable(); + $table->timestamp('rejected_at')->nullable(); + $table->unique(['group_id', 'profile_id']); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('group_members'); + } +} diff --git a/database/migrations/2021_08_04_095238_create_group_posts_table.php b/database/migrations/2021_08_04_095238_create_group_posts_table.php new file mode 100644 index 000000000..a5e637d8e --- /dev/null +++ b/database/migrations/2021_08_04_095238_create_group_posts_table.php @@ -0,0 +1,42 @@ +bigInteger('id')->unsigned()->primary(); + $table->bigInteger('group_id')->unsigned()->index(); + $table->bigInteger('profile_id')->unsigned()->nullable()->index(); + $table->string('type')->nullable()->index(); + $table->bigInteger('status_id')->unsigned()->unique(); + $table->string('remote_url')->unique()->nullable()->index(); + $table->bigInteger('reply_child_id')->unsigned()->nullable(); + $table->bigInteger('in_reply_to_id')->unsigned()->nullable(); + $table->bigInteger('reblog_of_id')->unsigned()->nullable(); + $table->unsignedInteger('reply_count')->nullable(); + $table->string('status')->nullable()->index(); + $table->json('metadata')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('group_posts'); + } +} diff --git a/database/migrations/2021_08_16_072457_create_group_invitations_table.php b/database/migrations/2021_08_16_072457_create_group_invitations_table.php new file mode 100644 index 000000000..aa13db23a --- /dev/null +++ b/database/migrations/2021_08_16_072457_create_group_invitations_table.php @@ -0,0 +1,38 @@ +bigIncrements('id'); + $table->bigInteger('group_id')->unsigned()->index(); + $table->bigInteger('from_profile_id')->unsigned()->index(); + $table->bigInteger('to_profile_id')->unsigned()->index(); + $table->string('role')->nullable(); + $table->boolean('to_local')->default(true)->index(); + $table->boolean('from_local')->default(true)->index(); + $table->unique(['group_id', 'to_profile_id']); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('group_invitations'); + } +} From 347e4f59a3a853088c28ec5d70861bb025dd4e4f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 23 Aug 2023 02:39:16 -0600 Subject: [PATCH 004/253] Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging --- app/Services/FollowerService.php | 8 ++++++-- app/Services/RelationshipService.php | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/app/Services/FollowerService.php b/app/Services/FollowerService.php index 9398fa53f..1c00a6f49 100644 --- a/app/Services/FollowerService.php +++ b/app/Services/FollowerService.php @@ -20,10 +20,14 @@ class FollowerService const FOLLOWING_KEY = 'pf:services:follow:following:id:'; const FOLLOWERS_KEY = 'pf:services:follow:followers:id:'; - public static function add($actor, $target) + public static function add($actor, $target, $refresh = true) { $ts = (int) microtime(true); - RelationshipService::refresh($actor, $target); + if($refresh) { + RelationshipService::refresh($actor, $target); + } else { + RelationshipService::forget($actor, $target); + } Redis::zadd(self::FOLLOWING_KEY . $actor, $ts, $target); Redis::zadd(self::FOLLOWERS_KEY . $target, $ts, $actor); Cache::forget('profile:following:' . $actor); diff --git a/app/Services/RelationshipService.php b/app/Services/RelationshipService.php index 3c6d2818f..476c9c9ae 100644 --- a/app/Services/RelationshipService.php +++ b/app/Services/RelationshipService.php @@ -66,6 +66,14 @@ class RelationshipService return self::get($aid, $tid); } + public static function forget($aid, $tid) + { + Cache::forget('pf:services:follower:audience:' . $aid); + Cache::forget('pf:services:follower:audience:' . $tid); + self::delete($tid, $aid); + self::delete($aid, $tid); + } + public static function defaultRelation($tid) { return [ From 93c7ad977930f290ba7404228dbffe49d9844881 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 23 Aug 2023 02:45:41 -0600 Subject: [PATCH 005/253] Update groups migration --- .../migrations/2021_08_04_095125_create_groups_table.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/database/migrations/2021_08_04_095125_create_groups_table.php b/database/migrations/2021_08_04_095125_create_groups_table.php index 0af73391c..29c63f73e 100644 --- a/database/migrations/2021_08_04_095125_create_groups_table.php +++ b/database/migrations/2021_08_04_095125_create_groups_table.php @@ -28,10 +28,6 @@ class CreateGroupsTable extends Migration $table->json('metadata')->nullable(); $table->timestamps(); }); - - Schema::table('statuses', function (Blueprint $table) { - $table->bigInteger('group_id')->unsigned()->nullable()->index(); - }); } /** @@ -42,9 +38,5 @@ class CreateGroupsTable extends Migration public function down() { Schema::dropIfExists('groups'); - - Schema::table('statuses', function (Blueprint $table) { - $table->dropColumn('group_id'); - }); } } From 61a6d904030df3077c07278d599dad7738d8c173 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 23 Aug 2023 03:45:51 -0600 Subject: [PATCH 006/253] Update FollowServiceWarmCache, improve handling larger following/follower lists --- .../FollowPipeline/FollowServiceWarmCache.php | 79 +++++++++++++---- ...lowServiceWarmCacheLargeIngestPipeline.php | 88 +++++++++++++++++++ 2 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php diff --git a/app/Jobs/FollowPipeline/FollowServiceWarmCache.php b/app/Jobs/FollowPipeline/FollowServiceWarmCache.php index cabea9958..990236f69 100644 --- a/app/Jobs/FollowPipeline/FollowServiceWarmCache.php +++ b/app/Jobs/FollowPipeline/FollowServiceWarmCache.php @@ -8,10 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\Middleware\WithoutOverlapping; use App\Services\AccountService; use App\Services\FollowerService; use Cache; use DB; +use Storage; +use App\Follower; use App\Profile; class FollowServiceWarmCache implements ShouldQueue @@ -23,6 +26,16 @@ class FollowServiceWarmCache implements ShouldQueue public $timeout = 5000; public $failOnTimeout = false; + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping($this->profileId))->dontRelease()]; + } + /** * Create a new job instance. * @@ -42,6 +55,10 @@ class FollowServiceWarmCache implements ShouldQueue { $id = $this->profileId; + if(Cache::has(FollowerService::FOLLOWERS_SYNC_KEY . $id) && Cache::has(FollowerService::FOLLOWING_SYNC_KEY . $id)) { + return; + } + $account = AccountService::get($id, true); if(!$account) { @@ -50,25 +67,43 @@ class FollowServiceWarmCache implements ShouldQueue return; } - DB::table('followers') - ->select('id', 'following_id', 'profile_id') - ->whereFollowingId($id) - ->orderBy('id') - ->chunk(200, function($followers) use($id) { - foreach($followers as $follow) { - FollowerService::add($follow->profile_id, $id); - } - }); + $hasFollowerPostProcessing = false; + $hasFollowingPostProcessing = false; - DB::table('followers') - ->select('id', 'following_id', 'profile_id') - ->whereProfileId($id) - ->orderBy('id') - ->chunk(200, function($followers) use($id) { - foreach($followers as $follow) { - FollowerService::add($id, $follow->following_id); - } - }); + if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) { + $following = []; + $followers = []; + foreach(Follower::lazy() as $follow) { + if($follow->following_id != $id && $follow->profile_id != $id) { + continue; + } + if($follow->profile_id == $id) { + $following[] = $follow->following_id; + } else { + $followers[] = $follow->profile_id; + } + } + + if(count($followers) > 100) { + // store follower ids and process in another job + Storage::put('follow-warm-cache/' . $id . '/followers.json', json_encode($followers)); + $hasFollowerPostProcessing = true; + } else { + foreach($followers as $follower) { + FollowerService::add($follower, $id); + } + } + + if(count($following) > 100) { + // store following ids and process in another job + Storage::put('follow-warm-cache/' . $id . '/following.json', json_encode($following)); + $hasFollowingPostProcessing = true; + } else { + foreach($following as $following) { + FollowerService::add($id, $following); + } + } + } Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $id, 1, 604800); Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $id, 1, 604800); @@ -82,6 +117,14 @@ class FollowServiceWarmCache implements ShouldQueue AccountService::del($id); + if($hasFollowingPostProcessing) { + FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'following')->onQueue('follow'); + } + + if($hasFollowerPostProcessing) { + FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'followers')->onQueue('follow'); + } + return; } } diff --git a/app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php b/app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php new file mode 100644 index 000000000..3299bf7a4 --- /dev/null +++ b/app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php @@ -0,0 +1,88 @@ +profileId = $profileId; + $this->followType = $followType; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $pid = $this->profileId; + $type = $this->followType; + + if($type === 'followers') { + $key = 'follow-warm-cache/' . $pid . '/followers.json'; + if(!Storage::exists($key)) { + return; + } + $file = Storage::get($key); + $json = json_decode($file, true); + + foreach($json as $id) { + FollowerService::add($id, $pid, false); + usleep(random_int(500, 3000)); + } + sleep(5); + Storage::delete($key); + } + + if($type === 'following') { + $key = 'follow-warm-cache/' . $pid . '/following.json'; + if(!Storage::exists($key)) { + return; + } + $file = Storage::get($key); + $json = json_decode($file, true); + + foreach($json as $id) { + FollowerService::add($pid, $id, false); + usleep(random_int(500, 3000)); + } + sleep(5); + Storage::delete($key); + } + + sleep(random_int(2, 5)); + $files = Storage::files('follow-warm-cache/' . $pid); + if(empty($files)) { + Storage::deleteDirectory('follow-warm-cache/' . $pid); + } + } +} From a3696dac95665df9d471c1e30b84d66a749094ab Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 23 Aug 2023 03:54:00 -0600 Subject: [PATCH 007/253] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 357f69f72..686e60576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Release Notes ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev) + +### Updates +- Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59)) +- Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From fc24630ebaeb18976b44cbe9150c1648e6d511a7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 24 Aug 2023 23:31:33 -0600 Subject: [PATCH 008/253] Update Privacy Settings, add support for Mastodon indexable search flag --- .../Controllers/Settings/PrivacySettings.php | 20 +++++++------ .../ActivityPub/ProfileTransformer.php | 7 +++++ ...add_indexable_column_to_profiles_table.php | 28 +++++++++++++++++++ .../views/settings/partial/sidebar.blade.php | 2 ++ resources/views/settings/privacy.blade.php | 22 ++++++++++----- 5 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php diff --git a/app/Http/Controllers/Settings/PrivacySettings.php b/app/Http/Controllers/Settings/PrivacySettings.php index 3d1cd4515..9a5febe83 100644 --- a/app/Http/Controllers/Settings/PrivacySettings.php +++ b/app/Http/Controllers/Settings/PrivacySettings.php @@ -20,13 +20,13 @@ trait PrivacySettings public function privacy() { - $user = Auth::user(); - $settings = $user->settings; - $profile = $user->profile; - $is_private = $profile->is_private; - $settings['is_private'] = (bool) $is_private; + $user = Auth::user(); + $settings = $user->settings; + $profile = $user->profile; + $is_private = $profile->is_private; + $settings['is_private'] = (bool) $is_private; - return view('settings.privacy', compact('settings', 'profile')); + return view('settings.privacy', compact('settings', 'profile')); } public function privacyStore(Request $request) @@ -39,11 +39,13 @@ trait PrivacySettings 'public_dm', 'show_profile_follower_count', 'show_profile_following_count', + 'indexable', 'show_atom', ]; - $profile->is_suggestable = $request->input('is_suggestable') == 'on'; - $profile->save(); + $profile->indexable = $request->input('indexable') == 'on'; + $profile->is_suggestable = $request->input('is_suggestable') == 'on'; + $profile->save(); foreach ($fields as $field) { $form = $request->input($field); @@ -70,6 +72,8 @@ trait PrivacySettings } else { $settings->{$field} = false; } + } elseif ($field == 'indexable') { + } else { if ($form == 'on') { $settings->{$field} = true; diff --git a/app/Transformer/ActivityPub/ProfileTransformer.php b/app/Transformer/ActivityPub/ProfileTransformer.php index 1df7b6100..92e59095f 100644 --- a/app/Transformer/ActivityPub/ProfileTransformer.php +++ b/app/Transformer/ActivityPub/ProfileTransformer.php @@ -16,6 +16,8 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'https://www.w3.org/ns/activitystreams', [ 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers', + 'pixelfed' => 'http://pixelfed.org/ns#', + 'schema' => 'http://schema.org/', 'alsoKnownAs' => [ '@id' => 'as:alsoKnownAs', '@type' => '@id' @@ -23,6 +25,10 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'movedTo' => [ '@id' => 'as:movedTo', '@type' => '@id' + ], + 'indexable' => [ + '@id' => 'pixelfed:indexable', + '@type' => 'schema:Boolean' ] ], ], @@ -37,6 +43,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'summary' => $profile->bio, 'url' => $profile->url(), 'manuallyApprovesFollowers' => (bool) $profile->is_private, + 'indexable' => (bool) $profile->indexable, 'publicKey' => [ 'id' => $profile->permalink().'#main-key', 'owner' => $profile->permalink(), diff --git a/database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php b/database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php new file mode 100644 index 000000000..f735366bd --- /dev/null +++ b/database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php @@ -0,0 +1,28 @@ +boolean('indexable')->default(false)->index(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('profiles', function (Blueprint $table) { + $table->dropColumn('indexable'); + }); + } +}; diff --git a/resources/views/settings/partial/sidebar.blade.php b/resources/views/settings/partial/sidebar.blade.php index b4acf8c9b..a3837066a 100644 --- a/resources/views/settings/partial/sidebar.blade.php +++ b/resources/views/settings/partial/sidebar.blade.php @@ -72,6 +72,8 @@ @media only screen and (min-width: 768px) { border-right: 1px solid #dee2e6 !important } + height: 100%; + flex-grow: 1; } @endpush diff --git a/resources/views/settings/privacy.blade.php b/resources/views/settings/privacy.blade.php index 78ead55ee..57f83c664 100644 --- a/resources/views/settings/privacy.blade.php +++ b/resources/views/settings/privacy.blade.php @@ -28,9 +28,17 @@
crawlable ? 'checked=""':''}} {{$settings->is_private ? 'disabled=""':''}}> -

When your account is visible to search engines, your information can be crawled and stored by search engines.

+

When your account is visible to search engines, your information can be crawled and stored by search engines. {!! $settings->is_private ? 'Not available when your account is private' : ''!!}

+
+ +
+ indexable ? 'checked=""':''}} {{$settings->is_private ? 'disabled=""':''}}> + +

Your public posts may appear in search results on Pixelfed and Mastodon. People who have interacted with your posts may be able to search them regardless. {!! $settings->is_private ? 'Not available when your account is private' : ''!!}

@@ -39,7 +47,7 @@ -

When this option is enabled, your profile is included in the Directory. Only public profiles are eligible.

+

When this option is enabled, your profile is included in the Directory. Only public profiles are eligible. {!! $settings->is_private ? 'Not available when your account is private' : ''!!}

@@ -97,10 +105,10 @@

Enable your profile atom feed. Only public profiles are eligible.

@if($settings->show_atom)

- - {{ $profile->permalink('.atom') }} - - + + {{ $profile->permalink('.atom') }} + +

@endif
From fbdcdd9dbc379de69e1315c9eceedce4a36b4d64 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 24 Aug 2023 23:36:50 -0600 Subject: [PATCH 009/253] Update AP Helpers, consume actor `indexable` attribute --- app/Util/ActivityPub/Helpers.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 7f47a8fea..c933820a5 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -276,7 +276,7 @@ class Helpers { } if(is_array($val)) { - return !empty($val) ? $val[0] : null; + return !empty($val) ? head($val) : null; } return null; @@ -785,6 +785,7 @@ class Helpers { 'inbox_url' => $res['inbox'], 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null, 'public_key' => $res['publicKey']['publicKeyPem'], + 'indexable' => isset($res['indexable']) && is_bool($res['indexable']) ? $res['indexable'] : false, ] ); From 23bc985b360630f774945bac1830913949c0d403 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 24 Aug 2023 23:37:11 -0600 Subject: [PATCH 010/253] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 686e60576..bc4c890df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev) +### Federation +- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) +- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d)) +- ([](https://github.com/pixelfed/pixelfed/commit/)) + ### Updates - Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59)) - Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904)) From 817b4947037fd08c7377486e618afaa2603b58c1 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 24 Aug 2023 23:54:08 -0600 Subject: [PATCH 011/253] Update Profile AP transformer, fix context --- app/Transformer/ActivityPub/ProfileTransformer.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/Transformer/ActivityPub/ProfileTransformer.php b/app/Transformer/ActivityPub/ProfileTransformer.php index 92e59095f..a9d6c4b9f 100644 --- a/app/Transformer/ActivityPub/ProfileTransformer.php +++ b/app/Transformer/ActivityPub/ProfileTransformer.php @@ -15,9 +15,8 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'https://w3id.org/security/v1', 'https://www.w3.org/ns/activitystreams', [ - 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers', 'pixelfed' => 'http://pixelfed.org/ns#', - 'schema' => 'http://schema.org/', + 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers', 'alsoKnownAs' => [ '@id' => 'as:alsoKnownAs', '@type' => '@id' @@ -26,10 +25,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract '@id' => 'as:movedTo', '@type' => '@id' ], - 'indexable' => [ - '@id' => 'pixelfed:indexable', - '@type' => 'schema:Boolean' - ] + 'indexable' => 'pixelfed:indexable', ], ], 'id' => $profile->permalink(), From 83900a3b00fc0fa5c755650b032279899bdab55a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 27 Aug 2023 21:43:11 -0600 Subject: [PATCH 012/253] Update ProfileTransformer, fix Mastodon indexable context --- app/Transformer/ActivityPub/ProfileTransformer.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Transformer/ActivityPub/ProfileTransformer.php b/app/Transformer/ActivityPub/ProfileTransformer.php index a9d6c4b9f..cdd4eb82d 100644 --- a/app/Transformer/ActivityPub/ProfileTransformer.php +++ b/app/Transformer/ActivityPub/ProfileTransformer.php @@ -15,7 +15,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'https://w3id.org/security/v1', 'https://www.w3.org/ns/activitystreams', [ - 'pixelfed' => 'http://pixelfed.org/ns#', + 'toot' => 'http://joinmastodon.org/ns#', 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers', 'alsoKnownAs' => [ '@id' => 'as:alsoKnownAs', @@ -25,7 +25,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract '@id' => 'as:movedTo', '@type' => '@id' ], - 'indexable' => 'pixelfed:indexable', + 'indexable' => 'toot:indexable', ], ], 'id' => $profile->permalink(), From 480394f3d876af0f1e10db35db8390b3e9d56f64 Mon Sep 17 00:00:00 2001 From: David Gabriel Date: Sat, 9 Sep 2023 19:50:45 +0200 Subject: [PATCH 013/253] [Bugfix] Fix for #4518: SQL query that generates the report list in the admin view needs to include the 'id' field --- app/Http/Controllers/Admin/AdminReportController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Controllers/Admin/AdminReportController.php b/app/Http/Controllers/Admin/AdminReportController.php index 4924acfa8..311eac220 100644 --- a/app/Http/Controllers/Admin/AdminReportController.php +++ b/app/Http/Controllers/Admin/AdminReportController.php @@ -643,7 +643,7 @@ trait AdminReportController $q->whereNull('admin_seen') : $q->whereNotNull('admin_seen'); }) - ->groupBy(['object_id', 'object_type']) + ->groupBy(['id', 'object_id', 'object_type']) ->cursorPaginate(6) ->withQueryString() ); From 2e5c141724a78fb9de1a6b2bce947f8e27506703 Mon Sep 17 00:00:00 2001 From: David Gabriel Date: Sat, 9 Sep 2023 20:46:50 +0200 Subject: [PATCH 014/253] Fix similar SQL error which triggers when mentioning people in new posts --- app/Http/Controllers/ComposeController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Controllers/ComposeController.php b/app/Http/Controllers/ComposeController.php index 54526ffe8..9be50f346 100644 --- a/app/Http/Controllers/ComposeController.php +++ b/app/Http/Controllers/ComposeController.php @@ -415,7 +415,7 @@ class ComposeController extends Controller $results = Profile::select('id','domain','username') ->whereNotIn('id', $blocked) ->where('username','like','%'.$q.'%') - ->groupBy('domain') + ->groupBy('id', 'domain') ->limit(15) ->get() ->map(function($profile) { From 74ad26fee64a99c8e37c14ab5c69956ada4edb68 Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Sat, 9 Sep 2023 22:54:11 +0200 Subject: [PATCH 015/253] Fix potential memory leak due to not calling imagedestroy on GdImage objects --- app/Services/InstanceService.php | 3 +++ app/Util/Media/Blurhash.php | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/Services/InstanceService.php b/app/Services/InstanceService.php index 1cead8d48..2ad991063 100644 --- a/app/Services/InstanceService.php +++ b/app/Services/InstanceService.php @@ -120,6 +120,9 @@ class InstanceService $pixels[] = $row; } + // Free the allocated GdImage object from memory: + imagedestroy($image); + $components_x = 4; $components_y = 4; $blurhash = Blurhash::encode($pixels, $components_x, $components_y); diff --git a/app/Util/Media/Blurhash.php b/app/Util/Media/Blurhash.php index c0cca59b9..8e232ea17 100644 --- a/app/Util/Media/Blurhash.php +++ b/app/Util/Media/Blurhash.php @@ -44,6 +44,9 @@ class Blurhash { $pixels[] = $row; } + // Free the allocated GdImage object from memory: + imagedestroy($image); + $components_x = 4; $components_y = 4; $blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y); @@ -53,4 +56,4 @@ class Blurhash { return $blurhash; } -} \ No newline at end of file +} From 941736ce6c29b657c3e7bed831b78d698b720fe4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 10 Sep 2023 01:12:27 -0600 Subject: [PATCH 016/253] Update StoryApiV1Controller, add viewers route to view story viewers --- .../Stories/StoryApiV1Controller.php | 23 +++++++++++++++++++ app/Http/Resources/StoryView.php | 20 ++++++++++++++++ app/Http/Resources/StoryViewerResource.php | 20 ++++++++++++++++ routes/api.php | 1 + 4 files changed, 64 insertions(+) create mode 100644 app/Http/Resources/StoryView.php create mode 100644 app/Http/Resources/StoryViewerResource.php diff --git a/app/Http/Controllers/Stories/StoryApiV1Controller.php b/app/Http/Controllers/Stories/StoryApiV1Controller.php index e32fffa26..db2b1f533 100644 --- a/app/Http/Controllers/Stories/StoryApiV1Controller.php +++ b/app/Http/Controllers/Stories/StoryApiV1Controller.php @@ -20,6 +20,7 @@ use App\Jobs\StoryPipeline\StoryViewDeliver; use App\Services\AccountService; use App\Services\MediaPathService; use App\Services\StoryService; +use App\Http\Resources\StoryView as StoryViewResource; class StoryApiV1Controller extends Controller { @@ -355,4 +356,26 @@ class StoryApiV1Controller extends Controller $path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension()); return $path; } + + public function viewers(Request $request) + { + abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404); + + $this->validate($request, [ + 'sid' => 'required|string|min:1|max:50' + ]); + + $pid = $request->user()->profile_id; + $sid = $request->input('sid'); + + $story = Story::whereProfileId($pid) + ->whereActive(true) + ->findOrFail($sid); + + $viewers = StoryView::whereStoryId($story->id) + ->orderByDesc('id') + ->cursorPaginate(10); + + return StoryViewResource::collection($viewers); + } } diff --git a/app/Http/Resources/StoryView.php b/app/Http/Resources/StoryView.php new file mode 100644 index 000000000..891bf2eee --- /dev/null +++ b/app/Http/Resources/StoryView.php @@ -0,0 +1,20 @@ + + */ + public function toArray(Request $request) + { + return AccountService::get($this->profile_id, true); + } +} diff --git a/app/Http/Resources/StoryViewerResource.php b/app/Http/Resources/StoryViewerResource.php new file mode 100644 index 000000000..8cae091bd --- /dev/null +++ b/app/Http/Resources/StoryViewerResource.php @@ -0,0 +1,20 @@ + + */ + public function toArray(Request $request): array + { + return AccountService::get($this->profile_id, false); + } +} diff --git a/routes/api.php b/routes/api.php index f305f277c..23abfc323 100644 --- a/routes/api.php +++ b/routes/api.php @@ -316,6 +316,7 @@ Route::group(['prefix' => 'api'], function() use($middleware) { Route::post('seen', 'Stories\StoryApiV1Controller@viewed')->middleware($middleware); Route::post('self-expire/{id}', 'Stories\StoryApiV1Controller@delete')->middleware($middleware); Route::post('comment', 'Stories\StoryApiV1Controller@comment')->middleware($middleware); + Route::get('viewers', 'Stories\StoryApiV1Controller@viewers')->middleware($middleware); }); }); }); From 8fa2afe016e7684daf414fecec158ce6ac076243 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 10 Sep 2023 01:13:45 -0600 Subject: [PATCH 017/253] Remove unused resource --- app/Http/Resources/StoryViewerResource.php | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 app/Http/Resources/StoryViewerResource.php diff --git a/app/Http/Resources/StoryViewerResource.php b/app/Http/Resources/StoryViewerResource.php deleted file mode 100644 index 8cae091bd..000000000 --- a/app/Http/Resources/StoryViewerResource.php +++ /dev/null @@ -1,20 +0,0 @@ - - */ - public function toArray(Request $request): array - { - return AccountService::get($this->profile_id, false); - } -} From 3979e33b5741128723e2c4e1d6b32cf236c7f45c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 10 Sep 2023 01:14:08 -0600 Subject: [PATCH 018/253] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc4c890df..45f5a80de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Updates - Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59)) - Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904)) +- Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From a510c3e89c686b49e51472c124c5f760e96bb7e4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 14 Sep 2023 22:23:46 -0600 Subject: [PATCH 019/253] Add AdminShadowFilter model/migration --- app/Models/AdminShadowFilter.php | 27 +++++++++++ ...4900_create_admin_shadow_filters_table.php | 47 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 app/Models/AdminShadowFilter.php create mode 100644 database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php diff --git a/app/Models/AdminShadowFilter.php b/app/Models/AdminShadowFilter.php new file mode 100644 index 000000000..f98086f7f --- /dev/null +++ b/app/Models/AdminShadowFilter.php @@ -0,0 +1,27 @@ + 'datetime' + ]; + + public function account() + { + if($this->item_type === 'App\Profile') { + return AccountService::get($this->item_id, true); + } + + return; + } +} diff --git a/database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php b/database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php new file mode 100644 index 000000000..6b62f32c2 --- /dev/null +++ b/database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php @@ -0,0 +1,47 @@ +id(); + $table->unsignedBigInteger('admin_id')->nullable(); + $table->morphs('item'); + $table->boolean('is_local')->default(true)->index(); + $table->text('note')->nullable(); + $table->boolean('active')->default(false)->index(); + $table->json('history')->nullable(); + $table->json('ruleset')->nullable(); + $table->boolean('prevent_ap_fanout')->default(false)->index(); + $table->boolean('prevent_new_dms')->default(false)->index(); + $table->boolean('ignore_reports')->default(false)->index(); + $table->boolean('ignore_mentions')->default(false)->index(); + $table->boolean('ignore_links')->default(false)->index(); + $table->boolean('ignore_hashtags')->default(false)->index(); + $table->boolean('hide_from_public_feeds')->default(false)->index(); + $table->boolean('hide_from_tag_feeds')->default(false)->index(); + $table->boolean('hide_embeds')->default(false)->index(); + $table->boolean('hide_from_story_carousel')->default(false)->index(); + $table->boolean('hide_from_search_autocomplete')->default(false)->index(); + $table->boolean('hide_from_search')->default(false)->index(); + $table->boolean('requires_login')->default(false)->index(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('admin_shadow_filters'); + } +}; From 33ed7a8c9182e4453e2d780abd5e32fc336004ce Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 14 Sep 2023 22:32:37 -0600 Subject: [PATCH 020/253] Add AdminShadowFilter feature --- .../AdminShadowFilterController.php | 122 ++++++++++++++++++ app/Jobs/StatusPipeline/StatusEntityLexer.php | 5 +- app/Services/AdminShadowFilterService.php | 51 ++++++++ app/Services/PublicTimelineService.php | 10 +- resources/views/admin/asf/create.blade.php | 64 +++++++++ resources/views/admin/asf/edit.blade.php | 64 +++++++++ resources/views/admin/asf/home.blade.php | 81 ++++++++++++ routes/web.php | 7 + 8 files changed, 399 insertions(+), 5 deletions(-) create mode 100644 app/Http/Controllers/AdminShadowFilterController.php create mode 100644 app/Services/AdminShadowFilterService.php create mode 100644 resources/views/admin/asf/create.blade.php create mode 100644 resources/views/admin/asf/edit.blade.php create mode 100644 resources/views/admin/asf/home.blade.php diff --git a/app/Http/Controllers/AdminShadowFilterController.php b/app/Http/Controllers/AdminShadowFilterController.php new file mode 100644 index 000000000..461e1d0c2 --- /dev/null +++ b/app/Http/Controllers/AdminShadowFilterController.php @@ -0,0 +1,122 @@ +middleware(['auth','admin']); + } + + public function home(Request $request) + { + $filter = $request->input('filter'); + $searchQuery = $request->input('q'); + $filters = AdminShadowFilter::when($filter, function($q, $filter) { + if($filter == 'all') { + return $q; + } else if($filter == 'inactive') { + return $q->whereActive(false); + } else { + return $q; + } + }, function($q, $filter) { + return $q->whereActive(true); + }) + ->when($searchQuery, function($q, $searchQuery) { + $ids = Profile::where('username', 'like', '%' . $searchQuery . '%') + ->limit(100) + ->pluck('id') + ->toArray(); + return $q->where('item_type', 'App\Profile')->whereIn('item_id', $ids); + }) + ->latest() + ->paginate(10) + ->withQueryString(); + + return view('admin.asf.home', compact('filters')); + } + + public function create(Request $request) + { + return view('admin.asf.create'); + } + + public function edit(Request $request, $id) + { + $filter = AdminShadowFilter::findOrFail($id); + $profile = AccountService::get($filter->item_id); + return view('admin.asf.edit', compact('filter', 'profile')); + } + + public function store(Request $request) + { + $this->validate($request, [ + 'username' => 'required', + 'active' => 'sometimes', + 'note' => 'sometimes', + 'hide_from_public_feeds' => 'sometimes' + ]); + + $profile = Profile::whereUsername($request->input('username'))->first(); + + if(!$profile) { + return back()->withErrors(['Invalid account']); + } + + if($profile->user && $profile->user->is_admin) { + return back()->withErrors(['Cannot filter an admin account']); + } + + $active = $request->has('active') && $request->has('hide_from_public_feeds'); + + AdminShadowFilter::updateOrCreate([ + 'item_id' => $profile->id, + 'item_type' => get_class($profile) + ], [ + 'is_local' => $profile->domain === null, + 'note' => $request->input('note'), + 'hide_from_public_feeds' => $request->has('hide_from_public_feeds'), + 'admin_id' => $request->user()->profile_id, + 'active' => $active + ]); + + AdminShadowFilterService::refresh(); + + return redirect('/i/admin/asf/home'); + } + + public function storeEdit(Request $request, $id) + { + $this->validate($request, [ + 'active' => 'sometimes', + 'note' => 'sometimes', + 'hide_from_public_feeds' => 'sometimes' + ]); + + $filter = AdminShadowFilter::findOrFail($id); + + $profile = Profile::findOrFail($filter->item_id); + + if($profile->user && $profile->user->is_admin) { + return back()->withErrors(['Cannot filter an admin account']); + } + + $active = $request->has('active'); + $filter->active = $active; + $filter->hide_from_public_feeds = $request->has('hide_from_public_feeds'); + $filter->note = $request->input('note'); + $filter->save(); + + AdminShadowFilterService::refresh(); + + return redirect('/i/admin/asf/home'); + } +} diff --git a/app/Jobs/StatusPipeline/StatusEntityLexer.php b/app/Jobs/StatusPipeline/StatusEntityLexer.php index d205f1e21..2bbc92102 100644 --- a/app/Jobs/StatusPipeline/StatusEntityLexer.php +++ b/app/Jobs/StatusPipeline/StatusEntityLexer.php @@ -20,6 +20,7 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use App\Services\UserFilterService; +use App\Services\AdminShadowFilterService; class StatusEntityLexer implements ShouldQueue { @@ -176,7 +177,9 @@ class StatusEntityLexer implements ShouldQueue $status->reblog_of_id === null && ($hideNsfw ? $status->is_nsfw == false : true) ) { - PublicTimelineService::add($status->id); + if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) { + PublicTimelineService::add($status->id); + } } if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') { diff --git a/app/Services/AdminShadowFilterService.php b/app/Services/AdminShadowFilterService.php new file mode 100644 index 000000000..a5933508a --- /dev/null +++ b/app/Services/AdminShadowFilterService.php @@ -0,0 +1,51 @@ +whereActive(1) + ->where('hide_from_public_feeds', true) + ->pluck('item_id') + ->toArray(); + } + + public static function getHideFromPublicFeedsList($refresh = false) + { + $key = self::CACHE_KEY . 'list:hide_from_public_feeds'; + if($refresh) { + Cache::forget($key); + } + return Cache::remember($key, 86400, function() { + return AdminShadowFilter::whereItemType('App\Profile') + ->whereActive(1) + ->where('hide_from_public_feeds', true) + ->pluck('item_id') + ->toArray(); + }); + } + + public static function canAddToPublicFeedByProfileId($profileId) + { + return !in_array($profileId, self::getHideFromPublicFeedsList()); + } + + public static function refresh() + { + $keys = [ + self::CACHE_KEY . 'list:hide_from_public_feeds' + ]; + + foreach($keys as $key) { + Cache::forget($key); + } + } +} diff --git a/app/Services/PublicTimelineService.php b/app/Services/PublicTimelineService.php index f2658e4b1..7cd6816b3 100644 --- a/app/Services/PublicTimelineService.php +++ b/app/Services/PublicTimelineService.php @@ -95,7 +95,7 @@ class PublicTimelineService { if(self::count() == 0 || $force == true) { $hideNsfw = config('instance.hide_nsfw_on_public_feeds'); Redis::del(self::CACHE_KEY); - $minId = SnowflakeService::byDate(now()->subDays(14)); + $minId = SnowflakeService::byDate(now()->subDays(90)); $ids = Status::where('id', '>', $minId) ->whereNull(['uri', 'in_reply_to_id', 'reblog_of_id']) ->when($hideNsfw, function($q, $hideNsfw) { @@ -105,9 +105,11 @@ class PublicTimelineService { ->whereScope('public') ->orderByDesc('id') ->limit($limit) - ->pluck('id'); - foreach($ids as $id) { - self::add($id); + ->pluck('id', 'profile_id'); + foreach($ids as $k => $id) { + if(AdminShadowFilterService::canAddToPublicFeedByProfileId($k)) { + self::add($id); + } } return 1; } diff --git a/resources/views/admin/asf/create.blade.php b/resources/views/admin/asf/create.blade.php new file mode 100644 index 000000000..8fc88e4bd --- /dev/null +++ b/resources/views/admin/asf/create.blade.php @@ -0,0 +1,64 @@ +@extends('admin.partial.template-full') + +@section('section') +
+
+
+
+
+

New Shadow Filters

+

Creating a new admin shadow filter

+
+
+
+
+
+
+
+
+
+ @if ($errors->any()) +
+
    + @foreach ($errors->all() as $error) +
  • {{ $error }}
  • + @endforeach +
+
+ @endif +
+
+ @csrf +
+ + +
+ +

Filters

+
+
+
+ + +
+
+ {{--
--}} +
+ +
+ + +
+
+ + +
+
+ +
+
+
+
+
+
+@endsection diff --git a/resources/views/admin/asf/edit.blade.php b/resources/views/admin/asf/edit.blade.php new file mode 100644 index 000000000..6d7a633f0 --- /dev/null +++ b/resources/views/admin/asf/edit.blade.php @@ -0,0 +1,64 @@ +@extends('admin.partial.template-full') + +@section('section') +
+
+
+
+
+

Edit Shadow Filters

+

Editing shadow filters

+
+
+
+
+
+
+
+
+
+ @if ($errors->any()) +
+
    + @foreach ($errors->all() as $error) +
  • {{ $error }}
  • + @endforeach +
+
+ @endif +
+
+ @csrf +
+ + +
+ +

Filters

+
+
+
+ hide_from_public_feeds ? 'checked=""' : '' !!}> + +
+
+ {{--
--}} +
+ +
+ + +
+
+ active ? 'checked=""' : ''}}> + +
+
+ +
+
+
+
+
+
+@endsection diff --git a/resources/views/admin/asf/home.blade.php b/resources/views/admin/asf/home.blade.php new file mode 100644 index 000000000..4fbb7730f --- /dev/null +++ b/resources/views/admin/asf/home.blade.php @@ -0,0 +1,81 @@ +@extends('admin.partial.template-full') + +@section('section') +
+
+
+
+
+

Admin Shadow Filters

+

Manage shadow filters across Accounts, Hashtags, Feeds and Stories

+
+
+
+
+
+
+
+
+
+ +
+ +
+
+ +
+
+
+ +
+ + + + + + + + + + + + @foreach($filters as $filter) + + + + + + + + @endforeach + +
IDUsernameHide FeedsActiveCreated
{{ $filter->id }} +
+ + +

+ @{{ $filter->account()['acct'] }} +

+
+
{{ $filter->hide_from_public_feeds ? '✅' : ''}}{{ $filter->active ? '✅' : ''}}{{ $filter->created_at->diffForHumans() }}
+ +
+ {{ $filters->links() }} +
+
+
+
+@endsection diff --git a/routes/web.php b/routes/web.php index bb091fce5..b823b8729 100644 --- a/routes/web.php +++ b/routes/web.php @@ -96,6 +96,13 @@ Route::domain(config('pixelfed.domain.admin'))->prefix('i/admin')->group(functio Route::get('autospam/home', 'AdminController@autospamHome')->name('admin.autospam'); + Route::redirect('asf/', 'asf/home'); + Route::get('asf/home', 'AdminShadowFilterController@home'); + Route::get('asf/create', 'AdminShadowFilterController@create'); + Route::get('asf/edit/{id}', 'AdminShadowFilterController@edit'); + Route::post('asf/edit/{id}', 'AdminShadowFilterController@storeEdit'); + Route::post('asf/create', 'AdminShadowFilterController@store'); + Route::prefix('api')->group(function() { Route::get('stats', 'AdminController@getStats'); Route::get('accounts', 'AdminController@getAccounts'); From 2496386d9b0403a47835cb8ac2fd896f0a3a7b0f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 17 Sep 2023 23:51:42 -0600 Subject: [PATCH 021/253] Update NotificationService, improve cache warming query --- app/Services/NotificationService.php | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/app/Services/NotificationService.php b/app/Services/NotificationService.php index 139b13a69..81ac0b912 100644 --- a/app/Services/NotificationService.php +++ b/app/Services/NotificationService.php @@ -16,6 +16,8 @@ use League\Fractal\Pagination\IlluminatePaginatorAdapter; class NotificationService { const CACHE_KEY = 'pf:services:notifications:ids:'; + const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:'; + const ITEM_CACHE_TTL = 86400; const MASTODON_TYPES = [ 'follow', 'follow_request', @@ -44,11 +46,18 @@ class NotificationService { return $res; } + public static function getEpochId($months = 6) + { + return Cache::remember(self::EPOCH_CACHE_KEY . $months, 1209600, function() use($months) { + return Notification::where('created_at', '>', now()->subMonths($months))->first()->id; + }); + } + public static function coldGet($id, $start = 0, $stop = 400) { $stop = $stop > 400 ? 400 : $stop; - $ids = Notification::whereProfileId($id) - ->latest() + $ids = Notification::where('id', '>', self::getEpochId()) + ->where('profile_id', $id) ->skip($start) ->take($stop) ->pluck('id'); @@ -227,7 +236,7 @@ class NotificationService { public static function getNotification($id) { - $notification = Cache::remember('service:notification:'.$id, 86400, function() use($id) { + $notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function() use($id) { $n = Notification::with('item')->find($id); if(!$n) { @@ -259,19 +268,19 @@ class NotificationService { public static function setNotification(Notification $notification) { - return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) { + return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function() use($notification) { $fractal = new Fractal\Manager(); $fractal->setSerializer(new ArraySerializer()); $resource = new Fractal\Resource\Item($notification, new NotificationTransformer()); return $fractal->createData($resource)->toArray(); }); - } + } public static function warmCache($id, $stop = 400, $force = false) { if(self::count($id) == 0 || $force == true) { - $ids = Notification::whereProfileId($id) - ->latest() + $ids = Notification::where('profile_id', $id) + ->where('id', '>', self::getEpochId()) ->limit($stop) ->pluck('id'); foreach($ids as $key) { From 223661ecb2555e068346a47a989d105cbbd0d782 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 17 Sep 2023 23:54:31 -0600 Subject: [PATCH 022/253] Update StatusService, hydrate accounts on request instead of caching them along with status objects --- app/Services/StatusService.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/app/Services/StatusService.php b/app/Services/StatusService.php index bdf2f31db..9730bb2d4 100644 --- a/app/Services/StatusService.php +++ b/app/Services/StatusService.php @@ -22,9 +22,9 @@ class StatusService return self::CACHE_KEY . $p . $id; } - public static function get($id, $publicOnly = true) + public static function get($id, $publicOnly = true, $mastodonMode = false) { - return Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) { + $res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) { if($publicOnly) { $status = Status::whereScope('public')->find($id); } else { @@ -36,13 +36,23 @@ class StatusService $fractal = new Fractal\Manager(); $fractal->setSerializer(new ArraySerializer()); $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer()); - return $fractal->createData($resource)->toArray(); + $res = $fractal->createData($resource)->toArray(); + $res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null; + if(isset($res['_pid'])) { + unset($res['account']); + } + return $res; }); + if($res) { + $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true); + unset($res['_pid']); + } + return $res; } public static function getMastodon($id, $publicOnly = true) { - $status = self::get($id, $publicOnly); + $status = self::get($id, $publicOnly, true); if(!$status) { return null; } From 5ab7f9958ccaebb1e7cd1b380a1fd48c7718252d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 17 Sep 2023 23:57:52 -0600 Subject: [PATCH 023/253] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f5a80de..024aa7423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ - Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59)) - Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904)) - Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce)) +- Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d)) +- Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 61d235b797973eb60acf6951c9b03dffa1890069 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Sep 2023 00:17:22 -0600 Subject: [PATCH 024/253] Update StatusService, fix logic check --- app/Services/StatusService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/StatusService.php b/app/Services/StatusService.php index 9730bb2d4..68bea8a13 100644 --- a/app/Services/StatusService.php +++ b/app/Services/StatusService.php @@ -43,7 +43,7 @@ class StatusService } return $res; }); - if($res) { + if($res && isset($res['_pid'])) { $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true); unset($res['_pid']); } From 0210f8aa2a541a40faafdec3d8d92058f51a6b26 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Sep 2023 00:25:01 -0600 Subject: [PATCH 025/253] Update NotificationService, fix order bug --- app/Services/NotificationService.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/Services/NotificationService.php b/app/Services/NotificationService.php index 81ac0b912..c068f8278 100644 --- a/app/Services/NotificationService.php +++ b/app/Services/NotificationService.php @@ -58,6 +58,7 @@ class NotificationService { $stop = $stop > 400 ? 400 : $stop; $ids = Notification::where('id', '>', self::getEpochId()) ->where('profile_id', $id) + ->orderByDesc('id') ->skip($start) ->take($stop) ->pluck('id'); @@ -281,6 +282,7 @@ class NotificationService { if(self::count($id) == 0 || $force == true) { $ids = Notification::where('profile_id', $id) ->where('id', '>', self::getEpochId()) + ->orderByDesc('id') ->limit($stop) ->pluck('id'); foreach($ids as $key) { From dc23c21db0848c0e32493540700f78b351760dca Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Sep 2023 06:42:10 -0600 Subject: [PATCH 026/253] Update profile embed, fix resize --- resources/views/profile/embed.blade.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/resources/views/profile/embed.blade.php b/resources/views/profile/embed.blade.php index 0050a90e2..d5921431a 100644 --- a/resources/views/profile/embed.blade.php +++ b/resources/views/profile/embed.blade.php @@ -73,7 +73,9 @@ - + From dcdfb28dcd86a743d499a792ed0aeafc14365997 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Sep 2023 06:47:59 -0600 Subject: [PATCH 027/253] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 024aa7423..1b0d55a00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce)) - Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d)) - Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec)) +- Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From fb1deb6e28e61738acdc35fcc928ee7222ae741d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 00:59:24 -0600 Subject: [PATCH 028/253] Add Resilient Media Storage --- app/Services/MediaStorageService.php | 11 ++-- app/Services/ResilientMediaStorageService.php | 66 +++++++++++++++++++ config/filesystems.php | 28 ++++++++ config/media.php | 38 ++++++----- 4 files changed, 119 insertions(+), 24 deletions(-) create mode 100644 app/Services/ResilientMediaStorageService.php diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index b52662d1f..b547ee39c 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -86,12 +86,11 @@ class MediaStorageService { $thumbname = array_pop($pt); $storagePath = implode('/', $p); - $disk = Storage::disk(config('filesystems.cloud')); - $file = $disk->putFileAs($storagePath, new File($path), $name, 'public'); - $url = $disk->url($file); - $thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public'); - $thumbUrl = $disk->url($thumbFile); - $media->thumbnail_url = $thumbUrl; + $url = ResilientMediaStorageService::store($storagePath, $path, $name); + if($thumb) { + $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); + $media->thumbnail_url = $thumbUrl; + } $media->cdn_url = $url; $media->optimized_url = $url; $media->replicated_at = now(); diff --git a/app/Services/ResilientMediaStorageService.php b/app/Services/ResilientMediaStorageService.php new file mode 100644 index 000000000..ac1b089af --- /dev/null +++ b/app/Services/ResilientMediaStorageService.php @@ -0,0 +1,66 @@ +putFileAs($storagePath, new File($path), $name, 'public'); + return $disk->url($file); + }, random_int(100, 500)); + } + + public static function handleResilientStore($storagePath, $path, $name) + { + $attempts = 0; + return retry(4, function() use($storagePath, $path, $name, $attempts) { + self::$attempts++; + usleep(100000); + $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud'); + try { + $disk = Storage::disk($baseDisk); + $file = $disk->putFileAs($storagePath, new File($path), $name, 'public'); + } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {} + return $disk->url($file); + }, function (int $attempt, Exception $exception) { + return $attempt * 200; + }); + } + + public static function getAltDriver() + { + $drivers = []; + if(config('filesystems.disks.alt-primary.enabled')) { + $drivers[] = 'alt-primary'; + } + if(config('filesystems.disks.alt-secondary.enabled')) { + $drivers[] = 'alt-secondary'; + } + if(empty($drivers)) { + return false; + } + $key = array_rand($drivers, 1); + return $drivers[$key]; + } +} diff --git a/config/filesystems.php b/config/filesystems.php index 6817d5e34..80e63ed99 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -79,6 +79,34 @@ return [ 'throw' => true, ], + 'alt-primary' => [ + 'enabled' => env('ALT_PRI_ENABLED', false), + 'driver' => 's3', + 'key' => env('ALT_PRI_AWS_ACCESS_KEY_ID'), + 'secret' => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'), + 'region' => env('ALT_PRI_AWS_DEFAULT_REGION'), + 'bucket' => env('ALT_PRI_AWS_BUCKET'), + 'visibility' => 'public', + 'url' => env('ALT_PRI_AWS_URL'), + 'endpoint' => env('ALT_PRI_AWS_ENDPOINT'), + 'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + ], + + 'alt-secondary' => [ + 'enabled' => env('ALT_SEC_ENABLED', false), + 'driver' => 's3', + 'key' => env('ALT_SEC_AWS_ACCESS_KEY_ID'), + 'secret' => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'), + 'region' => env('ALT_SEC_AWS_DEFAULT_REGION'), + 'bucket' => env('ALT_SEC_AWS_BUCKET'), + 'visibility' => 'public', + 'url' => env('ALT_SEC_AWS_URL'), + 'endpoint' => env('ALT_SEC_AWS_ENDPOINT'), + 'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + ], + 'spaces' => [ 'driver' => 's3', 'key' => env('DO_SPACES_KEY'), diff --git a/config/media.php b/config/media.php index b7d6e95cc..f550ff291 100644 --- a/config/media.php +++ b/config/media.php @@ -1,24 +1,26 @@ env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), + 'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), - 'exif' => [ - 'database' => env('MEDIA_EXIF_DATABASE', false), - ], + 'exif' => [ + 'database' => env('MEDIA_EXIF_DATABASE', false), + ], - 'storage' => [ - 'remote' => [ - /* - |-------------------------------------------------------------------------- - | Store remote media on cloud/S3 - |-------------------------------------------------------------------------- - | - | Set this to cache remote media on cloud/S3 filesystem drivers. - | Disabled by default. - | - */ - 'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false) - ], - ] + 'storage' => [ + 'remote' => [ + /* + |-------------------------------------------------------------------------- + | Store remote media on cloud/S3 + |-------------------------------------------------------------------------- + | + | Set this to cache remote media on cloud/S3 filesystem drivers. + | Disabled by default. + | + */ + 'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false), + + 'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false), + ], + ] ]; From 439638f7d78dc3907cf66bff6aacbf36b1edc250 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 01:13:14 -0600 Subject: [PATCH 029/253] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b0d55a00..d294f472c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev) +### Added +- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6)) + ### Federation - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) - Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d)) From d969a97360134eacd1462be0c65e5b4e12ef3cea Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 01:57:43 -0600 Subject: [PATCH 030/253] Update Status model, improve thumb logic --- app/Services/StatusService.php | 2 -- app/Status.php | 30 +++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/app/Services/StatusService.php b/app/Services/StatusService.php index 68bea8a13..4051bede4 100644 --- a/app/Services/StatusService.php +++ b/app/Services/StatusService.php @@ -161,8 +161,6 @@ class StatusService } Cache::forget('status:transformer:media:attachments:' . $id); MediaService::del($id); - Cache::forget('status:thumb:nsfw0' . $id); - Cache::forget('status:thumb:nsfw1' . $id); Cache::forget('pf:services:sh:id:' . $id); PublicTimelineService::rem($id); NetworkTimelineService::rem($id); diff --git a/app/Status.php b/app/Status.php index 77262597e..9b88c4903 100644 --- a/app/Status.php +++ b/app/Status.php @@ -9,7 +9,9 @@ use App\Http\Controllers\StatusController; use Illuminate\Database\Eloquent\SoftDeletes; use App\Models\Poll; use App\Services\AccountService; +use App\Services\StatusService; use App\Models\StatusEdit; +use Illuminate\Support\Str; class Status extends Model { @@ -95,16 +97,26 @@ class Status extends Model public function thumb($showNsfw = false) { - $key = $showNsfw ? 'status:thumb:nsfw1'.$this->id : 'status:thumb:nsfw0'.$this->id; - return Cache::remember($key, now()->addMinutes(15), function() use ($showNsfw) { - $type = $this->type ?? $this->setType(); - $is_nsfw = !$showNsfw ? $this->is_nsfw : false; - if ($this->media->count() == 0 || $is_nsfw || !in_array($type,['photo', 'photo:album', 'video'])) { - return url(Storage::url('public/no-preview.png')); - } + $entity = StatusService::get($this->id); - return url(Storage::url($this->firstMedia()->thumbnail_path)); - }); + if(!$entity || !isset($entity['media_attachments']) || empty($entity['media_attachments'])) { + return url(Storage::url('public/no-preview.png')); + } + + if((!isset($entity['sensitive']) || $entity['sensitive']) && !$showNsfw) { + return url(Storage::url('public/no-preview.png')); + } + + return collect($entity['media_attachments']) + ->filter(fn($media) => $media['type'] == 'image' && in_array($media['mime'], ['image/jpeg', 'image/png'])) + ->map(function($media) { + if(!Str::endsWith($media['preview_url'], ['no-preview.png', 'no-preview.jpg'])) { + return $media['preview_url']; + } + + return $media['url']; + }) + ->first() ?? url(Storage::url('public/no-preview.png')); } public function url($forceLocal = false) From 2d428f43e89a94865eb3c5f1b6456c60d6ac94d8 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 01:58:09 -0600 Subject: [PATCH 031/253] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d294f472c..36092276a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d)) - Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec)) - Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d)) +- Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 1f0a45b7f4bfa004cb75944121c1a64147e40958 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 02:10:19 -0600 Subject: [PATCH 032/253] Update Status model, allow unlisted thumbnails --- app/Status.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Status.php b/app/Status.php index 9b88c4903..d665464ae 100644 --- a/app/Status.php +++ b/app/Status.php @@ -97,7 +97,7 @@ class Status extends Model public function thumb($showNsfw = false) { - $entity = StatusService::get($this->id); + $entity = StatusService::get($this->id, false); if(!$entity || !isset($entity['media_attachments']) || empty($entity['media_attachments'])) { return url(Storage::url('public/no-preview.png')); @@ -107,6 +107,10 @@ class Status extends Model return url(Storage::url('public/no-preview.png')); } + if(!isset($entity['visibility']) || !in_array($entity['visibility'], ['public', 'unlisted'])) { + return url(Storage::url('public/no-preview.png')); + } + return collect($entity['media_attachments']) ->filter(fn($media) => $media['type'] == 'image' && in_array($media['mime'], ['image/jpeg', 'image/png'])) ->map(function($media) { From d295e6059b60c93fcb2bbc9a1f1a5e4408cb8d1c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 04:23:04 -0600 Subject: [PATCH 033/253] Update StatusTagsPipeline, fix object tags and slug normalization --- .../StatusPipeline/StatusTagsPipeline.php | 13 +- tests/Unit/ActivityPubTagObjectTest.php | 133 ++++++++++++++++++ 2 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 tests/Unit/ActivityPubTagObjectTest.php diff --git a/app/Jobs/StatusPipeline/StatusTagsPipeline.php b/app/Jobs/StatusPipeline/StatusTagsPipeline.php index a72e6d50e..6282370f8 100644 --- a/app/Jobs/StatusPipeline/StatusTagsPipeline.php +++ b/app/Jobs/StatusPipeline/StatusTagsPipeline.php @@ -45,6 +45,11 @@ class StatusTagsPipeline implements ShouldQueue { $res = $this->activity; $status = $this->status; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + $tags = collect($res['tag']); // Emoji @@ -73,19 +78,19 @@ class StatusTagsPipeline implements ShouldQueue if(config('database.default') === 'pgsql') { $hashtag = Hashtag::where('name', 'ilike', $name) - ->orWhere('slug', 'ilike', str_slug($name)) + ->orWhere('slug', 'ilike', str_slug($name, '-', false)) ->first(); if(!$hashtag) { $hashtag = new Hashtag; $hashtag->name = $name; - $hashtag->slug = str_slug($name); + $hashtag->slug = str_slug($name, '-', false); $hashtag->save(); } } else { $hashtag = Hashtag::firstOrCreate([ - 'slug' => str_slug($name) - ], [ + 'slug' => str_slug($name, '-', false), + ],[ 'name' => $name ]); } diff --git a/tests/Unit/ActivityPubTagObjectTest.php b/tests/Unit/ActivityPubTagObjectTest.php new file mode 100644 index 000000000..f402ff0da --- /dev/null +++ b/tests/Unit/ActivityPubTagObjectTest.php @@ -0,0 +1,133 @@ + [ + "href" => "https://gotosocial.example.org/users/GotosocialUser", + "name" => "@GotosocialUser@gotosocial.example.org", + "type" => "Mention" + ] + ]; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + + $tags = collect($res['tag']) + ->filter(function($tag) { + return $tag && + $tag['type'] == 'Mention' && + isset($tag['href']) && + substr($tag['href'], 0, 8) === 'https://'; + }); + $this->assertTrue($tags->count() === 1); + } + + public function test_pixelfed_hashtags(): void + { + $res = [ + "tag" => [ + [ + "type" => "Mention", + "href" => "https://pixelfed.social/dansup", + "name" => "@dansup@pixelfed.social" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/dogsofpixelfed", + "name" => "#dogsOfPixelFed" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/doggo", + "name" => "#doggo" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/dog", + "name" => "#dog" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/drake", + "name" => "#drake" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/blacklab", + "name" => "#blacklab" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/iconic", + "name" => "#Iconic" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/majestic", + "name" => "#majestic" + ] + ] + ]; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + + $tags = collect($res['tag']) + ->filter(function($tag) { + return $tag && + $tag['type'] == 'Hashtag' && + isset($tag['href']) && + substr($tag['href'], 0, 8) === 'https://'; + }); + $this->assertTrue($tags->count() === 7); + } + + + public function test_pixelfed_mentions(): void + { + $res = [ + "tag" => [ + [ + "type" => "Mention", + "href" => "https://pixelfed.social/dansup", + "name" => "@dansup@pixelfed.social" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/dogsofpixelfed", + "name" => "#dogsOfPixelFed" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/doggo", + "name" => "#doggo" + ], + ] + ]; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + + $tags = collect($res['tag']) + ->filter(function($tag) { + return $tag && + $tag['type'] == 'Mention' && + isset($tag['href']) && + substr($tag['href'], 0, 8) === 'https://'; + }); + $this->assertTrue($tags->count() === 1); + } +} From bf5b72f0829abffae8a8db202887884a08418a9d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 04:23:28 -0600 Subject: [PATCH 034/253] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36092276a..97d07ba2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ - Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec)) - Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d)) - Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973)) +- Update Status model, allow unlisted thumbnails ([1f0a45b7](https://github.com/pixelfed/pixelfed/commit/1f0a45b7)) +- Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605)) +- ([](https://github.com/pixelfed/pixelfed/commit/)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 9989d6c66f49833037266fd2951262650f7d0ea9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 05:13:09 -0600 Subject: [PATCH 035/253] Update StatusTagsPipeline, fix object tags slug query --- app/Jobs/StatusPipeline/StatusTagsPipeline.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/app/Jobs/StatusPipeline/StatusTagsPipeline.php b/app/Jobs/StatusPipeline/StatusTagsPipeline.php index 6282370f8..f0eb680c9 100644 --- a/app/Jobs/StatusPipeline/StatusTagsPipeline.php +++ b/app/Jobs/StatusPipeline/StatusTagsPipeline.php @@ -81,16 +81,17 @@ class StatusTagsPipeline implements ShouldQueue ->orWhere('slug', 'ilike', str_slug($name, '-', false)) ->first(); - if(!$hashtag) { - $hashtag = new Hashtag; - $hashtag->name = $name; - $hashtag->slug = str_slug($name, '-', false); - $hashtag->save(); - } + if(!$hashtag) { + $hashtag = Hashtag::updateOrCreate([ + 'slug' => str_slug($name, '-', false), + ],[ + 'name' => $name + ]); + } } else { - $hashtag = Hashtag::firstOrCreate([ + $hashtag = Hashtag::updateOrCreate([ 'slug' => str_slug($name, '-', false), - ],[ + ],[ 'name' => $name ]); } From 79b378cdb1653d2aa664fa74bad1c5ef29e7dbbb Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 05:20:04 -0600 Subject: [PATCH 036/253] Update StatusTagsPipeline, fix object tags slug query --- app/Jobs/StatusPipeline/StatusTagsPipeline.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Jobs/StatusPipeline/StatusTagsPipeline.php b/app/Jobs/StatusPipeline/StatusTagsPipeline.php index f0eb680c9..893fa6a83 100644 --- a/app/Jobs/StatusPipeline/StatusTagsPipeline.php +++ b/app/Jobs/StatusPipeline/StatusTagsPipeline.php @@ -84,14 +84,12 @@ class StatusTagsPipeline implements ShouldQueue if(!$hashtag) { $hashtag = Hashtag::updateOrCreate([ 'slug' => str_slug($name, '-', false), - ],[ 'name' => $name ]); } } else { $hashtag = Hashtag::updateOrCreate([ 'slug' => str_slug($name, '-', false), - ],[ 'name' => $name ]); } From ce1afe27112ea6c92926ea442cd42ce4e17d3b69 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 26 Sep 2023 23:10:20 -0600 Subject: [PATCH 037/253] Update Note and CreateNote transformers, include attachment blurhash, width and height --- app/Transformer/ActivityPub/Verb/CreateNote.php | 15 +++++++++++++-- app/Transformer/ActivityPub/Verb/Note.php | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/app/Transformer/ActivityPub/Verb/CreateNote.php b/app/Transformer/ActivityPub/Verb/CreateNote.php index a9d40d9ed..55fdfa8f4 100644 --- a/app/Transformer/ActivityPub/Verb/CreateNote.php +++ b/app/Transformer/ActivityPub/Verb/CreateNote.php @@ -81,7 +81,8 @@ class CreateNote extends Fractal\TransformerAbstract '@type' => '@id' ], 'toot' => 'http://joinmastodon.org/ns#', - 'Emoji' => 'toot:Emoji' + 'Emoji' => 'toot:Emoji', + 'blurhash' => 'toot:blurhash', ] ], 'id' => $status->permalink(), @@ -103,12 +104,22 @@ class CreateNote extends Fractal\TransformerAbstract 'cc' => $status->scopeToAudience('cc'), 'sensitive' => (bool) $status->is_nsfw, 'attachment' => $status->media()->orderBy('order')->get()->map(function ($media) { - return [ + $res = [ 'type' => $media->activityVerb(), 'mediaType' => $media->mime, 'url' => $media->url(), 'name' => $media->caption, ]; + if($media->blurhash) { + $res['blurhash'] = $media->blurhash; + } + if($media->width) { + $res['width'] = $media->width; + } + if($media->height) { + $res['height'] = $media->height; + } + return $res; })->toArray(), 'tag' => $tags, 'commentsEnabled' => (bool) !$status->comments_disabled, diff --git a/app/Transformer/ActivityPub/Verb/Note.php b/app/Transformer/ActivityPub/Verb/Note.php index 777bd22b0..1350641d4 100644 --- a/app/Transformer/ActivityPub/Verb/Note.php +++ b/app/Transformer/ActivityPub/Verb/Note.php @@ -82,7 +82,8 @@ class Note extends Fractal\TransformerAbstract '@type' => '@id' ], 'toot' => 'http://joinmastodon.org/ns#', - 'Emoji' => 'toot:Emoji' + 'Emoji' => 'toot:Emoji', + 'blurhash' => 'toot:blurhash', ] ], 'id' => $status->url(), @@ -97,12 +98,22 @@ class Note extends Fractal\TransformerAbstract 'cc' => $status->scopeToAudience('cc'), 'sensitive' => (bool) $status->is_nsfw, 'attachment' => $status->media()->orderBy('order')->get()->map(function ($media) { - return [ + $res = [ 'type' => $media->activityVerb(), 'mediaType' => $media->mime, 'url' => $media->url(), 'name' => $media->caption, ]; + if($media->blurhash) { + $res['blurhash'] = $media->blurhash; + } + if($media->width) { + $res['width'] = $media->width; + } + if($media->height) { + $res['height'] = $media->height; + } + return $res; })->toArray(), 'tag' => $tags, 'commentsEnabled' => (bool) !$status->comments_disabled, From 8c96919119b7293a419a97d78b6c3522929a8cfb Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 26 Sep 2023 23:14:19 -0600 Subject: [PATCH 038/253] Update ap helpers, store media attachment width and height if present --- app/Util/ActivityPub/Helpers.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index c933820a5..2fa98b8bd 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -108,7 +108,10 @@ class Helpers { 'string', Rule::in($mimeTypes) ], - '*.name' => 'sometimes|nullable|string' + '*.name' => 'sometimes|nullable|string', + '*.blurhash' => 'sometimes|nullable|string|min:6|max:164', + '*.width' => 'sometimes|nullable|integer|min:1|max:5000', + '*.height' => 'sometimes|nullable|integer|min:1|max:5000', ])->passes(); return $valid; @@ -684,6 +687,8 @@ class Helpers { $blurhash = isset($media['blurhash']) ? $media['blurhash'] : null; $license = isset($media['license']) ? License::nameToId($media['license']) : null; $caption = isset($media['name']) ? Purify::clean($media['name']) : null; + $width = isset($media['width']) ? $media['width'] : false; + $height = isset($media['height']) ? $media['height'] : false; $media = new Media(); $media->blurhash = $blurhash; @@ -695,6 +700,12 @@ class Helpers { $media->remote_url = $url; $media->caption = $caption; $media->order = $key + 1; + if($width) { + $media->width = $width; + } + if($height) { + $media->height = $height; + } if($license) { $media->license = $license; } From fcb4933369acf66aa68a97cb55db98fab8af7da1 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 26 Sep 2023 23:14:42 -0600 Subject: [PATCH 039/253] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97d07ba2e..5a2383513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ - Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973)) - Update Status model, allow unlisted thumbnails ([1f0a45b7](https://github.com/pixelfed/pixelfed/commit/1f0a45b7)) - Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605)) -- ([](https://github.com/pixelfed/pixelfed/commit/)) +- Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27)) +- Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 895dc4fa9e6769f9a7de9429d04462fa9b13ae58 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 27 Sep 2023 01:33:39 -0600 Subject: [PATCH 040/253] Update Sign-in with Mastodon, allow usage when registrations are closed --- app/Http/Controllers/RemoteAuthController.php | 131 ++++++++++++++++-- config/remote-auth.php | 1 + resources/views/auth/login.blade.php | 5 +- 3 files changed, 127 insertions(+), 10 deletions(-) diff --git a/app/Http/Controllers/RemoteAuthController.php b/app/Http/Controllers/RemoteAuthController.php index 72a2a08d5..e068f5d75 100644 --- a/app/Http/Controllers/RemoteAuthController.php +++ b/app/Http/Controllers/RemoteAuthController.php @@ -23,7 +23,13 @@ class RemoteAuthController extends Controller { public function start(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); if($request->user()) { return redirect('/'); } @@ -37,7 +43,13 @@ class RemoteAuthController extends Controller public function getAuthDomains(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); if(config('remote-auth.mastodon.domains.only_custom')) { $res = config('remote-auth.mastodon.domains.custom'); @@ -69,7 +81,14 @@ class RemoteAuthController extends Controller public function redirect(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); + $this->validate($request, ['domain' => 'required']); $domain = $request->input('domain'); @@ -158,6 +177,14 @@ class RemoteAuthController extends Controller public function preflight(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); + if(!$request->filled('d') || !$request->filled('dsh') || !$request->session()->exists('oauth_redirect_to')) { return redirect('/login'); } @@ -167,6 +194,14 @@ class RemoteAuthController extends Controller public function handleCallback(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); + $domain = $request->session()->get('oauth_domain'); if($request->filled('code')) { @@ -195,7 +230,13 @@ class RemoteAuthController extends Controller public function onboarding(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); if($request->user()) { return redirect('/'); } @@ -204,6 +245,13 @@ class RemoteAuthController extends Controller public function sessionCheck(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -248,6 +296,13 @@ class RemoteAuthController extends Controller public function sessionGetMastodonData(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -279,6 +334,13 @@ class RemoteAuthController extends Controller public function sessionValidateUsername(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -334,6 +396,13 @@ class RemoteAuthController extends Controller public function sessionValidateEmail(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -359,6 +428,13 @@ class RemoteAuthController extends Controller public function sessionGetMastodonFollowers(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); abort_unless($request->session()->exists('oauth_remasto_id'), 403); @@ -386,6 +462,13 @@ class RemoteAuthController extends Controller public function handleSubmit(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); abort_unless($request->session()->exists('oauth_remasto_id'), 403); @@ -464,7 +547,13 @@ class RemoteAuthController extends Controller public function storeBio(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->user(), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -483,7 +572,13 @@ class RemoteAuthController extends Controller public function accountToId(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -525,7 +620,13 @@ class RemoteAuthController extends Controller public function storeAvatar(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->user(), 404); $this->validate($request, [ 'avatar_url' => 'required|active_url', @@ -547,7 +648,13 @@ class RemoteAuthController extends Controller public function finishUp(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->user(), 404); $currentWebfinger = '@' . $request->user()->username . '@' . config('pixelfed.domain.app'); @@ -564,7 +671,13 @@ class RemoteAuthController extends Controller public function handleLogin(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); diff --git a/config/remote-auth.php b/config/remote-auth.php index 3f85b9d40..182bb99a7 100644 --- a/config/remote-auth.php +++ b/config/remote-auth.php @@ -3,6 +3,7 @@ return [ 'mastodon' => [ 'enabled' => env('PF_LOGIN_WITH_MASTODON_ENABLED', false), + 'ignore_closed_state' => env('PF_LOGIN_WITH_MASTODON_ENABLED_SKIP_CLOSED', false), 'contraints' => [ /* diff --git a/resources/views/auth/login.blade.php b/resources/views/auth/login.blade.php index 12b6b6f52..3403cd6b3 100644 --- a/resources/views/auth/login.blade.php +++ b/resources/views/auth/login.blade.php @@ -74,7 +74,10 @@ - @if(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled')) + @if( + (config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled')) || + (config('remote-auth.mastodon.ignore_closed_state') && config('remote-auth.mastodon.enabled')) + )
@csrf From a1e162f095ba3a84c1cc39d85e268909573be8b4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 27 Sep 2023 01:38:05 -0600 Subject: [PATCH 041/253] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2383513..df79210a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605)) - Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27)) - Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191)) +- Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 450869756380d12eee506fe6499c7e95a616e797 Mon Sep 17 00:00:00 2001 From: Vivianne Langdon Date: Wed, 27 Sep 2023 23:53:51 -0700 Subject: [PATCH 042/253] Update Post component, adding follow and unfollow methods. --- resources/assets/components/Post.vue | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/resources/assets/components/Post.vue b/resources/assets/components/Post.vue index 1cc57c84e..6b3361351 100644 --- a/resources/assets/components/Post.vue +++ b/resources/assets/components/Post.vue @@ -37,6 +37,8 @@ v-on:bookmark="handleBookmark()" v-on:share="shareStatus()" v-on:unshare="unshareStatus()" + v-on:follow="follow()" + v-on:unfollow="unfollow()" v-on:counter-change="counterChange" /> @@ -333,6 +335,30 @@ }) }, + follow() { + axios.post('/api/v1/accounts/' + this.post.account.id + '/follow') + .then(res => { + this.$store.commit('updateRelationship', [res.data]); + this.user.following_count++; + this.post.account.followers_count++; + }).catch(err => { + swal('Oops!', 'An error occurred when attempting to follow this account.', 'error'); + this.post.relationship.following = false; + }); + }, + + unfollow() { + axios.post('/api/v1/accounts/' + this.post.account.id + '/unfollow') + .then(res => { + this.$store.commit('updateRelationship', [res.data]); + this.user.following_count--; + this.post.account.followers_count--; + }).catch(err => { + swal('Oops!', 'An error occurred when attempting to unfollow this account.', 'error'); + this.post.relationship.following = true; + }); + }, + openContextMenu() { this.$nextTick(() => { this.$refs.contextMenu.open(); From ede5ec3bf4b571f6f354f894d2e9c7e07aa66f78 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 30 Sep 2023 14:45:24 -0600 Subject: [PATCH 043/253] Update profile embeds, filter sensitive posts --- resources/views/profile/embed.blade.php | 149 ++++++++++++------------ 1 file changed, 75 insertions(+), 74 deletions(-) diff --git a/resources/views/profile/embed.blade.php b/resources/views/profile/embed.blade.php index d5921431a..cc6097e3a 100644 --- a/resources/views/profile/embed.blade.php +++ b/resources/views/profile/embed.blade.php @@ -1,7 +1,7 @@ - + @@ -16,8 +16,8 @@ - - + + + } +
-
-
-
- - - {{$profile['username']}} - +
+ -
-
-
-

-

Posts

-
-
-

-

Followers

-
-
-

Follow

-
-
-
-
- +
+
+
+
+

+

Posts

+
+

+

Followers

- - - - - - + + + + + - + + }, 5000); +}) + From 6c1e56fcb2bc5ffc135ba328aef496b8761a54c2 Mon Sep 17 00:00:00 2001 From: mbliznikova Date: Wed, 4 Oct 2023 19:02:55 +0000 Subject: [PATCH 044/253] Provide the error message if a file to upload is too large --- resources/assets/js/components/ComposeModal.vue | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/resources/assets/js/components/ComposeModal.vue b/resources/assets/js/components/ComposeModal.vue index 578679d32..a4737294a 100644 --- a/resources/assets/js/components/ComposeModal.vue +++ b/resources/assets/js/components/ComposeModal.vue @@ -1068,6 +1068,16 @@ export default { return App.util.format.timeAgo(ts); }, + formatBytes(bytes, decimals = 2) { + if (!+bytes) { + return '0 Bytes' + } + const dec = decimals < 0 ? 0 : decimals + const units = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + const quotient = Math.floor(Math.log(bytes) / Math.log(1024)) + return `${parseFloat((bytes / Math.pow(1024, quotient)).toFixed(dec))} ${units[quotient]}` + }, + fetchProfile() { let tags = { public: 'Public', @@ -1178,6 +1188,13 @@ export default { }, 300); }).catch(function(e) { switch(e.response.status) { + case 413: + self.uploading = false; + io.value = null; + swal('File is too large', 'The file you uploaded has the size of ' + self.formatBytes(io.size) + '. Unfortunately, only images up to ' + self.formatBytes(self.config.uploader.max_photo_size * 1024) + ' are supported.\nPlease resize the file and try again.', 'error'); + self.page = 2; + break; + case 451: self.uploading = false; io.value = null; From 135798eb6806a09a4ca8c226e515705f61e66e36 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 9 Oct 2023 13:06:46 -0600 Subject: [PATCH 045/253] Update ApiV1Controller, hydrate reblog interactions. Fixes #4686 --- app/Http/Controllers/Api/ApiV1Controller.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index 92d5d43ee..635cba24e 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -2507,7 +2507,7 @@ class ApiV1Controller extends Controller { abort_if(!$request->user(), 403); - $user = $request->user(); + $pid = $request->user()->profile_id; $res = $request->has(self::PF_API_ENTITY_KEY) ? StatusService::get($id, false) : StatusService::getMastodon($id, false); if(!$res || !isset($res['visibility'])) { @@ -2517,17 +2517,23 @@ class ApiV1Controller extends Controller $scope = $res['visibility']; if(!in_array($scope, ['public', 'unlisted'])) { if($scope === 'private') { - if(intval($res['account']['id']) !== intval($user->profile_id)) { - abort_unless(FollowerService::follows($user->profile_id, $res['account']['id']), 403); + if(intval($res['account']['id']) !== intval($pid)) { + abort_unless(FollowerService::follows($pid, $res['account']['id']), 403); } } else { abort(400, 'Invalid request'); } } - $res['favourited'] = LikeService::liked($user->profile_id, $res['id']); - $res['reblogged'] = ReblogService::get($user->profile_id, $res['id']); - $res['bookmarked'] = BookmarkService::get($user->profile_id, $res['id']); + if(!empty($res['reblog']) && isset($res['reblog']['id'])) { + $res['reblog']['favourited'] = (bool) LikeService::liked($pid, $res['reblog']['id']); + $res['reblog']['reblogged'] = (bool) ReblogService::get($pid, $res['reblog']['id']); + $res['reblog']['bookmarked'] = BookmarkService::get($pid, $res['reblog']['id']); + } + + $res['favourited'] = LikeService::liked($pid, $res['id']); + $res['reblogged'] = ReblogService::get($pid, $res['id']); + $res['bookmarked'] = BookmarkService::get($pid, $res['id']); return $this->json($res); } From e4d3b1964200bde66d4de04befffa8a516f7de9e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 9 Oct 2023 13:44:25 -0600 Subject: [PATCH 046/253] Update AdminReportController, add `profile_id` to group by. Fixes #4685 --- app/Http/Controllers/Admin/AdminReportController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Controllers/Admin/AdminReportController.php b/app/Http/Controllers/Admin/AdminReportController.php index 311eac220..ac238f28c 100644 --- a/app/Http/Controllers/Admin/AdminReportController.php +++ b/app/Http/Controllers/Admin/AdminReportController.php @@ -643,7 +643,7 @@ trait AdminReportController $q->whereNull('admin_seen') : $q->whereNotNull('admin_seen'); }) - ->groupBy(['id', 'object_id', 'object_type']) + ->groupBy(['id', 'object_id', 'object_type', 'profile_id']) ->cursorPaginate(6) ->withQueryString() ); From 457d5454f84c96164384bf9bf91ae3942ca2b81a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 10 Oct 2023 19:13:19 -0600 Subject: [PATCH 047/253] Update NotificationService, handle empty epoch. Fixes #4689 --- app/Services/NotificationService.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/Services/NotificationService.php b/app/Services/NotificationService.php index c068f8278..d088c2015 100644 --- a/app/Services/NotificationService.php +++ b/app/Services/NotificationService.php @@ -49,6 +49,9 @@ class NotificationService { public static function getEpochId($months = 6) { return Cache::remember(self::EPOCH_CACHE_KEY . $months, 1209600, function() use($months) { + if(Notification::count() === 0) { + return 0; + } return Notification::where('created_at', '>', now()->subMonths($months))->first()->id; }); } From c6408fd79d3a36fc04fec58e2d47e57997969dd3 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 10 Oct 2023 20:00:12 -0600 Subject: [PATCH 048/253] Add user:2fa command to easily disable 2FA for given account --- app/Console/Commands/UserToggle2FA.php | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 app/Console/Commands/UserToggle2FA.php diff --git a/app/Console/Commands/UserToggle2FA.php b/app/Console/Commands/UserToggle2FA.php new file mode 100644 index 000000000..c3b368608 --- /dev/null +++ b/app/Console/Commands/UserToggle2FA.php @@ -0,0 +1,55 @@ + 'Which username should we disable 2FA for?', + ]; + } + /** + * Execute the console command. + */ + public function handle() + { + $user = User::whereUsername($this->argument('username'))->first(); + + if(!$user->{'2fa_enabled'}) { + $this->info('User did not have 2FA enabled!'); + return; + } + + $user->{'2fa_enabled'} = false; + $user->{'2fa_secret'} = null; + $user->{'2fa_backup_codes'} = null; + $user->save(); + + $this->info('Successfully disabled 2FA on this account!'); + } +} From 7bfe43095b9186240ddaf5cee172d6bc59444517 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 10 Oct 2023 20:08:16 -0600 Subject: [PATCH 049/253] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df79210a0..969c9cc99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6)) +- Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7)) ### Federation - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) @@ -23,6 +24,9 @@ - Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27)) - Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191)) - Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa)) +- Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b)) +- Update ApiV1Controller, hydrate reblog interactions. Fixes #4686 ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb)) +- Update AdminReportController, add `profile_id` to group by. Fixes #4685 ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 01bac51104f4efcc1dc34e0ac0edc01f00f8a1e7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 10 Oct 2023 20:20:18 -0600 Subject: [PATCH 050/253] Update user:admin command, improve logic. Fixes #2465 --- app/Console/Commands/UserAdmin.php | 26 ++++++++++++++------------ app/Console/Commands/UserToggle2FA.php | 6 ++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/app/Console/Commands/UserAdmin.php b/app/Console/Commands/UserAdmin.php index 8a485c52b..3b15d0dfa 100644 --- a/app/Console/Commands/UserAdmin.php +++ b/app/Console/Commands/UserAdmin.php @@ -3,16 +3,17 @@ namespace App\Console\Commands; use Illuminate\Console\Command; +use Illuminate\Contracts\Console\PromptsForMissingInput; use App\User; -class UserAdmin extends Command +class UserAdmin extends Command implements PromptsForMissingInput { /** * The name and signature of the console command. * * @var string */ - protected $signature = 'user:admin {id}'; + protected $signature = 'user:admin {username}'; /** * The console command description. @@ -22,13 +23,15 @@ class UserAdmin extends Command protected $description = 'Make a user an admin, or remove admin privileges.'; /** - * Create a new command instance. + * Prompt for missing input arguments using the returned questions. * - * @return void + * @return array */ - public function __construct() + protected function promptForMissingArgumentsUsing() { - parent::__construct(); + return [ + 'username' => 'Which username should we toggle admin privileges for?', + ]; } /** @@ -38,16 +41,15 @@ class UserAdmin extends Command */ public function handle() { - $id = $this->argument('id'); - if(ctype_digit($id) == true) { - $user = User::find($id); - } else { - $user = User::whereUsername($id)->first(); - } + $id = $this->argument('username'); + + $user = User::whereUsername($id)->first(); + if(!$user) { $this->error('Could not find any user with that username or id.'); exit; } + $this->info('Found username: ' . $user->username); $state = $user->is_admin ? 'Remove admin privileges from this user?' : 'Add admin privileges to this user?'; $confirmed = $this->confirm($state); diff --git a/app/Console/Commands/UserToggle2FA.php b/app/Console/Commands/UserToggle2FA.php index c3b368608..eed6843da 100644 --- a/app/Console/Commands/UserToggle2FA.php +++ b/app/Console/Commands/UserToggle2FA.php @@ -33,6 +33,7 @@ class UserToggle2FA extends Command implements PromptsForMissingInput 'username' => 'Which username should we disable 2FA for?', ]; } + /** * Execute the console command. */ @@ -40,6 +41,11 @@ class UserToggle2FA extends Command implements PromptsForMissingInput { $user = User::whereUsername($this->argument('username'))->first(); + if(!$user) { + $this->error('Could not find any user with that username'); + exit; + } + if(!$user->{'2fa_enabled'}) { $this->info('User did not have 2FA enabled!'); return; From 36b23fe34eae669cb7ce566a87c6a9df4c8653b0 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 10 Oct 2023 20:55:20 -0600 Subject: [PATCH 051/253] Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months --- app/Util/ActivityPub/Helpers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 2fa98b8bd..1304f0811 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -801,7 +801,7 @@ class Helpers { ); if( $profile->last_fetched_at == null || - $profile->last_fetched_at->lt(now()->subHours(24)) + $profile->last_fetched_at->lt(now()->subMonths(3)) ) { RemoteAvatarFetch::dispatch($profile); } From 82798b5ea3e77867cb06981ac412a3aee781752e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 00:48:30 -0600 Subject: [PATCH 052/253] Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars --- .../AvatarPipeline/AvatarStorageCleanup.php | 67 +++++++++ .../AvatarStorageLargePurge.php | 80 +++++++++++ app/Jobs/AvatarPipeline/RemoteAvatarFetch.php | 2 +- .../RemoteAvatarFetchFromUrl.php | 1 - app/Services/AvatarService.php | 128 ++++++++++++++++-- app/Services/MediaStorageService.php | 9 +- 6 files changed, 270 insertions(+), 17 deletions(-) create mode 100644 app/Jobs/AvatarPipeline/AvatarStorageCleanup.php create mode 100644 app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php diff --git a/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php new file mode 100644 index 000000000..230797bf6 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php @@ -0,0 +1,67 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + AvatarService::cleanup($this->avatar, true); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php new file mode 100644 index 000000000..f432e1e56 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php @@ -0,0 +1,80 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + $avatar = $this->avatar; + + $disk = AvatarService::disk(); + + $files = collect(AvatarService::storage($avatar)); + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php index df972dd38..4e4a1b2ec 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php @@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue $avatar->remote_url = $icon['url']; $avatar->save(); - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false); + MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; } diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php index 259058385..c8c6820e4 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php @@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue $avatar->save(); } - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; diff --git a/app/Services/AvatarService.php b/app/Services/AvatarService.php index 1c5e9e0c1..af578fdef 100644 --- a/app/Services/AvatarService.php +++ b/app/Services/AvatarService.php @@ -3,21 +3,125 @@ namespace App\Services; use Cache; +use Storage; +use Illuminate\Support\Str; +use App\Avatar; use App\Profile; +use App\Jobs\AvatarPipeline\AvatarStorageLargePurge; +use League\Flysystem\UnableToCheckDirectoryExistence; +use League\Flysystem\UnableToRetrieveMetadata; class AvatarService { - public static function get($profile_id) - { - $exists = Cache::get('avatar:' . $profile_id); - if($exists) { - return $exists; - } + public static function get($profile_id) + { + $exists = Cache::get('avatar:' . $profile_id); + if($exists) { + return $exists; + } - $profile = Profile::find($profile_id); - if(!$profile) { - return config('app.url') . '/storage/avatars/default.jpg'; - } - return $profile->avatarUrl(); - } + $profile = Profile::find($profile_id); + if(!$profile) { + return config('app.url') . '/storage/avatars/default.jpg'; + } + return $profile->avatarUrl(); + } + + public static function disk() + { + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return false; + } + + $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud'); + $disk = Storage::disk($driver); + + return $disk; + } + + public static function storage(Avatar $avatar) + { + $disk = self::disk(); + + if(!$disk) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + return $disk->allFiles($base . $avatar->profile_id); + } + + public static function cleanup($avatar, $confirm = false) + { + if(!$avatar || !$confirm) { + return; + } + + if($avatar->cdn_url == null) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return; + } + + $disk = self::disk(); + + if(!$disk) { + return; + } + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + try { + $exists = $disk->directoryExists($base . $avatar->profile_id); + } catch ( + UnableToRetrieveMetadata | + UnableToCheckDirectoryExistence | + Exception $e + ) { + return; + } + + if(!$exists) { + return; + } + + $files = collect($disk->allFiles($base . $avatar->profile_id)); + + if(!$files || !$files->count() || $files->count() === 1) { + return; + } + + if($files->count() > 5) { + AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo'); + return; + } + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } } diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index b547ee39c..128001de2 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController; use GuzzleHttp\Exception\RequestException; use App\Jobs\MediaPipeline\MediaDeletePipeline; use Illuminate\Support\Arr; +use App\Jobs\AvatarPipeline\AvatarStorageCleanup; class MediaStorageService { @@ -29,9 +30,9 @@ class MediaStorageService { return; } - public static function avatar($avatar, $local = false) + public static function avatar($avatar, $local = false, $skipRecentCheck = false) { - return (new self())->fetchAvatar($avatar, $local); + return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck); } public static function head($url) @@ -182,6 +183,7 @@ class MediaStorageService { protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false) { + $queue = random_int(1, 15) > 5 ? 'mmo' : 'low'; $url = $avatar->remote_url; $driver = $local ? 'local' : config('filesystems.cloud'); @@ -205,7 +207,7 @@ class MediaStorageService { $max_size = (int) config('pixelfed.max_avatar_size') * 1000; if(!$skipRecentCheck) { - if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) { + if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) { return; } } @@ -261,6 +263,7 @@ class MediaStorageService { Cache::forget('avatar:' . $avatar->profile_id); AccountService::del($avatar->profile_id); + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); unlink($tmpName); } From 95a1eddcb244420122b299af5110d480858bc3be Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 00:48:56 -0600 Subject: [PATCH 053/253] Update changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 969c9cc99..2ad8dbccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,11 @@ - Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191)) - Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa)) - Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b)) -- Update ApiV1Controller, hydrate reblog interactions. Fixes #4686 ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb)) -- Update AdminReportController, add `profile_id` to group by. Fixes #4685 ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196)) +- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb)) +- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196)) +- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511)) +- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3)) +- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 319ced405413bc2a2745b163301929857bf1a7d5 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 01:32:04 -0600 Subject: [PATCH 054/253] Update CreateAvatar job, add processing constraints and set is_remote attribute --- app/Jobs/AvatarPipeline/CreateAvatar.php | 59 +++++++++++++++++++----- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/app/Jobs/AvatarPipeline/CreateAvatar.php b/app/Jobs/AvatarPipeline/CreateAvatar.php index fd5f94cc7..f773d1590 100644 --- a/app/Jobs/AvatarPipeline/CreateAvatar.php +++ b/app/Jobs/AvatarPipeline/CreateAvatar.php @@ -2,19 +2,25 @@ namespace App\Jobs\AvatarPipeline; -use App\Avatar; -use App\Profile; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\Middleware\WithoutOverlapping; +use App\Avatar; +use App\Profile; -class CreateAvatar implements ShouldQueue +class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $profile; + public $profile; + public $tries = 3; + public $maxExceptions = 3; + public $timeout = 900; + public $failOnTimeout = true; /** * Delete the job if its models no longer exist. @@ -22,6 +28,31 @@ class CreateAvatar implements ShouldQueue * @var bool */ public $deleteWhenMissingModels = true; + + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 3600; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return 'avatar:create:' . $this->profile->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()]; + } /** * Create a new job instance. @@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue */ public function __construct(Profile $profile) { - $this->profile = $profile; + $this->profile = $profile->withoutRelations(); } /** @@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue public function handle() { $profile = $this->profile; + $isRemote = (bool) $profile->private_key == null; $path = 'public/avatars/default.jpg'; - $avatar = new Avatar(); - $avatar->profile_id = $profile->id; - $avatar->media_path = $path; - $avatar->change_count = 0; - $avatar->last_processed_at = \Carbon\Carbon::now(); - $avatar->save(); + Avatar::updateOrCreate( + [ + 'profile_id' => $profile->id, + ], + [ + 'media_path' => $path, + 'change_count' => 0, + 'is_remote' => $isRemote, + 'last_processed_at' => now() + ] + ); } } From c37b7cde30189200a28e2ae6a947f1345ce30ab7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 03:38:31 -0600 Subject: [PATCH 055/253] Add `avatar:storage-deep-clean` command to dispatch avatar storage cleanup jobs --- .../Commands/AvatarStorageDeepClean.php | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 app/Console/Commands/AvatarStorageDeepClean.php diff --git a/app/Console/Commands/AvatarStorageDeepClean.php b/app/Console/Commands/AvatarStorageDeepClean.php new file mode 100644 index 000000000..5840142f5 --- /dev/null +++ b/app/Console/Commands/AvatarStorageDeepClean.php @@ -0,0 +1,115 @@ +info(' ____ _ ______ __ '); + $this->info(' / __ \(_) _____ / / __/__ ____/ / '); + $this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / '); + $this->info(' / ____/ /> info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ '); + $this->info(' '); + $this->info(' Pixelfed Avatar Deep Cleaner'); + $this->line(' '); + $this->info(' Purge/delete old and outdated avatars from remote accounts'); + $this->line(' '); + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + $this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...'); + exit; + } + + $start = 0; + + if(!$this->confirm('Are you sure you want to proceed?')) { + $this->error('Aborting...'); + exit; + } + + if(!$this->activeCheck()) { + $this->info('Found existing deep cleaning job'); + if(!$this->confirm('Do you want to continue where you left off?')) { + $this->error('Aborting...'); + exit; + } else { + $start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json'); + + if($start && $start < 1 || $start > PHP_INT_MAX) { + $this->error('Error fetching cached value'); + $this->error('Aborting...'); + exit; + } + } + } + + $count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count(); + $bar = $this->output->createProgressBar($count); + + foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) { + usleep(random_int(50, 1000)); + $this->counter++; + $this->handleAvatar($avatar); + $bar->advance(); + } + $bar->finish(); + } + + protected function updateCache($id) + { + Cache::put('cmd:asdp', $id); + if($this->counter % 5 === 0) { + Storage::put('avatar-deep-clean.json', $id); + } + } + + protected function activeCheck() + { + if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) { + return false; + } + + return true; + } + + protected function handleAvatar($avatar) + { + $this->updateCache($avatar->id); + $queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low']; + $queue = $queues[random_int(0, 7)]; + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue); + } +} From 4e35f0d32ea7bee4f1a30db6d8d1a73ea88cea8f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 03:39:35 -0600 Subject: [PATCH 056/253] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad8dbccd..e1e446bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6)) - Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7)) +- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde)) ### Federation - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) @@ -30,6 +31,7 @@ - Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511)) - Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3)) - Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e)) +- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From edbcf3ed795d646c187323ae12e300c2631bd42e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 04:42:40 -0600 Subject: [PATCH 057/253] Update RemoteStatusDelete and DecrementPostCount pipelines --- app/Jobs/ProfilePipeline/DecrementPostCount.php | 12 +++--------- app/Jobs/StatusPipeline/RemoteStatusDelete.php | 11 ++++------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/app/Jobs/ProfilePipeline/DecrementPostCount.php b/app/Jobs/ProfilePipeline/DecrementPostCount.php index d6781d7a5..b463f1dda 100644 --- a/app/Jobs/ProfilePipeline/DecrementPostCount.php +++ b/app/Jobs/ProfilePipeline/DecrementPostCount.php @@ -43,15 +43,9 @@ class DecrementPostCount implements ShouldQueue return 1; } - if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) { - $profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count(); - $profile->save(); - AccountService::del($id); - } else { - $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0; - $profile->save(); - AccountService::del($id); - } + $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0; + $profile->save(); + AccountService::del($id); return 1; } diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index 19c17b54c..c334fd04e 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -37,6 +37,7 @@ use App\Services\AccountService; use App\Services\CollectionService; use App\Services\StatusService; use App\Jobs\MediaPipeline\MediaDeletePipeline; +use App\Jobs\ProfilePipeline\DecrementPostCount; class RemoteStatusDelete implements ShouldQueue { @@ -51,7 +52,7 @@ class RemoteStatusDelete implements ShouldQueue */ public $deleteWhenMissingModels = true; - public $timeout = 90; + public $timeout = 180; public $tries = 2; public $maxExceptions = 1; @@ -62,7 +63,7 @@ class RemoteStatusDelete implements ShouldQueue */ public function __construct(Status $status) { - $this->status = $status; + $this->status = $status->withoutRelations(); } /** @@ -77,14 +78,10 @@ class RemoteStatusDelete implements ShouldQueue if($status->deleted_at) { return; } - $profile = $this->status->profile; StatusService::del($status->id, true); - if($profile->status_count && $profile->status_count > 0) { - $profile->status_count = $profile->status_count - 1; - $profile->save(); - } + DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox'); return $this->unlinkRemoveMedia($status); } From f481f3d24819acf8b7c4a3f1b77062a703721944 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 04:56:39 -0600 Subject: [PATCH 058/253] Update RemoteStatusDelete pipeline --- .../StatusPipeline/RemoteStatusDelete.php | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index c334fd04e..5cf0b97b3 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -21,6 +21,7 @@ use App\{ }; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; @@ -39,7 +40,7 @@ use App\Services\StatusService; use App\Jobs\MediaPipeline\MediaDeletePipeline; use App\Jobs\ProfilePipeline\DecrementPostCount; -class RemoteStatusDelete implements ShouldQueue +class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; @@ -52,9 +53,35 @@ class RemoteStatusDelete implements ShouldQueue */ public $deleteWhenMissingModels = true; + public $tries = 3; + public $maxExceptions = 3; public $timeout = 180; - public $tries = 2; - public $maxExceptions = 1; + public $failOnTimeout = true; + + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 3600; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return 'status:remote:delete:' . $this->status->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("status-remote-delete-{$this->status->id}"))->shared()->dontRelease()]; + } /** * Create a new job instance. From 0d35f1a3e5db14673187b926aa2f06c39f1991bf Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 05:10:32 -0600 Subject: [PATCH 059/253] Update IncrementPostCount pipeline --- app/Jobs/ProfilePipeline/IncrementPostCount.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/app/Jobs/ProfilePipeline/IncrementPostCount.php b/app/Jobs/ProfilePipeline/IncrementPostCount.php index 9c7585e25..fe8d90648 100644 --- a/app/Jobs/ProfilePipeline/IncrementPostCount.php +++ b/app/Jobs/ProfilePipeline/IncrementPostCount.php @@ -43,17 +43,10 @@ class IncrementPostCount implements ShouldQueue return 1; } - if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) { - $profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count(); - $profile->last_status_at = now(); - $profile->save(); - AccountService::del($id); - } else { - $profile->status_count = $profile->status_count + 1; - $profile->last_status_at = now(); - $profile->save(); - AccountService::del($id); - } + $profile->status_count = $profile->status_count + 1; + $profile->last_status_at = now(); + $profile->save(); + AccountService::del($id); return 1; } From b76ad7cfe0cb9acbad3b7a34b7309001fa87f1d3 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 05:24:34 -0600 Subject: [PATCH 060/253] Update RemoteStatusDelete, fix include --- app/Jobs/StatusPipeline/RemoteStatusDelete.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index 5cf0b97b3..aabb81755 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -25,6 +25,7 @@ use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\Middleware\WithoutOverlapping; use League\Fractal; use Illuminate\Support\Str; use League\Fractal\Serializer\ArraySerializer; From e9d9c4d8cc2965fd0eb1ebfc31b96dfc9a6ff7d5 Mon Sep 17 00:00:00 2001 From: Andy Neillans Date: Wed, 11 Oct 2023 19:08:22 +0100 Subject: [PATCH 061/253] Strip tags from bio in embeds --- resources/views/profile/show.blade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/views/profile/show.blade.php b/resources/views/profile/show.blade.php index d16eeb7c0..5107c0ab3 100644 --- a/resources/views/profile/show.blade.php +++ b/resources/views/profile/show.blade.php @@ -20,7 +20,7 @@ @endsection -@push('meta') +@push('meta') @if(false == $settings['crawlable'] || $profile->remote_url) @else From 36df0d8373d87ef705ceafe63480ff3706230712 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 12 Oct 2023 21:29:51 -0600 Subject: [PATCH 062/253] Update nodeinfo --- app/Http/Controllers/Api/ApiV2Controller.php | 8 +- app/Services/LandingService.php | 8 +- app/Util/Site/Nodeinfo.php | 157 ++++++++++--------- 3 files changed, 89 insertions(+), 84 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV2Controller.php b/app/Http/Controllers/Api/ApiV2Controller.php index 757e14dce..2ca5b96c5 100644 --- a/app/Http/Controllers/Api/ApiV2Controller.php +++ b/app/Http/Controllers/Api/ApiV2Controller.php @@ -34,6 +34,7 @@ use App\Transformer\Api\Mastodon\v1\{ use App\Transformer\Api\{ RelationshipTransformer, }; +use App\Util\Site\Nodeinfo; class ApiV2Controller extends Controller { @@ -77,12 +78,7 @@ class ApiV2Controller extends Controller 'description' => config_cache('app.short_description'), 'usage' => [ 'users' => [ - 'active_month' => (int) Cache::remember('api:nodeinfo:am', 172800, function() { - return User::select('last_active_at', 'created_at') - ->where('last_active_at', '>', now()->subMonths(1)) - ->orWhere('created_at', '>', now()->subMonths(1)) - ->count(); - }) + 'active_month' => (int) Nodeinfo::activeUsersMonthly() ] ], 'thumbnail' => [ diff --git a/app/Services/LandingService.php b/app/Services/LandingService.php index 2a5acda07..ba16af5b6 100644 --- a/app/Services/LandingService.php +++ b/app/Services/LandingService.php @@ -9,17 +9,13 @@ use Illuminate\Support\Facades\Redis; use App\Status; use App\User; use App\Services\AccountService; +use App\Util\Site\Nodeinfo; class LandingService { public static function get($json = true) { - $activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() { - return User::select('last_active_at') - ->where('last_active_at', '>', now()->subMonths(1)) - ->orWhere('created_at', '>', now()->subMonths(1)) - ->count(); - }); + $activeMonth = Nodeinfo::activeUsersMonthly(); $totalUsers = Cache::remember('api:nodeinfo:users', 43200, function() { return User::count(); diff --git a/app/Util/Site/Nodeinfo.php b/app/Util/Site/Nodeinfo.php index 166b6bc6a..0458299c5 100644 --- a/app/Util/Site/Nodeinfo.php +++ b/app/Util/Site/Nodeinfo.php @@ -2,85 +2,98 @@ namespace App\Util\Site; -use Cache; -use App\{Like, Profile, Status, User}; +use Illuminate\Support\Facades\Cache; +use App\Like; +use App\Profile; +use App\Status; +use App\User; use Illuminate\Support\Str; -class Nodeinfo { +class Nodeinfo +{ + public static function get() + { + $res = Cache::remember('api:nodeinfo', 900, function () { + $activeHalfYear = self::activeUsersHalfYear(); + $activeMonth = self::activeUsersMonthly(); - public static function get() - { - $res = Cache::remember('api:nodeinfo', 300, function () { - $activeHalfYear = Cache::remember('api:nodeinfo:ahy', 172800, function() { - return User::select('last_active_at') - ->where('last_active_at', '>', now()->subMonths(6)) - ->orWhere('created_at', '>', now()->subMonths(6)) - ->count(); - }); + $users = Cache::remember('api:nodeinfo:users', 43200, function() { + return User::count(); + }); - $activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() { - return User::select('last_active_at') - ->where('last_active_at', '>', now()->subMonths(1)) - ->orWhere('created_at', '>', now()->subMonths(1)) - ->count(); - }); + $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() { + return Status::whereLocal(true)->count(); + }); - $users = Cache::remember('api:nodeinfo:users', 43200, function() { - return User::count(); - }); + $features = [ 'features' => \App\Util\Site\Config::get()['features'] ]; - $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() { - return Status::whereLocal(true)->count(); - }); + return [ + 'metadata' => [ + 'nodeName' => config_cache('app.name'), + 'software' => [ + 'homepage' => 'https://pixelfed.org', + 'repo' => 'https://github.com/pixelfed/pixelfed', + ], + 'config' => $features + ], + 'protocols' => [ + 'activitypub', + ], + 'services' => [ + 'inbound' => [], + 'outbound' => [], + ], + 'software' => [ + 'name' => 'pixelfed', + 'version' => config('pixelfed.version'), + ], + 'usage' => [ + 'localPosts' => (int) $statuses, + 'localComments' => 0, + 'users' => [ + 'total' => (int) $users, + 'activeHalfyear' => (int) $activeHalfYear, + 'activeMonth' => (int) $activeMonth, + ], + ], + 'version' => '2.0', + ]; + }); + $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration'); + return $res; + } - $features = [ 'features' => \App\Util\Site\Config::get()['features'] ]; + public static function wellKnown() + { + return [ + 'links' => [ + [ + 'href' => config('pixelfed.nodeinfo.url'), + 'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0', + ], + ], + ]; + } - return [ - 'metadata' => [ - 'nodeName' => config_cache('app.name'), - 'software' => [ - 'homepage' => 'https://pixelfed.org', - 'repo' => 'https://github.com/pixelfed/pixelfed', - ], - 'config' => $features - ], - 'protocols' => [ - 'activitypub', - ], - 'services' => [ - 'inbound' => [], - 'outbound' => [], - ], - 'software' => [ - 'name' => 'pixelfed', - 'version' => config('pixelfed.version'), - ], - 'usage' => [ - 'localPosts' => (int) $statuses, - 'localComments' => 0, - 'users' => [ - 'total' => (int) $users, - 'activeHalfyear' => (int) $activeHalfYear, - 'activeMonth' => (int) $activeMonth, - ], - ], - 'version' => '2.0', - ]; - }); - $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration'); - return $res; - } - - public static function wellKnown() - { - return [ - 'links' => [ - [ - 'href' => config('pixelfed.nodeinfo.url'), - 'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0', - ], - ], - ]; - } + public static function activeUsersMonthly() + { + return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() { + return User::withTrashed() + ->select('last_active_at, updated_at') + ->where('updated_at', '>', now()->subWeeks(5)) + ->orWhere('last_active_at', '>', now()->subWeeks(5)) + ->count(); + }); + } + public static function activeUsersHalfYear() + { + return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() { + return User::withTrashed() + ->select('last_active_at, updated_at') + ->where('last_active_at', '>', now()->subMonths(6)) + ->orWhere('updated_at', '>', now()->subMonths(6)) + ->count(); + }); + } } From 778e83d39822122a462671660fa315341d86256d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 15 Oct 2023 03:51:45 -0600 Subject: [PATCH 063/253] Update lexer regex, fix mention regex and add more tests --- app/Util/Lexer/Regex.php | 2 +- tests/Unit/Lexer/UsernameTest.php | 63 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/Util/Lexer/Regex.php b/app/Util/Lexer/Regex.php index ecc468d05..f8d77c95f 100755 --- a/app/Util/Lexer/Regex.php +++ b/app/Util/Lexer/Regex.php @@ -162,7 +162,7 @@ abstract class Regex // look-ahead capture here and don't append $after when we return. $tmp['valid_mention_preceding_chars'] = '([^a-zA-Z0-9_!#\$%&*@@\/]|^|(?:^|[^a-z0-9_+~.-])RT:?)'; - $re['valid_mentions_or_lists'] = '/'.$tmp['valid_mention_preceding_chars'].'(['.$tmp['at_signs'].'])([a-z0-9_\-.]{1,20})((\/[a-z][a-z0-9_\-]{0,24})?(?=(.*|$))(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i'; + $re['valid_mentions_or_lists'] = '/'.$tmp['valid_mention_preceding_chars'].'(['.$tmp['at_signs'].'])([\p{L}0-9_\-.]{1,20})((\/[a-z][a-z0-9_\-]{0,24})?(?=(.*|$))(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/iu'; $re['valid_reply'] = '/^(?:['.$tmp['spaces'].'])*['.$tmp['at_signs'].']([a-z0-9_\-.]{1,20})(?=(.*|$))/iu'; $re['end_mention_match'] = '/\A(?:['.$tmp['at_signs'].']|['.$tmp['latin_accents'].']|:\/\/)/iu'; diff --git a/tests/Unit/Lexer/UsernameTest.php b/tests/Unit/Lexer/UsernameTest.php index 0d21b6e00..64875df7e 100644 --- a/tests/Unit/Lexer/UsernameTest.php +++ b/tests/Unit/Lexer/UsernameTest.php @@ -175,4 +175,67 @@ class UsernameTest extends TestCase $this->assertEquals($expectedEntity, $entities); } + /** @test * */ + public function germanUmlatsAutolink() + { + $mentions = "@März and @königin and @Glück"; + $autolink = Autolink::create()->autolink($mentions); + + $expectedAutolink = '@März and @königin and @Glück'; + $this->assertEquals($expectedAutolink, $autolink); + } + + /** @test * */ + public function germanUmlatsExtractor() + { + $mentions = "@März and @königin and @Glück"; + $entities = Extractor::create()->extract($mentions); + + $expectedEntity = [ + "hashtags" => [], + "urls" => [], + "mentions" => [ + "märz", + "königin", + "glück", + ], + "replyto" => null, + "hashtags_with_indices" => [], + "urls_with_indices" => [], + "mentions_with_indices" => [ + [ + "screen_name" => "März", + "indices" => [ + 0, + 5, + ], + ], + [ + "screen_name" => "königin", + "indices" => [ + 10, + 18, + ], + ], + [ + "screen_name" => "Glück", + "indices" => [ + 23, + 29, + ], + ], + ], + ]; + $this->assertEquals($expectedEntity, $entities); + } + + /** @test * */ + public function germanUmlatsWebfingerAutolink() + { + $mentions = "hello @märz@example.org!"; + $autolink = Autolink::create()->autolink($mentions); + + $expectedAutolink = 'hello @märz@example.org!'; + $this->assertEquals($expectedAutolink, $autolink); + } } From 9677791bef6fcc32c5fe8c3452e433ef8c182ecc Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 15 Oct 2023 03:52:20 -0600 Subject: [PATCH 064/253] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1e446bde..a8a2c1be7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ - Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3)) - Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e)) - Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40)) +- Update RemoteStatusDelete and DecrementPostCount pipelines ([edbcf3ed](https://github.com/pixelfed/pixelfed/commit/edbcf3ed)) +- Update lexer regex, fix mention regex and add more tests ([778e83d3](https://github.com/pixelfed/pixelfed/commit/778e83d3)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From dfe2379b93c1990ce54663b4178e93d167d6f3c8 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 16 Oct 2023 06:28:57 -0600 Subject: [PATCH 065/253] Update StatusTransformer, generate autolink on request --- app/Transformer/Api/StatusStatelessTransformer.php | 4 +++- app/Transformer/Api/StatusTransformer.php | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/Transformer/Api/StatusStatelessTransformer.php b/app/Transformer/Api/StatusStatelessTransformer.php index c21720509..b15fd6e22 100644 --- a/app/Transformer/Api/StatusStatelessTransformer.php +++ b/app/Transformer/Api/StatusStatelessTransformer.php @@ -16,6 +16,7 @@ use App\Services\StatusLabelService; use App\Services\StatusMentionService; use App\Services\PollService; use App\Models\CustomEmoji; +use App\Util\Lexer\Autolink; class StatusStatelessTransformer extends Fractal\TransformerAbstract { @@ -23,6 +24,7 @@ class StatusStatelessTransformer extends Fractal\TransformerAbstract { $taggedPeople = MediaTagService::get($status->id); $poll = $status->type === 'poll' ? PollService::get($status->id) : null; + $rendered = $status->caption ? Autolink::create()->autolink($status->caption) : null; return [ '_v' => 1, @@ -34,7 +36,7 @@ class StatusStatelessTransformer extends Fractal\TransformerAbstract 'in_reply_to_id' => $status->in_reply_to_id ? (string) $status->in_reply_to_id : null, 'in_reply_to_account_id' => $status->in_reply_to_profile_id ? (string) $status->in_reply_to_profile_id : null, 'reblog' => $status->reblog_of_id ? StatusService::get($status->reblog_of_id, false) : null, - 'content' => $status->rendered ?? $status->caption, + 'content' => $rendered, 'content_text' => $status->caption, 'created_at' => str_replace('+00:00', 'Z', $status->created_at->format(DATE_RFC3339_EXTENDED)), 'emojis' => CustomEmoji::scan($status->caption), diff --git a/app/Transformer/Api/StatusTransformer.php b/app/Transformer/Api/StatusTransformer.php index d04b025f8..ca3e6783e 100644 --- a/app/Transformer/Api/StatusTransformer.php +++ b/app/Transformer/Api/StatusTransformer.php @@ -19,6 +19,7 @@ use Illuminate\Support\Str; use App\Services\PollService; use App\Models\CustomEmoji; use App\Services\BookmarkService; +use App\Util\Lexer\Autolink; class StatusTransformer extends Fractal\TransformerAbstract { @@ -27,6 +28,7 @@ class StatusTransformer extends Fractal\TransformerAbstract $pid = request()->user()->profile_id; $taggedPeople = MediaTagService::get($status->id); $poll = $status->type === 'poll' ? PollService::get($status->id, $pid) : null; + $rendered = $status->caption ? Autolink::create()->autolink($status->caption) : null; return [ '_v' => 1, @@ -37,7 +39,7 @@ class StatusTransformer extends Fractal\TransformerAbstract 'in_reply_to_id' => (string) $status->in_reply_to_id, 'in_reply_to_account_id' => (string) $status->in_reply_to_profile_id, 'reblog' => $status->reblog_of_id ? StatusService::get($status->reblog_of_id) : null, - 'content' => $status->rendered ?? $status->caption, + 'content' => $rendered, 'content_text' => $status->caption, 'created_at' => str_replace('+00:00', 'Z', $status->created_at->format(DATE_RFC3339_EXTENDED)), 'emojis' => CustomEmoji::scan($status->caption), From 65a048cdd5442375897351962e6fc9bdc9773c8a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 20 Oct 2023 00:17:24 -0600 Subject: [PATCH 066/253] Update StatusTransformer --- app/Transformer/Api/StatusStatelessTransformer.php | 4 +++- app/Transformer/Api/StatusTransformer.php | 4 +++- config/exp.php | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/Transformer/Api/StatusStatelessTransformer.php b/app/Transformer/Api/StatusStatelessTransformer.php index b15fd6e22..3c2c02d60 100644 --- a/app/Transformer/Api/StatusStatelessTransformer.php +++ b/app/Transformer/Api/StatusStatelessTransformer.php @@ -24,7 +24,9 @@ class StatusStatelessTransformer extends Fractal\TransformerAbstract { $taggedPeople = MediaTagService::get($status->id); $poll = $status->type === 'poll' ? PollService::get($status->id) : null; - $rendered = $status->caption ? Autolink::create()->autolink($status->caption) : null; + $rendered = config('exp.autolink') ? + ( $status->caption ? Autolink::create()->autolink($status->caption) : '' ) : + ( $status->rendered ?? $status->caption ); return [ '_v' => 1, diff --git a/app/Transformer/Api/StatusTransformer.php b/app/Transformer/Api/StatusTransformer.php index ca3e6783e..22a840ce0 100644 --- a/app/Transformer/Api/StatusTransformer.php +++ b/app/Transformer/Api/StatusTransformer.php @@ -28,7 +28,9 @@ class StatusTransformer extends Fractal\TransformerAbstract $pid = request()->user()->profile_id; $taggedPeople = MediaTagService::get($status->id); $poll = $status->type === 'poll' ? PollService::get($status->id, $pid) : null; - $rendered = $status->caption ? Autolink::create()->autolink($status->caption) : null; + $rendered = config('exp.autolink') ? + ( $status->caption ? Autolink::create()->autolink($status->caption) : '' ) : + ( $status->rendered ?? $status->caption ); return [ '_v' => 1, diff --git a/config/exp.php b/config/exp.php index 0ace5135b..e14463411 100644 --- a/config/exp.php +++ b/config/exp.php @@ -41,4 +41,6 @@ return [ // Post Update/Edits 'pue' => env('EXP_PUE', true), + + 'autolink' => env('EXP_AUTOLINK_V2', false), ]; From fdb51d1f5a66caf13b368b83f3c6650e5644b0c8 Mon Sep 17 00:00:00 2001 From: mbliznikova Date: Fri, 20 Oct 2023 21:09:29 +0000 Subject: [PATCH 067/253] Add check if collection is empty before publishing --- .../js/components/CollectionComponent.vue | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/components/CollectionComponent.vue b/resources/assets/js/components/CollectionComponent.vue index dd7ebf433..3f77cfc13 100644 --- a/resources/assets/js/components/CollectionComponent.vue +++ b/resources/assets/js/components/CollectionComponent.vue @@ -205,12 +205,20 @@
+ + + +
@@ -201,10 +205,10 @@
-
+
-
+
@@ -236,7 +240,7 @@
-
+
@@ -337,7 +341,7 @@
-
+
-
+
@@ -368,7 +372,9 @@ @@ -376,20 +382,21 @@
-
-
+
@@ -524,7 +531,7 @@
-
+
When you tag someone, they are sent a notification.
For more information on tagging, click here.

-
+

Tagging someone is like mentioning them, with the option to make it private between you.

You can choose to tag someone in public or private mode. Public mode will allow others to see who you tagged in the post and private mode tagged users will not be shown to others.

-
+

Add Location

-
+
@@ -910,6 +923,7 @@ export default { }, namedPages: [ + 'filteringMedia', 'cropPhoto', 'tagPeople', 'addLocation', @@ -943,7 +957,6 @@ export default { cb(res.data); }) .catch(err => { - console.log(err); }) }) }, @@ -957,7 +970,6 @@ export default { cb(res.data); }) .catch(err => { - console.log(err); }) }) } @@ -1032,6 +1044,10 @@ export default { collectionsPage: 1, collectionsCanLoadMore: false, spoilerText: undefined, + isFilteringMedia: false, + filteringMediaTimeout: undefined, + filteringRemainingCount: 0, + isPosting: false, } }, @@ -1242,6 +1258,50 @@ export default { }); }, + mediaReorder(dir) { + const m = this.media; + const cur = this.carouselCursor; + const pla = m[cur]; + let res = []; + let cursor = 0; + + if(dir == 'prev') { + if(cur == 0) { + for (let i = cursor; i < m.length - 1; i++) { + res[i] = m[i+1]; + } + res[m.length - 1] = pla; + cursor = 0; + } else { + res = this.handleSwap(m, cur, cur - 1); + cursor = cur - 1; + } + } else { + if(cur == m.length - 1) { + res = m; + let lastItem = res.pop(); + res.unshift(lastItem); + cursor = m.length - 1; + } else { + res = this.handleSwap(m, cur, cur + 1); + cursor = cur + 1; + } + } + this.$nextTick(() => { + this.media = res; + this.carouselCursor = cursor; + }) + }, + + handleSwap(arr, index1, index2) { + if (index1 >= 0 && index1 < arr.length && index2 >= 0 && index2 < arr.length) { + const temp = arr[index1]; + arr[index1] = arr[index2]; + arr[index2] = temp; + return arr; + } + }, + compose() { let state = this.composeState; @@ -1254,8 +1314,15 @@ export default { return; } + switch(state) { - case 'publish' : + case 'publish': + this.isPosting = true; + let count = this.media.filter(m => m.filter_class && !m.hasOwnProperty('is_filtered')).length; + if(count) { + this.applyFilterToMedia(); + return; + } if(this.composeSettings.media_descriptions === true) { let count = this.media.filter(m => { return !m.hasOwnProperty('alt') || m.alt.length < 2; @@ -1377,6 +1444,10 @@ export default { switch(this.mode) { case 'photo': switch(this.page) { + case 'filteringMedia': + this.page = 2; + break; + case 'addText': this.page = 1; break; @@ -1411,6 +1482,10 @@ export default { case 'video': switch(this.page) { + case 'filteringMedia': + this.page = 2; + break; + case 'licensePicker': this.page = 'video-2'; break; @@ -1431,6 +1506,10 @@ export default { this.page = 1; break; + case 'filteringMedia': + this.page = 2; + break; + case 'textOptions': this.page = 'addText'; break; @@ -1470,6 +1549,9 @@ export default { this.page = 2; break; + case 'filteringMedia': + break; + case 'cropPhoto': this.pageLoading = true; let self = this; @@ -1495,14 +1577,7 @@ export default { break; case 2: - if(this.currentFilter) { - if(window.confirm('Are you sure you want to apply this filter?')) { - this.applyFilterToMedia(); - this.page++; - } - } else { this.page++; - } break; case 3: this.page++; @@ -1649,43 +1724,73 @@ export default { // this is where the magic happens var ua = navigator.userAgent.toLowerCase(); if(ua.indexOf('firefox') == -1 && ua.indexOf('chrome') == -1) { + this.isPosting = false; swal('Oops!', 'Your browser does not support the filter feature.', 'error'); + this.page = 3; return; } - let medias = this.media; - let media = null; - const canvas = document.getElementById('pr_canvas'); - const ctx = canvas.getContext('2d'); - let image = document.getElementById('pr_img'); - let blob = null; - let data = null; - - for (var i = medias.length - 1; i >= 0; i--) { - media = medias[i]; - if(media.filter_class) { - image.src = media.url; - image.addEventListener('load', e => { - canvas.width = image.width; - canvas.height = image.height; - ctx.filter = App.util.filterCss[media.filter_class]; - ctx.drawImage(image, 0, 0, image.width, image.height); - ctx.save(); - canvas.toBlob(function(blob) { - data = new FormData(); - data.append('file', blob); - data.append('id', media.id); - axios.post('/api/compose/v0/media/update', data).then(res => { - }).catch(err => { - }); - }); - }, media.mime, 0.9); - ctx.clearRect(0, 0, image.width, image.height); - } - } - + let count = this.media.filter(m => m.filter_class).length; + if(count) { + this.page = 'filteringMedia'; + this.filteringRemainingCount = count; + this.$nextTick(() => { + this.isFilteringMedia = true; + this.media.forEach((media, idx) => this.applyFilterToMediaSave(media, idx)); + }) + } else { + this.page = 3; + } }, + applyFilterToMediaSave(media, idx) { + if(!media.filter_class) { + return; + } + + let self = this; + let data = null; + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + let image = document.createElement('img'); + image.src = media.url; + image.addEventListener('load', e => { + canvas.width = image.width; + canvas.height = image.height; + ctx.filter = App.util.filterCss[media.filter_class]; + ctx.drawImage(image, 0, 0, image.width, image.height); + ctx.save(); + canvas.toBlob(function(blob) { + data = new FormData(); + data.append('file', blob); + data.append('id', media.id); + axios.post('/api/compose/v0/media/update', data) + .then(res => { + self.media[idx].is_filtered = true; + self.updateFilteringMedia(); + }).catch(err => { + }); + }); + }, media.mime, 0.9); + ctx.clearRect(0, 0, image.width, image.height); + }, + + updateFilteringMedia() { + this.filteringRemainingCount--; + this.filteringMediaTimeout = setTimeout(() => this.filteringMediaTimeoutJob(), 500); + }, + + filteringMediaTimeoutJob() { + if(this.filteringRemainingCount === 0) { + this.isFilteringMedia = false; + clearTimeout(this.filteringMediaTimeout); + setTimeout(() => this.compose(), 500); + } else { + clearTimeout(this.filteringMediaTimeout); + this.filteringMediaTimeout = setTimeout(() => this.filteringMediaTimeoutJob(), 1000); + } + }, + tagSearch(input) { if (input.length < 1) { return []; } let self = this; @@ -1800,7 +1905,6 @@ export default { } window.location.href = res.data.url; }).catch(err => { - console.log(err.response.data.error); if(err.response.data.hasOwnProperty('error')) { if(err.response.data.error == 'Duplicate detected.') { this.postingPoll = false; From 00823545a5b9d3630989fad9c6a322b390e57068 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 22 Oct 2023 23:21:50 -0600 Subject: [PATCH 070/253] Add WebP2P support for Video --- config/media.php | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/config/media.php b/config/media.php index f550ff291..46c1719db 100644 --- a/config/media.php +++ b/config/media.php @@ -22,5 +22,39 @@ return [ 'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false), ], + ], + + 'hls' => [ + /* + |-------------------------------------------------------------------------- + | Enable HLS + |-------------------------------------------------------------------------- + | + | Enable optional HLS support, required for video p2p support. Requires FFMPEG + | Disabled by default. + | + */ + 'enabled' => env('MEDIA_HLS_ENABLED', false), + + 'debug' => env('MEDIA_HLS_DEBUG', false), + + /* + |-------------------------------------------------------------------------- + | Enable Video P2P support + |-------------------------------------------------------------------------- + | + | Enable optional video p2p support. Requires FFMPEG + HLS + | Disabled by default. + | + */ + 'p2p' => env('MEDIA_HLS_P2P', false), + + 'p2p_debug' => env('MEDIA_HLS_P2P_DEBUG', false), + + 'bitrate' => env('MEDIA_HLS_BITRATE', 1000), + + 'tracker' => env('MEDIA_HLS_P2P_TRACKER', 'wss://tracker.webtorrent.dev'), + + 'ice' => env('MEDIA_HLS_P2P_ICE_SERVER', 'stun:stun.l.google.com:19302'), ] ]; From 82fc36b2b377e45fe26439e5f5a72a895c46a21c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 22 Oct 2023 23:41:03 -0600 Subject: [PATCH 071/253] Update npm deps, add webp2p libs. Thanks @peertube <3 --- package-lock.json | 533 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 5 +- 2 files changed, 533 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1dde4df84..ad3df5977 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,9 +7,12 @@ "name": "pixelfed", "dependencies": { "@fancyapps/fancybox": "^3.5.7", + "@hcaptcha/vue-hcaptcha": "^1.3.0", + "@peertube/p2p-media-loader-core": "^1.0.14", + "@peertube/p2p-media-loader-hlsjs": "^1.0.14", "@trevoreyre/autocomplete-vue": "^2.2.0", "@web3-storage/parse-link-header": "^3.1.0", - "@zip.js/zip.js": "^2.7.14", + "@zip.js/zip.js": "^2.7.24", "animate.css": "^4.1.0", "bigpicture": "^2.6.2", "blurhash": "^1.1.3", @@ -1807,6 +1810,14 @@ "jquery": ">=1.9.0" } }, + "node_modules/@hcaptcha/vue-hcaptcha": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@hcaptcha/vue-hcaptcha/-/vue-hcaptcha-1.3.0.tgz", + "integrity": "sha512-aUSWyhRucgFeBOBUC3nWBZuE0TkeoSH5QIVFwiTLnNsYpIaxD1tKBbI5Tdoy0TdpkuXKsB4KqyElbvoMZ9reGw==", + "peerDependencies": { + "vue": "^2.0.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", @@ -1918,6 +1929,29 @@ "npm": ">=5.0.0" } }, + "node_modules/@peertube/p2p-media-loader-core": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/@peertube/p2p-media-loader-core/-/p2p-media-loader-core-1.0.14.tgz", + "integrity": "sha512-tjQv1CNziNY+zYzcL1h4q6AA2WuBUZnBIeVyjWR/EsO1EEC1VMdvPsL02cqYLz9yvIxgycjeTsWCm6XDqNgXRw==", + "dependencies": { + "bittorrent-tracker": "^9.19.0", + "debug": "^4.3.4", + "events": "^3.3.0", + "sha.js": "^2.4.11", + "simple-peer": "^9.11.1" + } + }, + "node_modules/@peertube/p2p-media-loader-hlsjs": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/@peertube/p2p-media-loader-hlsjs/-/p2p-media-loader-hlsjs-1.0.14.tgz", + "integrity": "sha512-ySUVgUvAFXCE5E94xxjfywQ8xzk3jy9UGVkgi5Oqq+QeY7uG+o7CZ+LsQ/RjXgWBD70tEnyyfADHtL+9FCnwyQ==", + "dependencies": { + "@peertube/p2p-media-loader-core": "^1.0.14", + "debug": "^4.3.4", + "events": "^3.3.0", + "m3u8-parser": "^4.7.1" + } + }, "node_modules/@trevoreyre/autocomplete-vue": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/@trevoreyre/autocomplete-vue/-/autocomplete-vue-2.4.1.tgz", @@ -2216,6 +2250,20 @@ "@types/node": "*" } }, + "node_modules/@videojs/vhs-utils": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@videojs/vhs-utils/-/vhs-utils-3.0.5.tgz", + "integrity": "sha512-PKVgdo8/GReqdx512F+ombhS+Bzogiofy1LgAj4tN8PfdBx3HSS7V5WfJotKTqtOWGwVfSWsrYN/t09/DSryrw==", + "dependencies": { + "@babel/runtime": "^7.12.5", + "global": "^4.4.0", + "url-toolkit": "^2.2.1" + }, + "engines": { + "node": ">=8", + "npm": ">=5" + } + }, "node_modules/@vue/compiler-sfc": { "version": "2.7.14", "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-2.7.14.tgz", @@ -2464,10 +2512,11 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" }, "node_modules/@zip.js/zip.js": { - "version": "2.7.15", - "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.7.15.tgz", - "integrity": "sha512-iuL2otty04U4YBfdpd22XJacKaNuR7AHWlQrE/L9zHxfunXNtIeSgCxi66T74pnzHdmmpGqlBlZoVRkWOT70aA==", + "version": "2.7.30", + "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.7.30.tgz", + "integrity": "sha512-nhMvQCj+TF1ATBqYzFds7v+yxPBhdDYHh8J341KtC1D2UrVBUIYcYK4Jy1/GiTsxOXEiKOXSUxvPG/XR+7jMqw==", "engines": { + "bun": ">=0.7.0", "deno": ">=1.0.0", "node": ">=16.5.0" } @@ -2503,6 +2552,11 @@ "acorn": "^8" } }, + "node_modules/addr-to-ip-port": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/addr-to-ip-port/-/addr-to-ip-port-1.5.4.tgz", + "integrity": "sha512-ByxmJgv8vjmDcl3IDToxL2yrWFrRtFpZAToY0f46XFXl8zS081t7El5MXIodwm7RC6DhHBRoOSMLFSPKCtHukg==" + }, "node_modules/adjust-sourcemap-loader": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-4.0.0.tgz", @@ -2826,6 +2880,11 @@ "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==" }, + "node_modules/bencode": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/bencode/-/bencode-2.0.3.tgz", + "integrity": "sha512-D/vrAD4dLVX23NalHwb8dSvsUsxeRPO8Y7ToKA015JQYq69MLDOMkC0uGZYA/MPpltLO8rt8eqFC2j8DxjTZ/w==" + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -2847,6 +2906,99 @@ "node": ">=8" } }, + "node_modules/bittorrent-peerid": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/bittorrent-peerid/-/bittorrent-peerid-1.3.6.tgz", + "integrity": "sha512-VyLcUjVMEOdSpHaCG/7odvCdLbAB1y3l9A2V6WIje24uV7FkJPrQrH/RrlFmKxP89pFVDEnE+YlHaFujlFIZsg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bittorrent-tracker": { + "version": "9.19.0", + "resolved": "https://registry.npmjs.org/bittorrent-tracker/-/bittorrent-tracker-9.19.0.tgz", + "integrity": "sha512-09d0aD2b+MC+zWvWajkUAKkYMynYW4tMbTKiRSthKtJZbafzEoNQSUHyND24SoCe3ZOb2fKfa6fu2INAESL9wA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "bencode": "^2.0.1", + "bittorrent-peerid": "^1.3.3", + "bn.js": "^5.2.0", + "chrome-dgram": "^3.0.6", + "clone": "^2.0.0", + "compact2string": "^1.4.1", + "debug": "^4.1.1", + "ip": "^1.1.5", + "lru": "^3.1.0", + "minimist": "^1.2.5", + "once": "^1.4.0", + "queue-microtask": "^1.2.3", + "random-iterate": "^1.0.1", + "randombytes": "^2.1.0", + "run-parallel": "^1.2.0", + "run-series": "^1.1.9", + "simple-get": "^4.0.0", + "simple-peer": "^9.11.0", + "simple-websocket": "^9.1.0", + "socks": "^2.0.0", + "string2compact": "^1.3.0", + "unordered-array-remove": "^1.0.2", + "ws": "^7.4.5" + }, + "bin": { + "bittorrent-tracker": "bin/cmd.js" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "bufferutil": "^4.0.3", + "utf-8-validate": "^5.0.5" + } + }, + "node_modules/bittorrent-tracker/node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -3125,6 +3277,19 @@ "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==" }, + "node_modules/bufferutil": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.8.tgz", + "integrity": "sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/builtin-status-codes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", @@ -3285,6 +3450,29 @@ "fsevents": "~2.3.2" } }, + "node_modules/chrome-dgram": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/chrome-dgram/-/chrome-dgram-3.0.6.tgz", + "integrity": "sha512-bqBsUuaOiXiqxXt/zA/jukNJJ4oaOtc7ciwqJpZVEaaXwwxqgI2/ZdG02vXYWUhHGziDlvGMQWk0qObgJwVYKA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "inherits": "^2.0.4", + "run-series": "^1.1.9" + } + }, "node_modules/chrome-trace-event": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", @@ -3349,6 +3537,14 @@ "node": ">=12" } }, + "node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/clone-deep": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", @@ -3406,6 +3602,14 @@ "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==" }, + "node_modules/compact2string": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/compact2string/-/compact2string-1.4.1.tgz", + "integrity": "sha512-3D+EY5nsRhqnOwDxveBv5T8wGo4DEvYxjDtPGmdOX+gfr5gE92c2RC0w2wa+xEefm07QuVqqcF3nZJUZ92l/og==", + "dependencies": { + "ipaddr.js": ">= 0.1.5" + } + }, "node_modules/compressible": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", @@ -3954,6 +4158,20 @@ } } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/default-gateway": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz", @@ -4266,6 +4484,11 @@ "node": ">=4" } }, + "node_modules/err-code": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", + "integrity": "sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA==" + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -4813,6 +5036,11 @@ "node": ">=6.9.0" } }, + "node_modules/get-browser-rtc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz", + "integrity": "sha512-MghbMJ61EJrRsDe7w1Bvqt3ZsBuqhce5nrn/XAwgwOXhcsz53/ltdxOse1h/8eKXj5slzxdsz56g5rzOFSGwfQ==" + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -5422,6 +5650,11 @@ "node": ">= 0.10" } }, + "node_modules/ip": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz", + "integrity": "sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg==" + }, "node_modules/ipaddr.js": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", @@ -5964,6 +6197,17 @@ "tslib": "^2.0.3" } }, + "node_modules/lru": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lru/-/lru-3.1.0.tgz", + "integrity": "sha512-5OUtoiVIGU4VXBOshidmtOsvBIvcQR6FD/RzWSvaeHyxCGB+PCUCu+52lqMfdc0h/2CLvHhZS4TwUmMQrrMbBQ==", + "dependencies": { + "inherits": "^2.0.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -5972,6 +6216,16 @@ "yallist": "^3.0.2" } }, + "node_modules/m3u8-parser": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/m3u8-parser/-/m3u8-parser-4.8.0.tgz", + "integrity": "sha512-UqA2a/Pw3liR6Df3gwxrqghCP17OpPlQj6RBPLYygf/ZSQ4MoSgvdvhvt35qV+3NaaA0FSZx93Ix+2brT1U7cA==", + "dependencies": { + "@babel/runtime": "^7.12.5", + "@videojs/vhs-utils": "^3.0.5", + "global": "^4.4.0" + } + }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -6140,6 +6394,17 @@ "node": ">=6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/min-document": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", @@ -6311,6 +6576,17 @@ "node": ">= 6.13.0" } }, + "node_modules/node-gyp-build": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz", + "integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -7507,6 +7783,11 @@ } ] }, + "node_modules/random-iterate": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/random-iterate/-/random-iterate-1.0.1.tgz", + "integrity": "sha512-Jdsdnezu913Ot8qgKgSgs63XkAjEsnMcS1z+cC6D6TNXsUXsMxy0RpclF2pzGZTEiTXL9BiArdGTEexcv4nqcA==" + }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -7844,6 +8125,25 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/run-series": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-series/-/run-series-1.1.9.tgz", + "integrity": "sha512-Arc4hUN896vjkqCYrUXquBFtRZdv1PfLbTYP71efP6butxyQ0kWpiNJyAgsxscmQg1cqvHY32/UCBzXedTpU2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -8183,6 +8483,172 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/simple-peer": { + "version": "9.11.1", + "resolved": "https://registry.npmjs.org/simple-peer/-/simple-peer-9.11.1.tgz", + "integrity": "sha512-D1SaWpOW8afq1CZGWB8xTfrT3FekjQmPValrqncJMX7QFl8YwhrPTZvMCANLtgBwwdS+7zURyqxDDEmY558tTw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "buffer": "^6.0.3", + "debug": "^4.3.2", + "err-code": "^3.0.1", + "get-browser-rtc": "^1.1.0", + "queue-microtask": "^1.2.3", + "randombytes": "^2.1.0", + "readable-stream": "^3.6.0" + } + }, + "node_modules/simple-peer/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/simple-peer/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/simple-websocket": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/simple-websocket/-/simple-websocket-9.1.0.tgz", + "integrity": "sha512-8MJPnjRN6A8UCp1I+H/dSFyjwJhp6wta4hsVRhjf8w9qBHRzxYt14RaOcjvQnhD1N4yKOddEjflwMnQM4VtXjQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "debug": "^4.3.1", + "queue-microtask": "^1.2.2", + "randombytes": "^2.1.0", + "readable-stream": "^3.6.0", + "ws": "^7.4.2" + } + }, + "node_modules/simple-websocket/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/simple-websocket/node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -8191,6 +8657,15 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/sockjs": { "version": "0.3.24", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", @@ -8201,6 +8676,24 @@ "websocket-driver": "^0.7.4" } }, + "node_modules/socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "dependencies": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks/node_modules/ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" + }, "node_modules/source-list-map": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", @@ -8333,6 +8826,15 @@ "node": ">=8" } }, + "node_modules/string2compact": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/string2compact/-/string2compact-1.3.2.tgz", + "integrity": "sha512-3XUxUgwhj7Eqh2djae35QHZZT4mN3fsO7kagZhSGmhhlrQagVvWSFuuFIWnpxFS0CdTB2PlQcaL16RDi14I8uw==", + "dependencies": { + "addr-to-ip-port": "^1.0.1", + "ipaddr.js": "^2.0.0" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -8677,6 +9179,11 @@ "node": ">= 10.0.0" } }, + "node_modules/unordered-array-remove": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unordered-array-remove/-/unordered-array-remove-1.0.2.tgz", + "integrity": "sha512-45YsfD6svkgaCBNyvD+dFHm4qFX9g3wRSIVgWVPtm2OCnphvPxzJoe20ATsiNpNJrmzHifnxm+BN5F7gFT/4gw==" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -8744,6 +9251,24 @@ "resolved": "https://registry.npmjs.org/url-polyfill/-/url-polyfill-1.1.12.tgz", "integrity": "sha512-mYFmBHCapZjtcNHW0MDq9967t+z4Dmg5CJ0KqysK3+ZbyoNOWQHksGCTWwDhxGXllkWlOc10Xfko6v4a3ucM6A==" }, + "node_modules/url-toolkit": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/url-toolkit/-/url-toolkit-2.2.5.tgz", + "integrity": "sha512-mtN6xk+Nac+oyJ/PrI7tzfmomRVNFIWKUbG8jdYFt52hxbiReFAXIjYskvu64/dvuW71IcB7lV8l0HvZMac6Jg==" + }, + "node_modules/utf-8-validate": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", + "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/util": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", diff --git a/package.json b/package.json index 9a188a1d4..6598175d9 100644 --- a/package.json +++ b/package.json @@ -34,9 +34,12 @@ }, "dependencies": { "@fancyapps/fancybox": "^3.5.7", + "@hcaptcha/vue-hcaptcha": "^1.3.0", + "@peertube/p2p-media-loader-core": "^1.0.14", + "@peertube/p2p-media-loader-hlsjs": "^1.0.14", "@trevoreyre/autocomplete-vue": "^2.2.0", "@web3-storage/parse-link-header": "^3.1.0", - "@zip.js/zip.js": "^2.7.14", + "@zip.js/zip.js": "^2.7.24", "animate.css": "^4.1.0", "bigpicture": "^2.6.2", "blurhash": "^1.1.3", From 4cd53247a66215ad7696b9bcab8571abccbe28c0 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 22 Oct 2023 23:42:25 -0600 Subject: [PATCH 072/253] Add MediaHlsService --- app/Services/Media/MediaHlsService.php | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 app/Services/Media/MediaHlsService.php diff --git a/app/Services/Media/MediaHlsService.php b/app/Services/Media/MediaHlsService.php new file mode 100644 index 000000000..04b5ac649 --- /dev/null +++ b/app/Services/Media/MediaHlsService.php @@ -0,0 +1,27 @@ +media_path; + if(!$path) { return; } + $parts = explode('/', $path); + $filename = array_pop($parts); + $dir = implode('/', $parts); + [$name, $ext] = explode('.', $filename); + + $files = Storage::files($dir); + + return collect($files) + ->filter(function($p) use($dir, $name) { + return str_starts_with($p, $dir . '/' . $name); + }) + ->values() + ->toArray(); + } +} From a144301085b83769132bb634161d7a97be4509f9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 00:11:46 -0600 Subject: [PATCH 073/253] Add RegisterForm component --- .../remote-auth/partials/RegisterForm.vue | 800 ++++++++++++++++++ 1 file changed, 800 insertions(+) create mode 100644 resources/assets/components/remote-auth/partials/RegisterForm.vue diff --git a/resources/assets/components/remote-auth/partials/RegisterForm.vue b/resources/assets/components/remote-auth/partials/RegisterForm.vue new file mode 100644 index 000000000..6b1c4047f --- /dev/null +++ b/resources/assets/components/remote-auth/partials/RegisterForm.vue @@ -0,0 +1,800 @@ + + + + + From 4e3e23db36fe28f34c36d2d55272e86d1598b442 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 00:15:53 -0600 Subject: [PATCH 074/253] Add js debounce util --- resources/assets/js/util/debounce.js | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 resources/assets/js/util/debounce.js diff --git a/resources/assets/js/util/debounce.js b/resources/assets/js/util/debounce.js new file mode 100644 index 000000000..b846d35e5 --- /dev/null +++ b/resources/assets/js/util/debounce.js @@ -0,0 +1,11 @@ +export function debounce (fn, delay) { + var timeoutID = null + return function () { + clearTimeout(timeoutID) + var args = arguments + var that = this + timeoutID = setTimeout(function () { + fn.apply(that, args) + }, delay) + } +} From fac7c3c5e79f4572d549d3767e02a577451dffad Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 00:38:37 -0600 Subject: [PATCH 075/253] Update MediaTransformer, add hls_manifest attribute --- app/Transformer/Api/MediaTransformer.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/Transformer/Api/MediaTransformer.php b/app/Transformer/Api/MediaTransformer.php index 0a4600bae..e1eeb1f6f 100644 --- a/app/Transformer/Api/MediaTransformer.php +++ b/app/Transformer/Api/MediaTransformer.php @@ -4,6 +4,7 @@ namespace App\Transformer\Api; use App\Media; use League\Fractal; +use Storage; class MediaTransformer extends Fractal\TransformerAbstract { @@ -28,6 +29,10 @@ class MediaTransformer extends Fractal\TransformerAbstract 'blurhash' => $media->blurhash ?? 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay' ]; + if(config('media.hls.enabled') && $media->hls_transcoded_at != null && $media->hls_path) { + $res['hls_manifest'] = url(Storage::url($media->hls_path)); + } + if($media->width && $media->height) { $res['meta'] = [ 'focus' => [ From f9bbb055755c4a985e51209bb0ffd66e852be866 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 01:09:16 -0600 Subject: [PATCH 076/253] Update MediaDeletePipeline, handle HLS deletion --- .../MediaPipeline/MediaDeletePipeline.php | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/app/Jobs/MediaPipeline/MediaDeletePipeline.php b/app/Jobs/MediaPipeline/MediaDeletePipeline.php index 4db76c9c7..55df84948 100644 --- a/app/Jobs/MediaPipeline/MediaDeletePipeline.php +++ b/app/Jobs/MediaPipeline/MediaDeletePipeline.php @@ -10,8 +10,11 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Storage; +use App\Services\Media\MediaHlsService; +use Illuminate\Queue\Middleware\WithoutOverlapping; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; -class MediaDeletePipeline implements ShouldQueue +class MediaDeletePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; @@ -20,8 +23,34 @@ class MediaDeletePipeline implements ShouldQueue public $timeout = 300; public $tries = 3; public $maxExceptions = 1; + public $failOnTimeout = true; public $deleteWhenMissingModels = true; + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 3600; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return 'media:purge-job:id-' . $this->media->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("media:purge-job:id-{$this->media->id}"))->shared()->dontRelease()]; + } + public function __construct(Media $media) { $this->media = $media; @@ -63,9 +92,17 @@ class MediaDeletePipeline implements ShouldQueue $disk->delete($thumb); } + if($media->hls_path != null) { + $files = MediaHlsService::allFiles($media); + if($files && count($files)) { + foreach($files as $file) { + $disk->delete($file); + } + } + } + $media->delete(); return 1; } - } From 3f292459ffaf5e851fa932010c2dcb3b117df10d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 01:12:40 -0600 Subject: [PATCH 077/253] Update VideoPipeline, add VideoHlsPipeline job for HLS generation --- app/Jobs/VideoPipeline/VideoHlsPipeline.php | 94 +++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 app/Jobs/VideoPipeline/VideoHlsPipeline.php diff --git a/app/Jobs/VideoPipeline/VideoHlsPipeline.php b/app/Jobs/VideoPipeline/VideoHlsPipeline.php new file mode 100644 index 000000000..eb91ed78b --- /dev/null +++ b/app/Jobs/VideoPipeline/VideoHlsPipeline.php @@ -0,0 +1,94 @@ +media->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("media:video-hls:id-{$this->media->id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct($media) + { + $this->media = $media; + } + + /** + * Execute the job. + */ + public function handle(): void + { + $media = $this->media; + + $bitrate = (new X264)->setKiloBitrate(config('media.hls.bitrate') ?? 1000); + + $mp4 = $media->media_path; + $man = str_replace('.mp4', '.m3u8', $mp4); + + FFMpeg::fromDisk('local') + ->open($mp4) + ->exportForHLS() + ->setSegmentLength(16) + ->setKeyFrameInterval(48) + ->addFormat($bitrate) + ->save($man); + + $media->hls_path = $man; + $media->hls_transcoded_at = now(); + $media->save(); + + MediaService::del($media->status_id); + usleep(50000); + StatusService::del($media->status_id); + + return; + } +} From f0ba2dfc69163b6e499ea1f2095c9a290d88db7d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 01:13:09 -0600 Subject: [PATCH 078/253] Update VideoThumbnail job, dispatch HLS job when applicable --- app/Jobs/VideoPipeline/VideoThumbnail.php | 40 +++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/app/Jobs/VideoPipeline/VideoThumbnail.php b/app/Jobs/VideoPipeline/VideoThumbnail.php index fed61e4fc..ebcb4cf7e 100644 --- a/app/Jobs/VideoPipeline/VideoThumbnail.php +++ b/app/Jobs/VideoPipeline/VideoThumbnail.php @@ -16,13 +16,46 @@ use App\Jobs\MediaPipeline\MediaStoragePipeline; use App\Util\Media\Blurhash; use App\Services\MediaService; use App\Services\StatusService; +use Illuminate\Queue\Middleware\WithoutOverlapping; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; -class VideoThumbnail implements ShouldQueue +class VideoThumbnail implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; protected $media; + public $timeout = 900; + public $tries = 3; + public $maxExceptions = 1; + public $failOnTimeout = true; + public $deleteWhenMissingModels = true; + + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 3600; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return 'media:video-thumb:id-' . $this->media->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("media:video-thumb:id-{$this->media->id}"))->shared()->dontRelease()]; + } + /** * Create a new job instance. * @@ -54,7 +87,7 @@ class VideoThumbnail implements ShouldQueue $path[$i] = $t; $save = implode('/', $path); $video = FFMpeg::open($base) - ->getFrameFromSeconds(0) + ->getFrameFromSeconds(1) ->export() ->toDisk('local') ->save($save); @@ -68,6 +101,9 @@ class VideoThumbnail implements ShouldQueue $media->save(); } + if(config('media.hls.enabled')) { + VideoHlsPipeline::dispatch($media)->onQueue('mmo'); + } } catch (Exception $e) { } From 6cf4363c507455782e03d300305c3d07e8405544 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 01:14:33 -0600 Subject: [PATCH 079/253] Update MediaService, remove hls_manifest attribute for MastoAPI entities --- app/Services/MediaService.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/Services/MediaService.php b/app/Services/MediaService.php index 8ca90118f..e0a2576e0 100644 --- a/app/Services/MediaService.php +++ b/app/Services/MediaService.php @@ -18,7 +18,7 @@ class MediaService public static function get($statusId) { - return Cache::remember(self::CACHE_KEY.$statusId, 86400, function() use($statusId) { + return Cache::remember(self::CACHE_KEY.$statusId, 21600, function() use($statusId) { $media = Media::whereStatusId($statusId)->orderBy('order')->get(); if(!$media) { return []; @@ -46,7 +46,8 @@ class MediaService $media['orientation'], $media['filter_name'], $media['filter_class'], - $media['mime'] + $media['mime'], + $media['hls_manifest'] ); $media['type'] = $mime ? strtolower($mime[0]) : 'unknown'; From 5c358010b088e5f72a6218e148c4414963644cf9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 01:15:02 -0600 Subject: [PATCH 080/253] Update Config util, add hls attributes --- app/Util/Site/Config.php | 156 +++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 71 deletions(-) diff --git a/app/Util/Site/Config.php b/app/Util/Site/Config.php index 90631159f..e0916591d 100644 --- a/app/Util/Site/Config.php +++ b/app/Util/Site/Config.php @@ -7,86 +7,100 @@ use Illuminate\Support\Str; class Config { - const CACHE_KEY = 'api:site:configuration:_v0.8'; + const CACHE_KEY = 'api:site:configuration:_v0.8'; - public static function get() { - return Cache::remember(self::CACHE_KEY, 900, function() { - return [ - 'version' => config('pixelfed.version'), - 'open_registration' => (bool) config_cache('pixelfed.open_registration'), - 'uploader' => [ - 'max_photo_size' => (int) config('pixelfed.max_photo_size'), - 'max_caption_length' => (int) config('pixelfed.max_caption_length'), - 'max_altext_length' => (int) config('pixelfed.max_altext_length', 150), - 'album_limit' => (int) config_cache('pixelfed.max_album_length'), - 'image_quality' => (int) config_cache('pixelfed.image_quality'), + public static function get() { + return Cache::remember(self::CACHE_KEY, 900, function() { + $hls = [ + 'enabled' => config('media.hls.enabled'), + ]; + if(config('media.hls.enabled')) { + $hls = [ + 'enabled' => true, + 'debug' => (bool) config('media.hls.debug'), + 'p2p' => (bool) config('media.hls.p2p'), + 'p2p_debug' => (bool) config('media.hls.p2p_debug'), + 'tracker' => config('media.hls.tracker'), + 'ice' => config('media.hls.ice') + ]; + } + return [ + 'version' => config('pixelfed.version'), + 'open_registration' => (bool) config_cache('pixelfed.open_registration'), + 'uploader' => [ + 'max_photo_size' => (int) config('pixelfed.max_photo_size'), + 'max_caption_length' => (int) config('pixelfed.max_caption_length'), + 'max_altext_length' => (int) config('pixelfed.max_altext_length', 150), + 'album_limit' => (int) config_cache('pixelfed.max_album_length'), + 'image_quality' => (int) config_cache('pixelfed.image_quality'), - 'max_collection_length' => (int) config('pixelfed.max_collection_length', 18), + 'max_collection_length' => (int) config('pixelfed.max_collection_length', 18), - 'optimize_image' => (bool) config('pixelfed.optimize_image'), - 'optimize_video' => (bool) config('pixelfed.optimize_video'), + 'optimize_image' => (bool) config('pixelfed.optimize_image'), + 'optimize_video' => (bool) config('pixelfed.optimize_video'), - 'media_types' => config_cache('pixelfed.media_types'), - 'mime_types' => config_cache('pixelfed.media_types') ? explode(',', config_cache('pixelfed.media_types')) : [], - 'enforce_account_limit' => (bool) config_cache('pixelfed.enforce_account_limit') - ], + 'media_types' => config_cache('pixelfed.media_types'), + 'mime_types' => config_cache('pixelfed.media_types') ? explode(',', config_cache('pixelfed.media_types')) : [], + 'enforce_account_limit' => (bool) config_cache('pixelfed.enforce_account_limit') + ], - 'activitypub' => [ - 'enabled' => (bool) config_cache('federation.activitypub.enabled'), - 'remote_follow' => config('federation.activitypub.remoteFollow') - ], + 'activitypub' => [ + 'enabled' => (bool) config_cache('federation.activitypub.enabled'), + 'remote_follow' => config('federation.activitypub.remoteFollow') + ], - 'ab' => config('exp'), + 'ab' => config('exp'), - 'site' => [ - 'name' => config_cache('app.name'), - 'domain' => config('pixelfed.domain.app'), - 'url' => config('app.url'), - 'description' => config_cache('app.short_description') - ], + 'site' => [ + 'name' => config_cache('app.name'), + 'domain' => config('pixelfed.domain.app'), + 'url' => config('app.url'), + 'description' => config_cache('app.short_description') + ], - 'account' => [ - 'max_avatar_size' => config('pixelfed.max_avatar_size'), - 'max_bio_length' => config('pixelfed.max_bio_length'), - 'max_name_length' => config('pixelfed.max_name_length'), - 'min_password_length' => config('pixelfed.min_password_length'), - 'max_account_size' => config('pixelfed.max_account_size') - ], + 'account' => [ + 'max_avatar_size' => config('pixelfed.max_avatar_size'), + 'max_bio_length' => config('pixelfed.max_bio_length'), + 'max_name_length' => config('pixelfed.max_name_length'), + 'min_password_length' => config('pixelfed.min_password_length'), + 'max_account_size' => config('pixelfed.max_account_size') + ], - 'username' => [ - 'remote' => [ - 'formats' => config('instance.username.remote.formats'), - 'format' => config('instance.username.remote.format'), - 'custom' => config('instance.username.remote.custom') - ] - ], + 'username' => [ + 'remote' => [ + 'formats' => config('instance.username.remote.formats'), + 'format' => config('instance.username.remote.format'), + 'custom' => config('instance.username.remote.custom') + ] + ], - 'features' => [ - 'timelines' => [ - 'local' => true, - 'network' => (bool) config('federation.network_timeline'), - ], - 'mobile_apis' => (bool) config_cache('pixelfed.oauth_enabled'), - 'stories' => (bool) config_cache('instance.stories.enabled'), - 'video' => Str::contains(config_cache('pixelfed.media_types'), 'video/mp4'), - 'import' => [ - 'instagram' => (bool) config_cache('pixelfed.import.instagram.enabled'), - 'mastodon' => false, - 'pixelfed' => false - ], - 'label' => [ - 'covid' => [ - 'enabled' => (bool) config('instance.label.covid.enabled'), - 'org' => config('instance.label.covid.org'), - 'url' => config('instance.label.covid.url'), - ] - ] - ] - ]; - }); - } + 'features' => [ + 'timelines' => [ + 'local' => true, + 'network' => (bool) config('federation.network_timeline'), + ], + 'mobile_apis' => (bool) config_cache('pixelfed.oauth_enabled'), + 'stories' => (bool) config_cache('instance.stories.enabled'), + 'video' => Str::contains(config_cache('pixelfed.media_types'), 'video/mp4'), + 'import' => [ + 'instagram' => (bool) config_cache('pixelfed.import.instagram.enabled'), + 'mastodon' => false, + 'pixelfed' => false + ], + 'label' => [ + 'covid' => [ + 'enabled' => (bool) config('instance.label.covid.enabled'), + 'org' => config('instance.label.covid.org'), + 'url' => config('instance.label.covid.url'), + ] + ], + 'hls' => $hls + ] + ]; + }); + } - public static function json() { - return json_encode(self::get(), JSON_FORCE_OBJECT); - } + public static function json() { + return json_encode(self::get(), JSON_FORCE_OBJECT); + } } From e3f8cfb49e1d8f2ead142de284e248085793bfe7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 01:27:11 -0600 Subject: [PATCH 081/253] Add hls/p2p video player --- .../components/presenter/VideoPlayer.vue | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 resources/assets/components/presenter/VideoPlayer.vue diff --git a/resources/assets/components/presenter/VideoPlayer.vue b/resources/assets/components/presenter/VideoPlayer.vue new file mode 100644 index 000000000..8f5ba0102 --- /dev/null +++ b/resources/assets/components/presenter/VideoPlayer.vue @@ -0,0 +1,198 @@ + + + From f11ce7009ff92d97b9868aaefa99702cc3b1d5ca Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 23 Oct 2023 01:28:06 -0600 Subject: [PATCH 082/253] Update PostContent, add new video-player component --- .../components/partials/post/PostContent.vue | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/resources/assets/components/partials/post/PostContent.vue b/resources/assets/components/partials/post/PostContent.vue index 057b07fea..0a88acb19 100644 --- a/resources/assets/components/partials/post/PostContent.vue +++ b/resources/assets/components/partials/post/PostContent.vue @@ -12,7 +12,7 @@
- +
@@ -108,27 +108,11 @@
- +
@@ -185,12 +169,14 @@ +@endpush From d3f032b2ec1cbef33066ec131961eb31dc4a76b6 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 16 Dec 2023 06:11:13 -0700 Subject: [PATCH 211/253] Update FollowerService, add quickCheck to follows method for non cold-boot checks --- app/Services/FollowerService.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Services/FollowerService.php b/app/Services/FollowerService.php index 8b5eeced9..dcb7a1158 100644 --- a/app/Services/FollowerService.php +++ b/app/Services/FollowerService.php @@ -89,12 +89,16 @@ class FollowerService return Redis::zCard(self::FOLLOWING_KEY . $id); } - public static function follows(string $actor, string $target) + public static function follows(string $actor, string $target, $quickCheck = false) { if($actor == $target) { return false; } + if($quickCheck) { + return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor); + } + if(self::followerCount($target, false) && self::followingCount($actor, false)) { self::cacheSyncCheck($target, 'followers'); return (bool) Redis::zScore(self::FOLLOWERS_KEY . $target, $actor); From 60e053c93671d0e790cde02aa1c51bc011cca17a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 16 Dec 2023 06:22:56 -0700 Subject: [PATCH 212/253] Update ApiV1Controller, update discoverAccountsPopular method --- app/Http/Controllers/Api/ApiV1Controller.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index f5e1202c0..e429a8681 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -3619,25 +3619,31 @@ class ApiV1Controller extends Controller $pid = $request->user()->profile_id; - $ids = Cache::remember('api:v1.1:discover:accounts:popular', 86400, function() { + $ids = Cache::remember('api:v1.1:discover:accounts:popular', 3600, function() { return DB::table('profiles') ->where('is_private', false) ->whereNull('status') ->orderByDesc('profiles.followers_count') - ->limit(20) + ->limit(30) ->get(); }); - + $filters = UserFilterService::filters($pid); $ids = $ids->map(function($profile) { return AccountService::get($profile->id, true); }) ->filter(function($profile) use($pid) { - return $profile && isset($profile['id']); + return $profile && isset($profile['id'], $profile['locked']) && !$profile['locked']; }) ->filter(function($profile) use($pid) { return $profile['id'] != $pid; }) - ->take(6) + ->filter(function($profile) use($pid) { + return !FollowerService::follows($pid, $profile['id'], true); + }) + ->filter(function($profile) use($filters) { + return !in_array($profile['id'], $filters); + }) + ->take(16) ->values(); return $this->json($ids); From 7016d195202e335253ab1143c1212434e23193ee Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sat, 16 Dec 2023 17:04:16 -0700 Subject: [PATCH 213/253] Update Privacy Settings view, change button to Blocked Domains and add l10n --- resources/lang/en/profile.php | 6 +++++- resources/views/settings/privacy.blade.php | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/resources/lang/en/profile.php b/resources/lang/en/profile.php index 3206a94bc..04a2bcd90 100644 --- a/resources/lang/en/profile.php +++ b/resources/lang/en/profile.php @@ -13,5 +13,9 @@ return [ 'status.disabled.header' => 'Profile Unavailable', 'status.disabled.body' => 'Sorry, this profile is not available at the moment. Please try again shortly.', - 'block.domain.max' => 'Max limit of domain blocks reached! You can only block :max domains at a time. Ask your admin to adjust this limit.' + 'block.domain.max' => 'Max limit of domain blocks reached! You can only block :max domains at a time. Ask your admin to adjust this limit.', + + 'mutedAccounts' => 'Muted Accounts', + 'blockedAccounts' => 'Blocked Accounts', + 'blockedDomains' => 'Blocked Domains', ]; diff --git a/resources/views/settings/privacy.blade.php b/resources/views/settings/privacy.blade.php index a1212141a..369ddbbb4 100644 --- a/resources/views/settings/privacy.blade.php +++ b/resources/views/settings/privacy.blade.php @@ -8,9 +8,9 @@
From e7c08fbbb2ed55acc04d2bd47a510c618319490e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:32:48 -0700 Subject: [PATCH 214/253] Update AccountService, add blocksDomain method --- app/Services/AccountService.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/Services/AccountService.php b/app/Services/AccountService.php index fa1613cae..98e878845 100644 --- a/app/Services/AccountService.php +++ b/app/Services/AccountService.php @@ -7,6 +7,7 @@ use App\Profile; use App\Status; use App\User; use App\UserSetting; +use App\Models\UserDomainBlock; use App\Transformer\Api\AccountTransformer; use League\Fractal; use League\Fractal\Serializer\ArraySerializer; @@ -234,4 +235,13 @@ class AccountService } return; } + + public static function blocksDomain($pid, $domain = false) + { + if(!$domain) { + return; + } + + return UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->exists(); + } } From a7f96d81949af8029edcbbaabc50ad63030485f9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:34:53 -0700 Subject: [PATCH 215/253] Update Inbox, add user domain blocks to Direct Message handler --- app/Util/ActivityPub/Inbox.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index 0dd4722e9..c0ad390b8 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -372,7 +372,11 @@ class Inbox ->whereUsername(array_last(explode('/', $activity['to'][0]))) ->firstOrFail(); - if(in_array($actor->id, $profile->blockedIds()->toArray())) { + if(!$actor || in_array($actor->id, $profile->blockedIds()->toArray())) { + return; + } + + if(AccountService::blocksDomain($profile->id, $actor->domain) == true) { return; } From c89dc45e8d8010282a5605537a665151d6c7790c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:37:34 -0700 Subject: [PATCH 216/253] Update Inbox, add user domain blocks to Follow handler --- app/Util/ActivityPub/Inbox.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index c0ad390b8..176890e70 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -514,6 +514,10 @@ class Inbox return; } + if(AccountService::blocksDomain($target->id, $actor->domain) == true) { + return; + } + if( Follower::whereProfileId($actor->id) ->whereFollowingId($target->id) From 279fb28e2a8d5749fb590b89ed47ff4c89ea9379 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:42:21 -0700 Subject: [PATCH 217/253] Update Inbox, add user domain blocks to Announce handler --- app/Util/ActivityPub/Inbox.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index 176890e70..6818a8e87 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -589,6 +589,10 @@ class Inbox return; } + if(AccountService::blocksDomain($parent->profile_id, $actor->domain) == true) { + return; + } + $blocks = UserFilterService::blocks($parent->profile_id); if($blocks && in_array($actor->id, $blocks)) { return; From 3fbf8f159ef24ba202cd727916724fe782eaa50d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:46:09 -0700 Subject: [PATCH 218/253] Update Inbox, add user domain blocks to Accept handler --- app/Util/ActivityPub/Inbox.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index 6818a8e87..15d7d3445 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -646,6 +646,10 @@ class Inbox return; } + if(AccountService::blocksDomain($target->id, $actor->domain) == true) { + return; + } + $request = FollowRequest::whereFollowerId($actor->id) ->whereFollowingId($target->id) ->whereIsRejected(false) From e32e50da7bc2a5c4d05379854752f3a780990628 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:47:03 -0700 Subject: [PATCH 219/253] Update Inbox, add user domain blocks to Like handler --- app/Util/ActivityPub/Inbox.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index 15d7d3445..defb75fa4 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -775,6 +775,10 @@ class Inbox return; } + if(AccountService::blocksDomain($status->profile_id, $profile->domain) == true) { + return; + } + $blocks = UserFilterService::blocks($status->profile_id); if($blocks && in_array($profile->id, $blocks)) { return; From 491468612f99dc5ae88fde79672767a74f943a8e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:49:31 -0700 Subject: [PATCH 220/253] Update Inbox, add user domain blocks to Undo handler --- app/Util/ActivityPub/Inbox.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index defb75fa4..09676301f 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -836,6 +836,9 @@ class Inbox if(!$status) { return; } + if(AccountService::blocksDomain($status->profile_id, $profile->domain) == true) { + return; + } FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed'); Status::whereProfileId($profile->id) ->whereReblogOfId($status->id) @@ -857,6 +860,9 @@ class Inbox if(!$following) { return; } + if(AccountService::blocksDomain($following->id, $profile->domain) == true) { + return; + } Follower::whereProfileId($profile->id) ->whereFollowingId($following->id) ->delete(); @@ -882,6 +888,9 @@ class Inbox if(!$status) { return; } + if(AccountService::blocksDomain($status->profile_id, $profile->domain) == true) { + return; + } Like::whereProfileId($profile->id) ->whereStatusId($status->id) ->forceDelete(); From 8a0ceaf801af614457ae65197d18bb0fe69372d2 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 18 Dec 2023 22:57:53 -0700 Subject: [PATCH 221/253] Update Inbox, add user domain blocks to Story reaction handlers --- app/Util/ActivityPub/Inbox.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index 09676301f..b8bb780c8 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -944,6 +944,10 @@ class Inbox return; } + if(AccountService::blocksDomain($story->profile_id, $profile->domain) == true) { + return; + } + if(!FollowerService::follows($profile->id, $story->profile_id)) { return; } @@ -1014,6 +1018,10 @@ class Inbox $actorProfile = Helpers::profileFetch($actor); + if(AccountService::blocksDomain($targetProfile->id, $actorProfile->domain) == true) { + return; + } + if(!FollowerService::follows($actorProfile->id, $targetProfile->id)) { return; } @@ -1132,6 +1140,11 @@ class Inbox $actorProfile = Helpers::profileFetch($actor); + + if(AccountService::blocksDomain($targetProfile->id, $actorProfile->domain) == true) { + return; + } + if(!FollowerService::follows($actorProfile->id, $targetProfile->id)) { return; } From 819e7d3b328e8a14361a4df78976d0cb41f27ce7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 01:10:48 -0700 Subject: [PATCH 222/253] Add FeedRemoveDomainPipeline --- .../FeedRemoveDomainPipeline.php | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php diff --git a/app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php b/app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php new file mode 100644 index 000000000..2168ee054 --- /dev/null +++ b/app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php @@ -0,0 +1,92 @@ +pid . ':d-' . $this->domain; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("hts:feed:remove:domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct($pid, $domain) + { + $this->pid = $pid; + $this->domain = $domain; + } + + /** + * Execute the job. + */ + public function handle(): void + { + if(!config('exp.cached_home_timeline')) { + return; + } + if(!$this->pid || !$this->domain) { + return; + } + $domain = strtolower($this->domain); + $pid = $this->pid; + $posts = HomeTimelineService::get($pid, '0', '-1'); + + foreach($posts as $post) { + $status = StatusService::get($post, false); + if(!$status || !isset($status['url'])) { + HomeTimelineService::rem($pid, $post); + continue; + } + $host = strtolower(parse_url($status['url'], PHP_URL_HOST)); + if($host === strtolower(config('pixelfed.domain.app')) || !$host) { + continue; + } + if($host === $domain) { + HomeTimelineService::rem($pid, $status['id']); + } + } + } +} From 5c1591fdffbdb1e842d7e2049be6ad967f485864 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 01:20:14 -0700 Subject: [PATCH 223/253] Add job batches migration --- ..._12_19_081928_create_job_batches_table.php | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 database/migrations/2023_12_19_081928_create_job_batches_table.php diff --git a/database/migrations/2023_12_19_081928_create_job_batches_table.php b/database/migrations/2023_12_19_081928_create_job_batches_table.php new file mode 100644 index 000000000..50e38c20f --- /dev/null +++ b/database/migrations/2023_12_19_081928_create_job_batches_table.php @@ -0,0 +1,35 @@ +string('id')->primary(); + $table->string('name'); + $table->integer('total_jobs'); + $table->integer('pending_jobs'); + $table->integer('failed_jobs'); + $table->longText('failed_job_ids'); + $table->mediumText('options')->nullable(); + $table->integer('cancelled_at')->nullable(); + $table->integer('created_at'); + $table->integer('finished_at')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('job_batches'); + } +}; From a492a95a0eb4e2a75d84d9da1cc807d8c3d85bb2 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 04:01:41 -0700 Subject: [PATCH 224/253] Update AdminShadowFilter, fix deleted profile bug --- app/Http/Controllers/AdminShadowFilterController.php | 3 ++- app/Models/AdminShadowFilter.php | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/Http/Controllers/AdminShadowFilterController.php b/app/Http/Controllers/AdminShadowFilterController.php index 461e1d0c2..e181be5c1 100644 --- a/app/Http/Controllers/AdminShadowFilterController.php +++ b/app/Http/Controllers/AdminShadowFilterController.php @@ -19,7 +19,8 @@ class AdminShadowFilterController extends Controller { $filter = $request->input('filter'); $searchQuery = $request->input('q'); - $filters = AdminShadowFilter::when($filter, function($q, $filter) { + $filters = AdminShadowFilter::whereHas('profile') + ->when($filter, function($q, $filter) { if($filter == 'all') { return $q; } else if($filter == 'inactive') { diff --git a/app/Models/AdminShadowFilter.php b/app/Models/AdminShadowFilter.php index f98086f7f..8a163feeb 100644 --- a/app/Models/AdminShadowFilter.php +++ b/app/Models/AdminShadowFilter.php @@ -5,6 +5,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use App\Services\AccountService; +use App\Profile; class AdminShadowFilter extends Model { @@ -24,4 +25,9 @@ class AdminShadowFilter extends Model return; } + + public function profile() + { + return $this->belongsTo(Profile::class, 'item_id'); + } } From 1664a5bc52fc62dc4f5c099cc75c189ccc4b0e27 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 05:46:06 -0700 Subject: [PATCH 225/253] Update FollowerService, add $silent param to remove method to more efficently purge relationships --- app/Services/FollowerService.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/Services/FollowerService.php b/app/Services/FollowerService.php index dcb7a1158..cec8f7068 100644 --- a/app/Services/FollowerService.php +++ b/app/Services/FollowerService.php @@ -35,16 +35,18 @@ class FollowerService Cache::forget('profile:following:' . $actor); } - public static function remove($actor, $target) + public static function remove($actor, $target, $silent = false) { Redis::zrem(self::FOLLOWING_KEY . $actor, $target); Redis::zrem(self::FOLLOWERS_KEY . $target, $actor); - Cache::forget('pf:services:follower:audience:' . $actor); - Cache::forget('pf:services:follower:audience:' . $target); - AccountService::del($actor); - AccountService::del($target); - RelationshipService::refresh($actor, $target); - Cache::forget('profile:following:' . $actor); + if($silent !== true) { + AccountService::del($actor); + AccountService::del($target); + RelationshipService::refresh($actor, $target); + Cache::forget('profile:following:' . $actor); + } else { + RelationshipService::forget($actor, $target); + } } public static function followers($id, $start = 0, $stop = 10) From 484a377a449c7d1c59e304c158e7c1514ec60649 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 06:02:58 -0700 Subject: [PATCH 226/253] Add ProfilePurgeFollowersByDomain pipeline job --- .../ProfilePurgeFollowersByDomain.php | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 app/Jobs/ProfilePipeline/ProfilePurgeFollowersByDomain.php diff --git a/app/Jobs/ProfilePipeline/ProfilePurgeFollowersByDomain.php b/app/Jobs/ProfilePipeline/ProfilePurgeFollowersByDomain.php new file mode 100644 index 000000000..24fcdc832 --- /dev/null +++ b/app/Jobs/ProfilePipeline/ProfilePurgeFollowersByDomain.php @@ -0,0 +1,119 @@ +pid . ':d-' . $this->domain; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("followers:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct($pid, $domain) + { + $this->pid = $pid; + $this->domain = $domain; + } + + /** + * Execute the job. + */ + public function handle(): void + { + if ($this->batch()->cancelled()) { + return; + } + + $pid = $this->pid; + $domain = $this->domain; + + $query = 'SELECT f.* + FROM followers f + JOIN profiles p ON p.id = f.profile_id OR p.id = f.following_id + WHERE (f.profile_id = ? OR f.following_id = ?) + AND p.domain = ?;'; + $params = [$pid, $pid, $domain]; + + foreach(DB::cursor($query, $params) as $n) { + if(!$n || !$n->id) { + continue; + } + $follower = Follower::find($n->id); + if($follower->following_id == $pid && $follower->profile_id) { + FollowerService::remove($follower->profile_id, $pid, true); + $follower->delete(); + } else if ($follower->profile_id == $pid && $follower->following_id) { + FollowerService::remove($follower->following_id, $pid, true); + $follower->delete(); + } + } + + $profile = Profile::find($pid); + + $followerCount = DB::table('profiles') + ->join('followers', 'profiles.id', '=', 'followers.following_id') + ->where('followers.following_id', $pid) + ->count(); + + $followingCount = DB::table('profiles') + ->join('followers', 'profiles.id', '=', 'followers.following_id') + ->where('followers.profile_id', $pid) + ->count(); + + $profile->followers_count = $followerCount; + $profile->following_count = $followingCount; + $profile->save(); + + AccountService::del($profile->id); + } +} From 9d621108b06d1470369f647d13f46c1af174d0c4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 06:03:36 -0700 Subject: [PATCH 227/253] Add ProfilePurgeNotificationsByDomain pipeline job --- .../ProfilePurgeNotificationsByDomain.php | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 app/Jobs/ProfilePipeline/ProfilePurgeNotificationsByDomain.php diff --git a/app/Jobs/ProfilePipeline/ProfilePurgeNotificationsByDomain.php b/app/Jobs/ProfilePipeline/ProfilePurgeNotificationsByDomain.php new file mode 100644 index 000000000..ea5a45e4a --- /dev/null +++ b/app/Jobs/ProfilePipeline/ProfilePurgeNotificationsByDomain.php @@ -0,0 +1,91 @@ +pid . ':d-' . $this->domain; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("notify:v1:purge-by-domain:{$this->pid}:d-{$this->domain}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct($pid, $domain) + { + $this->pid = $pid; + $this->domain = $domain; + } + + /** + * Execute the job. + */ + public function handle(): void + { + if ($this->batch()->cancelled()) { + return; + } + + $pid = $this->pid; + $domain = $this->domain; + + $query = 'SELECT notifications.* + FROM profiles + JOIN notifications on profiles.id = notifications.actor_id + WHERE notifications.profile_id = ? + AND profiles.domain = ?'; + $params = [$pid, $domain]; + + foreach(DB::cursor($query, $params) as $n) { + if(!$n || !$n->id) { + continue; + } + Notification::where('id', $n->id)->delete(); + NotificationService::del($pid, $n->id); + } + } +} From 54adbeb0592d4a3b1ad1d66f48227ca09f0c8d7b Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 06:04:03 -0700 Subject: [PATCH 228/253] Update FeedRemoveDomainPipeline, make batchable --- app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php b/app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php index 2168ee054..018ea3794 100644 --- a/app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php +++ b/app/Jobs/HomeFeedPipeline/FeedRemoveDomainPipeline.php @@ -2,6 +2,7 @@ namespace App\Jobs\HomeFeedPipeline; +use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; @@ -15,7 +16,7 @@ use App\Services\HomeTimelineService; class FeedRemoveDomainPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels; protected $pid; protected $domain; @@ -67,6 +68,11 @@ class FeedRemoveDomainPipeline implements ShouldQueue, ShouldBeUniqueUntilProces if(!config('exp.cached_home_timeline')) { return; } + + if ($this->batch()->cancelled()) { + return; + } + if(!$this->pid || !$this->domain) { return; } From 87bba03d23b60d7e1940b26ac0f633a0cfaaeb5d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 06:24:51 -0700 Subject: [PATCH 229/253] Update DomainBlockController, dispatch jobies --- .../Api/V1/DomainBlockController.php | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/app/Http/Controllers/Api/V1/DomainBlockController.php b/app/Http/Controllers/Api/V1/DomainBlockController.php index 5a6178e39..53a209ed9 100644 --- a/app/Http/Controllers/Api/V1/DomainBlockController.php +++ b/app/Http/Controllers/Api/V1/DomainBlockController.php @@ -6,6 +6,12 @@ use Illuminate\Http\Request; use App\Http\Controllers\Controller; use App\Models\UserDomainBlock; use App\Util\ActivityPub\Helpers; +use Illuminate\Bus\Batch; +use Illuminate\Support\Facades\Bus; +use Illuminate\Support\Facades\Cache; +use App\Jobs\HomeFeedPipeline\FeedRemoveDomainPipeline; +use App\Jobs\ProfilePipeline\ProfilePurgeNotificationsByDomain; +use App\Jobs\ProfilePipeline\ProfilePurgeFollowersByDomain; class DomainBlockController extends Controller { @@ -59,7 +65,7 @@ class DomainBlockController extends Controller return abort(500, 'Invalid domain or already blocked by server admins'); } - $domain = parse_url($domain, PHP_URL_HOST); + $domain = strtolower(parse_url($domain, PHP_URL_HOST)); abort_if(config_cache('pixelfed.domain.app') == $domain, 400, 'Cannot ban your own server'); @@ -69,11 +75,23 @@ class DomainBlockController extends Controller abort_if($existingCount >= $maxLimit, 400, $errorMsg); - $block = UserDomainBlock::updateOrInsert([ + $block = UserDomainBlock::updateOrCreate([ 'profile_id' => $pid, 'domain' => $domain ]); + if($block->wasRecentlyCreated) { + Bus::batch([ + [ + new FeedRemoveDomainPipeline($pid, $domain), + new ProfilePurgeNotificationsByDomain($pid, $domain), + new ProfilePurgeFollowersByDomain($pid, $domain) + ] + ])->allowFailures()->onQueue('feed')->dispatch(); + + Cache::forget('profile:following:' . $pid); + } + return $this->json([]); } @@ -87,7 +105,7 @@ class DomainBlockController extends Controller $pid = $request->user()->profile_id; - $domain = trim($request->input('domain')); + $domain = strtolower(trim($request->input('domain'))); $filters = UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->delete(); From 795132df1830dd99d5cb0d09de121ddbb3fbad39 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 19 Dec 2023 06:25:39 -0700 Subject: [PATCH 230/253] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b545a30..2f75a72a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,8 @@ - Update Inbox, improve tombstone query efficiency ([759a4393](https://github.com/pixelfed/pixelfed/commit/759a4393)) - Update AccountService, add setLastActive method ([ebbd98e7](https://github.com/pixelfed/pixelfed/commit/ebbd98e7)) - Update ApiV1Controller, set last_active_at ([b6419545](https://github.com/pixelfed/pixelfed/commit/b6419545)) +- Update AdminShadowFilter, fix deleted profile bug ([a492a95a](https://github.com/pixelfed/pixelfed/commit/a492a95a)) +- Update FollowerService, add $silent param to remove method to more efficently purge relationships ([1664a5bc](https://github.com/pixelfed/pixelfed/commit/1664a5bc)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From dd16189fc81ab0b0501fa61894d12985089b7316 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 20 Dec 2023 23:10:57 -0700 Subject: [PATCH 231/253] Update ImageResize job, add more logging --- app/Jobs/ImageOptimizePipeline/ImageResize.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/Jobs/ImageOptimizePipeline/ImageResize.php b/app/Jobs/ImageOptimizePipeline/ImageResize.php index 9bb896a40..c1b4ea7f0 100644 --- a/app/Jobs/ImageOptimizePipeline/ImageResize.php +++ b/app/Jobs/ImageOptimizePipeline/ImageResize.php @@ -9,6 +9,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Log; class ImageResize implements ShouldQueue { @@ -46,6 +47,7 @@ class ImageResize implements ShouldQueue } $path = storage_path('app/'.$media->media_path); if (!is_file($path) || $media->skip_optimize) { + Log::info('Tried to optimize media that does not exist or is not readable. ' . $path); return; } @@ -57,6 +59,7 @@ class ImageResize implements ShouldQueue $img = new Image(); $img->resizeImage($media); } catch (Exception $e) { + Log::error($e); } ImageThumbnail::dispatch($media)->onQueue('mmo'); From ae1db1e3ab079fc6c8965df6761031e3b4a49baf Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 20 Dec 2023 23:17:27 -0700 Subject: [PATCH 232/253] Update migration --- .../2023_12_16_052413_create_user_domain_blocks_table.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/migrations/2023_12_16_052413_create_user_domain_blocks_table.php b/database/migrations/2023_12_16_052413_create_user_domain_blocks_table.php index 4cacbfcae..16f8f3fb2 100644 --- a/database/migrations/2023_12_16_052413_create_user_domain_blocks_table.php +++ b/database/migrations/2023_12_16_052413_create_user_domain_blocks_table.php @@ -14,7 +14,7 @@ return new class extends Migration Schema::create('user_domain_blocks', function (Blueprint $table) { $table->id(); $table->unsignedBigInteger('profile_id')->index(); - $table->string('domain'); + $table->string('domain')->index(); $table->unique(['profile_id', 'domain'], 'user_domain_blocks_by_id'); }); } From 0455dd1996269eb4450f9f778f3ae4159e719c5a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 20 Dec 2023 23:55:26 -0700 Subject: [PATCH 233/253] Update UserFilter model, add user relation --- app/UserFilter.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/UserFilter.php b/app/UserFilter.php index b0af2d777..dfa0d4662 100644 --- a/app/UserFilter.php +++ b/app/UserFilter.php @@ -33,4 +33,9 @@ class UserFilter extends Model { return $this->belongsTo(Instance::class, 'filterable_id'); } + + public function user() + { + return $this->belongsTo(Profile::class, 'user_id'); + } } From 29aa87c282f141a788a5b95b7c8312a12966c14e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 00:17:20 -0700 Subject: [PATCH 234/253] Update HomeFeedPipeline jobs, add domain block filtering --- .../HomeFeedPipeline/FeedInsertPipeline.php | 22 +++++++++++++++++-- .../FeedInsertRemotePipeline.php | 22 +++++++++++++++++-- .../HashtagInsertFanoutPipeline.php | 18 +++++++++++++-- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/app/Jobs/HomeFeedPipeline/FeedInsertPipeline.php b/app/Jobs/HomeFeedPipeline/FeedInsertPipeline.php index 19a546e83..4237a7b1a 100644 --- a/app/Jobs/HomeFeedPipeline/FeedInsertPipeline.php +++ b/app/Jobs/HomeFeedPipeline/FeedInsertPipeline.php @@ -11,6 +11,7 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use App\UserFilter; +use App\Models\UserDomainBlock; use App\Services\FollowerService; use App\Services\HomeTimelineService; use App\Services\StatusService; @@ -69,7 +70,7 @@ class FeedInsertPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing $sid = $this->sid; $status = StatusService::get($sid, false); - if(!$status || !isset($status['account']) || !isset($status['account']['id'])) { + if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) { return; } @@ -85,7 +86,24 @@ class FeedInsertPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing return; } - $skipIds = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray(); + $domain = strtolower(parse_url($status['url'], PHP_URL_HOST)); + $skipIds = []; + + if(strtolower(config('pixelfed.domain.app')) !== $domain) { + $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray(); + } + + $filters = UserFilter::whereFilterableType('App\Profile') + ->whereFilterableId($status['account']['id']) + ->whereIn('filter_type', ['mute', 'block']) + ->pluck('user_id') + ->toArray(); + + if($filters && count($filters)) { + $skipIds = array_merge($skipIds, $filters); + } + + $skipIds = array_unique(array_values($skipIds)); foreach($ids as $id) { if(!in_array($id, $skipIds)) { diff --git a/app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php b/app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php index e24696bd8..6c4ce0c35 100644 --- a/app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php +++ b/app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php @@ -11,6 +11,7 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use App\UserFilter; +use App\Models\UserDomainBlock; use App\Services\FollowerService; use App\Services\HomeTimelineService; use App\Services\StatusService; @@ -69,7 +70,7 @@ class FeedInsertRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProces $sid = $this->sid; $status = StatusService::get($sid, false); - if(!$status || !isset($status['account']) || !isset($status['account']['id'])) { + if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) { return; } @@ -83,7 +84,24 @@ class FeedInsertRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProces return; } - $skipIds = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray(); + $domain = strtolower(parse_url($status['url'], PHP_URL_HOST)); + $skipIds = []; + + if(strtolower(config('pixelfed.domain.app')) !== $domain) { + $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray(); + } + + $filters = UserFilter::whereFilterableType('App\Profile') + ->whereFilterableId($status['account']['id']) + ->whereIn('filter_type', ['mute', 'block']) + ->pluck('user_id') + ->toArray(); + + if($filters && count($filters)) { + $skipIds = array_merge($skipIds, $filters); + } + + $skipIds = array_unique(array_values($skipIds)); foreach($ids as $id) { if(!in_array($id, $skipIds)) { diff --git a/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php b/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php index a200c06e8..eca598e49 100644 --- a/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php +++ b/app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php @@ -11,6 +11,7 @@ use Illuminate\Queue\SerializesModels; use App\Hashtag; use App\StatusHashtag; use App\UserFilter; +use App\Models\UserDomainBlock; use App\Services\HashtagFollowService; use App\Services\HomeTimelineService; use App\Services\StatusService; @@ -77,7 +78,7 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro $sid = $hashtag->status_id; $status = StatusService::get($sid, false); - if(!$status || !isset($status['account']) || !isset($status['account']['id'])) { + if(!$status || !isset($status['account']) || !isset($status['account']['id'], $status['url'])) { return; } @@ -85,7 +86,20 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro return; } - $skipIds = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray(); + $domain = strtolower(parse_url($status['url'], PHP_URL_HOST)); + $skipIds = []; + + if(strtolower(config('pixelfed.domain.app')) !== $domain) { + $skipIds = UserDomainBlock::where('domain', $domain)->pluck('profile_id')->toArray(); + } + + $filters = UserFilter::whereFilterableType('App\Profile')->whereFilterableId($status['account']['id'])->whereIn('filter_type', ['mute', 'block'])->pluck('user_id')->toArray(); + + if($filters && count($filters)) { + $skipIds = array_merge($skipIds, $filters); + } + + $skipIds = array_unique(array_values($skipIds)); $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id); From b3148b788eb81135e95c32e7631972a65e62f768 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 00:21:33 -0700 Subject: [PATCH 235/253] Update HomeTimelineService, add domain blocks filtering to warmCache method --- app/Services/HomeTimelineService.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/Services/HomeTimelineService.php b/app/Services/HomeTimelineService.php index 6a2db0482..08d990591 100644 --- a/app/Services/HomeTimelineService.php +++ b/app/Services/HomeTimelineService.php @@ -6,6 +6,7 @@ use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Redis; use App\Follower; use App\Status; +use App\Models\UserDomainBlock; class HomeTimelineService { @@ -81,6 +82,8 @@ class HomeTimelineService $following = array_diff($following, $filters); } + $domainBlocks = UserDomainBlock::whereProfileId($id)->pluck('domain')->toArray(); + $ids = Status::where('id', '>', $minId) ->whereIn('profile_id', $following) ->whereNull(['in_reply_to_id', 'reblog_of_id']) @@ -91,6 +94,16 @@ class HomeTimelineService ->pluck('id'); foreach($ids as $pid) { + $status = StatusService::get($pid, false); + if(!$status || !isset($status['account'], $status['url'])) { + continue; + } + if($domainBlocks && count($domainBlocks)) { + $domain = strtolower(parse_url($status['url'], PHP_URL_HOST)); + if(in_array($domain, $domainBlocks)) { + continue; + } + } self::add($id, $pid); } From 6d55cb27eed3bace3168e4350e4deac0192ed8dc Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 00:42:26 -0700 Subject: [PATCH 236/253] Update UserFilterService, add domainBlocks method --- app/Services/UserFilterService.php | 271 +++++++++++++++-------------- 1 file changed, 143 insertions(+), 128 deletions(-) diff --git a/app/Services/UserFilterService.php b/app/Services/UserFilterService.php index 1dcdc819a..5673db60c 100644 --- a/app/Services/UserFilterService.php +++ b/app/Services/UserFilterService.php @@ -4,145 +4,160 @@ namespace App\Services; use Cache; use App\UserFilter; +use App\Models\UserDomainBlock; use Illuminate\Support\Facades\Redis; class UserFilterService { - const USER_MUTES_KEY = 'pf:services:mutes:ids:'; - const USER_BLOCKS_KEY = 'pf:services:blocks:ids:'; + const USER_MUTES_KEY = 'pf:services:mutes:ids:'; + const USER_BLOCKS_KEY = 'pf:services:blocks:ids:'; + const USER_DOMAIN_KEY = 'pf:services:domain-blocks:ids:'; - public static function mutes(int $profile_id) - { - $key = self::USER_MUTES_KEY . $profile_id; - $warm = Cache::has($key . ':cached-v0'); - if($warm) { - return Redis::zrevrange($key, 0, -1) ?? []; - } else { - if(Redis::zrevrange($key, 0, -1)) { - return Redis::zrevrange($key, 0, -1); - } - $ids = UserFilter::whereFilterType('mute') - ->whereUserId($profile_id) - ->pluck('filterable_id') - ->map(function($id) { - $acct = AccountService::get($id, true); - if(!$acct) { - return false; - } - return $acct['id']; - }) - ->filter(function($res) { - return $res; - }) - ->values() - ->toArray(); - foreach ($ids as $muted_id) { - Redis::zadd($key, (int) $muted_id, (int) $muted_id); - } - Cache::set($key . ':cached-v0', 1, 7776000); - return $ids; - } - } + public static function mutes(int $profile_id) + { + $key = self::USER_MUTES_KEY . $profile_id; + $warm = Cache::has($key . ':cached-v0'); + if($warm) { + return Redis::zrevrange($key, 0, -1) ?? []; + } else { + if(Redis::zrevrange($key, 0, -1)) { + return Redis::zrevrange($key, 0, -1); + } + $ids = UserFilter::whereFilterType('mute') + ->whereUserId($profile_id) + ->pluck('filterable_id') + ->map(function($id) { + $acct = AccountService::get($id, true); + if(!$acct) { + return false; + } + return $acct['id']; + }) + ->filter(function($res) { + return $res; + }) + ->values() + ->toArray(); + foreach ($ids as $muted_id) { + Redis::zadd($key, (int) $muted_id, (int) $muted_id); + } + Cache::set($key . ':cached-v0', 1, 7776000); + return $ids; + } + } - public static function blocks(int $profile_id) - { - $key = self::USER_BLOCKS_KEY . $profile_id; - $warm = Cache::has($key . ':cached-v0'); - if($warm) { - return Redis::zrevrange($key, 0, -1) ?? []; - } else { - if(Redis::zrevrange($key, 0, -1)) { - return Redis::zrevrange($key, 0, -1); - } - $ids = UserFilter::whereFilterType('block') - ->whereUserId($profile_id) - ->pluck('filterable_id') - ->map(function($id) { - $acct = AccountService::get($id, true); - if(!$acct) { - return false; - } - return $acct['id']; - }) - ->filter(function($res) { - return $res; - }) - ->values() - ->toArray(); - foreach ($ids as $blocked_id) { - Redis::zadd($key, (int) $blocked_id, (int) $blocked_id); - } - Cache::set($key . ':cached-v0', 1, 7776000); - return $ids; - } - } + public static function blocks(int $profile_id) + { + $key = self::USER_BLOCKS_KEY . $profile_id; + $warm = Cache::has($key . ':cached-v0'); + if($warm) { + return Redis::zrevrange($key, 0, -1) ?? []; + } else { + if(Redis::zrevrange($key, 0, -1)) { + return Redis::zrevrange($key, 0, -1); + } + $ids = UserFilter::whereFilterType('block') + ->whereUserId($profile_id) + ->pluck('filterable_id') + ->map(function($id) { + $acct = AccountService::get($id, true); + if(!$acct) { + return false; + } + return $acct['id']; + }) + ->filter(function($res) { + return $res; + }) + ->values() + ->toArray(); + foreach ($ids as $blocked_id) { + Redis::zadd($key, (int) $blocked_id, (int) $blocked_id); + } + Cache::set($key . ':cached-v0', 1, 7776000); + return $ids; + } + } - public static function filters(int $profile_id) - { - return array_unique(array_merge(self::mutes($profile_id), self::blocks($profile_id))); - } + public static function filters(int $profile_id) + { + return array_unique(array_merge(self::mutes($profile_id), self::blocks($profile_id))); + } - public static function mute(int $profile_id, int $muted_id) - { - if($profile_id == $muted_id) { - return false; - } - $key = self::USER_MUTES_KEY . $profile_id; - $mutes = self::mutes($profile_id); - $exists = in_array($muted_id, $mutes); - if(!$exists) { - Redis::zadd($key, $muted_id, $muted_id); - } - return true; - } + public static function mute(int $profile_id, int $muted_id) + { + if($profile_id == $muted_id) { + return false; + } + $key = self::USER_MUTES_KEY . $profile_id; + $mutes = self::mutes($profile_id); + $exists = in_array($muted_id, $mutes); + if(!$exists) { + Redis::zadd($key, $muted_id, $muted_id); + } + return true; + } - public static function unmute(int $profile_id, string $muted_id) - { - if($profile_id == $muted_id) { - return false; - } - $key = self::USER_MUTES_KEY . $profile_id; - $mutes = self::mutes($profile_id); - $exists = in_array($muted_id, $mutes); - if($exists) { - Redis::zrem($key, $muted_id); - } - return true; - } + public static function unmute(int $profile_id, string $muted_id) + { + if($profile_id == $muted_id) { + return false; + } + $key = self::USER_MUTES_KEY . $profile_id; + $mutes = self::mutes($profile_id); + $exists = in_array($muted_id, $mutes); + if($exists) { + Redis::zrem($key, $muted_id); + } + return true; + } - public static function block(int $profile_id, int $blocked_id) - { - if($profile_id == $blocked_id) { - return false; - } - $key = self::USER_BLOCKS_KEY . $profile_id; - $exists = in_array($blocked_id, self::blocks($profile_id)); - if(!$exists) { - Redis::zadd($key, $blocked_id, $blocked_id); - } - return true; - } + public static function block(int $profile_id, int $blocked_id) + { + if($profile_id == $blocked_id) { + return false; + } + $key = self::USER_BLOCKS_KEY . $profile_id; + $exists = in_array($blocked_id, self::blocks($profile_id)); + if(!$exists) { + Redis::zadd($key, $blocked_id, $blocked_id); + } + return true; + } - public static function unblock(int $profile_id, string $blocked_id) - { - if($profile_id == $blocked_id) { - return false; - } - $key = self::USER_BLOCKS_KEY . $profile_id; - $exists = in_array($blocked_id, self::blocks($profile_id)); - if($exists) { - Redis::zrem($key, $blocked_id); - } - return $exists; - } + public static function unblock(int $profile_id, string $blocked_id) + { + if($profile_id == $blocked_id) { + return false; + } + $key = self::USER_BLOCKS_KEY . $profile_id; + $exists = in_array($blocked_id, self::blocks($profile_id)); + if($exists) { + Redis::zrem($key, $blocked_id); + } + return $exists; + } - public static function blockCount(int $profile_id) - { - return Redis::zcard(self::USER_BLOCKS_KEY . $profile_id); - } + public static function blockCount(int $profile_id) + { + return Redis::zcard(self::USER_BLOCKS_KEY . $profile_id); + } - public static function muteCount(int $profile_id) - { - return Redis::zcard(self::USER_MUTES_KEY . $profile_id); - } + public static function muteCount(int $profile_id) + { + return Redis::zcard(self::USER_MUTES_KEY . $profile_id); + } + + public static function domainBlocks($pid, $purge = false) + { + if($purge) { + Cache::forget(self::USER_DOMAIN_KEY . $pid); + } + return Cache::remember( + self::USER_DOMAIN_KEY . $pid, + 21600, + function() use($pid) { + return UserDomainBlock::whereProfileId($pid)->pluck('domain')->toArray(); + }); + } } From 6d8121413865c00f6ccc8e46ff2e367bb603bd66 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 00:44:54 -0700 Subject: [PATCH 237/253] Update DomainBlockController, purge domainBlocks cache --- app/Http/Controllers/Api/V1/DomainBlockController.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Http/Controllers/Api/V1/DomainBlockController.php b/app/Http/Controllers/Api/V1/DomainBlockController.php index 53a209ed9..2186c0936 100644 --- a/app/Http/Controllers/Api/V1/DomainBlockController.php +++ b/app/Http/Controllers/Api/V1/DomainBlockController.php @@ -6,6 +6,7 @@ use Illuminate\Http\Request; use App\Http\Controllers\Controller; use App\Models\UserDomainBlock; use App\Util\ActivityPub\Helpers; +use App\Services\UserFilterService; use Illuminate\Bus\Batch; use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\Cache; @@ -90,6 +91,7 @@ class DomainBlockController extends Controller ])->allowFailures()->onQueue('feed')->dispatch(); Cache::forget('profile:following:' . $pid); + UserFilterService::domainBlocks($pid, true); } return $this->json([]); @@ -109,6 +111,8 @@ class DomainBlockController extends Controller $filters = UserDomainBlock::whereProfileId($pid)->whereDomain($domain)->delete(); + UserFilterService::domainBlocks($pid, true); + return $this->json([]); } } From 21947835f8969a6100d275897bc84f337b958994 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 00:46:24 -0700 Subject: [PATCH 238/253] Update ApiV1Controller, use domainBlock filtering on public/network feeds --- app/Http/Controllers/Api/ApiV1Controller.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index e429a8681..be77b5606 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -31,6 +31,7 @@ use App\{ UserSetting, UserFilter, }; +use App\Models\UserDomainBlock; use League\Fractal; use App\Transformer\Api\Mastodon\v1\{ AccountTransformer, @@ -2422,6 +2423,7 @@ class ApiV1Controller extends Controller $local = $request->has('local'); $filtered = $user ? UserFilterService::filters($user->profile_id) : []; AccountService::setLastActive($user->id); + $domainBlocks = UserFilterService::domainBlocks($user->profile_id); if($remote && config('instance.timeline.network.cached')) { Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() { @@ -2496,6 +2498,13 @@ class ApiV1Controller extends Controller ->filter(function($s) use($filtered) { return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false; }) + ->filter(function($s) use($domainBlocks) { + if(!$domainBlocks || !count($domainBlocks)) { + return $s; + } + $domain = strtolower(parse_url($s['url'], PHP_URL_HOST)); + return !in_array($domain, $domainBlocks); + }) ->take($limit) ->values(); From c3f16c87a31eedb50e1070ec0f00c3d8cb6f6882 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 01:05:49 -0700 Subject: [PATCH 239/253] Update SearchApiV2Service, add user domain blocks filtering --- app/Services/SearchApiV2Service.php | 35 ++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/app/Services/SearchApiV2Service.php b/app/Services/SearchApiV2Service.php index 90691f0bd..f926c2c27 100644 --- a/app/Services/SearchApiV2Service.php +++ b/app/Services/SearchApiV2Service.php @@ -95,7 +95,15 @@ class SearchApiV2Service if(substr($webfingerQuery, 0, 1) !== '@') { $webfingerQuery = '@' . $webfingerQuery; } - $banned = InstanceService::getBannedDomains(); + $banned = InstanceService::getBannedDomains() ?? []; + $domainBlocks = UserFilterService::domainBlocks($user->profile_id); + if($domainBlocks && count($domainBlocks)) { + $banned = array_unique( + array_values( + array_merge($banned, $domainBlocks) + ) + ); + } $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like'; $results = Profile::select('username', 'id', 'followers_count', 'domain') ->where('username', $operator, $query) @@ -172,8 +180,18 @@ class SearchApiV2Service 'hashtags' => [], 'statuses' => [], ]; + $user = request()->user(); $mastodonMode = self::$mastodonMode; $query = urldecode($this->query->input('q')); + $banned = InstanceService::getBannedDomains(); + $domainBlocks = UserFilterService::domainBlocks($user->profile_id); + if($domainBlocks && count($domainBlocks)) { + $banned = array_unique( + array_values( + array_merge($banned, $domainBlocks) + ) + ); + } if(substr($query, 0, 1) === '@' && !Str::contains($query, '.')) { $default['accounts'] = $this->accounts(substr($query, 1)); return $default; @@ -197,7 +215,11 @@ class SearchApiV2Service } catch (\Exception $e) { return $default; } - if($res && isset($res['id'])) { + if($res && isset($res['id'], $res['url'])) { + $domain = strtolower(parse_url($res['url'], PHP_URL_HOST)); + if(in_array($domain, $banned)) { + return $default; + } $default['accounts'][] = $res; return $default; } else { @@ -212,6 +234,10 @@ class SearchApiV2Service return $default; } if($res && isset($res['id'])) { + $domain = strtolower(parse_url($res['url'], PHP_URL_HOST)); + if(in_array($domain, $banned)) { + return $default; + } $default['accounts'][] = $res; return $default; } else { @@ -221,6 +247,9 @@ class SearchApiV2Service if($sid = Status::whereUri($query)->first()) { $s = StatusService::get($sid->id, false); + if(!$s) { + return $default; + } if(in_array($s['visibility'], ['public', 'unlisted'])) { $default['statuses'][] = $s; return $default; @@ -229,7 +258,7 @@ class SearchApiV2Service try { $res = ActivityPubFetchService::get($query); - $banned = InstanceService::getBannedDomains(); + if($res) { $json = json_decode($res, true); From fcbcd7ec73bb89bb56156fbc0dc929994444751e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 01:53:49 -0700 Subject: [PATCH 240/253] Update Delete pipelines, delete status hashtags quietly --- app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php | 5 +---- app/Jobs/StatusPipeline/RemoteStatusDelete.php | 5 +---- app/Jobs/StatusPipeline/StatusDelete.php | 5 +---- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php index 353509c6c..824323cda 100644 --- a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php +++ b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php @@ -76,10 +76,7 @@ class DeleteRemoteStatusPipeline implements ShouldQueue }); Mention::whereStatusId($status->id)->forceDelete(); Report::whereObjectType('App\Status')->whereObjectId($status->id)->delete(); - $statusHashtags = StatusHashtag::whereStatusId($status->id)->get(); - foreach($statusHashtags as $stag) { - $stag->delete(); - } + StatusHashtag::whereStatusId($status->id)->deleteQuietly(); StatusView::whereStatusId($status->id)->delete(); Status::whereReblogOfId($status->id)->forceDelete(); $status->forceDelete(); diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index 78c41ed3d..9898d3c82 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -174,10 +174,7 @@ class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing ->whereObjectId($status->id) ->delete(); StatusArchived::whereStatusId($status->id)->delete(); - $statusHashtags = StatusHashtag::whereStatusId($status->id)->get(); - foreach($statusHashtags as $stag) { - $stag->delete(); - } + StatusHashtag::whereStatusId($status->id)->deleteQuietly(); StatusView::whereStatusId($status->id)->delete(); Status::whereInReplyToId($status->id)->update(['in_reply_to_id' => null]); diff --git a/app/Jobs/StatusPipeline/StatusDelete.php b/app/Jobs/StatusPipeline/StatusDelete.php index c0ced1368..a053bfe75 100644 --- a/app/Jobs/StatusPipeline/StatusDelete.php +++ b/app/Jobs/StatusPipeline/StatusDelete.php @@ -151,10 +151,7 @@ class StatusDelete implements ShouldQueue ->delete(); StatusArchived::whereStatusId($status->id)->delete(); - $statusHashtags = StatusHashtag::whereStatusId($status->id)->get(); - foreach($statusHashtags as $stag) { - $stag->delete(); - } + StatusHashtag::whereStatusId($status->id)->deleteQuietly(); StatusView::whereStatusId($status->id)->delete(); Status::whereInReplyToId($status->id)->update(['in_reply_to_id' => null]); From 89b8e87477caedfe4cec4534a370be184b8a9190 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 02:03:15 -0700 Subject: [PATCH 241/253] Update ApiV1Controller, apply user domain blocks filtering to hashtag timelines --- app/Http/Controllers/Api/ApiV1Controller.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index be77b5606..798d9ee55 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -3285,6 +3285,7 @@ class ApiV1Controller extends Controller $limit = $request->input('limit', 20); $onlyMedia = $request->input('only_media', true); $pe = $request->has(self::PF_API_ENTITY_KEY); + $pid = $request->user()->profile_id; if($min || $max) { $minMax = SnowflakeService::byDate(now()->subMonths(6)); @@ -3296,7 +3297,8 @@ class ApiV1Controller extends Controller } } - $filters = UserFilterService::filters($request->user()->profile_id); + $filters = UserFilterService::filters($pid); + $domainBlocks = UserFilterService::domainBlocks($pid); if(!$min && !$max) { $id = 1; @@ -3322,10 +3324,11 @@ class ApiV1Controller extends Controller if($onlyMedia && !isset($i['media_attachments']) || !count($i['media_attachments'])) { return false; } - return $i && isset($i['account']); + return $i && isset($i['account'], $i['url']); }) - ->filter(function($i) use($filters) { - return !in_array($i['account']['id'], $filters); + ->filter(function($i) use($filters, $domainBlocks) { + $domain = strtolower(parse_url($i['url'], PHP_URL_HOST)); + return !in_array($i['account']['id'], $filters) && !in_array($domain, $domainBlocks); }) ->values() ->toArray(); From 5169936062d801936ac688c0f4e220c4794f5e62 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 02:05:26 -0700 Subject: [PATCH 242/253] Update MarkerService, fix php deprecation warning --- app/Services/MarkerService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/MarkerService.php b/app/Services/MarkerService.php index 6b407b567..130f0b017 100644 --- a/app/Services/MarkerService.php +++ b/app/Services/MarkerService.php @@ -13,7 +13,7 @@ class MarkerService return Cache::get(self::CACHE_KEY . $timeline . ':' . $profileId); } - public static function set($profileId, $timeline = 'home', $entityId) + public static function set($profileId, $timeline = 'home', $entityId = false) { $existing = self::get($profileId, $timeline); $key = self::CACHE_KEY . $timeline . ':' . $profileId; From 6c39df7fb38781efd0b7464d96ace0cb0912add1 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 02:08:44 -0700 Subject: [PATCH 243/253] Update Inbox, import AccountService --- app/Util/ActivityPub/Inbox.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index b8bb780c8..e26f0a48c 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -39,6 +39,7 @@ use App\Util\ActivityPub\Validator\Like as LikeValidator; use App\Util\ActivityPub\Validator\UndoFollow as UndoFollowValidator; use App\Util\ActivityPub\Validator\UpdatePersonValidator; +use App\Services\AccountService; use App\Services\PollService; use App\Services\FollowerService; use App\Services\ReblogService; From e98df1196f079ccad4e44897457f035b8efeadcf Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 03:34:31 -0700 Subject: [PATCH 244/253] Add migration --- ...1_103223_purge_deleted_status_hashtags.php | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 database/migrations/2023_12_21_103223_purge_deleted_status_hashtags.php diff --git a/database/migrations/2023_12_21_103223_purge_deleted_status_hashtags.php b/database/migrations/2023_12_21_103223_purge_deleted_status_hashtags.php new file mode 100644 index 000000000..bf2acc34e --- /dev/null +++ b/database/migrations/2023_12_21_103223_purge_deleted_status_hashtags.php @@ -0,0 +1,25 @@ +lazyById(200)->each->deleteQuietly(); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; From 3e28cf661ba683a51256363de6deba4b372aa32f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 03:35:47 -0700 Subject: [PATCH 245/253] Add user domain block commands --- app/Console/Commands/AddUserDomainBlock.php | 99 +++++++++++++++++++ .../Commands/DeleteUserDomainBlock.php | 88 +++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 app/Console/Commands/AddUserDomainBlock.php create mode 100644 app/Console/Commands/DeleteUserDomainBlock.php diff --git a/app/Console/Commands/AddUserDomainBlock.php b/app/Console/Commands/AddUserDomainBlock.php new file mode 100644 index 000000000..33f441cdc --- /dev/null +++ b/app/Console/Commands/AddUserDomainBlock.php @@ -0,0 +1,99 @@ +validateDomain($domain); + $this->processBlocks($domain); + return; + } + + protected function validateDomain($domain) + { + if(!strpos($domain, '.')) { + $this->error('Invalid domain'); + return; + } + + if(str_starts_with($domain, 'https://')) { + $domain = str_replace('https://', '', $domain); + } + + if(str_starts_with($domain, 'http://')) { + $domain = str_replace('http://', '', $domain); + } + + $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE); + if(!$valid) { + $this->error('Invalid domain'); + return; + } + + $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST)); + + if($domain === config('pixelfed.domain.app')) { + $this->error('Invalid domain'); + return; + } + + $confirmed = confirm('Are you sure you want to block ' . $domain . '?'); + if(!$confirmed) { + return; + } + + return $domain; + } + + protected function processBlocks($domain) + { + progress( + label: 'Updating user domain blocks...', + steps: User::lazyById(500), + callback: fn ($user) => $this->performTask($user, $domain), + ); + } + + protected function performTask($user, $domain) + { + if(!$user->profile_id || $user->delete_after) { + return; + } + + if($user->status != null && $user->status != 'disabled') { + return; + } + + UserDomainBlock::updateOrCreate([ + 'profile_id' => $user->profile_id, + 'domain' => $domain + ]); + } +} diff --git a/app/Console/Commands/DeleteUserDomainBlock.php b/app/Console/Commands/DeleteUserDomainBlock.php new file mode 100644 index 000000000..80c139f2b --- /dev/null +++ b/app/Console/Commands/DeleteUserDomainBlock.php @@ -0,0 +1,88 @@ +validateDomain($domain); + $this->processUnblocks($domain); + return; + } + + protected function validateDomain($domain) + { + if(!strpos($domain, '.')) { + $this->error('Invalid domain'); + return; + } + + if(str_starts_with($domain, 'https://')) { + $domain = str_replace('https://', '', $domain); + } + + if(str_starts_with($domain, 'http://')) { + $domain = str_replace('http://', '', $domain); + } + + $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE); + if(!$valid) { + $this->error('Invalid domain'); + return; + } + + $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST)); + + if($domain === config('pixelfed.domain.app')) { + $this->error('Invalid domain'); + return; + } + + $confirmed = confirm('Are you sure you want to unblock ' . $domain . '?'); + if(!$confirmed) { + return; + } + + return $domain; + } + + protected function processUnblocks($domain) + { + progress( + label: 'Updating user domain blocks...', + steps: UserDomainBlock::whereDomain($domain)->lazyById(500), + callback: fn ($domainBlock) => $this->performTask($domainBlock), + ); + } + + protected function performTask($domainBlock) + { + $domainBlock->deleteQuietly(); + } +} From f3f0175c8494d9bf45931e02fa8c6ccb9558968d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 03:47:23 -0700 Subject: [PATCH 246/253] Add DefaultDomainBlock model + migration --- app/Models/DefaultDomainBlock.php | 13 +++++++++ ...103_create_default_domain_blocks_table.php | 29 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 app/Models/DefaultDomainBlock.php create mode 100644 database/migrations/2023_12_21_104103_create_default_domain_blocks_table.php diff --git a/app/Models/DefaultDomainBlock.php b/app/Models/DefaultDomainBlock.php new file mode 100644 index 000000000..d90816a32 --- /dev/null +++ b/app/Models/DefaultDomainBlock.php @@ -0,0 +1,13 @@ +id(); + $table->string('domain')->unique()->index(); + $table->text('note')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('default_domain_blocks'); + } +}; From 519c7a3735c7893cee93486dd5a27a43fee6e3d1 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 03:48:08 -0700 Subject: [PATCH 247/253] Update domain block commands --- app/Console/Commands/AddUserDomainBlock.php | 5 +++++ app/Console/Commands/DeleteUserDomainBlock.php | 3 +++ 2 files changed, 8 insertions(+) diff --git a/app/Console/Commands/AddUserDomainBlock.php b/app/Console/Commands/AddUserDomainBlock.php index 33f441cdc..7128879e1 100644 --- a/app/Console/Commands/AddUserDomainBlock.php +++ b/app/Console/Commands/AddUserDomainBlock.php @@ -4,6 +4,7 @@ namespace App\Console\Commands; use Illuminate\Console\Command; use App\User; +use App\Models\DefaultDomainBlock; use App\Models\UserDomainBlock; use function Laravel\Prompts\text; use function Laravel\Prompts\confirm; @@ -31,6 +32,7 @@ class AddUserDomainBlock extends Command public function handle() { $domain = text('Enter domain you want to block'); + $domain = strtolower($domain); $domain = $this->validateDomain($domain); $this->processBlocks($domain); return; @@ -74,6 +76,9 @@ class AddUserDomainBlock extends Command protected function processBlocks($domain) { + DefaultDomainBlock::updateOrCreate([ + 'domain' => $domain + ]); progress( label: 'Updating user domain blocks...', steps: User::lazyById(500), diff --git a/app/Console/Commands/DeleteUserDomainBlock.php b/app/Console/Commands/DeleteUserDomainBlock.php index 80c139f2b..9cc1c1ded 100644 --- a/app/Console/Commands/DeleteUserDomainBlock.php +++ b/app/Console/Commands/DeleteUserDomainBlock.php @@ -4,6 +4,7 @@ namespace App\Console\Commands; use Illuminate\Console\Command; use App\User; +use App\Models\DefaultDomainBlock; use App\Models\UserDomainBlock; use function Laravel\Prompts\text; use function Laravel\Prompts\confirm; @@ -31,6 +32,7 @@ class DeleteUserDomainBlock extends Command public function handle() { $domain = text('Enter domain you want to unblock'); + $domain = strtolower($domain); $domain = $this->validateDomain($domain); $this->processUnblocks($domain); return; @@ -74,6 +76,7 @@ class DeleteUserDomainBlock extends Command protected function processUnblocks($domain) { + DefaultDomainBlock::whereDomain($domain)->delete(); progress( label: 'Updating user domain blocks...', steps: UserDomainBlock::whereDomain($domain)->lazyById(500), From fa0380ac3be5534ef6f113347fd9feaec8e88d8c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 04:47:09 -0700 Subject: [PATCH 248/253] Update UserObserver, add default domain blocks logic --- app/Observers/UserObserver.php | 207 +++++++++++++++++++++------------ 1 file changed, 131 insertions(+), 76 deletions(-) diff --git a/app/Observers/UserObserver.php b/app/Observers/UserObserver.php index ec4ef9f34..d587bd7e8 100644 --- a/app/Observers/UserObserver.php +++ b/app/Observers/UserObserver.php @@ -7,90 +7,52 @@ use App\Follower; use App\Profile; use App\User; use App\UserSetting; +use App\Services\UserFilterService; +use App\Models\DefaultDomainBlock; +use App\Models\UserDomainBlock; use App\Jobs\FollowPipeline\FollowPipeline; use DB; use App\Services\FollowerService; class UserObserver { - /** - * Listen to the User created event. - * - * @param \App\User $user - * - * @return void - */ - public function saved(User $user) - { - if($user->status == 'deleted') { - return; - } + /** + * Handle the notification "created" event. + * + * @param \App\User $user + * @return void + */ + public function created(User $user): void + { + $this->handleUser($user); + } - if(Profile::whereUsername($user->username)->exists()) { - return; + /** + * Listen to the User saved event. + * + * @param \App\User $user + * + * @return void + */ + public function saved(User $user) + { + $this->handleUser($user); + } + + /** + * Listen to the User updated event. + * + * @param \App\User $user + * + * @return void + */ + public function updated(User $user): void + { + $this->handleUser($user); + if($user->profile) { + $this->applyDefaultDomainBlocks($user); } - - if (empty($user->profile)) { - $profile = DB::transaction(function() use($user) { - $profile = new Profile(); - $profile->user_id = $user->id; - $profile->username = $user->username; - $profile->name = $user->name; - $pkiConfig = [ - 'digest_alg' => 'sha512', - 'private_key_bits' => 2048, - 'private_key_type' => OPENSSL_KEYTYPE_RSA, - ]; - $pki = openssl_pkey_new($pkiConfig); - openssl_pkey_export($pki, $pki_private); - $pki_public = openssl_pkey_get_details($pki); - $pki_public = $pki_public['key']; - - $profile->private_key = $pki_private; - $profile->public_key = $pki_public; - $profile->save(); - return $profile; - }); - - DB::transaction(function() use($user, $profile) { - $user = User::findOrFail($user->id); - $user->profile_id = $profile->id; - $user->save(); - - CreateAvatar::dispatch($profile); - }); - - if(config_cache('account.autofollow') == true) { - $names = config_cache('account.autofollow_usernames'); - $names = explode(',', $names); - - if(!$names || !last($names)) { - return; - } - - $profiles = Profile::whereIn('username', $names)->get(); - - if($profiles) { - foreach($profiles as $p) { - $follower = new Follower; - $follower->profile_id = $profile->id; - $follower->following_id = $p->id; - $follower->save(); - - FollowPipeline::dispatch($follower); - } - } - } - } - - if (empty($user->settings)) { - DB::transaction(function() use($user) { - UserSetting::firstOrCreate([ - 'user_id' => $user->id - ]); - }); - } - } + } /** * Handle the user "deleted" event. @@ -102,4 +64,97 @@ class UserObserver { FollowerService::delCache($user->profile_id); } + + protected function handleUser($user) + { + if(in_array($user->status, ['deleted', 'delete'])) { + return; + } + + if(Profile::whereUsername($user->username)->exists()) { + return; + } + + if (empty($user->profile)) { + $profile = DB::transaction(function() use($user) { + $profile = new Profile(); + $profile->user_id = $user->id; + $profile->username = $user->username; + $profile->name = $user->name; + $pkiConfig = [ + 'digest_alg' => 'sha512', + 'private_key_bits' => 2048, + 'private_key_type' => OPENSSL_KEYTYPE_RSA, + ]; + $pki = openssl_pkey_new($pkiConfig); + openssl_pkey_export($pki, $pki_private); + $pki_public = openssl_pkey_get_details($pki); + $pki_public = $pki_public['key']; + + $profile->private_key = $pki_private; + $profile->public_key = $pki_public; + $profile->save(); + $this->applyDefaultDomainBlocks($user); + return $profile; + }); + + + DB::transaction(function() use($user, $profile) { + $user = User::findOrFail($user->id); + $user->profile_id = $profile->id; + $user->save(); + + CreateAvatar::dispatch($profile); + }); + + if(config_cache('account.autofollow') == true) { + $names = config_cache('account.autofollow_usernames'); + $names = explode(',', $names); + + if(!$names || !last($names)) { + return; + } + + $profiles = Profile::whereIn('username', $names)->get(); + + if($profiles) { + foreach($profiles as $p) { + $follower = new Follower; + $follower->profile_id = $profile->id; + $follower->following_id = $p->id; + $follower->save(); + + FollowPipeline::dispatch($follower); + } + } + } + } + + if (empty($user->settings)) { + DB::transaction(function() use($user) { + UserSetting::firstOrCreate([ + 'user_id' => $user->id + ]); + }); + } + } + + protected function applyDefaultDomainBlocks($user) + { + if($user->profile_id == null) { + return; + } + $defaultDomainBlocks = DefaultDomainBlock::pluck('domain')->toArray(); + + if(!$defaultDomainBlocks || !count($defaultDomainBlocks)) { + return; + } + + foreach($defaultDomainBlocks as $domain) { + UserDomainBlock::updateOrCreate([ + 'profile_id' => $user->profile_id, + 'domain' => strtolower(trim($domain)) + ]); + } + } } From d8f46f47a1106b5c5d78efaa4a67abb9696b965f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 04:48:56 -0700 Subject: [PATCH 249/253] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f75a72a2..8ab35fec3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6)) - Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7)) - Added Mutual Followers API endpoint ([33dbbe46](https://github.com/pixelfed/pixelfed/commit/33dbbe46)) +- Added User Domain Blocks ([#4834](https://github.com/pixelfed/pixelfed/pull/4834)) ([fa0380ac](https://github.com/pixelfed/pixelfed/commit/fa0380ac)) ### Federation - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) From 73a0f528ab6cc7a5e272ec2dff2c1df37e1f39d3 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 05:00:35 -0700 Subject: [PATCH 250/253] Update user domain block commands --- app/Console/Commands/AddUserDomainBlock.php | 10 ++++++---- app/Console/Commands/DeleteUserDomainBlock.php | 15 ++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/app/Console/Commands/AddUserDomainBlock.php b/app/Console/Commands/AddUserDomainBlock.php index 7128879e1..6d5c192bf 100644 --- a/app/Console/Commands/AddUserDomainBlock.php +++ b/app/Console/Commands/AddUserDomainBlock.php @@ -34,6 +34,10 @@ class AddUserDomainBlock extends Command $domain = text('Enter domain you want to block'); $domain = strtolower($domain); $domain = $this->validateDomain($domain); + if(!$domain || empty($domain)) { + $this->error('Invalid domain'); + return; + } $this->processBlocks($domain); return; } @@ -41,7 +45,6 @@ class AddUserDomainBlock extends Command protected function validateDomain($domain) { if(!strpos($domain, '.')) { - $this->error('Invalid domain'); return; } @@ -53,14 +56,13 @@ class AddUserDomainBlock extends Command $domain = str_replace('http://', '', $domain); } + $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST)); + $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE); if(!$valid) { - $this->error('Invalid domain'); return; } - $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST)); - if($domain === config('pixelfed.domain.app')) { $this->error('Invalid domain'); return; diff --git a/app/Console/Commands/DeleteUserDomainBlock.php b/app/Console/Commands/DeleteUserDomainBlock.php index 9cc1c1ded..405b6fe76 100644 --- a/app/Console/Commands/DeleteUserDomainBlock.php +++ b/app/Console/Commands/DeleteUserDomainBlock.php @@ -34,6 +34,10 @@ class DeleteUserDomainBlock extends Command $domain = text('Enter domain you want to unblock'); $domain = strtolower($domain); $domain = $this->validateDomain($domain); + if(!$domain || empty($domain)) { + $this->error('Invalid domain'); + return; + } $this->processUnblocks($domain); return; } @@ -41,7 +45,6 @@ class DeleteUserDomainBlock extends Command protected function validateDomain($domain) { if(!strpos($domain, '.')) { - $this->error('Invalid domain'); return; } @@ -53,16 +56,14 @@ class DeleteUserDomainBlock extends Command $domain = str_replace('http://', '', $domain); } + $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST)); + $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME|FILTER_NULL_ON_FAILURE); if(!$valid) { - $this->error('Invalid domain'); return; } - $domain = strtolower(parse_url('https://' . $domain, PHP_URL_HOST)); - if($domain === config('pixelfed.domain.app')) { - $this->error('Invalid domain'); return; } @@ -77,6 +78,10 @@ class DeleteUserDomainBlock extends Command protected function processUnblocks($domain) { DefaultDomainBlock::whereDomain($domain)->delete(); + if(!UserDomainBlock::whereDomain($domain)->count()) { + $this->info('No results found!'); + return; + } progress( label: 'Updating user domain blocks...', steps: UserDomainBlock::whereDomain($domain)->lazyById(500), From 1be21c76f38940509f2282c65db6288ba7dcf989 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 21 Dec 2023 06:18:51 -0700 Subject: [PATCH 251/253] Fix StatusHashtag delete bug --- app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php | 2 +- app/Jobs/StatusPipeline/RemoteStatusDelete.php | 2 +- app/Jobs/StatusPipeline/StatusDelete.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php index 824323cda..4969fca2f 100644 --- a/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php +++ b/app/Jobs/DeletePipeline/DeleteRemoteStatusPipeline.php @@ -76,7 +76,7 @@ class DeleteRemoteStatusPipeline implements ShouldQueue }); Mention::whereStatusId($status->id)->forceDelete(); Report::whereObjectType('App\Status')->whereObjectId($status->id)->delete(); - StatusHashtag::whereStatusId($status->id)->deleteQuietly(); + StatusHashtag::whereStatusId($status->id)->delete(); StatusView::whereStatusId($status->id)->delete(); Status::whereReblogOfId($status->id)->forceDelete(); $status->forceDelete(); diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index 9898d3c82..07a2f6236 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -174,7 +174,7 @@ class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing ->whereObjectId($status->id) ->delete(); StatusArchived::whereStatusId($status->id)->delete(); - StatusHashtag::whereStatusId($status->id)->deleteQuietly(); + StatusHashtag::whereStatusId($status->id)->delete(); StatusView::whereStatusId($status->id)->delete(); Status::whereInReplyToId($status->id)->update(['in_reply_to_id' => null]); diff --git a/app/Jobs/StatusPipeline/StatusDelete.php b/app/Jobs/StatusPipeline/StatusDelete.php index a053bfe75..dbbfad5ac 100644 --- a/app/Jobs/StatusPipeline/StatusDelete.php +++ b/app/Jobs/StatusPipeline/StatusDelete.php @@ -151,7 +151,7 @@ class StatusDelete implements ShouldQueue ->delete(); StatusArchived::whereStatusId($status->id)->delete(); - StatusHashtag::whereStatusId($status->id)->deleteQuietly(); + StatusHashtag::whereStatusId($status->id)->delete(); StatusView::whereStatusId($status->id)->delete(); Status::whereInReplyToId($status->id)->update(['in_reply_to_id' => null]); From adfaa2b1404e5a66cde66a563951fc24577e01f4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Dec 2023 00:30:05 -0700 Subject: [PATCH 252/253] Update AP ProfileTransformer, add published attribute --- app/Transformer/ActivityPub/ProfileTransformer.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Transformer/ActivityPub/ProfileTransformer.php b/app/Transformer/ActivityPub/ProfileTransformer.php index cdd4eb82d..45d22cd11 100644 --- a/app/Transformer/ActivityPub/ProfileTransformer.php +++ b/app/Transformer/ActivityPub/ProfileTransformer.php @@ -40,6 +40,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'url' => $profile->url(), 'manuallyApprovesFollowers' => (bool) $profile->is_private, 'indexable' => (bool) $profile->indexable, + 'published' => $profile->created_at->format('Y-m-d') . 'T00:00:00Z', 'publicKey' => [ 'id' => $profile->permalink().'#main-key', 'owner' => $profile->permalink(), From f66b9fe74e1bfc7842ab9d0fac21c05847bbf188 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Dec 2023 00:31:05 -0700 Subject: [PATCH 253/253] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab35fec3..10b13f261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ - Update ApiV1Controller, set last_active_at ([b6419545](https://github.com/pixelfed/pixelfed/commit/b6419545)) - Update AdminShadowFilter, fix deleted profile bug ([a492a95a](https://github.com/pixelfed/pixelfed/commit/a492a95a)) - Update FollowerService, add $silent param to remove method to more efficently purge relationships ([1664a5bc](https://github.com/pixelfed/pixelfed/commit/1664a5bc)) +- Update AP ProfileTransformer, add published attribute ([adfaa2b1](https://github.com/pixelfed/pixelfed/commit/adfaa2b1)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)