kopia lustrzana https://github.com/pixelfed/pixelfed
daniel
GitHub
commit
4050055e5e
|
|
@ -15,6 +15,7 @@
|
|||
- Added User Domain Blocks ([#4834](https://github.com/pixelfed/pixelfed/pull/4834)) ([fa0380ac](https://github.com/pixelfed/pixelfed/commit/fa0380ac))
|
||||
- Added Parental Controls ([#4862](https://github.com/pixelfed/pixelfed/pull/4862)) ([c91f1c59](https://github.com/pixelfed/pixelfed/commit/c91f1c59))
|
||||
- Added Forgot Email Feature ([67c650b1](https://github.com/pixelfed/pixelfed/commit/67c650b1))
|
||||
- Added S3 IG Import Media Storage support ([#4891](https://github.com/pixelfed/pixelfed/pull/4891)) ([081360b9](https://github.com/pixelfed/pixelfed/commit/081360b9))
|
||||
|
||||
### Federation
|
||||
- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use App\Models\ImportPost;
|
||||
use App\Jobs\ImportPipeline\ImportMediaToCloudPipeline;
|
||||
use function Laravel\Prompts\progress;
|
||||
|
||||
class ImportUploadMediaToCloudStorage extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'app:import-upload-media-to-cloud-storage {--limit=500}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Migrate media imported from Instagram to S3 cloud storage.';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
if(
|
||||
(bool) config('import.instagram.storage.cloud.enabled') === false ||
|
||||
(bool) config_cache('pixelfed.cloud_storage') === false
|
||||
) {
|
||||
$this->error('Aborted. Cloud storage is not enabled for IG imports.');
|
||||
return;
|
||||
}
|
||||
|
||||
$limit = $this->option('limit');
|
||||
|
||||
$progress = progress(label: 'Migrating import media', steps: $limit);
|
||||
|
||||
$progress->start();
|
||||
|
||||
$posts = ImportPost::whereUploadedToS3(false)->take($limit)->get();
|
||||
|
||||
foreach($posts as $post) {
|
||||
ImportMediaToCloudPipeline::dispatch($post)->onQueue('low');
|
||||
$progress->advance();
|
||||
}
|
||||
|
||||
$progress->finish();
|
||||
}
|
||||
}
|
||||
|
|
@ -42,6 +42,10 @@ class Kernel extends ConsoleKernel
|
|||
$schedule->command('app:import-upload-garbage-collection')->hourlyAt(51);
|
||||
$schedule->command('app:import-remove-deleted-accounts')->hourlyAt(37);
|
||||
$schedule->command('app:import-upload-clean-storage')->twiceDailyAt(1, 13, 32);
|
||||
|
||||
if(config('import.instagram.storage.cloud.enabled') && (bool) config_cache('pixelfed.cloud_storage')) {
|
||||
$schedule->command('app:import-upload-media-to-cloud-storage')->hourlyAt(39);
|
||||
}
|
||||
}
|
||||
$schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21');
|
||||
$schedule->command('app:hashtag-cached-count-update')->hourlyAt(25);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
<?php
|
||||
|
||||
namespace App\Jobs\ImportPipeline;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
|
||||
use App\Models\ImportPost;
|
||||
use App\Media;
|
||||
use App\Services\MediaStorageService;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use App\Jobs\VideoPipeline\VideoThumbnailToCloudPipeline;
|
||||
|
||||
class ImportMediaToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $importPost;
|
||||
|
||||
public $timeout = 900;
|
||||
public $tries = 3;
|
||||
public $maxExceptions = 1;
|
||||
public $failOnTimeout = true;
|
||||
|
||||
/**
|
||||
* The number of seconds after which the job's unique lock will be released.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $uniqueFor = 3600;
|
||||
|
||||
/**
|
||||
* Get the unique ID for the job.
|
||||
*/
|
||||
public function uniqueId(): string
|
||||
{
|
||||
return 'import-media-to-cloud-pipeline:ip-id:' . $this->importPost->id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the middleware the job should pass through.
|
||||
*
|
||||
* @return array<int, object>
|
||||
*/
|
||||
public function middleware(): array
|
||||
{
|
||||
return [(new WithoutOverlapping("import-media-to-cloud-pipeline:ip-id:{$this->importPost->id}"))->shared()->dontRelease()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete the job if its models no longer exist.
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
public $deleteWhenMissingModels = true;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(ImportPost $importPost)
|
||||
{
|
||||
$this->importPost = $importPost;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
$ip = $this->importPost;
|
||||
|
||||
if(
|
||||
$ip->status_id === null ||
|
||||
$ip->uploaded_to_s3 === true ||
|
||||
(bool) config_cache('pixelfed.cloud_storage') === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
$media = Media::whereStatusId($ip->status_id)->get();
|
||||
|
||||
if(!$media || !$media->count()) {
|
||||
$importPost = ImportPost::find($ip->id);
|
||||
$importPost->uploaded_to_s3 = true;
|
||||
$importPost->save();
|
||||
return;
|
||||
}
|
||||
|
||||
foreach($media as $mediaPart) {
|
||||
$this->handleMedia($mediaPart);
|
||||
}
|
||||
}
|
||||
|
||||
protected function handleMedia($media)
|
||||
{
|
||||
$ip = $this->importPost;
|
||||
|
||||
$importPost = ImportPost::find($ip->id);
|
||||
|
||||
if(!$importPost) {
|
||||
return;
|
||||
}
|
||||
|
||||
$res = MediaStorageService::move($media);
|
||||
|
||||
$importPost->uploaded_to_s3 = true;
|
||||
$importPost->save();
|
||||
|
||||
if(!$res) {
|
||||
return;
|
||||
}
|
||||
|
||||
if($res === 'invalid file') {
|
||||
return;
|
||||
}
|
||||
|
||||
if($res === 'success') {
|
||||
if($media->mime === 'video/mp4') {
|
||||
VideoThumbnailToCloudPipeline::dispatch($media)->onQueue('low');
|
||||
} else {
|
||||
Storage::disk('local')->delete($media->media_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
<?php
|
||||
|
||||
namespace App\Jobs\VideoPipeline;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
|
||||
use Illuminate\Http\File;
|
||||
use Cache;
|
||||
use FFMpeg;
|
||||
use Storage;
|
||||
use App\Media;
|
||||
use App\Jobs\MediaPipeline\MediaStoragePipeline;
|
||||
use App\Util\Media\Blurhash;
|
||||
use App\Services\MediaService;
|
||||
use App\Services\StatusService;
|
||||
use App\Services\ResilientMediaStorageService;
|
||||
|
||||
class VideoThumbnailToCloudPipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $media;
|
||||
|
||||
public $timeout = 900;
|
||||
public $tries = 3;
|
||||
public $maxExceptions = 1;
|
||||
public $failOnTimeout = true;
|
||||
public $deleteWhenMissingModels = true;
|
||||
|
||||
/**
|
||||
* The number of seconds after which the job's unique lock will be released.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $uniqueFor = 3600;
|
||||
|
||||
/**
|
||||
* Get the unique ID for the job.
|
||||
*/
|
||||
public function uniqueId(): string
|
||||
{
|
||||
return 'media:video-thumb-to-cloud:id-' . $this->media->id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the middleware the job should pass through.
|
||||
*
|
||||
* @return array<int, object>
|
||||
*/
|
||||
public function middleware(): array
|
||||
{
|
||||
return [(new WithoutOverlapping("media:video-thumb-to-cloud:id-{$this->media->id}"))->shared()->dontRelease()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(Media $media)
|
||||
{
|
||||
$this->media = $media;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
if((bool) config_cache('pixelfed.cloud_storage') === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
$media = $this->media;
|
||||
|
||||
if($media->mime != 'video/mp4') {
|
||||
return;
|
||||
}
|
||||
|
||||
if($media->profile_id === null || $media->status_id === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if($media->thumbnail_url) {
|
||||
return;
|
||||
}
|
||||
|
||||
$base = $media->media_path;
|
||||
$path = explode('/', $base);
|
||||
$name = last($path);
|
||||
|
||||
try {
|
||||
$t = explode('.', $name);
|
||||
$t = $t[0].'_thumb.jpeg';
|
||||
$i = count($path) - 1;
|
||||
$path[$i] = $t;
|
||||
$save = implode('/', $path);
|
||||
$video = FFMpeg::open($base)
|
||||
->getFrameFromSeconds(1)
|
||||
->export()
|
||||
->toDisk('local')
|
||||
->save($save);
|
||||
|
||||
if(!$save) {
|
||||
return;
|
||||
}
|
||||
|
||||
$media->thumbnail_path = $save;
|
||||
$p = explode('/', $media->media_path);
|
||||
array_pop($p);
|
||||
$pt = explode('/', $save);
|
||||
$thumbname = array_pop($pt);
|
||||
$storagePath = implode('/', $p);
|
||||
$thumb = storage_path('app/' . $save);
|
||||
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
|
||||
$media->thumbnail_url = $thumbUrl;
|
||||
$media->save();
|
||||
|
||||
$blurhash = Blurhash::generate($media);
|
||||
if($blurhash) {
|
||||
$media->blurhash = $blurhash;
|
||||
$media->save();
|
||||
}
|
||||
|
||||
if(str_starts_with($save, 'public/m/_v2/') && str_ends_with($save, '.jpeg')) {
|
||||
Storage::delete($save);
|
||||
}
|
||||
|
||||
if(str_starts_with($media->media_path, 'public/m/_v2/') && str_ends_with($media->media_path, '.mp4')) {
|
||||
Storage::disk('local')->delete($media->media_path);
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
}
|
||||
|
||||
if($media->status_id) {
|
||||
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
|
||||
MediaService::del($media->status_id);
|
||||
Cache::forget('status:thumb:nsfw0' . $media->status_id);
|
||||
Cache::forget('status:thumb:nsfw1' . $media->status_id);
|
||||
Cache::forget('pf:services:sh:id:' . $media->status_id);
|
||||
StatusService::del($media->status_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -21,28 +21,40 @@ use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
|
|||
|
||||
class MediaStorageService {
|
||||
|
||||
public static function store(Media $media)
|
||||
{
|
||||
if(config_cache('pixelfed.cloud_storage') == true) {
|
||||
(new self())->cloudStore($media);
|
||||
}
|
||||
public static function store(Media $media)
|
||||
{
|
||||
if(config_cache('pixelfed.cloud_storage') == true) {
|
||||
(new self())->cloudStore($media);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
public static function avatar($avatar, $local = false, $skipRecentCheck = false)
|
||||
{
|
||||
return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
|
||||
}
|
||||
public static function move(Media $media)
|
||||
{
|
||||
if($media->remote_media) {
|
||||
return;
|
||||
}
|
||||
|
||||
public static function head($url)
|
||||
{
|
||||
$c = new Client();
|
||||
try {
|
||||
$r = $c->request('HEAD', $url);
|
||||
} catch (RequestException $e) {
|
||||
return false;
|
||||
}
|
||||
if(config_cache('pixelfed.cloud_storage') == true) {
|
||||
return (new self())->cloudMove($media);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
public static function avatar($avatar, $local = false, $skipRecentCheck = false)
|
||||
{
|
||||
return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
|
||||
}
|
||||
|
||||
public static function head($url)
|
||||
{
|
||||
$c = new Client();
|
||||
try {
|
||||
$r = $c->request('HEAD', $url);
|
||||
} catch (RequestException $e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$h = Arr::mapWithKeys($r->getHeaders(), function($item, $key) {
|
||||
return [strtolower($key) => last($item)];
|
||||
|
|
@ -55,224 +67,261 @@ class MediaStorageService {
|
|||
$len = (int) $h['content-length'];
|
||||
$mime = $h['content-type'];
|
||||
|
||||
if($len < 10 || $len > ((config_cache('pixelfed.max_photo_size') * 1000))) {
|
||||
return false;
|
||||
}
|
||||
if($len < 10 || $len > ((config_cache('pixelfed.max_photo_size') * 1000))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return [
|
||||
'length' => $len,
|
||||
'mime' => $mime
|
||||
];
|
||||
}
|
||||
return [
|
||||
'length' => $len,
|
||||
'mime' => $mime
|
||||
];
|
||||
}
|
||||
|
||||
protected function cloudStore($media)
|
||||
{
|
||||
if($media->remote_media == true) {
|
||||
if(config('media.storage.remote.cloud')) {
|
||||
(new self())->remoteToCloud($media);
|
||||
}
|
||||
} else {
|
||||
(new self())->localToCloud($media);
|
||||
}
|
||||
}
|
||||
protected function cloudStore($media)
|
||||
{
|
||||
if($media->remote_media == true) {
|
||||
if(config('media.storage.remote.cloud')) {
|
||||
(new self())->remoteToCloud($media);
|
||||
}
|
||||
} else {
|
||||
(new self())->localToCloud($media);
|
||||
}
|
||||
}
|
||||
|
||||
protected function localToCloud($media)
|
||||
{
|
||||
$path = storage_path('app/'.$media->media_path);
|
||||
$thumb = storage_path('app/'.$media->thumbnail_path);
|
||||
protected function localToCloud($media)
|
||||
{
|
||||
$path = storage_path('app/'.$media->media_path);
|
||||
$thumb = storage_path('app/'.$media->thumbnail_path);
|
||||
|
||||
$p = explode('/', $media->media_path);
|
||||
$name = array_pop($p);
|
||||
$pt = explode('/', $media->thumbnail_path);
|
||||
$thumbname = array_pop($pt);
|
||||
$storagePath = implode('/', $p);
|
||||
$p = explode('/', $media->media_path);
|
||||
$name = array_pop($p);
|
||||
$pt = explode('/', $media->thumbnail_path);
|
||||
$thumbname = array_pop($pt);
|
||||
$storagePath = implode('/', $p);
|
||||
|
||||
$url = ResilientMediaStorageService::store($storagePath, $path, $name);
|
||||
if($thumb) {
|
||||
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
|
||||
$media->thumbnail_url = $thumbUrl;
|
||||
}
|
||||
$media->cdn_url = $url;
|
||||
$media->optimized_url = $url;
|
||||
$media->replicated_at = now();
|
||||
$media->save();
|
||||
if($media->status_id) {
|
||||
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
}
|
||||
$url = ResilientMediaStorageService::store($storagePath, $path, $name);
|
||||
if($thumb) {
|
||||
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
|
||||
$media->thumbnail_url = $thumbUrl;
|
||||
}
|
||||
$media->cdn_url = $url;
|
||||
$media->optimized_url = $url;
|
||||
$media->replicated_at = now();
|
||||
$media->save();
|
||||
if($media->status_id) {
|
||||
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
}
|
||||
|
||||
protected function remoteToCloud($media)
|
||||
{
|
||||
$url = $media->remote_url;
|
||||
protected function remoteToCloud($media)
|
||||
{
|
||||
$url = $media->remote_url;
|
||||
|
||||
if(!Helpers::validateUrl($url)) {
|
||||
return;
|
||||
}
|
||||
if(!Helpers::validateUrl($url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$head = $this->head($media->remote_url);
|
||||
$head = $this->head($media->remote_url);
|
||||
|
||||
if(!$head) {
|
||||
return;
|
||||
}
|
||||
if(!$head) {
|
||||
return;
|
||||
}
|
||||
|
||||
$mimes = [
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'video/mp4'
|
||||
];
|
||||
$mimes = [
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'video/mp4'
|
||||
];
|
||||
|
||||
$mime = $head['mime'];
|
||||
$max_size = (int) config_cache('pixelfed.max_photo_size') * 1000;
|
||||
$media->size = $head['length'];
|
||||
$media->remote_media = true;
|
||||
$media->save();
|
||||
$mime = $head['mime'];
|
||||
$max_size = (int) config_cache('pixelfed.max_photo_size') * 1000;
|
||||
$media->size = $head['length'];
|
||||
$media->remote_media = true;
|
||||
$media->save();
|
||||
|
||||
if(!in_array($mime, $mimes)) {
|
||||
return;
|
||||
}
|
||||
if(!in_array($mime, $mimes)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if($head['length'] >= $max_size) {
|
||||
return;
|
||||
}
|
||||
if($head['length'] >= $max_size) {
|
||||
return;
|
||||
}
|
||||
|
||||
switch ($mime) {
|
||||
case 'image/png':
|
||||
$ext = '.png';
|
||||
break;
|
||||
switch ($mime) {
|
||||
case 'image/png':
|
||||
$ext = '.png';
|
||||
break;
|
||||
|
||||
case 'image/gif':
|
||||
$ext = '.gif';
|
||||
break;
|
||||
case 'image/gif':
|
||||
$ext = '.gif';
|
||||
break;
|
||||
|
||||
case 'image/jpeg':
|
||||
$ext = '.jpg';
|
||||
break;
|
||||
case 'image/jpeg':
|
||||
$ext = '.jpg';
|
||||
break;
|
||||
|
||||
case 'video/mp4':
|
||||
$ext = '.mp4';
|
||||
break;
|
||||
}
|
||||
case 'video/mp4':
|
||||
$ext = '.mp4';
|
||||
break;
|
||||
}
|
||||
|
||||
$base = MediaPathService::get($media->profile);
|
||||
$path = Str::random(40) . $ext;
|
||||
$tmpBase = storage_path('app/remcache/');
|
||||
$tmpPath = $media->profile_id . '-' . $path;
|
||||
$tmpName = $tmpBase . $tmpPath;
|
||||
$data = file_get_contents($url, false, null, 0, $head['length']);
|
||||
file_put_contents($tmpName, $data);
|
||||
$hash = hash_file('sha256', $tmpName);
|
||||
$base = MediaPathService::get($media->profile);
|
||||
$path = Str::random(40) . $ext;
|
||||
$tmpBase = storage_path('app/remcache/');
|
||||
$tmpPath = $media->profile_id . '-' . $path;
|
||||
$tmpName = $tmpBase . $tmpPath;
|
||||
$data = file_get_contents($url, false, null, 0, $head['length']);
|
||||
file_put_contents($tmpName, $data);
|
||||
$hash = hash_file('sha256', $tmpName);
|
||||
|
||||
$disk = Storage::disk(config('filesystems.cloud'));
|
||||
$file = $disk->putFileAs($base, new File($tmpName), $path, 'public');
|
||||
$permalink = $disk->url($file);
|
||||
$disk = Storage::disk(config('filesystems.cloud'));
|
||||
$file = $disk->putFileAs($base, new File($tmpName), $path, 'public');
|
||||
$permalink = $disk->url($file);
|
||||
|
||||
$media->media_path = $file;
|
||||
$media->cdn_url = $permalink;
|
||||
$media->original_sha256 = $hash;
|
||||
$media->replicated_at = now();
|
||||
$media->save();
|
||||
$media->media_path = $file;
|
||||
$media->cdn_url = $permalink;
|
||||
$media->original_sha256 = $hash;
|
||||
$media->replicated_at = now();
|
||||
$media->save();
|
||||
|
||||
if($media->status_id) {
|
||||
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
|
||||
}
|
||||
if($media->status_id) {
|
||||
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
|
||||
}
|
||||
|
||||
unlink($tmpName);
|
||||
}
|
||||
unlink($tmpName);
|
||||
}
|
||||
|
||||
protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
|
||||
{
|
||||
$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
|
||||
$url = $avatar->remote_url;
|
||||
$driver = $local ? 'local' : config('filesystems.cloud');
|
||||
protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
|
||||
{
|
||||
$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
|
||||
$url = $avatar->remote_url;
|
||||
$driver = $local ? 'local' : config('filesystems.cloud');
|
||||
|
||||
if(empty($url) || Helpers::validateUrl($url) == false) {
|
||||
return;
|
||||
}
|
||||
if(empty($url) || Helpers::validateUrl($url) == false) {
|
||||
return;
|
||||
}
|
||||
|
||||
$head = $this->head($url);
|
||||
$head = $this->head($url);
|
||||
|
||||
if($head == false) {
|
||||
return;
|
||||
}
|
||||
if($head == false) {
|
||||
return;
|
||||
}
|
||||
|
||||
$mimes = [
|
||||
'application/octet-stream',
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
];
|
||||
$mimes = [
|
||||
'application/octet-stream',
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
];
|
||||
|
||||
$mime = $head['mime'];
|
||||
$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
|
||||
$mime = $head['mime'];
|
||||
$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
|
||||
|
||||
if(!$skipRecentCheck) {
|
||||
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if(!$skipRecentCheck) {
|
||||
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Cache::forget('avatar:' . $avatar->profile_id);
|
||||
AccountService::del($avatar->profile_id);
|
||||
Cache::forget('avatar:' . $avatar->profile_id);
|
||||
AccountService::del($avatar->profile_id);
|
||||
|
||||
// handle pleroma edge case
|
||||
if(Str::endsWith($mime, '; charset=utf-8')) {
|
||||
$mime = str_replace('; charset=utf-8', '', $mime);
|
||||
}
|
||||
// handle pleroma edge case
|
||||
if(Str::endsWith($mime, '; charset=utf-8')) {
|
||||
$mime = str_replace('; charset=utf-8', '', $mime);
|
||||
}
|
||||
|
||||
if(!in_array($mime, $mimes)) {
|
||||
return;
|
||||
}
|
||||
if(!in_array($mime, $mimes)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if($head['length'] >= $max_size) {
|
||||
return;
|
||||
}
|
||||
if($head['length'] >= $max_size) {
|
||||
return;
|
||||
}
|
||||
|
||||
$base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id;
|
||||
$ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png';
|
||||
$path = 'avatar_' . strtolower(Str::random(random_int(3,6))) . '.' . $ext;
|
||||
$tmpBase = storage_path('app/remcache/');
|
||||
$tmpPath = 'avatar_' . $avatar->profile_id . '-' . $path;
|
||||
$tmpName = $tmpBase . $tmpPath;
|
||||
$data = @file_get_contents($url, false, null, 0, $head['length']);
|
||||
if(!$data) {
|
||||
return;
|
||||
}
|
||||
file_put_contents($tmpName, $data);
|
||||
$base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id;
|
||||
$ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png';
|
||||
$path = 'avatar_' . strtolower(Str::random(random_int(3,6))) . '.' . $ext;
|
||||
$tmpBase = storage_path('app/remcache/');
|
||||
$tmpPath = 'avatar_' . $avatar->profile_id . '-' . $path;
|
||||
$tmpName = $tmpBase . $tmpPath;
|
||||
$data = @file_get_contents($url, false, null, 0, $head['length']);
|
||||
if(!$data) {
|
||||
return;
|
||||
}
|
||||
file_put_contents($tmpName, $data);
|
||||
|
||||
$mimeCheck = Storage::mimeType('remcache/' . $tmpPath);
|
||||
$mimeCheck = Storage::mimeType('remcache/' . $tmpPath);
|
||||
|
||||
if(!$mimeCheck || !in_array($mimeCheck, ['image/png', 'image/jpeg'])) {
|
||||
$avatar->last_fetched_at = now();
|
||||
$avatar->save();
|
||||
unlink($tmpName);
|
||||
return;
|
||||
}
|
||||
if(!$mimeCheck || !in_array($mimeCheck, ['image/png', 'image/jpeg'])) {
|
||||
$avatar->last_fetched_at = now();
|
||||
$avatar->save();
|
||||
unlink($tmpName);
|
||||
return;
|
||||
}
|
||||
|
||||
$disk = Storage::disk($driver);
|
||||
$file = $disk->putFileAs($base, new File($tmpName), $path, 'public');
|
||||
$permalink = $disk->url($file);
|
||||
$disk = Storage::disk($driver);
|
||||
$file = $disk->putFileAs($base, new File($tmpName), $path, 'public');
|
||||
$permalink = $disk->url($file);
|
||||
|
||||
$avatar->media_path = $base . '/' . $path;
|
||||
$avatar->is_remote = true;
|
||||
$avatar->cdn_url = $local ? config('app.url') . $permalink : $permalink;
|
||||
$avatar->size = $head['length'];
|
||||
$avatar->change_count = $avatar->change_count + 1;
|
||||
$avatar->last_fetched_at = now();
|
||||
$avatar->save();
|
||||
$avatar->media_path = $base . '/' . $path;
|
||||
$avatar->is_remote = true;
|
||||
$avatar->cdn_url = $local ? config('app.url') . $permalink : $permalink;
|
||||
$avatar->size = $head['length'];
|
||||
$avatar->change_count = $avatar->change_count + 1;
|
||||
$avatar->last_fetched_at = now();
|
||||
$avatar->save();
|
||||
|
||||
Cache::forget('avatar:' . $avatar->profile_id);
|
||||
AccountService::del($avatar->profile_id);
|
||||
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
|
||||
Cache::forget('avatar:' . $avatar->profile_id);
|
||||
AccountService::del($avatar->profile_id);
|
||||
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
|
||||
|
||||
unlink($tmpName);
|
||||
}
|
||||
unlink($tmpName);
|
||||
}
|
||||
|
||||
public static function delete(Media $media, $confirm = false)
|
||||
{
|
||||
if(!$confirm) {
|
||||
return;
|
||||
}
|
||||
MediaDeletePipeline::dispatch($media)->onQueue('mmo');
|
||||
}
|
||||
public static function delete(Media $media, $confirm = false)
|
||||
{
|
||||
if(!$confirm) {
|
||||
return;
|
||||
}
|
||||
MediaDeletePipeline::dispatch($media)->onQueue('mmo');
|
||||
}
|
||||
|
||||
protected function cloudMove($media)
|
||||
{
|
||||
if(!Storage::exists($media->media_path)) {
|
||||
return 'invalid file';
|
||||
}
|
||||
|
||||
$path = storage_path('app/'.$media->media_path);
|
||||
$thumb = false;
|
||||
if($media->thumbnail_path) {
|
||||
$thumb = storage_path('app/'.$media->thumbnail_path);
|
||||
$pt = explode('/', $media->thumbnail_path);
|
||||
$thumbname = array_pop($pt);
|
||||
}
|
||||
|
||||
$p = explode('/', $media->media_path);
|
||||
$name = array_pop($p);
|
||||
$storagePath = implode('/', $p);
|
||||
|
||||
$url = ResilientMediaStorageService::store($storagePath, $path, $name);
|
||||
if($thumb) {
|
||||
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
|
||||
$media->thumbnail_url = $thumbUrl;
|
||||
}
|
||||
$media->cdn_url = $url;
|
||||
$media->optimized_url = $url;
|
||||
$media->replicated_at = now();
|
||||
$media->save();
|
||||
|
||||
if($media->status_id) {
|
||||
Cache::forget('status:transformer:media:attachments:' . $media->status_id);
|
||||
MediaService::del($media->status_id);
|
||||
StatusService::del($media->status_id, false);
|
||||
}
|
||||
|
||||
return 'success';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ return [
|
|||
|
||||
// Limit to specific user ids, in comma separated format
|
||||
'user_ids' => env('PF_IMPORT_IG_PERM_ONLY_USER_IDS', null),
|
||||
],
|
||||
|
||||
'storage' => [
|
||||
'cloud' => [
|
||||
'enabled' => env('PF_IMPORT_IG_CLOUD_STORAGE', env('PF_ENABLE_CLOUD', false)),
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue