Skip to content

Commit 82798b5

Browse files
dansupdansup
committed
Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars
1 parent 36b23fe commit 82798b5

6 files changed

+271
-18
lines changed

‎app/Jobs/AvatarPipeline/AvatarStorageCleanup.php

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
<?php
2+
3+
namespace App\Jobs\AvatarPipeline;
4+
5+
use Illuminate\Bus\Queueable;
6+
use Illuminate\Contracts\Queue\ShouldQueue;
7+
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
8+
use Illuminate\Foundation\Bus\Dispatchable;
9+
use Illuminate\Queue\InteractsWithQueue;
10+
use Illuminate\Queue\SerializesModels;
11+
use Illuminate\Queue\Middleware\WithoutOverlapping;
12+
use App\Services\AvatarService;
13+
use App\Avatar;
14+
15+
class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
16+
{
17+
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
18+
19+
public $avatar;
20+
public $tries = 3;
21+
public $maxExceptions = 3;
22+
public $timeout = 900;
23+
public $failOnTimeout = true;
24+
25+
/**
26+
* The number of seconds after which the job's unique lock will be released.
27+
*
28+
* @var int
29+
*/
30+
public $uniqueFor = 3600;
31+
32+
/**
33+
* Get the unique ID for the job.
34+
*/
35+
public function uniqueId(): string
36+
{
37+
return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
38+
}
39+
40+
/**
41+
* Get the middleware the job should pass through.
42+
*
43+
* @return array<int, object>
44+
*/
45+
public function middleware(): array
46+
{
47+
return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
48+
}
49+
50+
/**
51+
* Create a new job instance.
52+
*/
53+
public function __construct(Avatar $avatar)
54+
{
55+
$this->avatar = $avatar->withoutRelations();
56+
}
57+
58+
/**
59+
* Execute the job.
60+
*/
61+
public function handle(): void
62+
{
63+
AvatarService::cleanup($this->avatar, true);
64+
65+
return;
66+
}
67+
}

‎app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
<?php
2+
3+
namespace App\Jobs\AvatarPipeline;
4+
5+
use Illuminate\Bus\Queueable;
6+
use Illuminate\Contracts\Queue\ShouldQueue;
7+
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
8+
use Illuminate\Foundation\Bus\Dispatchable;
9+
use Illuminate\Queue\InteractsWithQueue;
10+
use Illuminate\Queue\SerializesModels;
11+
use Illuminate\Queue\Middleware\WithoutOverlapping;
12+
use App\Services\AvatarService;
13+
use App\Avatar;
14+
use Illuminate\Support\Str;
15+
16+
class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
17+
{
18+
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
19+
20+
public $avatar;
21+
public $tries = 3;
22+
public $maxExceptions = 3;
23+
public $timeout = 900;
24+
public $failOnTimeout = true;
25+
26+
/**
27+
* The number of seconds after which the job's unique lock will be released.
28+
*
29+
* @var int
30+
*/
31+
public $uniqueFor = 3600;
32+
33+
/**
34+
* Get the unique ID for the job.
35+
*/
36+
public function uniqueId(): string
37+
{
38+
return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
39+
}
40+
41+
/**
42+
* Get the middleware the job should pass through.
43+
*
44+
* @return array<int, object>
45+
*/
46+
public function middleware(): array
47+
{
48+
return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
49+
}
50+
51+
/**
52+
* Create a new job instance.
53+
*/
54+
public function __construct(Avatar $avatar)
55+
{
56+
$this->avatar = $avatar->withoutRelations();
57+
}
58+
59+
/**
60+
* Execute the job.
61+
*/
62+
public function handle(): void
63+
{
64+
$avatar = $this->avatar;
65+
66+
$disk = AvatarService::disk();
67+
68+
$files = collect(AvatarService::storage($avatar));
69+
70+
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
71+
72+
$files = $files->filter(function($f) use($curFile) {
73+
return !$curFile || !str_ends_with($f, $curFile);
74+
})->each(function($name) use($disk) {
75+
$disk->delete($name);
76+
});
77+
78+
return;
79+
}
80+
}

‎app/Jobs/AvatarPipeline/RemoteAvatarFetch.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public function handle()
108108
$avatar->remote_url = $icon['url'];
109109
$avatar->save();
110110

111-
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
111+
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
112112

113113
return 1;
114114
}

‎app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php

-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,6 @@ public function handle()
8989
$avatar->save();
9090
}
9191

92-
9392
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
9493

9594
return 1;

‎app/Services/AvatarService.php

+117-13
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,125 @@
33
namespace App\Services;
44

55
use Cache;
6+
use Storage;
7+
use Illuminate\Support\Str;
8+
use App\Avatar;
69
use App\Profile;
10+
use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
11+
use League\Flysystem\UnableToCheckDirectoryExistence;
12+
use League\Flysystem\UnableToRetrieveMetadata;
713

814
class AvatarService
915
{
10-
public static function get($profile_id)
11-
{
12-
$exists = Cache::get('avatar:' . $profile_id);
13-
if($exists) {
14-
return $exists;
15-
}
16-
17-
$profile = Profile::find($profile_id);
18-
if(!$profile) {
19-
return config('app.url') . '/storage/avatars/default.jpg';
20-
}
21-
return $profile->avatarUrl();
22-
}
16+
public static function get($profile_id)
17+
{
18+
$exists = Cache::get('avatar:' . $profile_id);
19+
if($exists) {
20+
return $exists;
21+
}
22+
23+
$profile = Profile::find($profile_id);
24+
if(!$profile) {
25+
return config('app.url') . '/storage/avatars/default.jpg';
26+
}
27+
return $profile->avatarUrl();
28+
}
29+
30+
public static function disk()
31+
{
32+
$storage = [
33+
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
34+
'local' => boolval(config_cache('federation.avatars.store_local'))
35+
];
36+
37+
if(!$storage['cloud'] && !$storage['local']) {
38+
return false;
39+
}
40+
41+
$driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
42+
$disk = Storage::disk($driver);
43+
44+
return $disk;
45+
}
46+
47+
public static function storage(Avatar $avatar)
48+
{
49+
$disk = self::disk();
50+
51+
if(!$disk) {
52+
return;
53+
}
54+
55+
$storage = [
56+
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
57+
'local' => boolval(config_cache('federation.avatars.store_local'))
58+
];
59+
60+
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
61+
62+
return $disk->allFiles($base . $avatar->profile_id);
63+
}
64+
65+
public static function cleanup($avatar, $confirm = false)
66+
{
67+
if(!$avatar || !$confirm) {
68+
return;
69+
}
70+
71+
if($avatar->cdn_url == null) {
72+
return;
73+
}
74+
75+
$storage = [
76+
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
77+
'local' => boolval(config_cache('federation.avatars.store_local'))
78+
];
79+
80+
if(!$storage['cloud'] && !$storage['local']) {
81+
return;
82+
}
83+
84+
$disk = self::disk();
85+
86+
if(!$disk) {
87+
return;
88+
}
89+
90+
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
91+
92+
try {
93+
$exists = $disk->directoryExists($base . $avatar->profile_id);
94+
} catch (
95+
UnableToRetrieveMetadata |
96+
UnableToCheckDirectoryExistence |
97+
Exception $e
98+
) {
99+
return;
100+
}
101+
102+
if(!$exists) {
103+
return;
104+
}
105+
106+
$files = collect($disk->allFiles($base . $avatar->profile_id));
107+
108+
if(!$files || !$files->count() || $files->count() === 1) {
109+
return;
110+
}
111+
112+
if($files->count() > 5) {
113+
AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
114+
return;
115+
}
116+
117+
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
118+
119+
$files = $files->filter(function($f) use($curFile) {
120+
return !$curFile || !str_ends_with($f, $curFile);
121+
})->each(function($name) use($disk) {
122+
$disk->delete($name);
123+
});
124+
125+
return;
126+
}
23127
}

‎app/Services/MediaStorageService.php

+6-3
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
use GuzzleHttp\Exception\RequestException;
1818
use App\Jobs\MediaPipeline\MediaDeletePipeline;
1919
use Illuminate\Support\Arr;
20+
use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
2021

2122
class MediaStorageService {
2223

@@ -29,9 +30,9 @@ public static function store(Media $media)
2930
return;
3031
}
3132

32-
public static function avatar($avatar, $local = false)
33+
public static function avatar($avatar, $local = false, $skipRecentCheck = false)
3334
{
34-
return (new self())->fetchAvatar($avatar, $local);
35+
return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
3536
}
3637

3738
public static function head($url)
@@ -182,6 +183,7 @@ protected function remoteToCloud($media)
182183

183184
protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
184185
{
186+
$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
185187
$url = $avatar->remote_url;
186188
$driver = $local ? 'local' : config('filesystems.cloud');
187189

@@ -205,7 +207,7 @@ protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false
205207
$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
206208

207209
if(!$skipRecentCheck) {
208-
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
210+
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
209211
return;
210212
}
211213
}
@@ -261,6 +263,7 @@ protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false
261263

262264
Cache::forget('avatar:' . $avatar->profile_id);
263265
AccountService::del($avatar->profile_id);
266+
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
264267

265268
unlink($tmpName);
266269
}

0 commit comments

Comments
 (0)