In version 7 of Spaties popular Laravel Medialibrary package, the filename pattern for conversions on the filesystem changed. This means that when upgrading from version 6 to version 7, you must run a script to rename ALL of your conversion files on the filesystem.
Spatie has a repository on Github to help us out with this, but that script cannot handle large amount of folders when your filesystem is S3.
For us, running this script just crashed the script because the Filesystem adapter cannot handle that many directories (a classic "too much memory fatal error").
For our app, we had over 116 000 media items which means 116 000 * 2 (media folder + conversions folder) directories at S3 which the script tries to collect into a single collection.
So, we had to find another way of renaming all the files.
First, we tried to just divide the directories (based on the media table) and execute the rename command one directory at the time. This didn't crash our app, but sadly, this was very time consuming. We calculated that this would take over 53 hours to complete ?
Luckily Laravel has queues, so we converted the script into a dispatchable job in which we could send in a chunk of directories, and then we could just run a whole bunch of queue workers to rename the images in paralell.
The final script ran for about 9 hours with 5 concurrent queue workers on my local machine. Instead of pushing this to production, we could run this locally with the production database since no images were being added the app during this time.
We added a local converted column to our media table to keep track of which rows had already been converted so we could just keep executing the script avoiding duplicates issues.
This is the command that added all the jobs to the jobs table:
// app/Console/Commands/UpgradeMediaCommand.php
namespace App\Console\Commands;
use App\Jobs\UpgradeS3MediaJob;
use Illuminate\Console\Command;
use Illuminate\Console\ConfirmableTrait;
use Spatie\MediaLibrary\Models\Media;
class UpgradeMediaCommand extends Command
{
use ConfirmableTrait;
protected $signature = 'upgrade-media
{disk? : Disk to use}
{--d|dry-run : List files that will be renamed without renaming them}
{--f|force : Force the operation to run when in production}';
protected $description = 'Update the names of the version 6 files of spatie/laravel-medialibrary';
public function handle()
{
if (! $this->confirmToProceed()) {
return;
}
$isDryRun = $this->option('dry-run') ?? false;
$disk = $this->argument('disk') ?? config('medialibrary.disk_name');
Media::query()
->where('converted', false)
->orderBy('created_at', 'desc')
->chunk(100, function ($mediaCollection) use ($isDryRun, $disk) {
dispatch(new UpgradeS3MediaJob($mediaCollection->toArray(), $isDryRun, $disk));
});
}
}
This command chunked the entire media table into chunks which are then sent to the job which handled the renaming.
This is the job class that handled the renaming:
// app/Jobs/UpgradeS3MediaJob.php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Spatie\MediaLibrary\Models\Media;
class UpgradeS3MediaJob implements ShouldQueue
{
use Dispatchable, Queueable, SerializesModels;
private bool $isDryRun;
private array $mediaCollectionArray;
private array $mediaIdsToUpdate = [];
private Collection $mediaFilesToChange;
private string $disk;
public function __construct(array $mediaCollectionArray, $isDryRun, $disk)
{
$this->isDryRun = $isDryRun;
$this->mediaCollectionArray = $mediaCollectionArray;
$this->disk = $disk;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$this
->getMediaFilesToBeRenamed()
->renameMediaFiles();
if (! $this->isDryRun) {
Media::whereIn('id', $this->mediaIdsToUpdate)->update(['converted' => 1]);
info('Updated ' . count($this->mediaIdsToUpdate) . ' items as converted!');
} else {
info('Done dry run');
}
}
protected function getMediaFilesToBeRenamed(): self
{
$files = [];
foreach ($this->mediaCollectionArray as $mediaArray) {
$dir = './media/' . $mediaArray['id'];
$newFiles = $this->convert($dir)->toArray();
$files = array_merge($files, $newFiles);
}
$this->mediaFilesToChange = collect($files);
return $this;
}
protected function convert($directory)
{
return collect(Storage::disk($this->disk)->allFiles($directory))
->reject(function (string $file): bool {
return ! Str::startsWith($file, 'media') || strpos($file, '/') === false;
})
->filter(function (string $file): bool {
return $this->hasOriginal($file);
})
->filter(function (string $file): bool {
return $this->needsToBeConverted($file);
})
->map(function (string $file): array {
return $this->getReplaceArray($file);
});
}
protected function renameMediaFiles()
{
if ($this->mediaFilesToChange->count() === 0) {
info('There are no files to convert.');
}
if ($this->isDryRun) {
info('This is a dry-run and will not actually rename the files');
}
$this->mediaFilesToChange->each(function (array $filePaths) {
if (! $this->isDryRun) {
Storage::disk($this->disk)->move($filePaths['current'], $filePaths['replacement']);
$this->mediaIdsToUpdate[] = $this->getIdFromPath($filePaths['current']);
}
info("The file `{$filePaths['current']}` has become `{$filePaths['replacement']}`");
});
}
protected function hasOriginal(string $filePath): bool
{
$path = pathinfo($filePath, PATHINFO_DIRNAME);
$oneLevelHigher = dirname($path);
if ($oneLevelHigher === '.' || $oneLevelHigher === 'media') {
return false;
}
$original = Storage::disk($this->disk)->files($oneLevelHigher);
if (count($original) !== 1) {
return false;
}
return true;
}
protected function needsToBeConverted(string $file): bool
{
$currentFile = pathinfo($file);
$original = $this->getOriginal($currentFile['dirname']);
return strpos($currentFile['basename'], $original) === false;
}
protected function getReplaceArray(string $file): array
{
$currentFile = pathinfo($file);
$currentFilePath = $currentFile['dirname'];
$original = $this->getOriginal($currentFilePath);
return [
'current' => $file,
'replacement' => "{$currentFilePath}/{$original}-{$currentFile['basename']}",
];
}
protected function getOriginal(string $filePath): string
{
$oneLevelHigher = dirname($filePath);
$original = Storage::disk($this->disk)->files($oneLevelHigher);
return pathinfo($original[0], PATHINFO_FILENAME);
}
protected function getIdFromPath($currentPath)
{
$parts = explode('/', $currentPath);
return $parts[1];
}
}
