Add support for libvips in addition to ImageMagick (#30090)

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

.devcontainer/Dockerfile

@@ -9,7 +9,7 @@ RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSI
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends libicu-dev libidn11-dev ffmpeg imagemagick libpam-dev
+    && apt-get -y install --no-install-recommends libicu-dev libidn11-dev ffmpeg imagemagick libvips42 libpam-dev
 
 # [Optional] Uncomment this line to install additional gems.
 RUN gem install foreman

.github/actions/setup-ruby/action.yml

@@ -14,7 +14,7 @@ runs:
       shell: bash
       run: |
         sudo apt-get update
-        sudo apt-get install -y libicu-dev libidn11-dev ${{ inputs.additional-system-dependencies }}
+        sudo apt-get install -y libicu-dev libidn11-dev libvips42 ${{ inputs.additional-system-dependencies }}
 
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1

.github/workflows/test-ruby.yml

@@ -133,7 +133,7 @@ jobs:
         uses: ./.github/actions/setup-ruby
         with:
           ruby-version: ${{ matrix.ruby-version}}
-          additional-system-dependencies: ffmpeg imagemagick libpam-dev
+          additional-system-dependencies: ffmpeg libpam-dev
 
       - name: Load database schema
         run: './bin/rails db:create db:schema:load db:seed'
@@ -148,6 +148,93 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  test-libvips:
+    name: Libvips tests
+    runs-on: ubuntu-24.04
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: ${{ matrix.ruby-version != '.ruby-version' }}
+      RAILS_ENV: test
+      ALLOW_NOPAM: true
+      PAM_ENABLED: true
+      PAM_DEFAULT_SERVICE: pam_test
+      PAM_CONTROLLED_SERVICE: pam_test_controlled
+      OIDC_ENABLED: true
+      OIDC_SCOPE: read
+      SAML_ENABLED: true
+      CAS_ENABLED: true
+      BUNDLE_WITH: 'pam_authentication test'
+      GITHUB_RSPEC: ${{ matrix.ruby-version == '.ruby-version' && github.event.pull_request && 'true' }}
+      MASTODON_USE_LIBVIPS: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '3.1'
+          - '3.2'
+          - '.ruby-version'
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: './'
+          name: ${{ github.sha }}
+
+      - name: Expand archived asset artifacts
+        run: |
+          tar xvzf artifacts.tar.gz
+
+      - name: Set up Ruby environment
+        uses: ./.github/actions/setup-ruby
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          additional-system-dependencies: ffmpeg libpam-dev libyaml-dev
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bin/rspec --tag paperclip_processing
+
+      - name: Upload coverage reports to Codecov
+        if: matrix.ruby-version == '.ruby-version'
+        uses: codecov/codecov-action@v4
+        with:
+          files: coverage/lcov/mastodon.lcov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   test-e2e:
     name: End to End testing
     runs-on: ubuntu-latest
@@ -209,7 +296,7 @@ jobs:
         uses: ./.github/actions/setup-ruby
         with:
           ruby-version: ${{ matrix.ruby-version}}
-          additional-system-dependencies: ffmpeg imagemagick
+          additional-system-dependencies: ffmpeg
 
       - name: Set up Javascript environment
         uses: ./.github/actions/setup-javascript
@@ -329,7 +416,7 @@ jobs:
         uses: ./.github/actions/setup-ruby
         with:
           ruby-version: ${{ matrix.ruby-version}}
-          additional-system-dependencies: ffmpeg imagemagick
+          additional-system-dependencies: ffmpeg
 
       - name: Set up Javascript environment
         uses: ./.github/actions/setup-javascript

Dockerfile

@@ -43,6 +43,8 @@ ENV \
 # Apply Mastodon version information
   MASTODON_VERSION_PRERELEASE="${MASTODON_VERSION_PRERELEASE}" \
   MASTODON_VERSION_METADATA="${MASTODON_VERSION_METADATA}" \
+# Enable libvips
+  MASTODON_USE_LIBVIPS=true \
 # Apply Mastodon static files and YJIT options
   RAILS_SERVE_STATIC_FILES=${RAILS_SERVE_STATIC_FILES} \
   RUBY_YJIT_ENABLE=${RUBY_YJIT_ENABLE} \
@@ -97,7 +99,7 @@ RUN \
     curl \
     ffmpeg \
     file \
-    imagemagick \
+    libvips42 \
     libjemalloc2 \
     patchelf \
     procps \

Gemfile

@@ -23,6 +23,7 @@ gem 'fog-core', '<= 2.4.0'
 gem 'fog-openstack', '~> 1.0', require: false
 gem 'kt-paperclip', '~> 7.2'
 gem 'md-paperclip-azure', '~> 2.2', require: false
+gem 'ruby-vips', '~> 2.2', require: false
 
 gem 'active_model_serializers', '~> 0.10'
 gem 'addressable', '~> 2.8'

Gemfile.lock

@@ -763,6 +763,8 @@ GEM
     ruby-saml (1.16.0)
       nokogiri (>= 1.13.10)
       rexml
+    ruby-vips (2.2.1)
+      ffi (~> 1.12)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
     rufus-scheduler (3.9.1)
@@ -1023,6 +1025,7 @@ DEPENDENCIES
   rubocop-rspec
   ruby-prof
   ruby-progressbar (~> 1.13)
+  ruby-vips (~> 2.2)
   rubyzip (~> 2.3)
   sanitize (~> 6.0)
   scenic (~> 1.7)

app/lib/admin/metrics/dimension/software_versions_dimension.rb

@@ -10,7 +10,7 @@ def key
   protected
 
   def perform_query
-    [mastodon_version, ruby_version, postgresql_version, redis_version, elasticsearch_version].compact
+    [mastodon_version, ruby_version, postgresql_version, redis_version, elasticsearch_version, libvips_version].compact
   end
 
   def mastodon_version
@@ -71,6 +71,17 @@ def elasticsearch_version
     nil
   end
 
+  def libvips_version
+    return unless Rails.configuration.x.use_vips
+
+    {
+      key: 'libvips',
+      human_key: 'libvips',
+      value: Vips.version_string,
+      human_value: Vips.version_string,
+    }
+  end
+
   def redis_info
     @redis_info ||= if redis.is_a?(Redis::Namespace)
                       redis.redis.info

app/models/concerns/attachmentable.rb

@@ -69,7 +69,7 @@ def appropriate_extension(attachment)
     original_extension       = Paperclip::Interpolations.extension(attachment, :original)
     proper_extension         = extensions_for_mime_type.first.to_s
     extension                = extensions_for_mime_type.include?(original_extension) ? original_extension : proper_extension
-    extension                = 'jpeg' if extension == 'jpe'
+    extension                = 'jpeg' if ['jpe', 'jfif'].include?(extension)
 
     extension
   end

app/models/preview_card.rb

@@ -57,7 +57,11 @@ class PreviewCard < ApplicationRecord
   has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy
   belongs_to :author_account, class_name: 'Account', optional: true
 
-  has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' }, validate_media_type: false
+  has_attached_file :image,
+                    processors: [Rails.configuration.x.use_vips ? :lazy_thumbnail : :thumbnail, :blurhash_transcoder],
+                    styles: ->(f) { image_styles(f) },
+                    convert_options: { all: '-quality 90 +profile "!icc,*" +set date:modify +set date:create +set date:timestamp' },
+                    validate_media_type: false
 
   validates :url, presence: true, uniqueness: true, url: true
   validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES

config/application.rb

@@ -27,7 +27,7 @@
 require_relative '../lib/redis/namespace_extensions'
 require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
-require_relative '../lib/paperclip/lazy_thumbnail'
+
 require_relative '../lib/paperclip/gif_transcoder'
 require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
 require_relative '../lib/paperclip/transcoder'
@@ -100,6 +100,14 @@ class Application < Rails::Application
 
     config.before_configuration do
       require 'mastodon/redis_config'
+
+      config.x.use_vips = ENV['MASTODON_USE_LIBVIPS'] == 'true'
+
+      if config.x.use_vips
+        require_relative '../lib/paperclip/vips_lazy_thumbnail'
+      else
+        require_relative '../lib/paperclip/lazy_thumbnail'
+      end
     end
 
     config.to_prepare do

config/initializers/vips.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+if Rails.configuration.x.use_vips
+  ENV['VIPS_BLOCK_UNTRUSTED'] = 'true'
+
+  require 'vips'
+
+  abort('Incompatible libvips version, please install libvips >= 8.13') unless Vips.at_least_libvips?(8, 13)
+
+  Vips.block('VipsForeign', true)
+
+  %w(
+    VipsForeignLoadNsgif
+    VipsForeignLoadJpeg
+    VipsForeignLoadPng
+    VipsForeignLoadWebp
+    VipsForeignLoadHeif
+    VipsForeignSavePng
+    VipsForeignSaveSpng
+    VipsForeignSaveJpeg
+    VipsForeignSaveWebp
+  ).each do |operation|
+    Vips.block(operation, false)
+  end
+
+  Vips.block_untrusted(true)
+end

lib/paperclip/blurhash_transcoder.rb

@@ -5,12 +5,26 @@ class BlurhashTranscoder < Paperclip::Processor
     def make
       return @file unless options[:style] == :small || options[:blurhash]
 
-      pixels   = convert(':source -depth 8 RGB:-', source: File.expand_path(@file.path)).unpack('C*')
-      geometry = options.fetch(:file_geometry_parser).from_file(@file)
+      width, height, data = blurhash_params
+      # Guard against segfaults if data has unexpected size
+      raise RangeError("Invalid image data size (expected #{width * height * 3}, got #{data.size})") if data.size != width * height * 3 # TODO: should probably be another exception type
 
-      attachment.instance.blurhash = Blurhash.encode(geometry.width, geometry.height, pixels, **(options[:blurhash] || {}))
+      attachment.instance.blurhash = Blurhash.encode(width, height, data, **(options[:blurhash] || {}))
 
       @file
     end
+
+    private
+
+    def blurhash_params
+      if Rails.configuration.x.use_vips
+        image = Vips::Image.thumbnail(@file.path, 100)
+        [image.width, image.height, image.colourspace(:srgb).extract_band(0, n: 3).to_a.flatten]
+      else
+        pixels   = convert(':source -depth 8 RGB:-', source: File.expand_path(@file.path)).unpack('C*')
+        geometry = options.fetch(:file_geometry_parser).from_file(@file)
+        [geometry.width, geometry.height, pixels]
+      end
+    end
   end
 end