From ad36299b29283ee4b31dd8936c90f1b5908629b2 Mon Sep 17 00:00:00 2001 From: aidewoode Date: Thu, 14 Nov 2024 21:56:44 +0800 Subject: [PATCH] Replace Nginx with Thruster to send file --- Gemfile | 3 + Gemfile.lock | 2 + app/controllers/application_controller.rb | 20 -- app/controllers/concerns/stream_concern.rb | 7 +- .../concerns/transcoded_stream_concern.rb | 4 +- bin/thrust | 5 + config/application.rb | 1 - config/nginx/sendfile.conf | 9 - docker/production_start.sh | 2 +- docs/README_EDGE.md | 186 ++++++++++++++++++ .../api/v1/stream_controller_test.rb | 9 - .../v1/transcoded_stream_controller_test.rb | 13 -- test/lib/black_candy/config_test.rb | 9 - 13 files changed, 199 insertions(+), 71 deletions(-) create mode 100755 bin/thrust delete mode 100644 config/nginx/sendfile.conf create mode 100644 docs/README_EDGE.md diff --git a/Gemfile b/Gemfile index a8452ecd..b7eb0e71 100644 --- a/Gemfile +++ b/Gemfile @@ -73,6 +73,9 @@ gem "daemons", "~> 1.4.0" # Optional support for postgresql as database gem "pg", "~> 1.5.4" +# Add HTTP asset caching/compression and X-Sendfile acceleration to Puma [https://github.com/basecamp/thruster/] +gem "thruster", "~> 0.1.8", require: false + # Reduces boot times through caching; required in config/boot.rb gem "bootsnap", "~> 1.18.0", require: false diff --git a/Gemfile.lock b/Gemfile.lock index fb096811..6644aa60 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -361,6 +361,7 @@ GEM railties (>= 6.0.0) stringio (3.1.1) thor (1.3.2) + thruster (0.1.8) timeout (0.4.1) turbo-rails (1.5.0) actionpack (>= 6.0.0) @@ -428,6 +429,7 @@ DEPENDENCIES standard (~> 1.25.0) standard-rails stimulus-rails (~> 1.3.4) + thruster (~> 0.1.8) turbo-rails (~> 1.5.0) tzinfo-data wahwah (~> 1.6.0) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index f64be169..2c2db706 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -126,26 +126,6 @@ def render_json_error(error, status) render json: {type: error.type, message: error.message}, status: status end - def send_local_file(file_path, format, nginx_headers: {}) - if BlackCandy.config.nginx_sendfile? - nginx_headers.each { |name, value| response.headers[name] = value } - send_file file_path - - return - end - - # Use Rack::Files to support HTTP range without nginx. see https://github.com/rails/rails/issues/32193 - Rack::Files.new(nil).serving(request, file_path).tap do |(status, headers, body)| - self.status = status - self.response_body = body - - headers.each { |name, value| response.headers[name] = value } - - response.headers["Content-Type"] = Mime[format] - response.headers["Content-Disposition"] = "attachment" - end - end - def find_current_request_details Current.ip_address = request.ip Current.user_agent = request.user_agent diff --git a/app/controllers/concerns/stream_concern.rb b/app/controllers/concerns/stream_concern.rb index a7b7b08b..4ca668c8 100644 --- a/app/controllers/concerns/stream_concern.rb +++ b/app/controllers/concerns/stream_concern.rb @@ -8,12 +8,7 @@ module StreamConcern end def new - send_local_file @stream.file_path, @stream.format, nginx_headers: { - # Let nginx can get value of media_path dynamically in the nginx config - # when use X-Accel-Redirect header to send file. - "X-Media-Path" => Setting.media_path, - "X-Accel-Redirect" => File.join("/private_media", @stream.file_path.sub(File.expand_path(Setting.media_path), "")) - } + send_file @stream.file_path end private diff --git a/app/controllers/concerns/transcoded_stream_concern.rb b/app/controllers/concerns/transcoded_stream_concern.rb index 5f573115..377ddfda 100644 --- a/app/controllers/concerns/transcoded_stream_concern.rb +++ b/app/controllers/concerns/transcoded_stream_concern.rb @@ -36,9 +36,7 @@ def find_stream def find_cache return unless valid_cache? - send_local_file @stream.transcode_cache_file_path, @stream.format, nginx_headers: { - "X-Accel-Redirect" => File.join("/private_cache_media", @stream.transcode_cache_file_path.sub(Stream::TRANSCODE_CACHE_DIRECTORY.to_s, "")) - } + send_file @stream.transcode_cache_file_path end def valid_cache? diff --git a/bin/thrust b/bin/thrust new file mode 100755 index 00000000..ae871214 --- /dev/null +++ b/bin/thrust @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby +require "rubygems" +require "bundler/setup" + +load Gem.bin_path("thruster", "thrust") \ No newline at end of file diff --git a/config/application.rb b/config/application.rb index 66f1b164..89e5836d 100644 --- a/config/application.rb +++ b/config/application.rb @@ -33,7 +33,6 @@ module BlackCandy has_config :queue_db_url has_config :media_path has_config :db_adapter, default: "sqlite" - has_config :nginx_sendfile, default: false has_config :force_ssl, default: false has_config :demo_mode, default: false diff --git a/config/nginx/sendfile.conf b/config/nginx/sendfile.conf deleted file mode 100644 index 49219e73..00000000 --- a/config/nginx/sendfile.conf +++ /dev/null @@ -1,9 +0,0 @@ -location /private_media/ { - internal; - alias $upstream_http_x_media_path/; -} - -location /private_cache_media/ { - internal; - alias /app/tmp/cache/media_file/; -} diff --git a/docker/production_start.sh b/docker/production_start.sh index e034db73..17e2e2f0 100755 --- a/docker/production_start.sh +++ b/docker/production_start.sh @@ -1,4 +1,4 @@ #!/bin/sh rails db:prepare -bundle exec puma -C config/puma.rb \ No newline at end of file +./bin/thrust ./bin/rails server \ No newline at end of file diff --git a/docs/README_EDGE.md b/docs/README_EDGE.md new file mode 100644 index 00000000..ec4a2d04 --- /dev/null +++ b/docs/README_EDGE.md @@ -0,0 +1,186 @@ +

+ Black Candy logo +

+ +# Black Candy +[![CI](https://github.com/blackcandy-org/black_candy/actions/workflows/ci.yml/badge.svg)](https://github.com/blackcandy-org/black_candy/actions/workflows/ci.yml) +[![Coverage Status](https://coveralls.io/repos/github/blackcandy-org/blackcandy/badge.svg?branch=master)](https://coveralls.io/github/blackcandy-org/black_candy?branch=master) +[![Ruby Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://github.com/testdouble/standard) +![Docker Pulls](https://img.shields.io/docker/pulls/blackcandy/blackcandy) + +![Screenshot](https://raw.githubusercontent.com/blackcandy-org/blackcandy/master/docs/images/screenshot_main.png) + +Black Candy is a self-hosted music streaming server, your personal music center. + +## Try The Demo + +Please visit and use demo user (email: admin@admin.com, password: foobar) to log in. And feel free to try it. + +> [!NOTE] +> This demo user does not have administrator privileges. So you cannot experience all the features in Black Candy. And all music in the demo are from [Free Music Archive](https://freemusicarchive.org/). You can check their [licenses](https://github.com/blackcandy-org/blackcandy/blob/master/docs/demo_music_licenses.md). + +## Installation + +Black Candy uses docker image to install easily. You can run Black Candy like this. + +```shell +docker run -p 3000:3000 ghcr.io/blackcandy-org/blackcandy:latest + +# Or pull from Docker Hub. +docker run -p 3000:3000 blackcandy/blackcandy:latest +``` + +That's all. Now, you can access either http://localhost:3000 or http://host-ip:3000 in a browser, and use initial admin user to log in (email: admin@admin.com, password: foobar). + +## Upgrade + +> [!IMPORTANT] +> If you upgrade to a major version, you need to read the upgrade guide carefully before upgrade. Because there are some breaking changes in a major version. +> +> - See [V3 Upgrade](https://github.com/blackcandy-org/blackcandy/blob/master/docs/v3_upgrade.md) for upgrade from V2 release. +> - See [Edge Upgrade](https://github.com/blackcandy-org/blackcandy/blob/master/docs/edge_upgrade.md) for upgrade from edge release to latest stable release. + +Upgrade Black Candy is pull new image from remote. Then remove an old container and create a new one. + +```shell +docker pull ghcr.io/blackcandy-org/blackcandy:latest +docker stop +docker rm +docker run ghcr.io/blackcandy-org/blackcandy:latest +``` + +With docker compose, you can upgrade Black Candy like this: + +```shell +docker pull ghcr.io/blackcandy-org/blackcandy:latest +docker-compose down +docker-compose up +``` + +## Mobile Apps + +Black Candy mobile apps are available in the following app stores: + +[Get it on App Store](https://apps.apple.com/app/blackcandy/id6444304071) +[Get it on F-Droid](https://f-droid.org/packages/org.blackcandy.android/) + + +For Android app, you can also download APK from [GitHub Release](https://github.com/blackcandy-org/android/releases/latest) + +## Configuration + +### Port Mapping + +Black Candy exports the 3000 port. If you want to be able to access it from the host, you can use the `-p` option to map the port. + +```shell +docker run -p 3000:3000 ghcr.io/blackcandy-org/blackcandy:latest +``` + +### Media Files Mounts + +You can mount media files from host to container and use `MEDIA_PATH` environment variable to set the media path for black candy. + +```shell +docker run -v /media_data:/media_data -e MEDIA_PATH=/media_data ghcr.io/blackcandy-org/blackcandy:latest +``` + +### Use PostgreSQL As Database + +Black Candy use SQLite as database by default. Because SQLite can simplify the process of installation, and it's an ideal choice for self-hosted small server. If you think SQLite is not enough, or you are using some cloud service like heroku to host Black Candy, you can also use PostgreSQL as database. + +```shell +docker run -e DB_ADAPTER=postgresql -e DB_URL=postgresql://yourdatabaseurl ghcr.io/blackcandy-org/blackcandy:latest +``` + +### How to Persist Data + +All the data that need to persist in Black Candy are stored in `/app/storage`, So you can mount this directory to host to persist data. + +```shell +mkdir storage_data + +docker run -v ./storage_data:/app/storage ghcr.io/blackcandy-org/blackcandy:latest +``` + +### Logging + +Black Candy logs to `STDOUT` by default. So if you want to control the log, Docker already supports a lot of options to handle the log in the container. See: https://docs.docker.com/config/containers/logging/configure/. + +## Environment Variables + +| Name | Default | Description | +| --- | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| DB_URL | | The URL of PostgreSQL database. You must set this environment variable if you use PostgreSQL as database. | +| CABLE_DB_URL | | The URL of Pub/Sub database. You must set this environment variable if you use PostgreSQL as database. | +| QUEUE_DB_URL | | The URL of background job database. You must set this environment variable if you use PostgreSQL as database. | +| CACHE_DB_URL | | The URL of cache database. You must set this environment variable if you use PostgreSQL as database. | +| MEDIA_PATH | | You can use this environment variable to set media path for Black Candy, otherwise you can set media path in settings page. | +| DB_ADAPTER | "sqlite" | There are two adapters are supported, "sqlite" and "postgresql". | +| SECRET_KEY_BASE | | When the SECRET_KEY_BASE environment variable is not set, Black candy will generate SECRET_KEY_BASE environment variable every time when service start up. This will cause old sessions invalid, You can set your own SECRET_KEY_BASE environment variable on docker service to avoid it. | +| FORCE_SSL | false | Force all access to the app over SSL. | +| DEMO_MODE | false | Whether to enable demo mode, when demo mode is on, all users cannot access administrator privileges, even user is admin. And also users cannot change their profile. | + +## Edge Version + +The edge version of Black Candy base on master branch, which means it's not stable, you may encounter data loss or other issues. However, I don't recommend normal user using an edge version. But if you are a developer who wants to test or contribute to Black Candy, you can use the edge version. + +```shell +docker pull ghcr.io/blackcandy-org/blackcandy:edge +``` + +## Development + +### Requirements + +- Ruby 3.3 +- Node.js 20 +- libvips +- FFmpeg + +Make sure you have installed all those dependencies. + +### Install gem dependencies + +```shell +bundle install +``` + +### Install JavaScript dependencies + +```shell +npm install +``` + +### Database Configuration + +```shell +rails db:prepare +rails db:seed +``` + +### Start all services + +After you’ve set up everything, now you can run `./bin/dev` to start all services you need to develop. +Then visit use initial admin user to log in (email: admin@admin.com, password: foobar). + +### Running tests + +```shell +# Running all test +$ rails test:all + +# Running lint +$ rails lint:all +``` + +## Integrations + +Black Candy support get artist and album image from Discogs API. You can create an API token from Discogs and set Discogs token on Setting page to enable it. + +## Sponsorship + +This project is supported by: + + + diff --git a/test/controllers/api/v1/stream_controller_test.rb b/test/controllers/api/v1/stream_controller_test.rb index a56ef15e..cb297aba 100644 --- a/test/controllers/api/v1/stream_controller_test.rb +++ b/test/controllers/api/v1/stream_controller_test.rb @@ -12,15 +12,6 @@ class Api::V1::StreamControllerTest < ActionDispatch::IntegrationTest assert_response :success end - test "should set header for nginx send file" do - with_env("NGINX_SENDFILE" => "true") do - get new_api_v1_stream_url(song_id: songs(:mp3_sample).id), headers: api_token_header(@user) - - assert_equal Setting.media_path, @response.get_header("X-Media-Path") - assert_equal "/private_media/artist1_album2.mp3", @response.get_header("X-Accel-Redirect") - end - end - test "should respond file data" do get new_api_v1_stream_url(song_id: songs(:mp3_sample).id), headers: api_token_header(@user) assert_equal binary_data(file_fixture("artist1_album2.mp3")), response.body diff --git a/test/controllers/api/v1/transcoded_stream_controller_test.rb b/test/controllers/api/v1/transcoded_stream_controller_test.rb index 80218e79..1d668eb2 100644 --- a/test/controllers/api/v1/transcoded_stream_controller_test.rb +++ b/test/controllers/api/v1/transcoded_stream_controller_test.rb @@ -69,19 +69,6 @@ def close_tmp_cache_file end end - test "should send cached transcoded stream file when found cache and send file with nginx" do - Stream.stub(:new, @stream_mock) do - get new_api_v1_transcoded_stream_url(song_id: songs(:flac_sample).id), headers: api_token_header(@user) - assert_response :success - - with_env("NGINX_SENDFILE" => "true") do - get new_api_v1_transcoded_stream_url(song_id: songs(:flac_sample).id), headers: api_token_header(@user) - assert_equal "/private_cache_media#{@stream_mock.transcode_cache_file_path}", @response.get_header("X-Accel-Redirect") - assert_equal "audio/mpeg", @response.get_header("Content-Type") - end - end - end - test "should regenerate new cache when cache is invalid" do Stream.stub(:new, @stream_mock) do stream = Stream.new(songs(:flac_sample)) diff --git a/test/lib/black_candy/config_test.rb b/test/lib/black_candy/config_test.rb index dbf819a9..ac093c5c 100644 --- a/test/lib/black_candy/config_test.rb +++ b/test/lib/black_candy/config_test.rb @@ -37,15 +37,6 @@ class BlackCandy::ConfigTest < ActiveSupport::TestCase end end - test "should get nginx_sendfile value as a boolean" do - assert_nil ENV["NGINX_SENDFILE"] - assert_not BlackCandy.config.nginx_sendfile? - - with_env("NGINX_SENDFILE" => "true") do - assert BlackCandy.config.nginx_sendfile? - end - end - test "should raise error when database_adapter is not supported" do with_env("DB_ADAPTER" => "invalid_adapter") do assert_raises(BlackCandy::Config::ValidationError) do