diff --git a/config/config.exs b/config/config.exs index 04308869..7f552120 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,7 +12,7 @@ config :elixir, :time_zone_database, TimeZoneInfo.TimeZoneDatabase config :time_zone_info, update: :daily config :music_library, - ecto_repos: [MusicLibrary.Repo, MusicLibrary.ErrorRepo], + ecto_repos: [MusicLibrary.BackgroundRepo, MusicLibrary.Repo, MusicLibrary.ErrorRepo], generators: [timestamp_type: :utc_datetime, binary_id: true] config :music_library, MusicLibraryWeb, @@ -76,8 +76,15 @@ config :error_tracker, otp_app: :music_library, enabled: true +config :music_library, Oban, + engine: Oban.Engines.Lite, + queues: [default: 10], + repo: MusicLibrary.BackgroundRepo + config :music_library, MusicLibrary.ErrorRepo, priv: "priv/error_repo" +config :music_library, MusicLibrary.BackgroundRepo, priv: "priv/background_repo" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" diff --git a/config/dev.exs b/config/dev.exs index 3533fda7..cead805f 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -14,6 +14,12 @@ config :music_library, MusicLibrary.ErrorRepo, stacktrace: true, show_sensitive_data_on_connection_error: true +config :music_library, MusicLibrary.BackgroundRepo, + database: Path.expand("../data/music_library_background_dev.db", __DIR__), + pool_size: 5, + stacktrace: true, + show_sensitive_data_on_connection_error: true + # For development, we disable any cache and enable # debugging and code reloading. # diff --git a/config/runtime.exs b/config/runtime.exs index f4a185c2..bbe74d22 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -68,6 +68,19 @@ if config_env() == :prod do cache_size: -8000, pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5") + background_database_path = + System.get_env("BACKGROUND_DATABASE_PATH") || + raise """ + environment variable BACKGROUND_DATABASE_PATH is missing. + For example: /etc/music_library/music_library_background.db + """ + + config :music_library, MusicLibrary.BackgroundRepo, + database: background_database_path, + # 16MB * pool_size = base memory usage + cache_size: -8000, + pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5") + # The secret key base is used to sign/encrypt cookies and other secrets. # A default value is used in config/dev.exs and config/test.exs but you # want to use a different value for prod and you most likely don't want diff --git a/config/test.exs b/config/test.exs index 6f5ba780..48085727 100644 --- a/config/test.exs +++ b/config/test.exs @@ -16,6 +16,11 @@ config :music_library, MusicLibrary.ErrorRepo, pool_size: 32, pool: Ecto.Adapters.SQL.Sandbox +config :music_library, MusicLibrary.BackgroundRepo, + database: Path.expand("../data/music_library_background_test.db", __DIR__), + pool_size: 32, + pool: Ecto.Adapters.SQL.Sandbox + # We don't run a server during test. If one is required, # you can enable the server option below. config :music_library, MusicLibraryWeb.Endpoint, @@ -51,3 +56,5 @@ config :music_library, MusicBrainz, config :phoenix_test, :endpoint, MusicLibraryWeb.Endpoint config :floki, :html_parser, Floki.HTMLParser.FastHtml + +config :music_library, Oban, testing: :manual diff --git a/fly.toml b/fly.toml index e2ddb586..469e2203 100644 --- a/fly.toml +++ b/fly.toml @@ -12,6 +12,7 @@ kill_signal = 'SIGTERM' [env] DATABASE_PATH = '/mnt/music_library/music_library_prod.db' ERROR_DATABASE_PATH = '/mnt/music_library/music_library_prod_errors.db' +BACKGROUND_DATABASE_PATH = '/mnt/music_library/music_library_prod_background.db' PHX_HOST = 'music-library.claudio-ortolina.org' PORT = '8080' LAST_FM_USER = 'cloud8421' diff --git a/lib/music_library/application.ex b/lib/music_library/application.ex index 712ba37e..0437af83 100644 --- a/lib/music_library/application.ex +++ b/lib/music_library/application.ex @@ -11,6 +11,8 @@ defmodule MusicLibrary.Application do MusicLibraryWeb.Telemetry, MusicLibrary.Repo, MusicLibrary.ErrorRepo, + MusicLibrary.BackgroundRepo, + {Oban, Application.fetch_env!(:music_library, Oban)}, {Ecto.Migrator, repos: Application.fetch_env!(:music_library, :ecto_repos), skip: skip_migrations?()}, {DNSCluster, query: Application.get_env(:music_library, :dns_cluster_query) || :ignore}, diff --git a/lib/music_library/background_repo.ex b/lib/music_library/background_repo.ex new file mode 100644 index 00000000..89a49a36 --- /dev/null +++ b/lib/music_library/background_repo.ex @@ -0,0 +1,5 @@ +defmodule MusicLibrary.BackgroundRepo do + use Ecto.Repo, + otp_app: :music_library, + adapter: Ecto.Adapters.SQLite3 +end diff --git a/mix.exs b/mix.exs index 2fd288b0..cd8d648f 100644 --- a/mix.exs +++ b/mix.exs @@ -86,6 +86,9 @@ defmodule MusicLibrary.MixProject do # Data validation {:nimble_options, "~> 1.1"}, + # Background Jobs + {:oban, "~> 2.19"}, + # Test tooling {:phoenix_test, "~> 0.6.0", only: :test}, {:floki, ">= 0.30.0", only: :test}, diff --git a/mix.lock b/mix.lock index 424ee25b..982af6f5 100644 --- a/mix.lock +++ b/mix.lock @@ -30,6 +30,7 @@ "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "oban": {:hex, :oban, "2.19.4", "045adb10db1161dceb75c254782f97cdc6596e7044af456a59decb6d06da73c1", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5fcc6219e6464525b808d97add17896e724131f498444a292071bf8991c99f97"}, "phoenix": {:hex, :phoenix, "1.7.21", "14ca4f1071a5f65121217d6b57ac5712d1857e40a0833aff7a691b7870fc9a3b", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "336dce4f86cba56fed312a7d280bf2282c720abb6074bdb1b61ec8095bdd0bc9"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"}, "phoenix_html": {:hex, :phoenix_html, "4.2.1", "35279e2a39140068fc03f8874408d58eef734e488fc142153f055c5454fd1c08", [:mix], [], "hexpm", "cff108100ae2715dd959ae8f2a8cef8e20b593f8dfd031c9cba92702cf23e053"}, diff --git a/priv/background_repo/migrations/20250413170433_create_jobs_table.exs b/priv/background_repo/migrations/20250413170433_create_jobs_table.exs new file mode 100644 index 00000000..eff848c4 --- /dev/null +++ b/priv/background_repo/migrations/20250413170433_create_jobs_table.exs @@ -0,0 +1,13 @@ +defmodule MusicLibrary.BackgroundRepo.Migrations.CreateJobsTable do + use Ecto.Migration + + def up do + Oban.Migration.up(version: 12) + end + + # We specify `version: 1` in `down`, ensuring that we'll roll all the way back down if + # necessary, regardless of which version we've migrated `up` to. + def down do + Oban.Migration.down(version: 1) + end +end