Install Oban with dedicated repo

config/config.exs

@@ -12,7 +12,7 @@ config :elixir, :time_zone_database, TimeZoneInfo.TimeZoneDatabase
 config :time_zone_info, update: :daily
 
 config :music_library,
-  ecto_repos: [MusicLibrary.Repo, MusicLibrary.ErrorRepo],
+  ecto_repos: [MusicLibrary.BackgroundRepo, MusicLibrary.Repo, MusicLibrary.ErrorRepo],
   generators: [timestamp_type: :utc_datetime, binary_id: true]
 
 config :music_library, MusicLibraryWeb,
@@ -76,8 +76,15 @@ config :error_tracker,
   otp_app: :music_library,
   enabled: true
 
+config :music_library, Oban,
+  engine: Oban.Engines.Lite,
+  queues: [default: 10],
+  repo: MusicLibrary.BackgroundRepo
+
 config :music_library, MusicLibrary.ErrorRepo, priv: "priv/error_repo"
 
+config :music_library, MusicLibrary.BackgroundRepo, priv: "priv/background_repo"
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{config_env()}.exs"

config/dev.exs

@@ -14,6 +14,12 @@ config :music_library, MusicLibrary.ErrorRepo,
   stacktrace: true,
   show_sensitive_data_on_connection_error: true
 
+config :music_library, MusicLibrary.BackgroundRepo,
+  database: Path.expand("../data/music_library_background_dev.db", __DIR__),
+  pool_size: 5,
+  stacktrace: true,
+  show_sensitive_data_on_connection_error: true
+
 # For development, we disable any cache and enable
 # debugging and code reloading.
 #

config/runtime.exs

@@ -68,6 +68,19 @@ if config_env() == :prod do
     cache_size: -8000,
     pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5")
 
+  background_database_path =
+    System.get_env("BACKGROUND_DATABASE_PATH") ||
+      raise """
+      environment variable BACKGROUND_DATABASE_PATH is missing.
+      For example: /etc/music_library/music_library_background.db
+      """
+
+  config :music_library, MusicLibrary.BackgroundRepo,
+    database: background_database_path,
+    # 16MB * pool_size = base memory usage
+    cache_size: -8000,
+    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5")
+
   # The secret key base is used to sign/encrypt cookies and other secrets.
   # A default value is used in config/dev.exs and config/test.exs but you
   # want to use a different value for prod and you most likely don't want

config/test.exs

@@ -16,6 +16,11 @@ config :music_library, MusicLibrary.ErrorRepo,
   pool_size: 32,
   pool: Ecto.Adapters.SQL.Sandbox
 
+config :music_library, MusicLibrary.BackgroundRepo,
+  database: Path.expand("../data/music_library_background_test.db", __DIR__),
+  pool_size: 32,
+  pool: Ecto.Adapters.SQL.Sandbox
+
 # We don't run a server during test. If one is required,
 # you can enable the server option below.
 config :music_library, MusicLibraryWeb.Endpoint,
@@ -51,3 +56,5 @@ config :music_library, MusicBrainz,
 config :phoenix_test, :endpoint, MusicLibraryWeb.Endpoint
 
 config :floki, :html_parser, Floki.HTMLParser.FastHtml
+
+config :music_library, Oban, testing: :manual

fly.toml

@@ -12,6 +12,7 @@ kill_signal = 'SIGTERM'
 [env]
 DATABASE_PATH = '/mnt/music_library/music_library_prod.db'
 ERROR_DATABASE_PATH = '/mnt/music_library/music_library_prod_errors.db'
+BACKGROUND_DATABASE_PATH = '/mnt/music_library/music_library_prod_background.db'
 PHX_HOST = 'music-library.claudio-ortolina.org'
 PORT = '8080'
 LAST_FM_USER = 'cloud8421'

lib/music_library/application.ex

@@ -11,6 +11,8 @@ defmodule MusicLibrary.Application do
       MusicLibraryWeb.Telemetry,
       MusicLibrary.Repo,
       MusicLibrary.ErrorRepo,
+      MusicLibrary.BackgroundRepo,
+      {Oban, Application.fetch_env!(:music_library, Oban)},
       {Ecto.Migrator,
        repos: Application.fetch_env!(:music_library, :ecto_repos), skip: skip_migrations?()},
       {DNSCluster, query: Application.get_env(:music_library, :dns_cluster_query) || :ignore},

lib/music_library/background_repo.ex

@@ -0,0 +1,5 @@
+defmodule MusicLibrary.BackgroundRepo do
+  use Ecto.Repo,
+    otp_app: :music_library,
+    adapter: Ecto.Adapters.SQLite3
+end

mix.exs

@@ -86,6 +86,9 @@ defmodule MusicLibrary.MixProject do
       # Data validation
       {:nimble_options, "~> 1.1"},
 
+      # Background Jobs
+      {:oban, "~> 2.19"},
+
       # Test tooling
       {:phoenix_test, "~> 0.6.0", only: :test},
       {:floki, ">= 0.30.0", only: :test},

mix.lock

@@ -30,6 +30,7 @@
   "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"},
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
+  "oban": {:hex, :oban, "2.19.4", "045adb10db1161dceb75c254782f97cdc6596e7044af456a59decb6d06da73c1", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5fcc6219e6464525b808d97add17896e724131f498444a292071bf8991c99f97"},
   "phoenix": {:hex, :phoenix, "1.7.21", "14ca4f1071a5f65121217d6b57ac5712d1857e40a0833aff7a691b7870fc9a3b", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "336dce4f86cba56fed312a7d280bf2282c720abb6074bdb1b61ec8095bdd0bc9"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"},
   "phoenix_html": {:hex, :phoenix_html, "4.2.1", "35279e2a39140068fc03f8874408d58eef734e488fc142153f055c5454fd1c08", [:mix], [], "hexpm", "cff108100ae2715dd959ae8f2a8cef8e20b593f8dfd031c9cba92702cf23e053"},

priv/background_repo/migrations/20250413170433_create_jobs_table.exs

@@ -0,0 +1,13 @@
+defmodule MusicLibrary.BackgroundRepo.Migrations.CreateJobsTable do
+  use Ecto.Migration
+
+  def up do
+    Oban.Migration.up(version: 12)
+  end
+
+  # We specify `version: 1` in `down`, ensuring that we'll roll all the way back down if
+  # necessary, regardless of which version we've migrated `up` to.
+  def down do
+    Oban.Migration.down(version: 1)
+  end
+end