Introduce a Req.RateLimiter module to fix API rate limit issues

lib/discogs.ex

@@ -13,7 +13,5 @@ defmodule Discogs do
     API.get_artist_image(url, discogs_config)
   end
 
-  def api_cooldown, do: discogs_config().api_cooldown
-
   defp discogs_config, do: Discogs.Config.resolve(:music_library)
 end

lib/discogs/api.ex

@@ -34,6 +34,7 @@ defmodule Discogs.API do
       auth: "Discogs token=#{config.personal_access_token}"
     )
     |> Req.Request.merge_options(config.req_options)
+    |> Req.RateLimiter.attach(name: :discogs, cooldown: config.api_cooldown)
     |> Req.Request.append_request_steps(log_attempt: &log_attempt/1)
     |> Req.Request.append_response_steps(log_error: &log_error/1)
   end

lib/last_fm.ex

@@ -93,7 +93,5 @@ defmodule LastFm do
     "https://www.last.fm/api/auth/?api_key=" <> last_fm_config.api_key
   end
 
-  def api_cooldown, do: last_fm_config().api_cooldown
-
   defp last_fm_config, do: LastFm.Config.resolve(:music_library)
 end

lib/last_fm/api.ex

@@ -146,6 +146,7 @@ defmodule LastFm.API do
       user_agent: config.user_agent
     )
     |> Req.Request.merge_options(config.req_options)
+    |> Req.RateLimiter.attach(name: :last_fm, cooldown: config.api_cooldown)
     |> Req.Request.put_private(:api_key, config.api_key)
     |> Req.Request.append_request_steps(log_attempt: &log_attempt/1)
     |> Req.Request.append_response_steps(parse_error: &parse_error/1)

lib/music_brainz.ex

@@ -36,7 +36,5 @@ defmodule MusicBrainz do
     API.get_artist(musicbrainz_id, music_brainz_config())
   end
 
-  def api_cooldown, do: music_brainz_config().api_cooldown
-
   defp music_brainz_config, do: MusicBrainz.Config.resolve(:music_library)
 end

lib/music_brainz/api.ex

@@ -492,6 +492,7 @@ defmodule MusicBrainz.API do
       user_agent: config.user_agent
     )
     |> Req.Request.merge_options(config.req_options)
+    |> Req.RateLimiter.attach(name: :music_brainz, cooldown: config.api_cooldown)
     |> Req.Request.append_request_steps(log_attempt: &log_attempt/1)
   end
 

lib/music_library/application.ex

@@ -10,6 +10,7 @@ defmodule MusicLibrary.Application do
   @impl true
   def start(_type, _args) do
     _ = Assets.Cache.new()
+    _ = Req.RateLimiter.new()
 
     children = [
       MusicLibrary.Vault,

lib/music_library/worker/artist_refresh_discogs_data.ex

@@ -3,10 +3,6 @@ defmodule MusicLibrary.Worker.ArtistRefreshDiscogsData do
 
   @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => artist_info_id}}) do
-    result = MusicLibrary.Artists.refresh_discogs_data(artist_info_id)
-
-    Process.sleep(Discogs.api_cooldown())
-
-    result
+    MusicLibrary.Artists.refresh_discogs_data(artist_info_id)
   end
 end

lib/music_library/worker/artist_refresh_music_brainz_data.ex

@@ -3,10 +3,6 @@ defmodule MusicLibrary.Worker.ArtistRefreshMusicBrainzData do
 
   @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => artist_info_id}}) do
-    result = MusicLibrary.Artists.refresh_musicbrainz_data(artist_info_id)
-
-    Process.sleep(MusicBrainz.api_cooldown())
-
-    result
+    MusicLibrary.Artists.refresh_musicbrainz_data(artist_info_id)
   end
 end

lib/music_library/worker/fetch_artist_last_fm_data.ex

@@ -3,14 +3,9 @@ defmodule MusicLibrary.Worker.FetchArtistLastFmData do
 
   @impl Oban.Worker
   def perform(%Oban.Job{args: %{"id" => artist_id}}) do
-    result =
-      case MusicLibrary.Artists.fetch_lastfm_data(artist_id) do
-        {:ok, _artist_info} -> :ok
-        error -> error
-      end
-
-    Process.sleep(LastFm.api_cooldown())
-
-    result
+    case MusicLibrary.Artists.fetch_lastfm_data(artist_id) do
+      {:ok, _artist_info} -> :ok
+      error -> error
+    end
   end
 end

lib/music_library/worker/record_refresh_music_brainz_data.ex

@@ -5,13 +5,8 @@ defmodule MusicLibrary.Worker.RecordRefreshMusicBrainzData do
   def perform(%Oban.Job{args: %{"id" => record_id}}) do
     record = MusicLibrary.Records.get_record!(record_id)
 
-    result =
-      with {:ok, updated_record} <- MusicLibrary.Records.refresh_musicbrainz_data(record) do
-        MusicLibrary.Records.notify_update(updated_record)
-      end
-
-    Process.sleep(MusicBrainz.api_cooldown())
-
-    result
+    with {:ok, updated_record} <- MusicLibrary.Records.refresh_musicbrainz_data(record) do
+      MusicLibrary.Records.notify_update(updated_record)
+    end
   end
 end

lib/req/rate_limiter.ex

@@ -0,0 +1,71 @@
+defmodule Req.RateLimiter do
+  @moduledoc """
+  A Req request step that enforces minimum intervals between requests per API.
+
+  Uses an ETS table to track the last request timestamp for each named API.
+  When a request is made before the cooldown has elapsed, the calling process
+  sleeps for the remaining time.
+
+  ## Usage
+
+      Req.new(...)
+      |> Req.RateLimiter.attach(name: :music_brainz, cooldown: 500)
+
+  When `cooldown` is 0, the step is a no-op.
+  """
+
+  @table __MODULE__
+
+  @doc """
+  Creates the ETS table used to track request timestamps.
+  Call once at application startup.
+  """
+  def new do
+    :ets.new(@table, [:set, :public, :named_table])
+  end
+
+  @doc """
+  Attaches the rate limiter as a request step on the given Req request.
+
+  ## Options
+
+    * `:name` - atom identifying the API (e.g. `:music_brainz`)
+    * `:cooldown` - minimum milliseconds between requests
+
+  """
+  def attach(request, opts) do
+    name = Keyword.fetch!(opts, :name)
+    cooldown = Keyword.fetch!(opts, :cooldown)
+
+    request
+    |> Req.Request.put_private(:rate_limiter_name, name)
+    |> Req.Request.put_private(:rate_limiter_cooldown, cooldown)
+    |> Req.Request.prepend_request_steps(rate_limiter: &throttle/1)
+  end
+
+  defp throttle(request) do
+    cooldown = Req.Request.get_private(request, :rate_limiter_cooldown)
+
+    if cooldown > 0 do
+      name = Req.Request.get_private(request, :rate_limiter_name)
+      now = System.monotonic_time(:millisecond)
+
+      case :ets.lookup(@table, name) do
+        [{^name, last_at}] ->
+          elapsed = now - last_at
+          remaining = cooldown - elapsed
+
+          if remaining > 0 do
+            Process.sleep(remaining)
+          end
+
+        [] ->
+          :ok
+      end
+
+      :ets.insert(@table, {name, System.monotonic_time(:millisecond)})
+    end
+
+    request
+  end
+end