From 0295e58e57d68c692027ec61e0f36d905620810e Mon Sep 17 00:00:00 2001 From: Claudio Ortolina Date: Wed, 25 Mar 2026 11:38:17 +0000 Subject: [PATCH] Add rate limiting to Wikipedia and BraveSearch APIs Closes #133 --- config/config.exs | 7 +++++-- config/test.exs | 6 ++++-- docs/production-infrastructure.md | 4 ++-- lib/brave_search/api.ex | 1 + lib/brave_search/config.ex | 11 +++++++++-- lib/wikipedia/api.ex | 3 +++ lib/wikipedia/config.ex | 11 +++++++++-- 7 files changed, 33 insertions(+), 10 deletions(-) diff --git a/config/config.exs b/config/config.exs index e1894c39..e6fe950e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -68,9 +68,12 @@ config :music_library, Discogs, user_agent: user_agent, api_cooldown: 1000 -config :music_library, Wikipedia, user_agent: user_agent +config :music_library, Wikipedia, user_agent: user_agent, api_cooldown: 1000 -config :music_library, BraveSearch, api_key: "change me", user_agent: user_agent +config :music_library, BraveSearch, + api_key: "change me", + user_agent: user_agent, + api_cooldown: 1000 config :music_library, OpenAI, api_key: "change me" diff --git a/config/test.exs b/config/test.exs index 77ba0b02..a8fe55e0 100644 --- a/config/test.exs +++ b/config/test.exs @@ -79,14 +79,16 @@ config :music_library, Wikipedia, req_options: [ plug: {Req.Test, Wikipedia.API}, max_retries: 0 - ] + ], + api_cooldown: 0 config :music_library, BraveSearch, api_key: "test_key", req_options: [ plug: {Req.Test, BraveSearch.API}, max_retries: 0 - ] + ], + api_cooldown: 0 config :music_library, OpenAI, api_key: "test_key", diff --git a/docs/production-infrastructure.md b/docs/production-infrastructure.md index 654586e0..cefa186e 100644 --- a/docs/production-infrastructure.md +++ b/docs/production-infrastructure.md @@ -229,8 +229,8 @@ All HTTP clients use `Req` with per-API rate limiting (`Req.RateLimiter`, ETS-ba | MusicBrainz | 500 ms cooldown | Release/artist metadata, search | | Last.fm | 500 ms cooldown | Scrobbling, listening history, artist tags | | Discogs | 1000 ms cooldown | Artist profiles, images | -| Wikipedia | — | Artist biographies | -| Brave Search | — | Cover art, artist image search | +| Wikipedia | 1000 ms cooldown | Artist biographies | +| Brave Search | 1000 ms cooldown | Cover art, artist image search | | OpenAI | — | Text embeddings (similarity), streaming chat | --- diff --git a/lib/brave_search/api.ex b/lib/brave_search/api.ex index 559018a5..5fa4f4f0 100644 --- a/lib/brave_search/api.ex +++ b/lib/brave_search/api.ex @@ -57,6 +57,7 @@ defmodule BraveSearch.API do headers: %{"x-subscription-token" => config.api_key} ) |> Req.Request.merge_options(config.req_options) + |> Req.RateLimiter.attach(name: :brave_search, cooldown: config.api_cooldown) |> Req.Request.append_request_steps(log_attempt: &log_attempt/1) |> Req.Request.append_response_steps(log_error: &log_error/1) end diff --git a/lib/brave_search/config.ex b/lib/brave_search/config.ex index 0748ce4a..0901db59 100644 --- a/lib/brave_search/config.ex +++ b/lib/brave_search/config.ex @@ -2,13 +2,15 @@ defmodule BraveSearch.Config do @type t :: %__MODULE__{ api_key: String.t(), user_agent: String.t(), - req_options: Keyword.t() + req_options: Keyword.t(), + api_cooldown: non_neg_integer() } @enforce_keys [:api_key] defstruct api_key: "", user_agent: "change me", - req_options: [] + req_options: [], + api_cooldown: 1000 @schema NimbleOptions.new!( api_key: [ @@ -24,6 +26,11 @@ defmodule BraveSearch.Config do type: :keyword_list, required: false, default: [] + ], + api_cooldown: [ + type: :integer, + required: false, + default: 1000 ] ) diff --git a/lib/wikipedia/api.ex b/lib/wikipedia/api.ex index 627301f9..d1866147 100644 --- a/lib/wikipedia/api.ex +++ b/lib/wikipedia/api.ex @@ -15,6 +15,7 @@ defmodule Wikipedia.API do user_agent: config.user_agent ) |> Req.Request.merge_options(config.req_options) + |> Req.RateLimiter.attach(name: :wikipedia, cooldown: config.api_cooldown) |> Req.Request.append_request_steps(log_attempt: &log_attempt/1) |> Req.Request.append_response_steps(log_error: &log_error/1) |> Req.merge( @@ -52,6 +53,7 @@ defmodule Wikipedia.API do user_agent: config.user_agent ) |> Req.Request.merge_options(config.req_options) + |> Req.RateLimiter.attach(name: :wikipedia, cooldown: config.api_cooldown) |> Req.Request.append_request_steps(log_attempt: &log_attempt/1) |> Req.Request.append_response_steps(log_error: &log_error/1) |> Req.merge(url: "/api/rest_v1/page/summary/#{URI.encode(title)}") @@ -78,6 +80,7 @@ defmodule Wikipedia.API do user_agent: config.user_agent ) |> Req.Request.merge_options(config.req_options) + |> Req.RateLimiter.attach(name: :wikipedia, cooldown: config.api_cooldown) |> Req.Request.append_request_steps(log_attempt: &log_attempt/1) |> Req.Request.append_response_steps(log_error: &log_error/1) |> Req.merge( diff --git a/lib/wikipedia/config.ex b/lib/wikipedia/config.ex index f361e8d4..b0467dc7 100644 --- a/lib/wikipedia/config.ex +++ b/lib/wikipedia/config.ex @@ -1,11 +1,13 @@ defmodule Wikipedia.Config do @type t :: %__MODULE__{ user_agent: String.t(), - req_options: Keyword.t() + req_options: Keyword.t(), + api_cooldown: non_neg_integer() } defstruct user_agent: "change me", - req_options: [] + req_options: [], + api_cooldown: 1000 @schema NimbleOptions.new!( user_agent: [ @@ -17,6 +19,11 @@ defmodule Wikipedia.Config do type: :keyword_list, required: false, default: [] + ], + api_cooldown: [ + type: :integer, + required: false, + default: 1000 ] )