cloud/music_library
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ML-21: Classify API errors as transient vs permanent

Browse Source
This commit is contained in:
Claudio Ortolina
2026-04-24 13:55:00 +01:00
parent be4f4686f8
commit a1c665b490
43 changed files with 1429 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/brave_search/api.ex
+24 -6
View File
@@ -3,12 +3,13 @@ defmodule BraveSearch.API do
Interface to the Brave Search API.
"""
alias BraveSearch.API.ErrorResponse
alias Req.Request
require Logger
@spec search_images(String.t(), keyword(), BraveSearch.Config.t()) ::
{:ok, [map()]} | {:error, term()}
{:ok, [map()]} | {:error, ErrorResponse.t() | Exception.t()}
def search_images(query, opts, config) do
params = [q: query, count: Keyword.get(opts, :count, 20)]
@@ -61,16 +62,20 @@ defmodule BraveSearch.API do
|> Request.merge_options(config.req_options)
|> Req.RateLimiter.attach(name: :brave_search, cooldown: config.api_cooldown)
|> Request.append_request_steps(log_attempt: &log_attempt/1)
|> Request.append_response_steps(log_error: &log_error/1)
|> Request.append_response_steps(parse_error: &parse_error/1)
end
defp get_request(request) do
case Req.get(request) do
{:ok, response} when response.status == 200 ->
{:ok, response.body}
{:ok, %{status: status, body: body}} when status in 200..299 ->
{:ok, body}
{:ok, response} ->
{:error, response.body}
{:ok, %{body: %ErrorResponse{} = error}} ->
{:error, error}
# Image download path does not attach parse_error; fall back to raw body.
{:ok, %{body: body}} ->
{:error, body}
error ->
error
@@ -93,4 +98,17 @@ defmodule BraveSearch.API do
{request, response}
end
defp parse_error({request, %{status: status} = response}) when status not in 200..299 do
error = ErrorResponse.from_response(response)
Logger.error(fn ->
url = URI.to_string(request.url)
"Failed to fetch data from #{url}, status: #{status}, reason: #{inspect(response.body)}"
end)
Request.halt(request, %{response | body: error})
end
defp parse_error(tuple), do: tuple
end
lib/brave_search/api/error_response.ex
+60
View File
@@ -0,0 +1,60 @@
defmodule BraveSearch.API.ErrorResponse do
@moduledoc """
Structured error response for Brave Search API calls.
Brave Search returns a consistent JSON envelope on errors:
%{"type" => "ErrorResponse",
"error" => %{"status" => 422, "code" => "SUBSCRIPTION_TOKEN_INVALID",
"detail" => "...", "meta" => %{...}}}
HTTP status codes are the primary classifier (429 for rate limit, 5xx for
server errors). Brave uses 422 for most validation failures including
authentication errors that other APIs would return as 401.
"""
@behaviour MusicLibrary.ErrorResponse
alias MusicLibrary.HttpError
@type t :: %__MODULE__{
status: integer() | nil,
code: String.t() | nil,
message: String.t() | nil,
kind: HttpError.kind(),
body: term()
}
defstruct [:status, :code, :message, :kind, :body]
@spec from_response(Req.Response.t() | map()) :: t()
def from_response(%{status: status, body: body} = _response) do
%__MODULE__{
status: status,
code: extract_code(body),
message: extract_message(body),
kind: HttpError.default_kind(status),
body: body
}
end
@impl MusicLibrary.ErrorResponse
@spec retryable?(t()) :: boolean()
def retryable?(%__MODULE__{kind: kind}) when kind in [:rate_limit, :server_error, :timeout],
do: true
def retryable?(%__MODULE__{}), do: false
@impl MusicLibrary.ErrorResponse
@spec retry_delay_seconds(t()) :: pos_integer()
def retry_delay_seconds(%__MODULE__{kind: :rate_limit}), do: 60
def retry_delay_seconds(%__MODULE__{kind: :server_error}), do: 30
def retry_delay_seconds(%__MODULE__{kind: :timeout}), do: 10
def retry_delay_seconds(%__MODULE__{}), do: 30
defp extract_code(%{"error" => %{"code" => code}}) when is_binary(code), do: code
defp extract_code(_), do: nil
defp extract_message(%{"error" => %{"detail" => msg}}) when is_binary(msg), do: msg
defp extract_message(_), do: nil
end
lib/discogs/api.ex
+25 -7
View File
@@ -3,11 +3,13 @@ defmodule Discogs.API do
Interface to the Discogs API.
"""
alias Discogs.API.ErrorResponse
alias Req.Request
require Logger
@spec get_artist(integer() | String.t(), Discogs.Config.t()) :: {:ok, map()} | {:error, term()}
@spec get_artist(integer() | String.t(), Discogs.Config.t()) ::
{:ok, map()} | {:error, ErrorResponse.t() | Exception.t()}
def get_artist(id, config) do
config
|> new_request()
@@ -41,17 +43,20 @@ defmodule Discogs.API do
|> Request.merge_options(config.req_options)
|> Req.RateLimiter.attach(name: :discogs, cooldown: config.api_cooldown)
|> Request.append_request_steps(log_attempt: &log_attempt/1)
|> Request.append_response_steps(log_error: &log_error/1)
|> Request.append_response_steps(parse_error: &parse_error/1)
end
defp get_request(request) do
case Req.get(request) do
{:ok, response} when response.status == 200 ->
{:ok, response.body}
{:ok, %{status: status, body: body}} when status in 200..299 ->
{:ok, body}
# all non-success responses can be treated as errors
{:ok, response} ->
{:error, response.body}
{:ok, %{body: %ErrorResponse{} = error}} ->
{:error, error}
# Image download path does not attach parse_error; fall back to raw body.
{:ok, %{body: body}} ->
{:error, body}
error ->
error
@@ -74,4 +79,17 @@ defmodule Discogs.API do
{request, response}
end
defp parse_error({request, %{status: status} = response}) when status not in 200..299 do
error = ErrorResponse.from_response(response)
Logger.error(fn ->
url = URI.to_string(request.url)
"Failed to fetch data from #{url}, status: #{status}, reason: #{inspect(response.body)}"
end)
Request.halt(request, %{response | body: error})
end
defp parse_error(tuple), do: tuple
end
lib/discogs/api/error_response.ex
+48
View File
@@ -0,0 +1,48 @@
defmodule Discogs.API.ErrorResponse do
@moduledoc """
Structured error response for Discogs API calls.
Discogs uses HTTP status codes as the error channel. Error bodies are JSON with
a single `"message"` field and no application-level error codes.
"""
@behaviour MusicLibrary.ErrorResponse
alias MusicLibrary.HttpError
@type t :: %__MODULE__{
status: integer() | nil,
message: String.t() | nil,
kind: HttpError.kind(),
body: term()
}
defstruct [:status, :message, :kind, :body]
@spec from_response(Req.Response.t() | map()) :: t()
def from_response(%{status: status, body: body} = _response) do
%__MODULE__{
status: status,
message: extract_message(body),
kind: HttpError.default_kind(status),
body: body
}
end
@impl MusicLibrary.ErrorResponse
@spec retryable?(t()) :: boolean()
def retryable?(%__MODULE__{kind: kind}) when kind in [:rate_limit, :server_error, :timeout],
do: true
def retryable?(%__MODULE__{}), do: false
@impl MusicLibrary.ErrorResponse
@spec retry_delay_seconds(t()) :: pos_integer()
def retry_delay_seconds(%__MODULE__{kind: :rate_limit}), do: 60
def retry_delay_seconds(%__MODULE__{kind: :server_error}), do: 30
def retry_delay_seconds(%__MODULE__{kind: :timeout}), do: 10
def retry_delay_seconds(%__MODULE__{}), do: 30
defp extract_message(%{"message" => message}) when is_binary(message), do: message
defp extract_message(_), do: nil
end
lib/last_fm/api/error_response.ex
+39
View File
@@ -1,4 +1,19 @@
defmodule LastFm.API.ErrorResponse do
@moduledoc """
Structured error response for Last.fm API calls.
Last.fm is unusual among the APIs this project consumes: it returns HTTP 200
with a `%{"error" => N, "message" => "..."}` JSON body on failure, carrying a
numeric application-level error code in the range 2–29. Those codes map to
the atoms declared in `t:error_atom/0`.
The struct-based helpers (`retryable?/1`, `retry_delay_seconds/1`) exist to
share a uniform protocol with the per-HTTP-status `ErrorResponse` modules
used by the other APIs — see `MusicLibrary.Worker.ErrorHandler`.
"""
@behaviour MusicLibrary.ErrorResponse
defstruct [:error, :message]
@type error_atom ::
@@ -66,4 +81,28 @@ defmodule LastFm.API.ErrorResponse do
# 5 seconds
def retry_delay(:operation_failed), do: 5_000
def retry_delay(_), do: nil
@doc """
Struct-based retryability predicate shared with other API `ErrorResponse` modules.
Enables `MusicLibrary.Worker.ErrorHandler.to_oban_result/1` to treat Last.fm
uniformly alongside HTTP-status-based errors without requiring callers to
unwrap the atom first.
"""
@impl MusicLibrary.ErrorResponse
@spec retryable?(t()) :: boolean()
def retryable?(%__MODULE__{error: error}), do: retryable_error?(error)
@doc """
Struct-based retry delay in seconds. Falls back to 30 s when the underlying
atom has no specific delay (see `retry_delay/1`).
"""
@impl MusicLibrary.ErrorResponse
@spec retry_delay_seconds(t()) :: pos_integer()
def retry_delay_seconds(%__MODULE__{error: error}) do
case retry_delay(error) do
ms when is_integer(ms) -> div(ms, 1000)
nil -> 30
end
end
end
lib/music_brainz/api.ex
+34 -11
View File
@@ -6,6 +6,7 @@ defmodule MusicBrainz.API do
- Extend the metadata associated with existing records
"""
alias MusicBrainz.API.ErrorResponse
alias MusicBrainz.{Artist, ReleaseGroupSearchResult, ReleaseSearchResult}
alias Req.Request
@@ -207,7 +208,8 @@ defmodule MusicBrainz.API do
}
"""
@spec get_release_group(String.t(), MusicBrainz.Config.t()) :: {:ok, map()} | {:error, term()}
@spec get_release_group(String.t(), MusicBrainz.Config.t()) ::
{:ok, map()} | {:error, ErrorResponse.t() | Exception.t()}
def get_release_group(id, config) do
config
|> new_request()
@@ -284,7 +286,8 @@ defmodule MusicBrainz.API do
"title": "Clark (Soundtrack From the Netflix Series)"
}
"""
@spec get_release(String.t(), MusicBrainz.Config.t()) :: {:ok, map()} | {:error, term()}
@spec get_release(String.t(), MusicBrainz.Config.t()) ::
{:ok, map()} | {:error, ErrorResponse.t() | Exception.t()}
def get_release(id, config) do
config
|> new_request()
@@ -299,7 +302,7 @@ defmodule MusicBrainz.API do
end
@spec get_releases(String.t(), keyword(), MusicBrainz.Config.t()) ::
{:ok, map()} | {:error, term()}
{:ok, map()} | {:error, ErrorResponse.t() | Exception.t()}
def get_releases(release_group_id, opts, config) do
Keyword.validate!(opts, [:limit, :offset])
@@ -320,7 +323,7 @@ defmodule MusicBrainz.API do
end
@spec search_release_by_barcode(String.t(), MusicBrainz.Config.t()) ::
{:ok, [ReleaseSearchResult.t()]} | {:error, term()}
{:ok, [ReleaseSearchResult.t()]} | {:error, ErrorResponse.t() | Exception.t()}
def search_release_by_barcode(barcode, config) do
config
|> new_request()
@@ -439,7 +442,7 @@ defmodule MusicBrainz.API do
"""
@spec search_release_group(String.t(), keyword(), MusicBrainz.Config.t()) ::
{:ok, %{total_count: non_neg_integer(), release_groups: [ReleaseGroupSearchResult.t()]}}
| {:error, term()}
| {:error, ErrorResponse.t() | Exception.t()}
def search_release_group(query, opts, config) do
Keyword.validate!(opts, [:limit, :offset])
@@ -461,7 +464,8 @@ defmodule MusicBrainz.API do
|> get_request()
end
@spec get_artist(String.t(), MusicBrainz.Config.t()) :: {:ok, Artist.t()} | {:error, term()}
@spec get_artist(String.t(), MusicBrainz.Config.t()) ::
{:ok, Artist.t()} | {:error, ErrorResponse.t() | Exception.t()}
def get_artist(musicbrainz_id, config) do
config
|> new_request()
@@ -507,22 +511,41 @@ defmodule MusicBrainz.API do
|> Request.merge_options(config.req_options)
|> Req.RateLimiter.attach(name: :music_brainz, cooldown: config.api_cooldown)
|> Request.append_request_steps(log_attempt: &log_attempt/1)
|> Request.append_response_steps(parse_error: &parse_error/1)
end
defp get_request(request) do
case Req.get(request) do
{:ok, response} when response.status == 200 ->
{:ok, response.body}
{:ok, %{status: status, body: body}} when status in 200..299 ->
{:ok, body}
# all non-success responses can be treated as errors
{:ok, response} ->
{:error, response.body}
{:ok, %{body: %ErrorResponse{} = error}} ->
{:error, error}
# Fallback for pipelines without parse_error (e.g. cover-art binary path)
# or Req exhausting retries before the step runs. Callers like
# get_cover_art/2 normalise this further.
{:ok, %{body: body}} ->
{:error, body}
error ->
error
end
end
defp parse_error({request, %{status: status} = response}) when status not in 200..299 do
error = ErrorResponse.from_response(response)
Logger.error(fn ->
url = URI.to_string(request.url)
"Failed to fetch data from #{url}, status: #{status}, reason: #{inspect(response.body)}"
end)
Request.halt(request, %{response | body: error})
end
defp parse_error(tuple), do: tuple
defp log_attempt(request) do
url = URI.to_string(request.url)
Logger.debug("Fetching data from #{url}")
lib/music_brainz/api/error_response.ex
+64
View File
@@ -0,0 +1,64 @@
defmodule MusicBrainz.API.ErrorResponse do
@moduledoc """
Structured error response for MusicBrainz API calls.
MusicBrainz uses classic HTTP status codes as the error channel. The body is a
flat JSON `{"error": "message"}` with no numeric application codes.
## Rate limiting
MusicBrainz signals rate limiting with **HTTP 503**, not 429. The service does
not use 429 at all — a 503 response with a `Retry-After` header is the rate
limit signal. `from_response/1` therefore maps 503 to `:rate_limit` (not the
generic `:server_error` classification from `MusicLibrary.HttpError`).
"""
@behaviour MusicLibrary.ErrorResponse
alias MusicLibrary.HttpError
@type t :: %__MODULE__{
status: integer() | nil,
message: String.t() | nil,
kind: HttpError.kind(),
body: term()
}
defstruct [:status, :message, :kind, :body]
@spec from_response(Req.Response.t() | map()) :: t()
def from_response(%{status: 503, body: body} = _response) do
%__MODULE__{
status: 503,
message: extract_message(body),
kind: :rate_limit,
body: body
}
end
def from_response(%{status: status, body: body} = _response) do
%__MODULE__{
status: status,
message: extract_message(body),
kind: HttpError.default_kind(status),
body: body
}
end
@impl MusicLibrary.ErrorResponse
@spec retryable?(t()) :: boolean()
def retryable?(%__MODULE__{kind: kind}) when kind in [:rate_limit, :server_error, :timeout],
do: true
def retryable?(%__MODULE__{}), do: false
@impl MusicLibrary.ErrorResponse
@spec retry_delay_seconds(t()) :: pos_integer()
def retry_delay_seconds(%__MODULE__{kind: :rate_limit}), do: 60
def retry_delay_seconds(%__MODULE__{kind: :server_error}), do: 30
def retry_delay_seconds(%__MODULE__{kind: :timeout}), do: 10
def retry_delay_seconds(%__MODULE__{}), do: 30
defp extract_message(%{"error" => message}) when is_binary(message), do: message
defp extract_message(_), do: nil
end
lib/music_library/error_response.ex
+23
View File
@@ -0,0 +1,23 @@
defmodule MusicLibrary.ErrorResponse do
@moduledoc """
Behaviour shared by all per-API `ErrorResponse` modules.
Each API (MusicBrainz, Discogs, Wikipedia, Brave Search, OpenAI, Last.fm)
returns a struct implementing this behaviour on HTTP failure, so
`MusicLibrary.Worker.ErrorHandler.to_oban_result/1` can dispatch uniformly
to produce the right Oban tuple (`{:snooze, n}` / `{:cancel, reason}`)
without needing to know which API raised the error.
"""
@doc """
Returns `true` if the error is transient and the worker should retry after
a delay; `false` if the error is permanent and the worker should cancel.
"""
@callback retryable?(struct()) :: boolean()
@doc """
Returns the snooze delay in seconds for retryable errors. Implementations
may return any positive integer; common defaults are 30–60 s.
"""
@callback retry_delay_seconds(struct()) :: pos_integer()
end
lib/music_library/http_error.ex
+37
View File
@@ -0,0 +1,37 @@
defmodule MusicLibrary.HttpError do
@moduledoc """
Default HTTP status → error kind mapping shared by per-API `ErrorResponse` modules.
Each API's `ErrorResponse.from_response/1` uses this as a baseline before applying
API-specific overrides (e.g. MusicBrainz treats 503 as a rate limit, not a server
error; OpenAI splits HTTP 429 into `:rate_limit` vs `:auth_error` based on the
body `code`).
## Kinds
* `:rate_limit` — back off and retry (transient)
* `:server_error` — retry with backoff (transient)
* `:timeout` — retry with shorter backoff (transient)
* `:auth_error` — permanent until credentials change
* `:not_found` — permanent
* `:client_error` — permanent (malformed request)
* `:unknown` — unclassified; treated as permanent
"""
@type kind ::
:rate_limit
| :server_error
| :timeout
| :auth_error
| :not_found
| :client_error
| :unknown
@spec default_kind(integer()) :: kind()
def default_kind(429), do: :rate_limit
def default_kind(status) when status in 500..599, do: :server_error
def default_kind(status) when status in [401, 403], do: :auth_error
def default_kind(404), do: :not_found
def default_kind(status) when status in 400..499, do: :client_error
def default_kind(_), do: :unknown
end
lib/music_library/worker/artist_refresh_discogs_data.ex
+7 -1
View File
@@ -1,8 +1,14 @@
defmodule MusicLibrary.Worker.ArtistRefreshDiscogsData do
@moduledoc false
use Oban.Worker, queue: :discogs, max_attempts: 3
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => artist_info_id}}) do
MusicLibrary.Artists.refresh_discogs_data(artist_info_id)
artist_info_id
|> MusicLibrary.Artists.refresh_discogs_data()
|> ErrorHandler.to_oban_result()
end
end
lib/music_library/worker/artist_refresh_music_brainz_data.ex
+7 -1
View File
@@ -1,8 +1,14 @@
defmodule MusicLibrary.Worker.ArtistRefreshMusicBrainzData do
@moduledoc false
use Oban.Worker, queue: :music_brainz, max_attempts: 3
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => artist_info_id}}) do
MusicLibrary.Artists.refresh_musicbrainz_data(artist_info_id)
artist_info_id
|> MusicLibrary.Artists.refresh_musicbrainz_data()
|> ErrorHandler.to_oban_result()
end
end
lib/music_library/worker/artist_refresh_wikipedia_data.ex
+7 -3
View File
@@ -1,11 +1,15 @@
defmodule MusicLibrary.Worker.ArtistRefreshWikipediaData do
@moduledoc false
use Oban.Worker, queue: :wikipedia, max_attempts: 3
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => artist_info_id}}) do
with {:error, :no_english_wikipedia} <-
MusicLibrary.Artists.refresh_wikipedia_data(artist_info_id) do
{:cancel, :no_english_wikipedia}
case MusicLibrary.Artists.refresh_wikipedia_data(artist_info_id) do
{:error, :no_english_wikipedia} -> {:cancel, :no_english_wikipedia}
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/error_handler.ex
+49
View File
@@ -0,0 +1,49 @@
defmodule MusicLibrary.Worker.ErrorHandler do
@moduledoc """
Converts context-layer results into Oban worker return values.
Workers call external APIs that return `{:error, %ErrorResponse{}}` (per-API
structs) on HTTP failures. This helper recognises any of the known error
response structs and translates them into the correct Oban tuple:
* retryable errors → `{:snooze, seconds}` so the attempt isn't consumed
* non-retryable errors → `{:cancel, reason}` so Oban stops retrying
* unknown `{:error, reason}` → passed through for Oban's default backoff
Workers that have app-layer atom-cancel reasons (e.g. `:no_english_wikipedia`,
`:cover_not_available`) must match those **before** calling this helper, since
atoms fall through to the generic `{:error, reason}` branch here.
"""
@error_structs [
LastFm.API.ErrorResponse,
MusicBrainz.API.ErrorResponse,
Discogs.API.ErrorResponse,
Wikipedia.API.ErrorResponse,
BraveSearch.API.ErrorResponse,
OpenAI.API.ErrorResponse
]
@type oban_result ::
:ok
| {:ok, term()}
| {:error, term()}
| {:cancel, term()}
| {:snooze, pos_integer()}
@spec to_oban_result(term()) :: oban_result()
def to_oban_result(:ok), do: :ok
def to_oban_result({:ok, _} = result), do: result
def to_oban_result({:error, %mod{} = response}) when mod in @error_structs do
if mod.retryable?(response) do
{:snooze, mod.retry_delay_seconds(response)}
else
{:cancel, response}
end
end
def to_oban_result({:error, reason}), do: {:error, reason}
def to_oban_result({:cancel, _} = result), do: result
def to_oban_result({:snooze, _} = result), do: result
end
lib/music_library/worker/fetch_artist_image.ex
+8 -11
View File
@@ -1,20 +1,17 @@
defmodule MusicLibrary.Worker.FetchArtistImage do
@moduledoc false
use Oban.Worker, queue: :heavy_writes, max_attempts: 3
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => artist_id}}) do
case MusicLibrary.Artists.refresh_image(artist_id) do
{:ok, _artist_info} ->
:ok
{:error, :image_not_found} ->
{:cancel, :image_not_found}
{:error, :no_discogs_data} ->
{:cancel, :no_discogs_data}
error ->
error
{:ok, _artist_info} -> :ok
{:error, :image_not_found} -> {:cancel, :image_not_found}
{:error, :no_discogs_data} -> {:cancel, :no_discogs_data}
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/fetch_artist_info.ex
+4 -1
View File
@@ -1,8 +1,11 @@
defmodule MusicLibrary.Worker.FetchArtistInfo do
@moduledoc false
use Oban.Worker, queue: :default, max_attempts: 3
alias MusicLibrary.Artists
alias MusicLibrary.Records.Similarity
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => artist_id}}) do
@@ -13,7 +16,7 @@ defmodule MusicLibrary.Worker.FetchArtistInfo do
Similarity.regenerate_artist_embeddings(artist_id)
else
{:error, :no_english_wikipedia} -> {:cancel, :no_english_wikipedia}
error -> error
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/fetch_artist_last_fm_data.ex
+5 -1
View File
@@ -1,11 +1,15 @@
defmodule MusicLibrary.Worker.FetchArtistLastFmData do
@moduledoc false
use Oban.Worker, queue: :last_fm, max_attempts: 3
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => artist_id}}) do
case MusicLibrary.Artists.refresh_lastfm_data(artist_id) do
{:ok, _artist_info} -> :ok
error -> error
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/generate_record_embedding.ex
+4 -1
View File
@@ -1,8 +1,11 @@
defmodule MusicLibrary.Worker.GenerateRecordEmbedding do
@moduledoc false
use Oban.Worker, queue: :heavy_writes, max_attempts: 3
alias MusicLibrary.Records
alias MusicLibrary.Records.Similarity
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"record_id" => record_id}}) do
@@ -11,7 +14,7 @@ defmodule MusicLibrary.Worker.GenerateRecordEmbedding do
case Similarity.generate_embedding(record) do
:noop -> :ok
{:ok, _} -> Records.notify_update(record)
{:error, _} = error -> error
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/import_from_musicbrainz_release.ex
+2 -1
View File
@@ -8,6 +8,7 @@ defmodule MusicLibrary.Worker.ImportFromMusicbrainzRelease do
use Oban.Worker, queue: :music_brainz, max_attempts: 3
alias MusicLibrary.Records.Record
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"release_id" => release_id} = args}) do
@@ -19,7 +20,7 @@ defmodule MusicLibrary.Worker.ImportFromMusicbrainzRelease do
case MusicLibrary.Records.import_from_musicbrainz_release(release_id, opts) do
{:ok, _record} -> :ok
{:error, reason} -> {:error, reason}
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/import_from_musicbrainz_release_group.ex
+2 -1
View File
@@ -13,6 +13,7 @@ defmodule MusicLibrary.Worker.ImportFromMusicbrainzReleaseGroup do
alias MusicLibrary.Records
alias MusicLibrary.Records.Record
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"release_group_id" => release_group_id} = args}) do
@@ -23,7 +24,7 @@ defmodule MusicLibrary.Worker.ImportFromMusicbrainzReleaseGroup do
case Records.import_from_musicbrainz_release_group(release_group_id, opts) do
{:ok, _record} -> :ok
{:error, reason} -> {:error, reason}
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/populate_genres.ex
+6 -2
View File
@@ -1,16 +1,20 @@
defmodule MusicLibrary.Worker.PopulateGenres do
@moduledoc false
use Oban.Worker, queue: :heavy_writes, max_attempts: 10
alias MusicLibrary.Records
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => record_id}}) do
record = Records.get_record!(record_id)
with {:ok, updated_record} <- Records.populate_genres(record),
{:ok, _worker} <-
Records.Similarity.generate_embedding_async(updated_record) do
{:ok, _worker} <- Records.Similarity.generate_embedding_async(updated_record) do
Records.notify_update(updated_record)
else
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/record_refresh_music_brainz_data.ex
+6 -2
View File
@@ -1,14 +1,18 @@
defmodule MusicLibrary.Worker.RecordRefreshMusicBrainzData do
@moduledoc false
use Oban.Worker, queue: :music_brainz, max_attempts: 3
alias MusicLibrary.Records
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => record_id}}) do
record = Records.get_record!(record_id)
with {:ok, updated_record} <- Records.refresh_musicbrainz_data(record) do
Records.notify_update(updated_record)
case Records.refresh_musicbrainz_data(record) do
{:ok, updated_record} -> Records.notify_update(updated_record)
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library/worker/refresh_cover.ex
+6 -8
View File
@@ -1,21 +1,19 @@
defmodule MusicLibrary.Worker.RefreshCover do
@moduledoc false
use Oban.Worker, queue: :heavy_writes, max_attempts: 3
alias MusicLibrary.Records
alias MusicLibrary.Worker.ErrorHandler
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => record_id}}) do
record = Records.get_record!(record_id)
case Records.refresh_cover(record) do
{:ok, updated_record} ->
Records.notify_update(updated_record)
{:error, :cover_not_available} ->
{:cancel, :cover_not_available}
error ->
error
{:ok, updated_record} -> Records.notify_update(updated_record)
{:error, :cover_not_available} -> {:cancel, :cover_not_available}
other -> ErrorHandler.to_oban_result(other)
end
end
end
lib/music_library_web/components/chat.ex
+1 -1
View File
@@ -399,7 +399,7 @@ defmodule MusicLibraryWeb.Components.Chat do
)
{:error, reason} ->
Logger.error("Chat streaming error: #{reason}")
Logger.error("Chat streaming error: #{inspect(reason)}")
LiveView.send_update(parent_pid, __MODULE__,
id: component_id,
lib/open_ai.ex
+1 -1
View File
@@ -17,7 +17,7 @@ defmodule OpenAI do
API.gpt(completion, config())
end
@spec chat_stream([map()], chat_stream_opts()) :: :ok | {:error, String.t()}
@spec chat_stream([map()], chat_stream_opts()) :: :ok | {:error, term()}
def chat_stream(messages, opts \\ []) when is_list(messages) do
model = Keyword.get(opts, :model, "gpt-4.1")
temperature = Keyword.get(opts, :temperature, 0.7)
lib/open_ai/api.ex
+37 -25
View File
@@ -1,13 +1,23 @@
defmodule OpenAI.API do
@moduledoc """
Low-level HTTP client for the OpenAI API (chat completions, streaming responses, embeddings).
HTTP errors are returned as `OpenAI.API.ErrorResponse` structs, which classify
rate-limit failures (retryable) separately from billing-quota failures
(`insufficient_quota` — non-retryable) despite both using HTTP 429.
Mid-stream failures in `chat_stream/6` come from parsed SSE event payloads
rather than HTTP responses and remain as message strings passed through the
stream callback plumbing.
"""
alias OpenAI.API.ErrorResponse
alias Req.Request
require Logger
@spec gpt(OpenAI.Completion.t(), OpenAI.Config.t()) :: {:ok, map()} | {:error, term()}
@spec gpt(OpenAI.Completion.t(), OpenAI.Config.t()) ::
{:ok, map()} | {:error, ErrorResponse.t() | Exception.t()}
def gpt(completion, config) do
case config
|> new_request()
@@ -27,17 +37,17 @@ defmodule OpenAI.API do
content = get_in(body, ["choices", Access.at(0), "message", "content"])
JSON.decode(content)
{:ok, %{body: body}} ->
{:error, "OpenAI API error: #{inspect(body)}"}
{:ok, response} ->
{:error, ErrorResponse.from_response(response)}
{:error, exception} ->
{:error, "Connection error: #{Exception.message(exception)}"}
{:error, exception}
end
end
@spec chat_stream([map()], String.t(), String.t(), float(), OpenAI.Config.t(), (String.t() ->
any())) ::
:ok | {:error, String.t()}
:ok | {:error, ErrorResponse.t() | Exception.t() | String.t()}
def chat_stream(messages, instructions, model, temperature, config, cb) do
config
|> new_request()
@@ -86,33 +96,35 @@ defmodule OpenAI.API do
:ok
end
{:ok, %{body: body}} ->
{:error, "OpenAI API error: #{inspect(body)}"}
{:ok, response} ->
{:error, ErrorResponse.from_response(response)}
{:error, exception} ->
{:error, "Connection error: #{Exception.message(exception)}"}
{:error, exception}
end
end
@spec get_embeddings(String.t(), OpenAI.Config.t()) :: {:ok, [float()]} | {:error, term()}
@spec get_embeddings(String.t(), OpenAI.Config.t()) ::
{:ok, [float()]} | {:error, ErrorResponse.t() | Exception.t()}
def get_embeddings(text, config) do
resp =
config
|> new_request()
|> Req.merge(
url: "/v1/embeddings",
json: %{
input: text,
model: "text-embedding-3-small"
}
)
|> Req.post!()
case config
|> new_request()
|> Req.merge(
url: "/v1/embeddings",
json: %{
input: text,
model: "text-embedding-3-small"
}
)
|> Req.post() do
{:ok, %{status: 200, body: body}} ->
{:ok, get_in(body, ["data", Access.at(0), "embedding"])}
if resp.status == 200 do
embeddings = get_in(resp.body, ["data", Access.at(0), "embedding"])
{:ok, embeddings}
else
{:error, resp.body}
{:ok, response} ->
{:error, ErrorResponse.from_response(response)}
{:error, exception} ->
{:error, exception}
end
end
lib/open_ai/api/error_response.ex
+83
View File
@@ -0,0 +1,83 @@
defmodule OpenAI.API.ErrorResponse do
@moduledoc """
Structured error response for OpenAI API calls.
OpenAI uses HTTP status codes as the primary classifier, with one critical
body-peek exception:
* HTTP **429** covers both `code: "rate_limit_exceeded"` (transient — retry)
and `code: "insufficient_quota"` (permanent — billing failure, cancel).
`from_response/1` reads `error.code` to disambiguate.
Error body shape (non-streaming):
%{"error" => %{"message" => ..., "type" => ..., "code" => ..., "param" => ...}}
Mid-stream SSE `error` / `response.failed` events are handled separately in
`OpenAI.API.chat_stream/6` and do not flow through this module.
"""
@behaviour MusicLibrary.ErrorResponse
alias MusicLibrary.HttpError
@type t :: %__MODULE__{
status: integer() | nil,
code: String.t() | nil,
type: String.t() | nil,
message: String.t() | nil,
kind: HttpError.kind(),
body: term()
}
defstruct [:status, :code, :type, :message, :kind, :body]
@spec from_response(Req.Response.t() | map()) :: t()
def from_response(%{status: 429, body: %{"error" => %{"code" => "insufficient_quota"} = e}} = r) do
%__MODULE__{
status: 429,
code: "insufficient_quota",
type: e["type"],
message: e["message"],
kind: :auth_error,
body: r.body
}
end
def from_response(%{status: status, body: %{"error" => err} = body} = _response)
when is_map(err) do
%__MODULE__{
status: status,
code: err["code"],
type: err["type"],
message: err["message"],
kind: HttpError.default_kind(status),
body: body
}
end
def from_response(%{status: status, body: body} = _response) do
%__MODULE__{
status: status,
code: nil,
type: nil,
message: nil,
kind: HttpError.default_kind(status),
body: body
}
end
@impl MusicLibrary.ErrorResponse
@spec retryable?(t()) :: boolean()
def retryable?(%__MODULE__{kind: kind}) when kind in [:rate_limit, :server_error, :timeout],
do: true
def retryable?(%__MODULE__{}), do: false
@impl MusicLibrary.ErrorResponse
@spec retry_delay_seconds(t()) :: pos_integer()
def retry_delay_seconds(%__MODULE__{kind: :rate_limit}), do: 60
def retry_delay_seconds(%__MODULE__{kind: :server_error}), do: 30
def retry_delay_seconds(%__MODULE__{kind: :timeout}), do: 10
def retry_delay_seconds(%__MODULE__{}), do: 30
end
lib/wikipedia/api.ex
+50 -16
View File
@@ -1,14 +1,23 @@
defmodule Wikipedia.API do
@moduledoc """
Interface to the Wikidata and Wikipedia APIs.
Two API surfaces are used, with different error handling:
* **Action API** (`/w/api.php`) returns HTTP 200 with an `{"error": {...}}`
payload on failure. `parse_error/1` promotes those bodies into
`Wikipedia.API.ErrorResponse` errors.
* **REST v1 API** (`/api/rest_v1/page/summary/:title`) uses classic HTTP
status codes. `parse_error/1` handles non-2xx statuses the same way.
"""
alias Req.Request
alias Wikipedia.API.ErrorResponse
require Logger
@spec get_wikipedia_title(String.t(), Wikipedia.Config.t()) ::
{:ok, String.t() | nil} | {:error, term()}
{:ok, String.t() | nil} | {:error, ErrorResponse.t() | Exception.t()}
def get_wikipedia_title(wikidata_id, config) do
case config
|> new_request("https://www.wikidata.org")
@@ -32,7 +41,8 @@ defmodule Wikipedia.API do
end
end
@spec get_article_summary(String.t(), Wikipedia.Config.t()) :: {:ok, map()} | {:error, term()}
@spec get_article_summary(String.t(), Wikipedia.Config.t()) ::
{:ok, map()} | {:error, ErrorResponse.t() | Exception.t()}
def get_article_summary(title, config) do
config
|> new_request("https://en.wikipedia.org")
@@ -41,7 +51,7 @@ defmodule Wikipedia.API do
end
@spec get_article_extract(String.t(), Wikipedia.Config.t()) ::
{:ok, String.t() | nil} | {:error, term()}
{:ok, String.t() | nil} | {:error, ErrorResponse.t() | Exception.t()}
def get_article_extract(title, config) do
case config
|> new_request("https://en.wikipedia.org")
@@ -80,16 +90,22 @@ defmodule Wikipedia.API do
|> Request.merge_options(config.req_options)
|> Req.RateLimiter.attach(name: :wikipedia, cooldown: config.api_cooldown)
|> Request.append_request_steps(log_attempt: &log_attempt/1)
|> Request.append_response_steps(log_error: &log_error/1)
|> Request.append_response_steps(parse_error: &parse_error/1)
end
defp get_request(request) do
case Req.get(request) do
{:ok, response} when response.status == 200 ->
{:ok, response.body}
{:ok, %{status: 200, body: %ErrorResponse{} = error}} ->
{:error, error}
{:ok, response} ->
{:error, response.body}
{:ok, %{status: status, body: body}} when status in 200..299 ->
{:ok, body}
{:ok, %{body: %ErrorResponse{} = error}} ->
{:error, error}
{:ok, %{body: body}} ->
{:error, body}
error ->
error
@@ -102,14 +118,32 @@ defmodule Wikipedia.API do
request
end
defp log_error({request, response}) do
if response.status in 400..499 or response.status in 500..599 do
Logger.error(fn ->
url = URI.to_string(request.url)
"Failed to fetch data from #{url}, reason: #{inspect(response.body)}"
end)
end
# Action API: HTTP 200 with an error envelope in the body.
defp parse_error(
{request,
%{status: 200, body: %{"error" => %{"code" => _, "info" => _}} = body} = response}
) do
error = ErrorResponse.from_action_api_body(body)
{request, response}
Logger.error(fn ->
url = URI.to_string(request.url)
"Failed to fetch data from #{url}, code: #{error.code}, info: #{error.message}"
end)
Request.halt(request, %{response | body: error})
end
# REST v1 API and other non-2xx responses.
defp parse_error({request, %{status: status} = response}) when status not in 200..299 do
error = ErrorResponse.from_response(response)
Logger.error(fn ->
url = URI.to_string(request.url)
"Failed to fetch data from #{url}, status: #{status}, reason: #{inspect(response.body)}"
end)
Request.halt(request, %{response | body: error})
end
defp parse_error(tuple), do: tuple
end
lib/wikipedia/api/error_response.ex
+94
View File
@@ -0,0 +1,94 @@
defmodule Wikipedia.API.ErrorResponse do
@moduledoc """
Structured error response for Wikipedia/Wikidata API calls.
Wikipedia exposes two API surfaces with different error paradigms:
* **Action API** (`/w/api.php`, used by `get_wikipedia_title/2` and
`get_article_extract/2`) returns **HTTP 200 with an error in the body**:
`%{"error" => %{"code" => ..., "info" => ...}}`. Without body inspection
the error would be silently returned as `{:ok, body}`.
* **REST v1 API** (`/api/rest_v1/page/summary/:title`, used by
`get_article_summary/2`) uses classic HTTP status codes.
`from_response/1` handles both paths. `from_action_api_body/1` is a dedicated
entry point for the HTTP 200 + body-error case.
## Non-error body shapes
These are NOT errors and should continue to return `{:ok, body}`:
* `%{"entities" => %{_id => %{"missing" => ""}}}` from `wbgetentities` —
the facade converts this to `{:error, :no_english_wikipedia}` at a
higher level.
* `get_in(body, ["query", "pages", _, "extract"])` returning `nil` — a
title exists but has no extract.
The discriminator for an actual error is the literal top-level `"error"` key
whose value is a map with both `"code"` and `"info"`.
"""
@behaviour MusicLibrary.ErrorResponse
alias MusicLibrary.HttpError
@type t :: %__MODULE__{
status: integer() | nil,
code: String.t() | nil,
message: String.t() | nil,
kind: HttpError.kind(),
body: term()
}
defstruct [:status, :code, :message, :kind, :body]
@spec from_response(Req.Response.t() | map()) :: t()
def from_response(%{status: status, body: body} = _response) do
%__MODULE__{
status: status,
code: extract_rest_code(body),
message: extract_rest_message(body),
kind: HttpError.default_kind(status),
body: body
}
end
@spec from_action_api_body(map()) :: t()
def from_action_api_body(%{"error" => %{"code" => code, "info" => info}} = body) do
%__MODULE__{
status: 200,
code: code,
message: info,
kind: action_api_kind(code),
body: body
}
end
@impl MusicLibrary.ErrorResponse
@spec retryable?(t()) :: boolean()
def retryable?(%__MODULE__{kind: kind}) when kind in [:rate_limit, :server_error, :timeout],
do: true
def retryable?(%__MODULE__{}), do: false
@impl MusicLibrary.ErrorResponse
@spec retry_delay_seconds(t()) :: pos_integer()
def retry_delay_seconds(%__MODULE__{kind: :rate_limit}), do: 30
def retry_delay_seconds(%__MODULE__{kind: :server_error}), do: 30
def retry_delay_seconds(%__MODULE__{kind: :timeout}), do: 10
def retry_delay_seconds(%__MODULE__{}), do: 30
defp action_api_kind("ratelimited"), do: :rate_limit
defp action_api_kind("maxlag"), do: :rate_limit
defp action_api_kind("readonly"), do: :server_error
defp action_api_kind("internal_api_error_" <> _), do: :server_error
defp action_api_kind(_), do: :client_error
defp extract_rest_code(%{"httpCode" => code}), do: to_string(code)
defp extract_rest_code(_), do: nil
defp extract_rest_message(%{"messageTranslations" => %{"en" => msg}}), do: msg
defp extract_rest_message(%{"detail" => msg}) when is_binary(msg), do: msg
defp extract_rest_message(_), do: nil
end