ML-21: Classify API errors as transient vs permanent

2026-04-24 13:55:00 +01:00
parent be4f4686f8
commit a1c665b490
43 changed files with 1429 additions and 135 deletions
@@ -1,14 +1,23 @@
 defmodule Wikipedia.API do
  @moduledoc """
  Interface to the Wikidata and Wikipedia APIs.
+
+  Two API surfaces are used, with different error handling:
+
+    * **Action API** (`/w/api.php`) returns HTTP 200 with an `{"error": {...}}`
+      payload on failure. `parse_error/1` promotes those bodies into
+      `Wikipedia.API.ErrorResponse` errors.
+    * **REST v1 API** (`/api/rest_v1/page/summary/:title`) uses classic HTTP
+      status codes. `parse_error/1` handles non-2xx statuses the same way.
  """

  alias Req.Request
+  alias Wikipedia.API.ErrorResponse

  require Logger

  @spec get_wikipedia_title(String.t(), Wikipedia.Config.t()) ::
-          {:ok, String.t() | nil} | {:error, term()}
+          {:ok, String.t() | nil} | {:error, ErrorResponse.t() | Exception.t()}
  def get_wikipedia_title(wikidata_id, config) do
    case config
         |> new_request("https://www.wikidata.org")
@@ -32,7 +41,8 @@ defmodule Wikipedia.API do
    end
  end

-  @spec get_article_summary(String.t(), Wikipedia.Config.t()) :: {:ok, map()} | {:error, term()}
+  @spec get_article_summary(String.t(), Wikipedia.Config.t()) ::
+          {:ok, map()} | {:error, ErrorResponse.t() | Exception.t()}
  def get_article_summary(title, config) do
    config
    |> new_request("https://en.wikipedia.org")
@@ -41,7 +51,7 @@ defmodule Wikipedia.API do
  end

  @spec get_article_extract(String.t(), Wikipedia.Config.t()) ::
-          {:ok, String.t() | nil} | {:error, term()}
+          {:ok, String.t() | nil} | {:error, ErrorResponse.t() | Exception.t()}
  def get_article_extract(title, config) do
    case config
         |> new_request("https://en.wikipedia.org")
@@ -80,16 +90,22 @@ defmodule Wikipedia.API do
    |> Request.merge_options(config.req_options)
    |> Req.RateLimiter.attach(name: :wikipedia, cooldown: config.api_cooldown)
    |> Request.append_request_steps(log_attempt: &log_attempt/1)
-    |> Request.append_response_steps(log_error: &log_error/1)
+    |> Request.append_response_steps(parse_error: &parse_error/1)
  end

  defp get_request(request) do
    case Req.get(request) do
-      {:ok, response} when response.status == 200 ->
-        {:ok, response.body}
+      {:ok, %{status: 200, body: %ErrorResponse{} = error}} ->
+        {:error, error}

-      {:ok, response} ->
-        {:error, response.body}
+      {:ok, %{status: status, body: body}} when status in 200..299 ->
+        {:ok, body}
+
+      {:ok, %{body: %ErrorResponse{} = error}} ->
+        {:error, error}
+
+      {:ok, %{body: body}} ->
+        {:error, body}

      error ->
        error
@@ -102,14 +118,32 @@ defmodule Wikipedia.API do
    request
  end

-  defp log_error({request, response}) do
-    if response.status in 400..499 or response.status in 500..599 do
-      Logger.error(fn ->
-        url = URI.to_string(request.url)
-        "Failed to fetch data from #{url}, reason: #{inspect(response.body)}"
-      end)
-    end
+  # Action API: HTTP 200 with an error envelope in the body.
+  defp parse_error(
+         {request,
+          %{status: 200, body: %{"error" => %{"code" => _, "info" => _}} = body} = response}
+       ) do
+    error = ErrorResponse.from_action_api_body(body)

-    {request, response}
+    Logger.error(fn ->
+      url = URI.to_string(request.url)
+      "Failed to fetch data from #{url}, code: #{error.code}, info: #{error.message}"
+    end)
+
+    Request.halt(request, %{response | body: error})
  end
+
+  # REST v1 API and other non-2xx responses.
+  defp parse_error({request, %{status: status} = response}) when status not in 200..299 do
+    error = ErrorResponse.from_response(response)
+
+    Logger.error(fn ->
+      url = URI.to_string(request.url)
+      "Failed to fetch data from #{url}, status: #{status}, reason: #{inspect(response.body)}"
+    end)
+
+    Request.halt(request, %{response | body: error})
+  end
+
+  defp parse_error(tuple), do: tuple
 end
@@ -0,0 +1,94 @@
+defmodule Wikipedia.API.ErrorResponse do
+  @moduledoc """
+  Structured error response for Wikipedia/Wikidata API calls.
+
+  Wikipedia exposes two API surfaces with different error paradigms:
+
+    * **Action API** (`/w/api.php`, used by `get_wikipedia_title/2` and
+      `get_article_extract/2`) returns **HTTP 200 with an error in the body**:
+      `%{"error" => %{"code" => ..., "info" => ...}}`. Without body inspection
+      the error would be silently returned as `{:ok, body}`.
+
+    * **REST v1 API** (`/api/rest_v1/page/summary/:title`, used by
+      `get_article_summary/2`) uses classic HTTP status codes.
+
+  `from_response/1` handles both paths. `from_action_api_body/1` is a dedicated
+  entry point for the HTTP 200 + body-error case.
+
+  ## Non-error body shapes
+
+  These are NOT errors and should continue to return `{:ok, body}`:
+
+    * `%{"entities" => %{_id => %{"missing" => ""}}}` from `wbgetentities` —
+      the facade converts this to `{:error, :no_english_wikipedia}` at a
+      higher level.
+    * `get_in(body, ["query", "pages", _, "extract"])` returning `nil` — a
+      title exists but has no extract.
+
+  The discriminator for an actual error is the literal top-level `"error"` key
+  whose value is a map with both `"code"` and `"info"`.
+  """
+
+  @behaviour MusicLibrary.ErrorResponse
+
+  alias MusicLibrary.HttpError
+
+  @type t :: %__MODULE__{
+          status: integer() | nil,
+          code: String.t() | nil,
+          message: String.t() | nil,
+          kind: HttpError.kind(),
+          body: term()
+        }
+
+  defstruct [:status, :code, :message, :kind, :body]
+
+  @spec from_response(Req.Response.t() | map()) :: t()
+  def from_response(%{status: status, body: body} = _response) do
+    %__MODULE__{
+      status: status,
+      code: extract_rest_code(body),
+      message: extract_rest_message(body),
+      kind: HttpError.default_kind(status),
+      body: body
+    }
+  end
+
+  @spec from_action_api_body(map()) :: t()
+  def from_action_api_body(%{"error" => %{"code" => code, "info" => info}} = body) do
+    %__MODULE__{
+      status: 200,
+      code: code,
+      message: info,
+      kind: action_api_kind(code),
+      body: body
+    }
+  end
+
+  @impl MusicLibrary.ErrorResponse
+  @spec retryable?(t()) :: boolean()
+  def retryable?(%__MODULE__{kind: kind}) when kind in [:rate_limit, :server_error, :timeout],
+    do: true
+
+  def retryable?(%__MODULE__{}), do: false
+
+  @impl MusicLibrary.ErrorResponse
+  @spec retry_delay_seconds(t()) :: pos_integer()
+  def retry_delay_seconds(%__MODULE__{kind: :rate_limit}), do: 30
+  def retry_delay_seconds(%__MODULE__{kind: :server_error}), do: 30
+  def retry_delay_seconds(%__MODULE__{kind: :timeout}), do: 10
+  def retry_delay_seconds(%__MODULE__{}), do: 30
+
+  defp action_api_kind("ratelimited"), do: :rate_limit
+  defp action_api_kind("maxlag"), do: :rate_limit
+  defp action_api_kind("readonly"), do: :server_error
+  defp action_api_kind("internal_api_error_" <> _), do: :server_error
+  defp action_api_kind(_), do: :client_error
+
+  defp extract_rest_code(%{"httpCode" => code}), do: to_string(code)
+  defp extract_rest_code(_), do: nil
+
+  defp extract_rest_message(%{"messageTranslations" => %{"en" => msg}}), do: msg
+  defp extract_rest_message(%{"detail" => msg}) when is_binary(msg), do: msg
+  defp extract_rest_message(_), do: nil
+end