Support getting artist bios from Wikipedia (experimental)

2026-02-09 15:19:35 +00:00
parent f82e764c94
commit bcae291cff
23 changed files with 639 additions and 31 deletions
@@ -32,6 +32,14 @@ defmodule MusicBrainz.Artist do
    end
  end

+  def get_wikidata_id(r) do
+    Enum.find_value(r.relations, fn relation ->
+      if relation.type == "wikidata" do
+        parse_wikidata_id(relation.url["resource"])
+      end
+    end)
+  end
+
  def url(id) do
    "https://musicbrainz.org/artist/#{id}"
  end
@@ -47,4 +55,7 @@ defmodule MusicBrainz.Artist do

  defp parse_discogs_id("https://www.discogs.com/artist/" <> id), do: String.to_integer(id)
  defp parse_discogs_id(_other), do: nil
+
+  defp parse_wikidata_id("https://www.wikidata.org/wiki/" <> id), do: id
+  defp parse_wikidata_id(_other), do: nil
 end
@@ -134,6 +134,32 @@ defmodule MusicLibrary.Artists do
    enqueue_worker(Worker.ArtistRefreshDiscogsData, %{"id" => artist_info.id})
  end

+  def fetch_wikipedia_data(artist_id) do
+    artist_info = get_artist_info!(artist_id)
+
+    if wikidata_id = ArtistInfo.wikidata_id(artist_info) do
+      case Wikipedia.get_artist_summary(wikidata_id) do
+        {:ok, summary} ->
+          artist_info
+          |> ArtistInfo.changeset(%{wikipedia_data: summary})
+          |> Repo.update()
+
+        error ->
+          error
+      end
+    else
+      {:ok, artist_info}
+    end
+  end
+
+  def refresh_wikipedia_data(artist_id) do
+    fetch_wikipedia_data(artist_id)
+  end
+
+  def refresh_wikipedia_data_async(artist_info) do
+    enqueue_worker(Worker.ArtistRefreshWikipediaData, %{"id" => artist_info.id})
+  end
+
  def create_artist_info(attrs) do
    %ArtistInfo{}
    |> ArtistInfo.changeset(attrs)
@@ -10,6 +10,7 @@ defmodule MusicLibrary.Artists.ArtistInfo do
  schema "artist_infos" do
    field :musicbrainz_data, :map, default: %{}
    field :discogs_data, :map, default: %{}
+    field :wikipedia_data, :map, default: %{}
    field :image_data_hash, :string

    has_one :note, Note, foreign_key: :musicbrainz_id
@@ -23,6 +24,7 @@ defmodule MusicLibrary.Artists.ArtistInfo do
      :id,
      :musicbrainz_data,
      :discogs_data,
+      :wikipedia_data,
      :image_data_hash
    ])
    |> validate_required([:musicbrainz_data])
@@ -82,4 +84,33 @@ defmodule MusicLibrary.Artists.ArtistInfo do
      _ -> nil
    end
  end
+
+  def wikidata_id(artist_info) do
+    relations = get_in(artist_info.musicbrainz_data, ["relations"]) || []
+
+    Enum.find_value(relations, fn
+      %{"type" => "wikidata", "url" => %{"resource" => "https://www.wikidata.org/wiki/" <> id}} ->
+        id
+
+      _ ->
+        nil
+    end)
+  end
+
+  def wikipedia_bio(artist_info) do
+    get_in(artist_info.wikipedia_data, ["intro_html"]) ||
+      get_in(artist_info.wikipedia_data, ["extract_html"])
+  end
+
+  def wikipedia_summary(artist_info) do
+    get_in(artist_info.wikipedia_data, ["extract"])
+  end
+
+  def wikipedia_url(artist_info) do
+    get_in(artist_info.wikipedia_data, ["content_urls", "desktop", "page"])
+  end
+
+  def wikipedia_description(artist_info) do
+    get_in(artist_info.wikipedia_data, ["description"])
+  end
 end
@@ -0,0 +1,12 @@
+defmodule MusicLibrary.Worker.ArtistRefreshWikipediaData do
+  use Oban.Worker, queue: :wikipedia, max_attempts: 3
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"id" => artist_info_id}}) do
+    result = MusicLibrary.Artists.refresh_wikipedia_data(artist_info_id)
+
+    Process.sleep(1_000)
+
+    result
+  end
+end
@@ -4,6 +4,7 @@ defmodule MusicLibrary.Worker.FetchArtistInfo do
  @impl Oban.Worker
  def perform(%Oban.Job{args: %{"id" => artist_id}}) do
    with {:ok, _artist_info} <- MusicLibrary.Artists.fetch_artist_info(artist_id),
+         {:ok, _artist_info} <- MusicLibrary.Artists.fetch_wikipedia_data(artist_id),
         {:ok, _artist_info} <- MusicLibrary.Artists.fetch_image(artist_id) do
      :ok
    end
@@ -101,6 +101,7 @@ defmodule MusicLibraryWeb.ArtistLive.Show do
        {:noreply,
         socket
         |> assign(:artist_info, artist_info)
+         |> assign(:biography, build_biography(artist_info))
         |> put_toast(:info, gettext("Artist info refreshed successfully"))}

      {:error, reason} ->
@@ -113,6 +114,25 @@ defmodule MusicLibraryWeb.ArtistLive.Show do
    end
  end

+  def handle_event("refresh_wikipedia_data", %{"id" => id}, socket) do
+    case Artists.refresh_wikipedia_data(id) do
+      {:ok, artist_info} ->
+        {:noreply,
+         socket
+         |> assign(:artist_info, artist_info)
+         |> assign(:biography, build_biography(artist_info))
+         |> put_toast(:info, gettext("Wikipedia data refreshed successfully"))}
+
+      {:error, reason} ->
+        {:noreply,
+         socket
+         |> put_toast(
+           :error,
+           gettext("Error refreshing Wikipedia data") <> "," <> inspect(reason)
+         )}
+    end
+  end
+
  def handle_event("refresh_artist_image", %{"id" => id}, socket) do
    case Artists.fetch_image(id) do
      {:ok, artist_info} ->
@@ -171,6 +191,7 @@ defmodule MusicLibraryWeb.ArtistLive.Show do
    |> assign(:current_section, :artists)
    |> assign(:artist, artist)
    |> assign(:artist_info, artist_info)
+    |> assign(:biography, build_biography(artist_info))
    |> assign(:external_links, ArtistInfo.external_links(artist_info))
    |> assign(:country, ArtistInfo.country(artist_info))
    |> assign_async(:lastfm_artist_info, fn ->
@@ -250,6 +271,20 @@ defmodule MusicLibraryWeb.ArtistLive.Show do
    }
  end

+  defp build_biography(artist_info) do
+    bio_html = ArtistInfo.wikipedia_bio(artist_info)
+
+    if bio_html do
+      %{
+        source: "Wikipedia",
+        summary_html: ArtistInfo.wikipedia_summary(artist_info),
+        bio_html: bio_html,
+        url: ArtistInfo.wikipedia_url(artist_info),
+        description: ArtistInfo.wikipedia_description(artist_info)
+      }
+    end
+  end
+
  # Bios start with text, then a link to read more on Last.fm, followed by a license text.
  # We split the bio at the read more link in order to render the license separately.
  defp render_bio(bio) do
@@ -73,6 +73,20 @@
                  />
                  {gettext("Refresh info")}
                </.dropdown_link>
+                <.dropdown_link
+                  id={"actions-#{@artist.musicbrainz_id}-refresh-wikipedia"}
+                  phx-click={
+                    JS.push("refresh_wikipedia_data", value: %{id: @artist.musicbrainz_id})
+                  }
+                >
+                  <.icon
+                    name="hero-arrow-path"
+                    class="h-4 w-4 mr-1 phx-click-loading:animate-spin"
+                    aria-hidden="true"
+                    data-slot="icon"
+                  />
+                  {gettext("Refresh Wikipedia")}
+                </.dropdown_link>
              </.focus_wrap>
            </.dropdown>
          </.button_group>
@@ -141,34 +155,19 @@
            />
          </dd>
        </dl>
-        <.async_result :let={lastfm_artist_info} assign={@lastfm_artist_info}>
-          <:loading>
-            <div class="mt-4 text-sm leading-5 text-zinc-500 dark:text-zinc-400">
-              {gettext("Loading biography")}
-            </div>
-          </:loading>
-          <:failed :let={_failure}>
-            <div class="mt-4 text-sm leading-5 text-zinc-500 dark:text-zinc-400">
-              <.icon
-                name="hero-exclamation-triangle"
-                class="-mt-1 mr-1 h-5 w-5"
-                aria-hidden="true"
-                data-slot="icon"
-              />
-              {gettext("Error loading biography")}
-            </div>
-          </:failed>
-          <dt
-            :if={lastfm_artist_info.bio not in [nil, ""]}
-            class="mt-4 text-sm font-medium leading-6 text-zinc-900 dark:text-zinc-400"
-          >
+        <%= if @biography do %>
+          <dt class="mt-4 text-sm font-medium leading-6 text-zinc-900 dark:text-zinc-400">
            {gettext("Biography")}
+            <.badge variant="soft" class="ml-1">{@biography.source}</.badge>
          </dt>
-          <dd
-            :if={lastfm_artist_info.bio not in [nil, ""]}
-            class="text-zinc-700 dark:text-zinc-300"
-          >
-            {remove_read_more_link(lastfm_artist_info.summary)}
+          <dd class="text-zinc-700 dark:text-zinc-300">
+            <p
+              :if={@biography.description}
+              class="mt-2 text-sm italic text-zinc-500 dark:text-zinc-400"
+            >
+              {@biography.description}
+            </p>
+            <p class="mt-2 text-sm/7">{@biography.summary_html}</p>
            <.link
              class="block mt-2 text-sm font-medium text-zinc-900 dark:text-zinc-400"
              phx-click={Fluxon.open_dialog("bio")}
@@ -183,14 +182,80 @@
            </.link>
          </dd>
          <.sheet
-            :if={lastfm_artist_info.bio not in [nil, ""]}
            id="bio"
            class="max-w-2xl text-zinc-700 dark:text-zinc-300"
            placement="left"
          >
-            {render_bio(lastfm_artist_info.bio)}
+            <div class="prose prose-sm dark:prose-invert">
+              {Phoenix.HTML.raw(@biography.bio_html)}
+            </div>
+            <a
+              :if={@biography.url}
+              href={@biography.url}
+              target="_blank"
+              class="mt-4 block text-sm font-medium text-zinc-900 dark:text-zinc-400 hover:text-zinc-500"
+            >
+              {gettext("Read full article on Wikipedia")}
+              <.icon
+                name="hero-arrow-top-right-on-square"
+                class="-mt-1 ml-1 h-4 w-4"
+                aria-hidden="true"
+                data-slot="icon"
+              />
+            </a>
          </.sheet>
-        </.async_result>
+        <% else %>
+          <.async_result :let={lastfm_artist_info} assign={@lastfm_artist_info}>
+            <:loading>
+              <div class="mt-4 text-sm leading-5 text-zinc-500 dark:text-zinc-400">
+                {gettext("Loading biography")}
+              </div>
+            </:loading>
+            <:failed :let={_failure}>
+              <div class="mt-4 text-sm leading-5 text-zinc-500 dark:text-zinc-400">
+                <.icon
+                  name="hero-exclamation-triangle"
+                  class="-mt-1 mr-1 h-5 w-5"
+                  aria-hidden="true"
+                  data-slot="icon"
+                />
+                {gettext("Error loading biography")}
+              </div>
+            </:failed>
+            <dt
+              :if={lastfm_artist_info.bio not in [nil, ""]}
+              class="mt-4 text-sm font-medium leading-6 text-zinc-900 dark:text-zinc-400"
+            >
+              {gettext("Biography")}
+            </dt>
+            <dd
+              :if={lastfm_artist_info.bio not in [nil, ""]}
+              class="text-zinc-700 dark:text-zinc-300"
+            >
+              {remove_read_more_link(lastfm_artist_info.summary)}
+              <.link
+                class="block mt-2 text-sm font-medium text-zinc-900 dark:text-zinc-400"
+                phx-click={Fluxon.open_dialog("lastfm-bio")}
+              >
+                <.icon
+                  name="hero-arrow-right-end-on-rectangle"
+                  class="-mt-1 mr-1 h-5 w-5"
+                  aria-hidden="true"
+                  data-slot="icon"
+                />
+                {gettext("Read more")}
+              </.link>
+            </dd>
+            <.sheet
+              :if={lastfm_artist_info.bio not in [nil, ""]}
+              id="lastfm-bio"
+              class="max-w-2xl text-zinc-700 dark:text-zinc-300"
+              placement="left"
+            >
+              {render_bio(lastfm_artist_info.bio)}
+            </.sheet>
+          </.async_result>
+        <% end %>
        <.external_links external_links={@external_links} />
      </div>
      <div class="md:col-span-7 md:order-1">
@@ -251,6 +316,12 @@
      title={gettext("Discogs data")}
      data={@artist_info.discogs_data}
    />
+
+    <.json_viewer
+      :if={@artist_info.wikipedia_data != %{}}
+      title={gettext("Wikipedia data")}
+      data={@artist_info.wikipedia_data}
+    />
  </div>

  <.live_component
@@ -0,0 +1,18 @@
+defmodule Wikipedia do
+  alias Wikipedia.API
+
+  def get_artist_summary(wikidata_id) do
+    config = wikipedia_config()
+
+    with {:ok, title} when not is_nil(title) <- API.get_wikipedia_title(wikidata_id, config),
+         {:ok, summary} <- API.get_article_summary(title, config),
+         {:ok, intro_html} <- API.get_article_extract(title, config) do
+      {:ok, Map.put(summary, "intro_html", intro_html)}
+    else
+      {:ok, nil} -> {:error, :no_english_wikipedia}
+      error -> error
+    end
+  end
+
+  defp wikipedia_config, do: Wikipedia.Config.resolve(:music_library)
+end
@@ -0,0 +1,124 @@
+defmodule Wikipedia.API do
+  @moduledoc """
+  Interface to the Wikidata and Wikipedia APIs.
+  """
+
+  require Logger
+
+  def get_wikipedia_title(wikidata_id, config) do
+    request =
+      Req.new(
+        base_url: "https://www.wikidata.org",
+        max_retries: 1,
+        user_agent: config.user_agent
+      )
+      |> Req.Request.merge_options(config.req_options)
+      |> Req.Request.append_request_steps(log_attempt: &log_attempt/1)
+      |> Req.Request.append_response_steps(log_error: &log_error/1)
+      |> Req.merge(
+        url: "/w/api.php",
+        params: [
+          action: "wbgetentities",
+          ids: wikidata_id,
+          props: "sitelinks",
+          sitefilter: "enwiki",
+          format: "json"
+        ]
+      )
+
+    case Req.get(request) do
+      {:ok, response} when response.status == 200 ->
+        title =
+          get_in(response.body, ["entities", wikidata_id, "sitelinks", "enwiki", "title"])
+
+        {:ok, title}
+
+      {:ok, response} ->
+        {:error, response.body}
+
+      error ->
+        error
+    end
+  end
+
+  def get_article_summary(title, config) do
+    request =
+      Req.new(
+        base_url: "https://en.wikipedia.org",
+        max_retries: 1,
+        user_agent: config.user_agent
+      )
+      |> Req.Request.merge_options(config.req_options)
+      |> Req.Request.append_request_steps(log_attempt: &log_attempt/1)
+      |> Req.Request.append_response_steps(log_error: &log_error/1)
+      |> Req.merge(url: "/api/rest_v1/page/summary/#{URI.encode(title)}")
+
+    case Req.get(request) do
+      {:ok, response} when response.status == 200 ->
+        {:ok, response.body}
+
+      {:ok, response} ->
+        {:error, response.body}
+
+      error ->
+        error
+    end
+  end
+
+  def get_article_extract(title, config) do
+    request =
+      Req.new(
+        base_url: "https://en.wikipedia.org",
+        max_retries: 1,
+        user_agent: config.user_agent
+      )
+      |> Req.Request.merge_options(config.req_options)
+      |> Req.Request.append_request_steps(log_attempt: &log_attempt/1)
+      |> Req.Request.append_response_steps(log_error: &log_error/1)
+      |> Req.merge(
+        url: "/w/api.php",
+        params: [
+          action: "query",
+          titles: title,
+          prop: "extracts",
+          exintro: "1",
+          format: "json"
+        ]
+      )
+
+    case Req.get(request) do
+      {:ok, response} when response.status == 200 ->
+        extract =
+          response.body
+          |> get_in(["query", "pages"])
+          |> Map.values()
+          |> List.first()
+          |> Map.get("extract")
+
+        {:ok, extract}
+
+      {:ok, response} ->
+        {:error, response.body}
+
+      error ->
+        error
+    end
+  end
+
+  defp log_attempt(request) do
+    url = URI.to_string(request.url)
+    Logger.debug("Fetching data from #{url}")
+    request
+  end
+
+  defp log_error({request, response}) do
+    if response.status in 400..499 or response.status in 500..599 do
+      Logger.error(fn ->
+        url = URI.to_string(request.url)
+        "Failed to fetch data from #{url}, reason: #{inspect(response.body)}"
+      end)
+    end
+
+    {request, response}
+  end
+end
@@ -0,0 +1,32 @@
+defmodule Wikipedia.Config do
+  @type t :: %__MODULE__{
+          user_agent: String.t(),
+          req_options: Keyword.t()
+        }
+
+  defstruct user_agent: "change me",
+            req_options: []
+
+  @schema NimbleOptions.new!(
+            user_agent: [
+              type: :string,
+              required: false,
+              default: "change me"
+            ],
+            req_options: [
+              type: :keyword_list,
+              required: false,
+              default: []
+            ]
+          )
+
+  @doc NimbleOptions.docs(@schema)
+  @spec resolve(Application.app()) :: t | no_return
+  def resolve(otp_app) do
+    app_config =
+      Application.get_env(otp_app, Wikipedia)
+      |> NimbleOptions.validate!(@schema)
+
+    struct(__MODULE__, app_config)
+  end
+end