Use Wikipedia data for record similarity and raise similarity threshold

config/config.exs

@@ -17,6 +17,8 @@ config :music_library,
 
 config :music_library, default_timezone: "Europe/London"
 
+config :music_library, :similarity, max_distance: 0.45
+
 config :music_library, :pagination,
   default_page_size: 20,
   stats_limit: 30,

lib/music_library/records/similarity.ex

@@ -6,11 +6,15 @@ defmodule MusicLibrary.Records.Similarity do
   import Ecto.Query
   import SqliteVec.Ecto.Query
 
-  alias MusicLibrary.{Artists, Records}
+  alias MusicLibrary.Artists
+  alias MusicLibrary.Artists.ArtistInfo
+  alias MusicLibrary.Records
   alias MusicLibrary.Records.{Record, RecordEmbedding}
   alias MusicLibrary.Repo
   alias MusicLibrary.Worker.GenerateRecordEmbedding
 
+  @max_distance Application.compile_env!(:music_library, :similarity)[:max_distance]
+
   @doc """
   Generates a text representation of a record for embedding generation.
 
@@ -20,6 +24,7 @@ defmodule MusicLibrary.Records.Similarity do
   - Genres
   - Release year
   - Type (album, EP, etc.)
+  - Artist musical style summaries (from Wikipedia, falling back to Discogs)
   """
   def text_representation(%Record{} = record) do
     artist_infos =
@@ -47,32 +52,75 @@ defmodule MusicLibrary.Records.Similarity do
   defp artist_infos_summary([]), do: ""
 
   defp artist_infos_summary(artist_infos) do
-    Enum.map_join(artist_infos, "\n\n", &artist_info_summary/1)
+    artist_infos
+    |> Enum.map(&artist_info_summary/1)
+    |> Enum.reject(&(&1 == ""))
+    |> Enum.join("\n\n")
   end
 
-  defp artist_info_summary(artist_info) when is_nil(artist_info.discogs_data), do: ""
+  defp artist_info_summary(artist_info) do
+    cond do
+      wikipedia_available?(artist_info) ->
+        wikipedia_artist_summary(artist_info)
 
-  defp artist_info_summary(%{discogs_data: discogs_data}) do
+      discogs_available?(artist_info) ->
-    # if available, use the plain text version of the profile to avoid pollution by BBTags
+        discogs_artist_summary(artist_info)
-    profile_data = Map.get(discogs_data, "profile_plaintext") || Map.get(discogs_data, "profile")
 
-    profile_section =
+      true ->
-      if profile_data do
-        "Profile for #{discogs_data["name"]}: #{profile_data}"
-      else
         ""
-      end
+    end
+  end
 
-    members_section =
+  defp wikipedia_available?(artist_info) do
-      if members = Map.get(discogs_data, "members") do
+    ArtistInfo.wikipedia_description(artist_info) != nil ||
-        members_list = Enum.map_join(members, ", ", fn member -> member["name"] end)
+      ArtistInfo.wikipedia_summary(artist_info) != nil
+  end
 
-        "Members (past and present): #{members_list}"
+  defp discogs_available?(artist_info) do
-      else
+    artist_info.discogs_data != nil &&
+      (Map.get(artist_info.discogs_data, "profile_plaintext") != nil ||
+         Map.get(artist_info.discogs_data, "profile") != nil)
+  end
+
+  defp wikipedia_artist_summary(artist_info) do
+    description = ArtistInfo.wikipedia_description(artist_info) || ""
+    summary = ArtistInfo.wikipedia_summary(artist_info) || ""
+    truncated_summary = truncate_to_sentence(summary, 200)
+
+    [description, truncated_summary]
+    |> Enum.reject(&(&1 == ""))
+    |> Enum.join(". ")
+  end
+
+  defp discogs_artist_summary(artist_info) do
+    profile =
+      Map.get(artist_info.discogs_data, "profile_plaintext") ||
+        Map.get(artist_info.discogs_data, "profile") ||
         ""
-      end
 
-    Enum.join([profile_section, members_section], "\n")
+    truncate_to_sentence(profile, 200)
+  end
+
+  @doc false
+  def truncate_to_sentence(text, max_length) when byte_size(text) <= max_length, do: text
+
+  def truncate_to_sentence(text, max_length) do
+    truncated = String.slice(text, 0, max_length)
+
+    case String.split(truncated, ~r/[.!?]\s/, include_captures: true) |> Enum.count() do
+      count when count > 1 ->
+        # Find the last sentence boundary within the limit
+        truncated
+        |> String.replace(~r/[^.!?]*$/, "")
+        |> String.trim()
+        |> case do
+          "" -> String.trim(truncated)
+          result -> result
+        end
+
+      _ ->
+        String.trim(truncated)
+    end
   end
 
   @doc """
@@ -82,6 +130,8 @@ defmodule MusicLibrary.Records.Similarity do
 
   - `:limit` - Maximum number of similar records to return (default: 10)
   - `:scope` - Filter by :collection or :wishlist (default: no filter)
+  - `:max_distance` - Maximum cosine distance threshold (default: #{@max_distance}).
+    Results with distance above this are excluded.
 
   ## Examples
 
@@ -94,6 +144,7 @@ defmodule MusicLibrary.Records.Similarity do
   def find_similar(record_id, opts \\ []) do
     limit = Keyword.get(opts, :limit, 10)
     scope = Keyword.get(opts, :scope)
+    max_distance = Keyword.get(opts, :max_distance, @max_distance)
 
     record = Records.get_record!(record_id)
     record_musicbrainz_id = record.musicbrainz_id
@@ -113,6 +164,7 @@ defmodule MusicLibrary.Records.Similarity do
                 |> selected_as(:similarity)
             },
             group_by: r.musicbrainz_id,
+            having: vec_distance_cosine(re.embedding, vec_f32(source_embedding)) <= ^max_distance,
             limit: ^limit
 
         query = apply_scope_filter(query, scope)

lib/music_library_web/components/record_components.ex

@@ -472,7 +472,7 @@ defmodule MusicLibraryWeb.RecordComponents do
           {gettext("Similar Records")}
         </h2>
         <span class="ml-2 text-xs font-normal text-zinc-500 dark:text-zinc-400">
-          {gettext("Based on genres, artists, and release year")}
+          {gettext("Based on genres, artists, and musical style")}
         </span>
       </header>
 

priv/gettext/default.pot

@@ -1446,11 +1446,6 @@ msgstr ""
 msgid "External Links"
 msgstr ""
 
-#: lib/music_library_web/components/record_components.ex
-#, elixir-autogen, elixir-format
-msgid "Based on genres, artists, and release year"
-msgstr ""
-
 #: lib/music_library_web/components/record_components.ex
 #, elixir-autogen, elixir-format
 msgid "Similar Records"
@@ -1957,3 +1952,8 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "Retry"
 msgstr ""
+
+#: lib/music_library_web/components/record_components.ex
+#, elixir-autogen, elixir-format
+msgid "Based on genres, artists, and musical style"
+msgstr ""

priv/gettext/en/LC_MESSAGES/default.po

@@ -1446,11 +1446,6 @@ msgstr ""
 msgid "External Links"
 msgstr ""
 
-#: lib/music_library_web/components/record_components.ex
-#, elixir-autogen, elixir-format
-msgid "Based on genres, artists, and release year"
-msgstr ""
-
 #: lib/music_library_web/components/record_components.ex
 #, elixir-autogen, elixir-format
 msgid "Similar Records"
@@ -1957,3 +1952,8 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "Retry"
 msgstr ""
+
+#: lib/music_library_web/components/record_components.ex
+#, elixir-autogen, elixir-format, fuzzy
+msgid "Based on genres, artists, and musical style"
+msgstr ""

test/music_library/records/similarity_test.exs

@@ -3,7 +3,9 @@ defmodule MusicLibrary.Records.SimilarityTest do
 
   import MusicLibrary.Fixtures.Records
 
+  alias MusicLibrary.Artists.ArtistInfo
   alias MusicLibrary.Records.{Record, Similarity}
+  alias MusicLibrary.Repo
 
   describe "text_representation/1" do
     test "generates text representation for a record" do
@@ -32,6 +34,78 @@ defmodule MusicLibrary.Records.SimilarityTest do
       assert text =~ "Type: Album"
     end
 
+    test "includes Wikipedia data when available" do
+      artist_id = Ecto.UUID.generate()
+
+      Repo.insert!(%ArtistInfo{
+        id: artist_id,
+        musicbrainz_data: %{"name" => "Radiohead"},
+        wikipedia_data: %{
+          "description" => "English rock band",
+          "extract" =>
+            "Radiohead are an English rock band formed in Abingdon. They are known for experimental music."
+        }
+      })
+
+      record = %Record{
+        title: "OK Computer",
+        artists: [
+          %{
+            name: "Radiohead",
+            sort_name: "Radiohead",
+            musicbrainz_id: artist_id,
+            disambiguation: "",
+            joinphrase: ""
+          }
+        ],
+        genres: ["alternative rock"],
+        release_date: "1997-05-21",
+        type: :album
+      }
+
+      text = Similarity.text_representation(record)
+
+      assert text =~ "English rock band"
+      assert text =~ "experimental music"
+      refute text =~ "Members"
+    end
+
+    test "falls back to truncated Discogs profile when Wikipedia unavailable" do
+      artist_id = Ecto.UUID.generate()
+
+      Repo.insert!(%ArtistInfo{
+        id: artist_id,
+        musicbrainz_data: %{"name" => "Some Artist"},
+        discogs_data: %{
+          "name" => "Some Artist",
+          "profile_plaintext" => "Some Artist is a funk band. They have released many albums.",
+          "members" => [%{"name" => "Member One"}, %{"name" => "Member Two"}]
+        }
+      })
+
+      record = %Record{
+        title: "Funky Album",
+        artists: [
+          %{
+            name: "Some Artist",
+            sort_name: "Some Artist",
+            musicbrainz_id: artist_id,
+            disambiguation: "",
+            joinphrase: ""
+          }
+        ],
+        genres: ["funk"],
+        release_date: "2000",
+        type: :album
+      }
+
+      text = Similarity.text_representation(record)
+
+      assert text =~ "funk band"
+      refute text =~ "Members"
+      refute text =~ "Member One"
+    end
+
     test "handles records with no release date" do
       record = %Record{
         title: "Unknown Album",
@@ -107,6 +181,40 @@ defmodule MusicLibrary.Records.SimilarityTest do
     end
   end
 
+  describe "truncate_to_sentence/2" do
+    test "returns text unchanged when within limit" do
+      assert Similarity.truncate_to_sentence("Short text.", 200) == "Short text."
+    end
+
+    test "truncates at sentence boundary" do
+      text = "First sentence. Second sentence. Third sentence that is very long."
+
+      result = Similarity.truncate_to_sentence(text, 40)
+
+      assert result == "First sentence. Second sentence."
+    end
+
+    test "truncates at character limit when no sentence boundary" do
+      text = String.duplicate("a", 300)
+
+      result = Similarity.truncate_to_sentence(text, 200)
+
+      assert byte_size(result) <= 200
+    end
+
+    test "handles empty string" do
+      assert Similarity.truncate_to_sentence("", 200) == ""
+    end
+
+    test "handles text with exclamation and question marks" do
+      text = "What a band! They play great music? Yes indeed. More text that goes over the limit."
+
+      result = Similarity.truncate_to_sentence(text, 55)
+
+      assert result == "What a band! They play great music? Yes indeed."
+    end
+  end
+
   describe "store_embedding/3 and get_embedding/1" do
     test "stores and retrieves an embedding" do
       record = record()
@@ -158,7 +266,7 @@ defmodule MusicLibrary.Records.SimilarityTest do
     end
 
     test "finds similar records", %{record1: record1, record2: record2} do
-      similar = Similarity.find_similar(record1.id, limit: 5)
+      similar = Similarity.find_similar(record1.id, limit: 5, max_distance: 1.0)
 
       refute Enum.empty?(similar)
       # record2 should be most similar to record1
@@ -168,7 +276,7 @@ defmodule MusicLibrary.Records.SimilarityTest do
     end
 
     test "respects limit option", %{record1: record1} do
-      similar = Similarity.find_similar(record1.id, limit: 1)
+      similar = Similarity.find_similar(record1.id, limit: 1, max_distance: 1.0)
 
       assert length(similar) == 1
     end
@@ -185,11 +293,21 @@ defmodule MusicLibrary.Records.SimilarityTest do
       # Mark record2 as purchased
       {:ok, _} = MusicLibrary.Records.update_record(record2, %{purchased_at: DateTime.utc_now()})
 
-      similar = Similarity.find_similar(record1.id, scope: :wishlist)
+      similar = Similarity.find_similar(record1.id, scope: :wishlist, max_distance: 1.0)
 
       # Should not include record2 since it's in collection
-      record_ids = Enum.map(similar, fn {record, _} -> record.id end)
+      record_ids = Enum.map(similar, fn %{record: record} -> record.id end)
       refute record2.id in record_ids
     end
+
+    test "filters results by max_distance threshold", %{record1: record1} do
+      # With a very low threshold, only very similar records should be returned
+      similar_strict = Similarity.find_similar(record1.id, max_distance: 0.001)
+
+      # With a very high threshold, all records should be returned
+      similar_loose = Similarity.find_similar(record1.id, max_distance: 2.0)
+
+      assert length(similar_strict) <= length(similar_loose)
+    end
   end
 end