From 2e2000fe01cfda658be4ce43ef957aa25eacd130 Mon Sep 17 00:00:00 2001 From: Claudio Ortolina Date: Fri, 22 Nov 2024 10:01:48 +0000 Subject: [PATCH] Introduce separate search index Uses a fts5 virtual table, kept up to date via triggers on the records table. --- lib/music_library/collection.ex | 13 +- lib/music_library/records.ex | 24 +-- lib/music_library/records/search_index.ex | 30 ++++ lib/music_library/wishlist.ex | 4 +- ...1122094655_create_records_search_index.exs | 151 ++++++++++++++++++ test/music_library/records_test.exs | 20 +-- 6 files changed, 215 insertions(+), 27 deletions(-) create mode 100644 lib/music_library/records/search_index.ex create mode 100644 priv/repo/migrations/20241122094655_create_records_search_index.exs diff --git a/lib/music_library/collection.ex b/lib/music_library/collection.ex index 0a294fa0..eb0d6c1f 100644 --- a/lib/music_library/collection.ex +++ b/lib/music_library/collection.ex @@ -3,7 +3,7 @@ defmodule MusicLibrary.Collection do alias MusicLibrary.Repo alias MusicLibrary.Records - alias MusicLibrary.Records.Record + alias MusicLibrary.Records.{Record, SearchIndex} def search_records(query, opts \\ []) do limit = Keyword.get(opts, :limit, 20) @@ -18,7 +18,8 @@ defmodule MusicLibrary.Collection do def count_records_by_format do q = - from r in base_search(), + from r in Record, + where: not is_nil(r.purchased_at), group_by: r.format, order_by: [desc: count(r.id)], select: {r.format, count(r.id)} @@ -28,7 +29,8 @@ defmodule MusicLibrary.Collection do def count_records_by_type do q = - from r in base_search(), + from r in Record, + where: not is_nil(r.purchased_at), group_by: r.type, order_by: [desc: count(r.id)], select: {r.type, count(r.id)} @@ -38,7 +40,8 @@ defmodule MusicLibrary.Collection do def get_latest_record! do q = - from r in base_search(), + from r in Record, + where: not is_nil(r.purchased_at), order_by: [desc: r.purchased_at], limit: 1, select: ^Records.essential_fields() @@ -57,7 +60,7 @@ defmodule MusicLibrary.Collection do end defp base_search do - from r in Record, + from r in SearchIndex, where: not is_nil(r.purchased_at) end end diff --git a/lib/music_library/records.ex b/lib/music_library/records.ex index d1dc96ec..be038bca 100644 --- a/lib/music_library/records.ex +++ b/lib/music_library/records.ex @@ -12,14 +12,15 @@ defmodule MusicLibrary.Records do [ :id, :type, - :artists, :format, :title, - :release, + :artists, :genres, :musicbrainz_id, + :release_ids, :included_release_group_ids, - :cover_hash + :cover_hash, + :release ] end @@ -56,13 +57,16 @@ defmodule MusicLibrary.Records do Enum.reduce(parsed_query, search_with_order, fn {:artist, artist}, search -> - search |> where([r], like(r.artists, ^"%#{artist}%")) + search + |> where(fragment("records_search_index match 'artists : ?*'", literal(^artist))) {:album, album}, search -> - search |> where([r], like(r.title, ^"%#{album}%")) + search + |> where(fragment("records_search_index match 'title : ?*'", literal(^album))) {:mbid, mbid}, search -> - search |> where([r], r.musicbrainz_id == ^mbid or like(r.artists, ^"%#{mbid}%")) + search + |> where(fragment("records_search_index = '?*'", literal(^mbid))) {:format, format}, search -> search |> where([r], r.format == ^format) @@ -70,12 +74,12 @@ defmodule MusicLibrary.Records do {:type, type}, search -> search |> where([r], r.type == ^type) + {:query, ""}, search -> + search + {:query, raw_query}, search -> search - |> where( - [r], - like(r.title, ^"%#{raw_query}%") or like(r.artists, ^"%#{raw_query}%") - ) + |> where(fragment("records_search_index = '?*'", literal(^raw_query))) end) end diff --git a/lib/music_library/records/search_index.ex b/lib/music_library/records/search_index.ex new file mode 100644 index 00000000..098603de --- /dev/null +++ b/lib/music_library/records/search_index.ex @@ -0,0 +1,30 @@ +defmodule MusicLibrary.Records.SearchIndex do + use Ecto.Schema + + @formats [:cd, :vinyl, :blu_ray, :dvd, :multi] + @types [:album, :ep, :live, :compilation, :single, :other] + + # q = from s in MusicLibrary.Records.SearchIndex, + # where: fragment("records_index = 'lex*'"), select: s.id + + @primary_key {:id, :binary_id, autogenerate: false} + schema "records_search_index" do + field :type, Ecto.Enum, values: @types + field :format, Ecto.Enum, values: @formats + field :title, :string + field :musicbrainz_id, Ecto.UUID + field :genres, {:array, :string} + field :release, :string + field :purchased_at, :utc_datetime + field :cover_hash, :string + field :release_ids, {:array, :string}, default: [] + field :included_release_group_ids, {:array, :string}, default: [] + + embeds_many :artists, Artist do + field :name, :string + field :sort_name, :string + field :disambiguation, :string + field :musicbrainz_id, Ecto.UUID + end + end +end diff --git a/lib/music_library/wishlist.ex b/lib/music_library/wishlist.ex index b1fb9669..bcb19192 100644 --- a/lib/music_library/wishlist.ex +++ b/lib/music_library/wishlist.ex @@ -3,7 +3,7 @@ defmodule MusicLibrary.Wishlist do alias MusicLibrary.Repo alias MusicLibrary.Records - alias MusicLibrary.Records.Record + alias MusicLibrary.Records.SearchIndex def search_records(query, opts \\ []) do limit = Keyword.get(opts, :limit, 20) @@ -31,7 +31,7 @@ defmodule MusicLibrary.Wishlist do end defp base_search do - from r in Record, + from r in SearchIndex, where: is_nil(r.purchased_at) end end diff --git a/priv/repo/migrations/20241122094655_create_records_search_index.exs b/priv/repo/migrations/20241122094655_create_records_search_index.exs new file mode 100644 index 00000000..a2b59f3c --- /dev/null +++ b/priv/repo/migrations/20241122094655_create_records_search_index.exs @@ -0,0 +1,151 @@ +defmodule MusicLibrary.Repo.Migrations.CreateRecordsSearchIndex do + use Ecto.Migration + + def up do + execute """ + CREATE VIRTUAL TABLE records_search_index USING fts5( + id UNINDEXED, + type, + format, + title, + artists, + genres, + musicbrainz_id, + release_ids UNINDEXED, + included_release_group_ids UNINDEXED, + cover_hash UNINDEXED, + purchased_at UNINDEXED, + release + ); + """ + + flush() + + execute """ + CREATE TRIGGER records_search_index_before_update + BEFORE UPDATE ON records + BEGIN + DELETE FROM records_search_index WHERE id=OLD.id; + END; + """ + + execute """ + CREATE TRIGGER records_search_index_before_delete + BEFORE DELETE ON records + BEGIN + DELETE FROM records_search_index WHERE id=OLD.id; + END; + """ + + execute """ + CREATE TRIGGER records_after_insert + AFTER INSERT ON records + BEGIN + INSERT INTO records_search_index( + id, + type, + format, + title, + artists, + genres, + musicbrainz_id, + release_ids, + included_release_group_ids, + cover_hash, + purchased_at, + release + ) SELECT + id, + type, + format, + title, + artists, + genres, + musicbrainz_id, + release_ids, + included_release_group_ids, + cover_hash, + purchased_at, + release + FROM records + WHERE NEW.id = records.id; + END; + """ + + execute """ + CREATE TRIGGER records_after_update + AFTER UPDATE ON records + BEGIN + INSERT INTO records_search_index( + id, + type, + format, + title, + artists, + genres, + musicbrainz_id, + release_ids, + included_release_group_ids, + cover_hash, + purchased_at, + release + ) SELECT + id, + type, + format, + title, + artists, + genres, + musicbrainz_id, + release_ids, + included_release_group_ids, + cover_hash, + purchased_at, + release + FROM records + WHERE NEW.id = records.id; + END; + """ + + flush() + + execute """ + INSERT INTO records_search_index( + id, + type, + format, + title, + artists, + genres, + musicbrainz_id, + release_ids, + included_release_group_ids, + cover_hash, + purchased_at, + release + ) SELECT + id, + type, + format, + title, + artists, + genres, + musicbrainz_id, + release_ids, + included_release_group_ids, + cover_hash, + purchased_at, + release + FROM records; + """ + + flush() + end + + def down do + execute "DROP TRIGGER IF EXISTS records_search_index_before_update" + execute "DROP TRIGGER IF EXISTS records_search_index_before_delete" + execute "DROP TRIGGER IF EXISTS records_after_update" + execute "DROP TABLE records_search_index" + end +end diff --git a/test/music_library/records_test.exs b/test/music_library/records_test.exs index f5cac231..26732880 100644 --- a/test/music_library/records_test.exs +++ b/test/music_library/records_test.exs @@ -3,7 +3,7 @@ defmodule MusicLibrary.RecordsTest do use MusicLibrary.DataCase alias MusicLibrary.Records - alias MusicLibrary.Records.Record + alias MusicLibrary.Records.SearchIndex alias MusicBrainz.APIBehaviourMock import MusicLibrary.RecordsFixtures import MusicLibrary.ReleaseGroupsFixtures @@ -26,7 +26,7 @@ defmodule MusicLibrary.RecordsTest do # when searching we do not return all record fields (e.g. cover data) # so we rely on record ids to compare results defp search(query, limit, offset) do - Record + SearchIndex |> Records.search_records(query, limit: limit, offset: offset) |> Enum.map(& &1.id) end @@ -115,24 +115,24 @@ defmodule MusicLibrary.RecordsTest do setup [:create_records] test "untagged search" do - assert 2 == Records.search_records_count(Record, "brave") + assert 2 == Records.search_records_count(SearchIndex, "brave") end test "tagged search - album" do - assert 1 == Records.search_records_count(Record, ~s(album:"Brave \(Live\)")) + assert 1 == Records.search_records_count(SearchIndex, ~s(album:"Brave \(Live\)")) end test "tagged search - artist" do - assert 2 == Records.search_records_count(Record, "artist:airbag") - assert 1 == Records.search_records_count(Record, ~s(artist:"airbag \(AU\)")) + assert 2 == Records.search_records_count(SearchIndex, "artist:airbag") + assert 1 == Records.search_records_count(SearchIndex, ~s(artist:"airbag \(AU\)")) end test "tagged search - format" do - assert 1 == Records.search_records_count(Record, "brave format:cd") + assert 1 == Records.search_records_count(SearchIndex, "brave format:cd") end test "tagged search - type" do - assert 1 == Records.search_records_count(Record, "brave type:live") + assert 1 == Records.search_records_count(SearchIndex, "brave type:live") end test "tagged search - mbid", %{records: [_, _, _, greatest_show_on_earth, libertad]} do @@ -140,9 +140,9 @@ defmodule MusicLibrary.RecordsTest do [airbag_au_mbid] = Enum.map(libertad.artists, fn a -> a.musicbrainz_id end) assert 1 == - Records.search_records_count(Record, "mbid:#{airbag_mbid}") + Records.search_records_count(SearchIndex, "mbid:#{airbag_mbid}") - assert 1 == Records.search_records_count(Record, "mbid:#{airbag_au_mbid}") + assert 1 == Records.search_records_count(SearchIndex, "mbid:#{airbag_au_mbid}") end end