cloud/music_library
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce separate search index

Browse Source 
Uses a fts5 virtual table, kept up to date via triggers on the records
table.
This commit is contained in:
Claudio Ortolina
2024-11-22 10:01:48 +00:00
parent be36c29f76
commit 2e2000fe01
6 changed files with 215 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/music_library/collection.ex
+8 -5
View File
@@ -3,7 +3,7 @@ defmodule MusicLibrary.Collection do
alias MusicLibrary.Repo
alias MusicLibrary.Records
alias MusicLibrary.Records.Record
alias MusicLibrary.Records.{Record, SearchIndex}
def search_records(query, opts \\ []) do
limit = Keyword.get(opts, :limit, 20)
@@ -18,7 +18,8 @@ defmodule MusicLibrary.Collection do
def count_records_by_format do
q =
from r in base_search(),
from r in Record,
where: not is_nil(r.purchased_at),
group_by: r.format,
order_by: [desc: count(r.id)],
select: {r.format, count(r.id)}
@@ -28,7 +29,8 @@ defmodule MusicLibrary.Collection do
def count_records_by_type do
q =
from r in base_search(),
from r in Record,
where: not is_nil(r.purchased_at),
group_by: r.type,
order_by: [desc: count(r.id)],
select: {r.type, count(r.id)}
@@ -38,7 +40,8 @@ defmodule MusicLibrary.Collection do
def get_latest_record! do
q =
from r in base_search(),
from r in Record,
where: not is_nil(r.purchased_at),
order_by: [desc: r.purchased_at],
limit: 1,
select: ^Records.essential_fields()
@@ -57,7 +60,7 @@ defmodule MusicLibrary.Collection do
end
defp base_search do
from r in Record,
from r in SearchIndex,
where: not is_nil(r.purchased_at)
end
end
lib/music_library/records.ex
+14 -10
View File
@@ -12,14 +12,15 @@ defmodule MusicLibrary.Records do
[
:id,
:type,
:artists,
:format,
:title,
:release,
:artists,
:genres,
:musicbrainz_id,
:release_ids,
:included_release_group_ids,
:cover_hash
:cover_hash,
:release
]
end
@@ -56,13 +57,16 @@ defmodule MusicLibrary.Records do
Enum.reduce(parsed_query, search_with_order, fn
{:artist, artist}, search ->
search |> where([r], like(r.artists, ^"%#{artist}%"))
search
|> where(fragment("records_search_index match 'artists : ?*'", literal(^artist)))
{:album, album}, search ->
search |> where([r], like(r.title, ^"%#{album}%"))
search
|> where(fragment("records_search_index match 'title : ?*'", literal(^album)))
{:mbid, mbid}, search ->
search |> where([r], r.musicbrainz_id == ^mbid or like(r.artists, ^"%#{mbid}%"))
search
|> where(fragment("records_search_index = '?*'", literal(^mbid)))
{:format, format}, search ->
search |> where([r], r.format == ^format)
@@ -70,12 +74,12 @@ defmodule MusicLibrary.Records do
{:type, type}, search ->
search |> where([r], r.type == ^type)
{:query, ""}, search ->
search
{:query, raw_query}, search ->
search
|> where(
[r],
like(r.title, ^"%#{raw_query}%") or like(r.artists, ^"%#{raw_query}%")
)
|> where(fragment("records_search_index = '?*'", literal(^raw_query)))
end)
end
lib/music_library/records/search_index.ex
+30
View File
@@ -0,0 +1,30 @@
defmodule MusicLibrary.Records.SearchIndex do
use Ecto.Schema
@formats [:cd, :vinyl, :blu_ray, :dvd, :multi]
@types [:album, :ep, :live, :compilation, :single, :other]
# q = from s in MusicLibrary.Records.SearchIndex,
# where: fragment("records_index = 'lex*'"), select: s.id
@primary_key {:id, :binary_id, autogenerate: false}
schema "records_search_index" do
field :type, Ecto.Enum, values: @types
field :format, Ecto.Enum, values: @formats
field :title, :string
field :musicbrainz_id, Ecto.UUID
field :genres, {:array, :string}
field :release, :string
field :purchased_at, :utc_datetime
field :cover_hash, :string
field :release_ids, {:array, :string}, default: []
field :included_release_group_ids, {:array, :string}, default: []
embeds_many :artists, Artist do
field :name, :string
field :sort_name, :string
field :disambiguation, :string
field :musicbrainz_id, Ecto.UUID
end
end
end
lib/music_library/wishlist.ex
+2 -2
View File
@@ -3,7 +3,7 @@ defmodule MusicLibrary.Wishlist do
alias MusicLibrary.Repo
alias MusicLibrary.Records
alias MusicLibrary.Records.Record
alias MusicLibrary.Records.SearchIndex
def search_records(query, opts \\ []) do
limit = Keyword.get(opts, :limit, 20)
@@ -31,7 +31,7 @@ defmodule MusicLibrary.Wishlist do
end
defp base_search do
from r in Record,
from r in SearchIndex,
where: is_nil(r.purchased_at)
end
end
priv/repo/migrations/20241122094655_create_records_search_index.exs
+151
View File
@@ -0,0 +1,151 @@
defmodule MusicLibrary.Repo.Migrations.CreateRecordsSearchIndex do
use Ecto.Migration
def up do
execute """
CREATE VIRTUAL TABLE records_search_index USING fts5(
id UNINDEXED,
type,
format,
title,
artists,
genres,
musicbrainz_id,
release_ids UNINDEXED,
included_release_group_ids UNINDEXED,
cover_hash UNINDEXED,
purchased_at UNINDEXED,
release
);
"""
flush()
execute """
CREATE TRIGGER records_search_index_before_update
BEFORE UPDATE ON records
BEGIN
DELETE FROM records_search_index WHERE id=OLD.id;
END;
"""
execute """
CREATE TRIGGER records_search_index_before_delete
BEFORE DELETE ON records
BEGIN
DELETE FROM records_search_index WHERE id=OLD.id;
END;
"""
execute """
CREATE TRIGGER records_after_insert
AFTER INSERT ON records
BEGIN
INSERT INTO records_search_index(
id,
type,
format,
title,
artists,
genres,
musicbrainz_id,
release_ids,
included_release_group_ids,
cover_hash,
purchased_at,
release
) SELECT
id,
type,
format,
title,
artists,
genres,
musicbrainz_id,
release_ids,
included_release_group_ids,
cover_hash,
purchased_at,
release
FROM records
WHERE NEW.id = records.id;
END;
"""
execute """
CREATE TRIGGER records_after_update
AFTER UPDATE ON records
BEGIN
INSERT INTO records_search_index(
id,
type,
format,
title,
artists,
genres,
musicbrainz_id,
release_ids,
included_release_group_ids,
cover_hash,
purchased_at,
release
) SELECT
id,
type,
format,
title,
artists,
genres,
musicbrainz_id,
release_ids,
included_release_group_ids,
cover_hash,
purchased_at,
release
FROM records
WHERE NEW.id = records.id;
END;
"""
flush()
execute """
INSERT INTO records_search_index(
id,
type,
format,
title,
artists,
genres,
musicbrainz_id,
release_ids,
included_release_group_ids,
cover_hash,
purchased_at,
release
) SELECT
id,
type,
format,
title,
artists,
genres,
musicbrainz_id,
release_ids,
included_release_group_ids,
cover_hash,
purchased_at,
release
FROM records;
"""
flush()
end
def down do
execute "DROP TRIGGER IF EXISTS records_search_index_before_update"
execute "DROP TRIGGER IF EXISTS records_search_index_before_delete"
execute "DROP TRIGGER IF EXISTS records_after_update"
execute "DROP TABLE records_search_index"
end
end
test/music_library/records_test.exs
+10 -10
View File
@@ -3,7 +3,7 @@ defmodule MusicLibrary.RecordsTest do
use MusicLibrary.DataCase
alias MusicLibrary.Records
alias MusicLibrary.Records.Record
alias MusicLibrary.Records.SearchIndex
alias MusicBrainz.APIBehaviourMock
import MusicLibrary.RecordsFixtures
import MusicLibrary.ReleaseGroupsFixtures
@@ -26,7 +26,7 @@ defmodule MusicLibrary.RecordsTest do
# when searching we do not return all record fields (e.g. cover data)
# so we rely on record ids to compare results
defp search(query, limit, offset) do
Record
SearchIndex
|> Records.search_records(query, limit: limit, offset: offset)
|> Enum.map(& &1.id)
end
@@ -115,24 +115,24 @@ defmodule MusicLibrary.RecordsTest do
setup [:create_records]
test "untagged search" do
assert 2 == Records.search_records_count(Record, "brave")
assert 2 == Records.search_records_count(SearchIndex, "brave")
end
test "tagged search - album" do
assert 1 == Records.search_records_count(Record, ~s(album:"Brave \(Live\)"))
assert 1 == Records.search_records_count(SearchIndex, ~s(album:"Brave \(Live\)"))
end
test "tagged search - artist" do
assert 2 == Records.search_records_count(Record, "artist:airbag")
assert 1 == Records.search_records_count(Record, ~s(artist:"airbag \(AU\)"))
assert 2 == Records.search_records_count(SearchIndex, "artist:airbag")
assert 1 == Records.search_records_count(SearchIndex, ~s(artist:"airbag \(AU\)"))
end
test "tagged search - format" do
assert 1 == Records.search_records_count(Record, "brave format:cd")
assert 1 == Records.search_records_count(SearchIndex, "brave format:cd")
end
test "tagged search - type" do
assert 1 == Records.search_records_count(Record, "brave type:live")
assert 1 == Records.search_records_count(SearchIndex, "brave type:live")
end
test "tagged search - mbid", %{records: [_, _, _, greatest_show_on_earth, libertad]} do
@@ -140,9 +140,9 @@ defmodule MusicLibrary.RecordsTest do
[airbag_au_mbid] = Enum.map(libertad.artists, fn a -> a.musicbrainz_id end)
assert 1 ==
Records.search_records_count(Record, "mbid:#{airbag_mbid}")
Records.search_records_count(SearchIndex, "mbid:#{airbag_mbid}")
assert 1 == Records.search_records_count(Record, "mbid:#{airbag_au_mbid}")
assert 1 == Records.search_records_count(SearchIndex, "mbid:#{airbag_au_mbid}")
end
end