Add audit tooling for scrobbled tracks data quality (#89)
This commit is contained in:
@@ -44,3 +44,6 @@ npm-debug.log
|
|||||||
|
|
||||||
# LSP Servers
|
# LSP Servers
|
||||||
/.lexical
|
/.lexical
|
||||||
|
|
||||||
|
# Database files in priv directory
|
||||||
|
priv/**/*.db
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
- [Setup](#setup)
|
- [Setup](#setup)
|
||||||
- [Environment configuration](#environment-configuration)
|
- [Environment configuration](#environment-configuration)
|
||||||
- [Running the application](#running-the-application)
|
- [Running the application](#running-the-application)
|
||||||
|
- [Auditing Scrobble Data Quality](#auditing-scrobble-data-quality)
|
||||||
- [Deployment](#deployment)
|
- [Deployment](#deployment)
|
||||||
- [CI](#ci)
|
- [CI](#ci)
|
||||||
- [Architecture](#architecture)
|
- [Architecture](#architecture)
|
||||||
@@ -33,6 +34,7 @@
|
|||||||
connect them with records in the collection or wishlist
|
connect them with records in the collection or wishlist
|
||||||
- scrobble a record
|
- scrobble a record
|
||||||
- store a local copy of the complete scrobble history, and setup rules to fix its data as needed
|
- store a local copy of the complete scrobble history, and setup rules to fix its data as needed
|
||||||
|
- audit scrobble data quality and identify tracks with missing MusicBrainz IDs
|
||||||
- Some basic stats
|
- Some basic stats
|
||||||
- All data stored in a single SQLite database for portability and ease of backup/restore
|
- All data stored in a single SQLite database for portability and ease of backup/restore
|
||||||
|
|
||||||
@@ -106,6 +108,57 @@ Start the Phoenix endpoint with `mise run console` (along with an attached IEx s
|
|||||||
Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
|
Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
|
||||||
The default password for development is `change me`.
|
The default password for development is `change me`.
|
||||||
|
|
||||||
|
## Auditing Scrobble Data Quality
|
||||||
|
|
||||||
|
The application includes a Mix task to audit scrobbled tracks and identify data quality issues such as missing MusicBrainz IDs for artists and albums.
|
||||||
|
|
||||||
|
### Running the Audit
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Audit all tracks
|
||||||
|
mix scrobble.audit
|
||||||
|
|
||||||
|
# Audit with detailed output including sample tracks
|
||||||
|
mix scrobble.audit --verbose
|
||||||
|
|
||||||
|
# Audit only artist issues
|
||||||
|
mix scrobble.audit --type artist
|
||||||
|
|
||||||
|
# Audit only album issues
|
||||||
|
mix scrobble.audit --type album
|
||||||
|
|
||||||
|
# Output as JSON for processing
|
||||||
|
mix scrobble.audit --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
### Understanding the Audit Report
|
||||||
|
|
||||||
|
The audit report shows:
|
||||||
|
- Total number of scrobbled tracks
|
||||||
|
- Artists with missing MusicBrainz IDs (grouped by artist name)
|
||||||
|
- Albums with missing MusicBrainz IDs (grouped by album title and artist)
|
||||||
|
- Track counts for each issue
|
||||||
|
|
||||||
|
### Fixing Data Quality Issues
|
||||||
|
|
||||||
|
After identifying issues, you can:
|
||||||
|
|
||||||
|
1. **Create Scrobble Rules**: Navigate to the Scrobble Rules page in the web interface and add rules to map artist or album names to their correct MusicBrainz IDs.
|
||||||
|
|
||||||
|
2. **Apply Rules**: Use the "Apply Rules" button in the Scrobble Rules page to update existing tracks, or run in IEx:
|
||||||
|
```elixir
|
||||||
|
MusicLibrary.ScrobbleRules.apply_all_rules()
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Re-audit**: Run the audit again to verify the fixes worked.
|
||||||
|
|
||||||
|
The application also provides helper functions in the `MusicLibrary.ScrobbleActivity` context:
|
||||||
|
- `count_tracks_missing_artist_musicbrainz_id/0`
|
||||||
|
- `count_tracks_missing_album_musicbrainz_id/0`
|
||||||
|
- `get_artists_missing_musicbrainz_id/1`
|
||||||
|
- `get_albums_missing_musicbrainz_id/1`
|
||||||
|
|
||||||
|
|
||||||
## Deployment
|
## Deployment
|
||||||
|
|
||||||
The application is deployed via Coolify, using a Docker Compose strategy.
|
The application is deployed via Coolify, using a Docker Compose strategy.
|
||||||
|
|||||||
@@ -0,0 +1,379 @@
|
|||||||
|
defmodule Mix.Tasks.Scrobble.Audit do
|
||||||
|
@shortdoc "Audit scrobbled tracks for data quality issues"
|
||||||
|
@moduledoc """
|
||||||
|
Audit scrobbled tracks to identify data quality issues such as:
|
||||||
|
- Missing or empty MusicBrainz IDs for artists
|
||||||
|
- Missing or empty MusicBrainz IDs for albums
|
||||||
|
- Tracks that could benefit from scrobble rules
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
# Audit all tracks
|
||||||
|
mix scrobble.audit
|
||||||
|
|
||||||
|
# Audit and output detailed report
|
||||||
|
mix scrobble.audit --verbose
|
||||||
|
|
||||||
|
# Audit and output as JSON
|
||||||
|
mix scrobble.audit --format json
|
||||||
|
|
||||||
|
# Audit only tracks with missing artist IDs
|
||||||
|
mix scrobble.audit --type artist
|
||||||
|
|
||||||
|
# Audit only tracks with missing album IDs
|
||||||
|
mix scrobble.audit --type album
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
The task generates a report showing:
|
||||||
|
- Total scrobbled tracks
|
||||||
|
- Tracks with missing artist MusicBrainz IDs (grouped by artist name)
|
||||||
|
- Tracks with missing album MusicBrainz IDs (grouped by album title + artist)
|
||||||
|
- Suggested scrobble rules to fix the issues
|
||||||
|
"""
|
||||||
|
|
||||||
|
use Mix.Task
|
||||||
|
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
alias LastFm.Track
|
||||||
|
alias MusicLibrary.{Repo, ScrobbleActivity}
|
||||||
|
|
||||||
|
@impl Mix.Task
|
||||||
|
def run(args) do
|
||||||
|
Mix.Task.run("app.start")
|
||||||
|
|
||||||
|
opts = parse_args(args)
|
||||||
|
audit_type = Keyword.get(opts, :type, :all)
|
||||||
|
format = Keyword.get(opts, :format, :text)
|
||||||
|
verbose = Keyword.get(opts, :verbose, false)
|
||||||
|
|
||||||
|
report = generate_audit_report(audit_type, verbose)
|
||||||
|
|
||||||
|
case format do
|
||||||
|
:json -> output_json(report)
|
||||||
|
:text -> output_text(report, verbose)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp parse_args(args) do
|
||||||
|
{opts, _, _} =
|
||||||
|
OptionParser.parse(args,
|
||||||
|
switches: [type: :string, format: :string, verbose: :boolean],
|
||||||
|
aliases: [t: :type, f: :format, v: :verbose]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate and convert type option
|
||||||
|
type =
|
||||||
|
case Keyword.get(opts, :type) do
|
||||||
|
nil ->
|
||||||
|
:all
|
||||||
|
|
||||||
|
"artist" ->
|
||||||
|
:artist
|
||||||
|
|
||||||
|
"album" ->
|
||||||
|
:album
|
||||||
|
|
||||||
|
other ->
|
||||||
|
Mix.raise("Invalid type: #{other}. Valid types are: artist, album")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Validate and convert format option
|
||||||
|
format =
|
||||||
|
case Keyword.get(opts, :format) do
|
||||||
|
nil ->
|
||||||
|
:text
|
||||||
|
|
||||||
|
"json" ->
|
||||||
|
:json
|
||||||
|
|
||||||
|
"text" ->
|
||||||
|
:text
|
||||||
|
|
||||||
|
other ->
|
||||||
|
Mix.raise("Invalid format: #{other}. Valid formats are: json, text")
|
||||||
|
end
|
||||||
|
|
||||||
|
verbose = Keyword.get(opts, :verbose, false)
|
||||||
|
|
||||||
|
[type: type, format: format, verbose: verbose]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp generate_audit_report(:all, verbose) do
|
||||||
|
%{
|
||||||
|
total_tracks: ScrobbleActivity.count_tracks(),
|
||||||
|
artist_issues: audit_artist_musicbrainz_ids(verbose),
|
||||||
|
album_issues: audit_album_musicbrainz_ids(verbose)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp generate_audit_report(:artist, verbose) do
|
||||||
|
%{
|
||||||
|
total_tracks: ScrobbleActivity.count_tracks(),
|
||||||
|
artist_issues: audit_artist_musicbrainz_ids(verbose)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp generate_audit_report(:album, verbose) do
|
||||||
|
%{
|
||||||
|
total_tracks: ScrobbleActivity.count_tracks(),
|
||||||
|
album_issues: audit_album_musicbrainz_ids(verbose)
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp audit_artist_musicbrainz_ids(verbose) do
|
||||||
|
results = ScrobbleActivity.get_artists_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
total_tracks_affected =
|
||||||
|
Enum.reduce(results, 0, fn %{track_count: count}, acc -> acc + count end)
|
||||||
|
|
||||||
|
report = %{
|
||||||
|
total_artists: length(results),
|
||||||
|
total_tracks_affected: total_tracks_affected,
|
||||||
|
artists:
|
||||||
|
Enum.map(results, fn %{artist_name: name, track_count: count} ->
|
||||||
|
%{name: name, track_count: count}
|
||||||
|
end)
|
||||||
|
}
|
||||||
|
|
||||||
|
if verbose do
|
||||||
|
Map.put(report, :sample_tracks, get_sample_tracks_by_artist(Enum.take(results, 5)))
|
||||||
|
else
|
||||||
|
report
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp audit_album_musicbrainz_ids(verbose) do
|
||||||
|
results = ScrobbleActivity.get_albums_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
total_tracks_affected =
|
||||||
|
Enum.reduce(results, 0, fn %{track_count: count}, acc -> acc + count end)
|
||||||
|
|
||||||
|
report = %{
|
||||||
|
total_albums: length(results),
|
||||||
|
total_tracks_affected: total_tracks_affected,
|
||||||
|
albums:
|
||||||
|
Enum.map(results, fn %{album_title: title, artist_name: artist, track_count: count} ->
|
||||||
|
%{title: title, artist: artist, track_count: count}
|
||||||
|
end)
|
||||||
|
}
|
||||||
|
|
||||||
|
if verbose do
|
||||||
|
Map.put(report, :sample_tracks, get_sample_tracks_by_album(Enum.take(results, 5)))
|
||||||
|
else
|
||||||
|
report
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp get_sample_tracks_by_artist(artists) do
|
||||||
|
artist_names = Enum.map(artists, & &1.artist_name)
|
||||||
|
|
||||||
|
query =
|
||||||
|
from t in Track,
|
||||||
|
where:
|
||||||
|
fragment("json_extract(?, '$.name')", t.artist) in ^artist_names and
|
||||||
|
(fragment("json_extract(?, '$.musicbrainz_id') IS NULL", t.artist) or
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') = ''", t.artist)),
|
||||||
|
select: %{
|
||||||
|
artist_name: fragment("json_extract(?, '$.name')", t.artist),
|
||||||
|
title: t.title,
|
||||||
|
album: fragment("json_extract(?, '$.title')", t.album),
|
||||||
|
scrobbled_at: t.scrobbled_at_label
|
||||||
|
}
|
||||||
|
|
||||||
|
tracks_by_artist =
|
||||||
|
query
|
||||||
|
|> Repo.all()
|
||||||
|
|> Enum.group_by(& &1.artist_name, fn track -> Map.delete(track, :artist_name) end)
|
||||||
|
|
||||||
|
Enum.map(artists, fn %{artist_name: artist_name} ->
|
||||||
|
%{
|
||||||
|
artist: artist_name,
|
||||||
|
sample_tracks:
|
||||||
|
tracks_by_artist
|
||||||
|
|> Map.get(artist_name, [])
|
||||||
|
|> Enum.take(3)
|
||||||
|
}
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp get_sample_tracks_by_album(albums) do
|
||||||
|
album_keys =
|
||||||
|
Enum.map(albums, fn %{album_title: album_title, artist_name: artist_name} ->
|
||||||
|
{album_title, artist_name}
|
||||||
|
end)
|
||||||
|
|
||||||
|
case album_keys do
|
||||||
|
[] ->
|
||||||
|
[]
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
base_query =
|
||||||
|
from t in Track,
|
||||||
|
where:
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') IS NULL", t.album) or
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') = ''", t.album),
|
||||||
|
select: %{
|
||||||
|
title: t.title,
|
||||||
|
scrobbled_at: t.scrobbled_at_label,
|
||||||
|
album_title: fragment("json_extract(?, '$.title')", t.album),
|
||||||
|
artist_name: fragment("json_extract(?, '$.name')", t.artist)
|
||||||
|
}
|
||||||
|
|
||||||
|
album_match_dynamic =
|
||||||
|
Enum.reduce(album_keys, false, fn {album_title, artist_name}, dynamic_acc ->
|
||||||
|
pair_dynamic =
|
||||||
|
dynamic(
|
||||||
|
[t],
|
||||||
|
fragment("json_extract(?, '$.title') = ?", t.album, ^album_title) and
|
||||||
|
fragment("json_extract(?, '$.name') = ?", t.artist, ^artist_name)
|
||||||
|
)
|
||||||
|
|
||||||
|
dynamic([t], ^dynamic_acc or ^pair_dynamic)
|
||||||
|
end)
|
||||||
|
|
||||||
|
query =
|
||||||
|
from t in base_query,
|
||||||
|
where: ^album_match_dynamic
|
||||||
|
|
||||||
|
tracks = Repo.all(query)
|
||||||
|
|
||||||
|
tracks_by_key =
|
||||||
|
Enum.group_by(tracks, fn %{album_title: album_title, artist_name: artist_name} ->
|
||||||
|
{album_title, artist_name}
|
||||||
|
end)
|
||||||
|
|
||||||
|
Enum.map(albums, fn %{album_title: album_title, artist_name: artist_name} ->
|
||||||
|
key = {album_title, artist_name}
|
||||||
|
|
||||||
|
sample_tracks =
|
||||||
|
tracks_by_key
|
||||||
|
|> Map.get(key, [])
|
||||||
|
|> Enum.take(3)
|
||||||
|
|> Enum.map(fn %{title: title, scrobbled_at: scrobbled_at} ->
|
||||||
|
%{title: title, scrobbled_at: scrobbled_at}
|
||||||
|
end)
|
||||||
|
|
||||||
|
%{
|
||||||
|
album: album_title,
|
||||||
|
artist: artist_name,
|
||||||
|
sample_tracks: sample_tracks
|
||||||
|
}
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp output_json(report) do
|
||||||
|
report
|
||||||
|
|> Jason.encode!(pretty: true)
|
||||||
|
|> Mix.Shell.IO.info()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp output_text(report, verbose) do
|
||||||
|
Mix.Shell.IO.info("\n=== Scrobbled Tracks Data Quality Audit ===\n")
|
||||||
|
Mix.Shell.IO.info("Total scrobbled tracks: #{report.total_tracks}\n")
|
||||||
|
|
||||||
|
if Map.has_key?(report, :artist_issues) do
|
||||||
|
output_artist_issues(report.artist_issues, verbose)
|
||||||
|
end
|
||||||
|
|
||||||
|
if Map.has_key?(report, :album_issues) do
|
||||||
|
output_album_issues(report.album_issues, verbose)
|
||||||
|
end
|
||||||
|
|
||||||
|
output_summary(report)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp output_artist_issues(artist_issues, verbose) do
|
||||||
|
Mix.Shell.IO.info("--- Artists with Missing MusicBrainz IDs ---")
|
||||||
|
Mix.Shell.IO.info("Unique artists: #{artist_issues.total_artists}")
|
||||||
|
Mix.Shell.IO.info("Affected tracks: #{artist_issues.total_tracks_affected}\n")
|
||||||
|
|
||||||
|
if artist_issues.total_artists > 0 do
|
||||||
|
Mix.Shell.IO.info("Top artists by track count:")
|
||||||
|
|
||||||
|
artist_issues.artists
|
||||||
|
|> Enum.take(10)
|
||||||
|
|> Enum.each(fn %{name: name, track_count: count} ->
|
||||||
|
Mix.Shell.IO.info(" • #{name} (#{count} tracks)")
|
||||||
|
end)
|
||||||
|
|
||||||
|
Mix.Shell.IO.info("")
|
||||||
|
|
||||||
|
if verbose and Map.has_key?(artist_issues, :sample_tracks) do
|
||||||
|
Mix.Shell.IO.info("Sample tracks:")
|
||||||
|
|
||||||
|
Enum.each(artist_issues.sample_tracks, fn %{artist: artist, sample_tracks: tracks} ->
|
||||||
|
Mix.Shell.IO.info("\n Artist: #{artist}")
|
||||||
|
|
||||||
|
Enum.each(tracks, fn track ->
|
||||||
|
Mix.Shell.IO.info(" - #{track.title} (from #{track.album})")
|
||||||
|
end)
|
||||||
|
end)
|
||||||
|
|
||||||
|
Mix.Shell.IO.info("")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp output_album_issues(album_issues, verbose) do
|
||||||
|
Mix.Shell.IO.info("--- Albums with Missing MusicBrainz IDs ---")
|
||||||
|
Mix.Shell.IO.info("Unique albums: #{album_issues.total_albums}")
|
||||||
|
Mix.Shell.IO.info("Affected tracks: #{album_issues.total_tracks_affected}\n")
|
||||||
|
|
||||||
|
if album_issues.total_albums > 0 do
|
||||||
|
Mix.Shell.IO.info("Top albums by track count:")
|
||||||
|
|
||||||
|
album_issues.albums
|
||||||
|
|> Enum.take(10)
|
||||||
|
|> Enum.each(fn %{title: title, artist: artist, track_count: count} ->
|
||||||
|
Mix.Shell.IO.info(" • #{title} by #{artist} (#{count} tracks)")
|
||||||
|
end)
|
||||||
|
|
||||||
|
Mix.Shell.IO.info("")
|
||||||
|
|
||||||
|
if verbose and Map.has_key?(album_issues, :sample_tracks) do
|
||||||
|
Mix.Shell.IO.info("Sample tracks:")
|
||||||
|
|
||||||
|
Enum.each(album_issues.sample_tracks, fn %{
|
||||||
|
album: album,
|
||||||
|
artist: artist,
|
||||||
|
sample_tracks: tracks
|
||||||
|
} ->
|
||||||
|
Mix.Shell.IO.info("\n Album: #{album} by #{artist}")
|
||||||
|
|
||||||
|
Enum.each(tracks, fn track ->
|
||||||
|
Mix.Shell.IO.info(" - #{track.title}")
|
||||||
|
end)
|
||||||
|
end)
|
||||||
|
|
||||||
|
Mix.Shell.IO.info("")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp output_summary(report) do
|
||||||
|
total_issues =
|
||||||
|
(Map.get(report, :artist_issues, %{}) |> Map.get(:total_tracks_affected, 0)) +
|
||||||
|
(Map.get(report, :album_issues, %{}) |> Map.get(:total_tracks_affected, 0))
|
||||||
|
|
||||||
|
Mix.Shell.IO.info("--- Summary ---")
|
||||||
|
Mix.Shell.IO.info("Total tracks needing enrichment: #{total_issues}")
|
||||||
|
|
||||||
|
Mix.Shell.IO.info("""
|
||||||
|
|
||||||
|
To fix these issues:
|
||||||
|
1. Create scrobble rules for artists with missing MusicBrainz IDs:
|
||||||
|
- Navigate to the Scrobble Rules page in the app
|
||||||
|
- Add artist rules with the correct MusicBrainz ID
|
||||||
|
|
||||||
|
2. Create scrobble rules for albums with missing MusicBrainz IDs:
|
||||||
|
- Add album rules with the correct MusicBrainz ID
|
||||||
|
|
||||||
|
3. Apply the rules to update existing tracks:
|
||||||
|
- Use the "Apply Rules" button in the Scrobble Rules page
|
||||||
|
- Or run: MusicLibrary.ScrobbleRules.apply_all_rules()
|
||||||
|
""")
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -506,6 +506,20 @@ defmodule MusicLibrary.ScrobbleActivity do
|
|||||||
|> Repo.all()
|
|> Repo.all()
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Returns the total number of scrobbled tracks.
|
||||||
|
|
||||||
|
This counts all tracks stored in the database using their `scrobbled_at_uts` as the primary key.
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
iex> MusicLibrary.ScrobbleActivity.count_tracks()
|
||||||
|
42
|
||||||
|
"""
|
||||||
|
def count_tracks do
|
||||||
|
Repo.aggregate(Track, :count, :scrobbled_at_uts)
|
||||||
|
end
|
||||||
|
|
||||||
@doc """
|
@doc """
|
||||||
Gets a single track by scrobbled_at_uts.
|
Gets a single track by scrobbled_at_uts.
|
||||||
"""
|
"""
|
||||||
@@ -556,4 +570,120 @@ defmodule MusicLibrary.ScrobbleActivity do
|
|||||||
|
|
||||||
Repo.aggregate(search_query, :count, :scrobbled_at_uts)
|
Repo.aggregate(search_query, :count, :scrobbled_at_uts)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Counts tracks with missing or empty artist MusicBrainz IDs.
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
iex> count_tracks_missing_artist_musicbrainz_id()
|
||||||
|
42
|
||||||
|
|
||||||
|
"""
|
||||||
|
def count_tracks_missing_artist_musicbrainz_id do
|
||||||
|
query =
|
||||||
|
from t in Track,
|
||||||
|
where:
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') IS NULL", t.artist) or
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') = ''", t.artist),
|
||||||
|
select: count(t.scrobbled_at_uts)
|
||||||
|
|
||||||
|
Repo.one(query) || 0
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Counts tracks with missing or empty album MusicBrainz IDs.
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
iex> count_tracks_missing_album_musicbrainz_id()
|
||||||
|
37
|
||||||
|
|
||||||
|
"""
|
||||||
|
def count_tracks_missing_album_musicbrainz_id do
|
||||||
|
query =
|
||||||
|
from t in Track,
|
||||||
|
where:
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') IS NULL", t.album) or
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') = ''", t.album),
|
||||||
|
select: count(t.scrobbled_at_uts)
|
||||||
|
|
||||||
|
Repo.one(query) || 0
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Gets artists with missing MusicBrainz IDs, grouped by artist name.
|
||||||
|
|
||||||
|
Returns a list of maps with artist name and track count.
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
iex> get_artists_missing_musicbrainz_id()
|
||||||
|
[%{artist_name: "Some Artist", track_count: 5}, ...]
|
||||||
|
|
||||||
|
"""
|
||||||
|
def get_artists_missing_musicbrainz_id(opts \\ []) do
|
||||||
|
limit = Keyword.get(opts, :limit)
|
||||||
|
|
||||||
|
query =
|
||||||
|
from t in Track,
|
||||||
|
where:
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') IS NULL", t.artist) or
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') = ''", t.artist),
|
||||||
|
select: %{
|
||||||
|
artist_name: fragment("json_extract(?, '$.name')", t.artist),
|
||||||
|
track_count: count(t.scrobbled_at_uts)
|
||||||
|
},
|
||||||
|
group_by: fragment("json_extract(?, '$.name')", t.artist),
|
||||||
|
order_by: [desc: count(t.scrobbled_at_uts)]
|
||||||
|
|
||||||
|
query =
|
||||||
|
if limit do
|
||||||
|
from q in query, limit: ^limit
|
||||||
|
else
|
||||||
|
query
|
||||||
|
end
|
||||||
|
|
||||||
|
Repo.all(query)
|
||||||
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Gets albums with missing MusicBrainz IDs, grouped by album title and artist.
|
||||||
|
|
||||||
|
Returns a list of maps with album title, artist name, and track count.
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
iex> get_albums_missing_musicbrainz_id()
|
||||||
|
[%{album_title: "Some Album", artist_name: "Some Artist", track_count: 3}, ...]
|
||||||
|
|
||||||
|
"""
|
||||||
|
def get_albums_missing_musicbrainz_id(opts \\ []) do
|
||||||
|
limit = Keyword.get(opts, :limit)
|
||||||
|
|
||||||
|
query =
|
||||||
|
from t in Track,
|
||||||
|
where:
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') IS NULL", t.album) or
|
||||||
|
fragment("json_extract(?, '$.musicbrainz_id') = ''", t.album),
|
||||||
|
select: %{
|
||||||
|
album_title: fragment("json_extract(?, '$.title')", t.album),
|
||||||
|
artist_name: fragment("json_extract(?, '$.name')", t.artist),
|
||||||
|
track_count: count(t.scrobbled_at_uts)
|
||||||
|
},
|
||||||
|
group_by: [
|
||||||
|
fragment("json_extract(?, '$.title')", t.album),
|
||||||
|
fragment("json_extract(?, '$.name')", t.artist)
|
||||||
|
],
|
||||||
|
order_by: [desc: count(t.scrobbled_at_uts)]
|
||||||
|
|
||||||
|
query =
|
||||||
|
if limit do
|
||||||
|
from q in query, limit: ^limit
|
||||||
|
else
|
||||||
|
query
|
||||||
|
end
|
||||||
|
|
||||||
|
Repo.all(query)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
defmodule Mix.Tasks.Scrobble.AuditTest do
|
||||||
|
use MusicLibrary.DataCase
|
||||||
|
|
||||||
|
import ExUnit.CaptureIO
|
||||||
|
import MusicLibrary.ScrobbledTracksFixtures
|
||||||
|
|
||||||
|
alias Mix.Tasks.Scrobble.Audit
|
||||||
|
|
||||||
|
describe "run/1" do
|
||||||
|
test "generates audit report with no tracks" do
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run([])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ "Scrobbled Tracks Data Quality Audit"
|
||||||
|
assert output =~ "Total scrobbled tracks: 0"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "raises error for invalid type" do
|
||||||
|
assert_raise Mix.Error, ~r/Invalid type/, fn ->
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run(["--type", "invalid"])
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "raises error for invalid format" do
|
||||||
|
assert_raise Mix.Error, ~r/Invalid format/, fn ->
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run(["--format", "invalid"])
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "generates audit report with tracks having missing artist musicbrainz_ids" do
|
||||||
|
# Create tracks with empty artist musicbrainz_id
|
||||||
|
track_fixture(%{
|
||||||
|
artist_name: "Test Artist 1",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
title: "Track 1"
|
||||||
|
})
|
||||||
|
|
||||||
|
track_fixture(%{
|
||||||
|
artist_name: "Test Artist 1",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
title: "Track 2"
|
||||||
|
})
|
||||||
|
|
||||||
|
track_fixture(%{
|
||||||
|
artist_name: "Test Artist 2",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
title: "Track 3"
|
||||||
|
})
|
||||||
|
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run([])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ "Artists with Missing MusicBrainz IDs"
|
||||||
|
assert output =~ "Unique artists: 2"
|
||||||
|
assert output =~ "Affected tracks: 3"
|
||||||
|
assert output =~ "Test Artist 1"
|
||||||
|
assert output =~ "Test Artist 2"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "generates audit report with tracks having missing album musicbrainz_ids" do
|
||||||
|
# Create tracks with empty album musicbrainz_id
|
||||||
|
track_fixture(%{
|
||||||
|
album_title: "Test Album 1",
|
||||||
|
album_musicbrainz_id: "",
|
||||||
|
artist_name: "Test Artist",
|
||||||
|
title: "Track 1"
|
||||||
|
})
|
||||||
|
|
||||||
|
track_fixture(%{
|
||||||
|
album_title: "Test Album 1",
|
||||||
|
album_musicbrainz_id: "",
|
||||||
|
artist_name: "Test Artist",
|
||||||
|
title: "Track 2"
|
||||||
|
})
|
||||||
|
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run([])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ "Albums with Missing MusicBrainz IDs"
|
||||||
|
assert output =~ "Unique albums: 1"
|
||||||
|
assert output =~ "Affected tracks: 2"
|
||||||
|
assert output =~ "Test Album 1"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "generates audit report with --verbose flag" do
|
||||||
|
track_fixture(%{
|
||||||
|
artist_name: "Test Artist",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
title: "Test Track",
|
||||||
|
album_title: "Test Album"
|
||||||
|
})
|
||||||
|
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run(["--verbose"])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ "Sample tracks:"
|
||||||
|
assert output =~ "Test Artist"
|
||||||
|
assert output =~ "Test Track"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "generates audit report for artists only with --type artist" do
|
||||||
|
track_fixture(%{
|
||||||
|
artist_name: "Test Artist",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
album_musicbrainz_id: "",
|
||||||
|
title: "Track 1"
|
||||||
|
})
|
||||||
|
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run(["--type", "artist"])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ "Artists with Missing MusicBrainz IDs"
|
||||||
|
refute output =~ "Albums with Missing MusicBrainz IDs"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "generates audit report for albums only with --type album" do
|
||||||
|
track_fixture(%{
|
||||||
|
album_title: "Test Album",
|
||||||
|
album_musicbrainz_id: "",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
title: "Track 1"
|
||||||
|
})
|
||||||
|
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run(["--type", "album"])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ "Albums with Missing MusicBrainz IDs"
|
||||||
|
refute output =~ "Artists with Missing MusicBrainz IDs"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "generates JSON output with --format json" do
|
||||||
|
track_fixture(%{
|
||||||
|
artist_name: "Test Artist",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
title: "Track 1"
|
||||||
|
})
|
||||||
|
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run(["--format", "json"])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ ~s("total_tracks")
|
||||||
|
assert output =~ ~s("artist_issues")
|
||||||
|
assert output =~ ~s("album_issues")
|
||||||
|
|
||||||
|
# Validate it's valid JSON
|
||||||
|
assert {:ok, _parsed} = Jason.decode(output)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "audit report shows summary with remediation steps" do
|
||||||
|
track_fixture(%{
|
||||||
|
artist_name: "Test Artist",
|
||||||
|
artist_musicbrainz_id: "",
|
||||||
|
title: "Track 1"
|
||||||
|
})
|
||||||
|
|
||||||
|
output =
|
||||||
|
capture_io(fn ->
|
||||||
|
Audit.run([])
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert output =~ "Summary"
|
||||||
|
assert output =~ "To fix these issues:"
|
||||||
|
assert output =~ "Create scrobble rules"
|
||||||
|
assert output =~ "MusicLibrary.ScrobbleRules.apply_all_rules()"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -414,6 +414,172 @@ defmodule MusicLibrary.ScrobbleActivityTest do
|
|||||||
|> Enum.map(fn r -> r.track end)
|
|> Enum.map(fn r -> r.track end)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "count_tracks_missing_artist_musicbrainz_id/0" do
|
||||||
|
test "returns zero when no tracks exist" do
|
||||||
|
count = ScrobbleActivity.count_tracks_missing_artist_musicbrainz_id()
|
||||||
|
|
||||||
|
assert count == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns count of tracks with empty artist musicbrainz_id" do
|
||||||
|
track_fixture(%{artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_musicbrainz_id: "valid-id"})
|
||||||
|
|
||||||
|
count = ScrobbleActivity.count_tracks_missing_artist_musicbrainz_id()
|
||||||
|
|
||||||
|
assert count == 2
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "count_tracks_missing_album_musicbrainz_id/0" do
|
||||||
|
test "returns zero when no tracks exist" do
|
||||||
|
count = ScrobbleActivity.count_tracks_missing_album_musicbrainz_id()
|
||||||
|
|
||||||
|
assert count == 0
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns count of tracks with empty album musicbrainz_id" do
|
||||||
|
track_fixture(%{album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_musicbrainz_id: "valid-id"})
|
||||||
|
|
||||||
|
count = ScrobbleActivity.count_tracks_missing_album_musicbrainz_id()
|
||||||
|
|
||||||
|
assert count == 2
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "get_artists_missing_musicbrainz_id/1" do
|
||||||
|
test "returns empty list when no tracks exist" do
|
||||||
|
result = ScrobbleActivity.get_artists_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert result == []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns artists grouped by name with track counts" do
|
||||||
|
track_fixture(%{artist_name: "Artist A", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist A", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist B", artist_musicbrainz_id: ""})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_artists_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert length(result) == 2
|
||||||
|
|
||||||
|
artist_a = Enum.find(result, fn r -> r.artist_name == "Artist A" end)
|
||||||
|
artist_b = Enum.find(result, fn r -> r.artist_name == "Artist B" end)
|
||||||
|
|
||||||
|
assert artist_a.track_count == 2
|
||||||
|
assert artist_b.track_count == 1
|
||||||
|
end
|
||||||
|
|
||||||
|
test "orders artists by track count descending" do
|
||||||
|
track_fixture(%{artist_name: "Artist A", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist B", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist B", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist B", artist_musicbrainz_id: ""})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_artists_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert length(result) == 2
|
||||||
|
assert List.first(result).artist_name == "Artist B"
|
||||||
|
assert List.first(result).track_count == 3
|
||||||
|
end
|
||||||
|
|
||||||
|
test "limits results when limit option provided" do
|
||||||
|
track_fixture(%{artist_name: "Artist A", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist B", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist C", artist_musicbrainz_id: ""})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_artists_missing_musicbrainz_id(limit: 2)
|
||||||
|
|
||||||
|
assert length(result) == 2
|
||||||
|
end
|
||||||
|
|
||||||
|
test "excludes artists with valid musicbrainz_id" do
|
||||||
|
track_fixture(%{artist_name: "Artist A", artist_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{artist_name: "Artist B", artist_musicbrainz_id: "valid-id"})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_artists_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert length(result) == 1
|
||||||
|
assert List.first(result).artist_name == "Artist A"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "get_albums_missing_musicbrainz_id/1" do
|
||||||
|
test "returns empty list when no tracks exist" do
|
||||||
|
result = ScrobbleActivity.get_albums_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert result == []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns albums grouped by title and artist with track counts" do
|
||||||
|
track_fixture(%{
|
||||||
|
album_title: "Album A",
|
||||||
|
artist_name: "Artist X",
|
||||||
|
album_musicbrainz_id: ""
|
||||||
|
})
|
||||||
|
|
||||||
|
track_fixture(%{
|
||||||
|
album_title: "Album A",
|
||||||
|
artist_name: "Artist X",
|
||||||
|
album_musicbrainz_id: ""
|
||||||
|
})
|
||||||
|
|
||||||
|
track_fixture(%{
|
||||||
|
album_title: "Album B",
|
||||||
|
artist_name: "Artist Y",
|
||||||
|
album_musicbrainz_id: ""
|
||||||
|
})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_albums_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert length(result) == 2
|
||||||
|
|
||||||
|
album_a = Enum.find(result, fn r -> r.album_title == "Album A" end)
|
||||||
|
album_b = Enum.find(result, fn r -> r.album_title == "Album B" end)
|
||||||
|
|
||||||
|
assert album_a.track_count == 2
|
||||||
|
assert album_a.artist_name == "Artist X"
|
||||||
|
assert album_b.track_count == 1
|
||||||
|
assert album_b.artist_name == "Artist Y"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "orders albums by track count descending" do
|
||||||
|
track_fixture(%{album_title: "Album A", album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_title: "Album B", album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_title: "Album B", album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_title: "Album B", album_musicbrainz_id: ""})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_albums_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert length(result) == 2
|
||||||
|
assert List.first(result).album_title == "Album B"
|
||||||
|
assert List.first(result).track_count == 3
|
||||||
|
end
|
||||||
|
|
||||||
|
test "limits results when limit option provided" do
|
||||||
|
track_fixture(%{album_title: "Album A", album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_title: "Album B", album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_title: "Album C", album_musicbrainz_id: ""})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_albums_missing_musicbrainz_id(limit: 2)
|
||||||
|
|
||||||
|
assert length(result) == 2
|
||||||
|
end
|
||||||
|
|
||||||
|
test "excludes albums with valid musicbrainz_id" do
|
||||||
|
track_fixture(%{album_title: "Album A", album_musicbrainz_id: ""})
|
||||||
|
track_fixture(%{album_title: "Album B", album_musicbrainz_id: "valid-id"})
|
||||||
|
|
||||||
|
result = ScrobbleActivity.get_albums_missing_musicbrainz_id()
|
||||||
|
|
||||||
|
assert length(result) == 1
|
||||||
|
assert List.first(result).album_title == "Album A"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
defp list_tracks(params) do
|
defp list_tracks(params) do
|
||||||
ScrobbleActivity.list_tracks(params)
|
ScrobbleActivity.list_tracks(params)
|
||||||
|> Enum.map(fn r -> r.track end)
|
|> Enum.map(fn r -> r.track end)
|
||||||
|
|||||||
Reference in New Issue
Block a user