Remove obsolete color extraction strategies

This commit is contained in:
Claudio Ortolina
2026-03-03 16:01:25 +00:00
parent 1d3361c425
commit 6d451481b7
15 changed files with 81 additions and 488 deletions
+1 -3
View File
@@ -117,8 +117,7 @@ Last.fm schemas (separate, not Ecto-persisted to main DB):
| `Req.RateLimiter` | ETS-backed Req request step enforcing per-API minimum intervals between requests |
| `Assets.Cache` | ETS-based asset cache with TTL |
| `Assets.Image` / `Assets.Transform` | Image processing via Vix (libvips) |
| `Colors.ColorFrequencyExtractor` | Color extraction via pixel sampling/histogram |
| `Colors.EdgeWeightedExtractor` | Color extraction weighted by Sobel edge detection |
| `Colors.KMeansExtractor` | Color extraction via K-Means clustering (dominant_colors library) |
| `Chat` | Behaviour for streaming AI chat (`stream_response/3` callback) |
| `RecordChat` | Chat implementation for records (OpenAI streaming, web search enabled) |
| `ArtistChat` | Chat implementation for artists (OpenAI streaming, uses Wikipedia/artist context) |
@@ -164,7 +163,6 @@ stubbed via `Req.Test` (configured in `config/test.exs`).
| `FetchArtistLastFmData` | last_fm | Manual / batch |
| `FetchArtistImage` | heavy_writes | Artist info fetched |
| `RefreshCover` | heavy_writes | Manual action / import |
| `ExtractColors` | heavy_writes | Manual action / import |
| `PopulateGenres` | heavy_writes | Manual action (chains → GenerateRecordEmbedding) |
| `GenerateRecordEmbedding` | heavy_writes | Manual / after genre population |
| `RecordRefreshMusicBrainzData` | music_brainz | Manual / batch |
@@ -1,132 +0,0 @@
defmodule MusicLibrary.Colors.ColorFrequencyExtractor do
@moduledoc """
Extracts dominant colors from images using Vix.
Uses a fast but naive approach based on color sampling and histogram
analysis.
Initially by Claude, using Sonnet 4.
"""
alias Vix.Vips.{Image, Operation}
@doc """
Extracts the n most dominant colors from image data (defaults to 5)
and returns them in hex format, e.g. ["#FF5733", "#33C3FF", "#75FF33"].
"""
@spec extract_dominant_colors(binary(), pos_integer()) :: {:ok, [String.t()]} | {:error, term()}
def extract_dominant_colors(image_data, num_colors \\ 5) do
with {:ok, image} <- Image.new_from_buffer(image_data),
{:ok, processed_image} <- prepare_image_for_analysis(image),
{:ok, colors} <- extract_colors_via_sampling(processed_image, num_colors) do
hex_colors = Enum.map(colors, &rgb_to_hex/1)
{:ok, hex_colors}
end
end
@doc """
Same as `extract-dominant_colors/2`, but raises an error if extraction fails.
"""
@spec extract_dominant_colors!(binary(), pos_integer()) :: [String.t()] | no_return
def extract_dominant_colors!(image_data, num_colors \\ 5) do
case extract_dominant_colors(image_data, num_colors) do
{:ok, colors} -> colors
{:error, reason} -> raise "Failed to extract dominant colors: #{inspect(reason)}"
end
end
defp prepare_image_for_analysis(image) do
with {:ok, resized} <- Operation.thumbnail_image(image, 1000) do
ensure_rgb_channels(resized)
end
end
defp ensure_rgb_channels(image) do
bands = Image.bands(image)
cond do
bands >= 3 ->
# Image already has 3+ channels (RGB or RGBA), use as-is
{:ok, image}
bands == 1 ->
# Grayscale image, convert to 3-channel by copying the single channel
Operation.bandjoin([image, image, image])
true ->
{:error, "Unsupported image format with #{bands} bands"}
end
end
defp extract_colors_via_sampling(image, num_colors) do
width = Image.width(image)
height = Image.height(image)
# Sample every nth pixel to get a good distribution
sample_step = max(1, div(min(width, height), 10))
pixels =
for y <- 0..(height - 1)//sample_step,
x <- 0..(width - 1)//sample_step do
case Operation.getpoint(image, x, y) do
{:ok, [r, g, b | _]} ->
{trunc(r), trunc(g), trunc(b)}
{:ok, [gray]} ->
gray_val = trunc(gray)
{gray_val, gray_val, gray_val}
{:ok, [r, g]} ->
# Handle 2-channel images
{trunc(r), trunc(g), 0}
_ ->
nil
end
end
|> Enum.reject(fn
{r, g, b} ->
# Filter out very dark or very light colors
brightness = (r + g + b) / 3
brightness < 20 || brightness > 235
nil ->
true
end)
if Enum.empty?(pixels) do
{:error, "No valid pixels found for color extraction"}
else
colors = analyze_color_histogram(pixels, num_colors)
{:ok, colors}
end
end
defp analyze_color_histogram(pixels, num_colors) do
# Simple frequency-based approach with color grouping
pixels
|> group_similar_colors()
|> Enum.frequencies()
|> Enum.sort_by(fn {_color, count} -> count end, :desc)
|> Enum.take(num_colors)
|> Enum.map(fn {{r, g, b}, _count} -> {r, g, b} end)
end
defp group_similar_colors(pixels) do
Enum.map(pixels, fn {r, g, b} ->
# Group colors into buckets to reduce similar colors
bucket_size = 64
grouped_r = div(r, bucket_size) * bucket_size
grouped_g = div(g, bucket_size) * bucket_size
grouped_b = div(b, bucket_size) * bucket_size
{grouped_r, grouped_g, grouped_b}
end)
end
defp rgb_to_hex({r, g, b}) do
"#" <>
(Integer.to_string(r, 16) |> String.pad_leading(2, "0") |> String.upcase()) <>
(Integer.to_string(g, 16) |> String.pad_leading(2, "0") |> String.upcase()) <>
(Integer.to_string(b, 16) |> String.pad_leading(2, "0") |> String.upcase())
end
end
@@ -1,123 +0,0 @@
defmodule MusicLibrary.Colors.EdgeWeightedExtractor do
@moduledoc """
Simple edge-weighted color extraction using Vix.
This approach weights colors based on their proximity to edges,
giving more importance to colors from visually significant regions
without requiring complex algorithms.
Extraction is slow, so it should always be done asynchronously.
Generated by Claude, using Sonnet 4.
"""
alias Vix.Vips.{Image, Operation}
@doc """
Extracts dominant colors weighted by edge proximity.
Colors near edges (important features) are given higher importance.
"""
def extract_dominant_colors(image_data, num_colors \\ 5) do
with {:ok, image} <- Image.new_from_buffer(image_data),
{:ok, resized} <- Operation.thumbnail_image(image, 200),
{:ok, rgb_image} <- ensure_rgb_channels(resized),
{:ok, edge_map} <- create_edge_map(rgb_image),
{:ok, colors} <- extract_edge_weighted_colors(rgb_image, edge_map, num_colors) do
hex_colors = Enum.map(colors, &rgb_to_hex/1)
{:ok, hex_colors}
end
end
@doc """
Same as `extract_dominant_colors/2`, but raises an error on failure.
"""
@spec extract_dominant_colors!(binary(), integer()) :: [String.t()] | no_return
def extract_dominant_colors!(image_data, num_colors \\ 5) do
case extract_dominant_colors(image_data, num_colors) do
{:ok, colors} -> colors
{:error, reason} -> raise "Failed to extract dominant colors: #{reason}"
end
end
defp create_edge_map(image) do
# Convert to grayscale for edge detection
# Apply Sobel edge detection - built into Vix!
with {:ok, gray} <- Operation.colourspace(image, :VIPS_INTERPRETATION_B_W) do
Operation.sobel(gray)
end
end
defp ensure_rgb_channels(image) do
bands = Image.bands(image)
cond do
bands >= 3 -> {:ok, image}
bands == 1 -> Operation.bandjoin([image, image, image])
true -> {:error, "Unsupported image format"}
end
end
defp extract_edge_weighted_colors(rgb_image, edge_map, num_colors) do
width = Image.width(rgb_image)
height = Image.height(rgb_image)
# Sample pixels with edge-based weighting
# Sample every 3rd pixel for performance
weighted_pixels =
for y <- 0..(height - 1)//3,
x <- 0..(width - 1)//3 do
with {:ok, [r, g, b | _]} <- Operation.getpoint(rgb_image, x, y),
{:ok, [edge_strength]} <- Operation.getpoint(edge_map, x, y) do
# Weight this pixel by edge strength
# Normalize edge strength
weight = max(1, trunc(edge_strength / 50))
color = {trunc(r), trunc(g), trunc(b)}
# Return the color multiple times based on edge weight
List.duplicate(color, weight)
else
_ -> []
end
end
|> List.flatten()
|> Enum.reject(fn {r, g, b} ->
# Filter out very dark/light colors
brightness = (r + g + b) / 3
brightness < 15 || brightness > 240
end)
if Enum.empty?(weighted_pixels) do
{:error, "No valid edge-weighted pixels found"}
else
colors = cluster_weighted_colors(weighted_pixels, num_colors)
{:ok, colors}
end
end
defp cluster_weighted_colors(pixels, num_colors) do
pixels
# Slightly larger buckets for better clustering
|> group_similar_colors(40)
|> Enum.frequencies()
|> Enum.sort_by(fn {_color, count} -> count end, :desc)
|> Enum.take(num_colors)
|> Enum.map(fn {{r, g, b}, _count} -> {r, g, b} end)
end
defp group_similar_colors(pixels, bucket_size) do
Enum.map(pixels, fn {r, g, b} ->
{
div(r, bucket_size) * bucket_size,
div(g, bucket_size) * bucket_size,
div(b, bucket_size) * bucket_size
}
end)
end
defp rgb_to_hex({r, g, b}) do
"#" <>
(r |> Integer.to_string(16) |> String.pad_leading(2, "0") |> String.upcase()) <>
(g |> Integer.to_string(16) |> String.pad_leading(2, "0") |> String.upcase()) <>
(b |> Integer.to_string(16) |> String.pad_leading(2, "0") |> String.upcase())
end
end
+8 -7
View File
@@ -7,6 +7,7 @@ defmodule MusicLibrary.Records do
alias MusicLibrary.Artists
alias MusicLibrary.Assets
alias MusicLibrary.Colors.KMeansExtractor
alias MusicLibrary.Records.{ArtistRecord, Record, SearchIndex, SearchParser}
alias MusicLibrary.{Repo, Worker}
@@ -304,12 +305,12 @@ defmodule MusicLibrary.Records do
enqueue_worker(Worker.RefreshCover, %{"id" => record.id}, record_meta(record))
end
def extract_colors_async(record, method) do
enqueue_worker(
Worker.ExtractColors,
%{"id" => record.id, "method" => method},
record_meta(record)
)
def extract_colors(record) do
asset = Assets.get!(record.cover_hash)
with {:ok, colors} <- KMeansExtractor.extract_dominant_colors(asset.content) do
update_record(record, %{dominant_colors: colors})
end
end
def generate_embedding_async(record) do
@@ -375,7 +376,7 @@ defmodule MusicLibrary.Records do
def create_record(attrs \\ %{}) do
with {:ok, record} <- do_create_record(attrs) do
extract_colors_async(record, :accurate)
{:ok, record} = extract_colors(record)
generate_embedding_async(record)
record
@@ -1,27 +0,0 @@
defmodule MusicLibrary.Worker.ExtractColors do
use Oban.Worker, queue: :heavy_writes, max_attempts: 3
alias MusicLibrary.{Assets, Records}
alias MusicLibrary.Colors.{ColorFrequencyExtractor, EdgeWeightedExtractor, KMeansExtractor}
@impl Oban.Worker
def perform(%Oban.Job{args: %{"id" => record_id, "method" => method}}) do
record = MusicLibrary.Records.get_record!(record_id)
asset = Assets.get!(record.cover_hash)
method = String.to_existing_atom(method)
with {:ok, colors} <- extract_colors(asset.content, method),
{:ok, updated_record} <- Records.update_record(record, %{dominant_colors: colors}) do
MusicLibrary.Records.notify_update(updated_record)
end
end
defp extract_colors(image_data, :fast),
do: ColorFrequencyExtractor.extract_dominant_colors(image_data)
defp extract_colors(image_data, :slow),
do: EdgeWeightedExtractor.extract_dominant_colors(image_data)
defp extract_colors(image_data, :accurate),
do: KMeansExtractor.extract_dominant_colors(image_data)
end
@@ -170,8 +170,8 @@ defmodule MusicLibraryWeb.CollectionLive.Show do
</.dropdown_link>
<.dropdown_link
id={"actions-#{@record.id}-extract-colors-fast"}
phx-click={JS.push("extract_colors", value: %{id: @record.id, method: :fast})}
id={"actions-#{@record.id}-extract-colors"}
phx-click={JS.push("extract_colors", value: %{id: @record.id})}
>
<.icon
name="hero-paint-brush"
@@ -179,35 +179,7 @@ defmodule MusicLibraryWeb.CollectionLive.Show do
aria-hidden="true"
data-slot="icon"
/>
{gettext("Extract colors (fast)")}
</.dropdown_link>
<.dropdown_link
id={"actions-#{@record.id}-extract-colors-slow"}
phx-click={JS.push("extract_colors", value: %{id: @record.id, method: :slow})}
>
<.icon
name="hero-paint-brush"
class="h-4 w-4 mr-1 phx-click-loading:animate-shake"
aria-hidden="true"
data-slot="icon"
/>
{gettext("Extract colors (slow)")}
</.dropdown_link>
<.dropdown_link
id={"actions-#{@record.id}-extract-colors-accurate"}
phx-click={
JS.push("extract_colors", value: %{id: @record.id, method: :accurate})
}
>
<.icon
name="hero-paint-brush"
class="h-4 w-4 mr-1 phx-click-loading:animate-shake"
aria-hidden="true"
data-slot="icon"
/>
{gettext("Extract colors (accurate)")}
{gettext("Extract colors")}
</.dropdown_link>
<.dropdown_separator />
@@ -442,22 +414,22 @@ defmodule MusicLibraryWeb.CollectionLive.Show do
end
end
def handle_event("extract_colors", %{"id" => id, "method" => method}, socket) do
def handle_event("extract_colors", %{"id" => id}, socket) do
record = Records.get_record!(id)
method = String.to_existing_atom(method)
case Records.extract_colors_async(record, method) do
{:ok, _worker} ->
case Records.extract_colors(record) do
{:ok, updated_record} ->
{:noreply,
socket
|> put_toast(:info, gettext("In progress - record will update automatically"))}
|> assign(:record, updated_record)
|> put_toast(:info, gettext("Colors extracted"))}
{:error, reason} ->
{:noreply,
socket
|> put_toast(
:error,
gettext("Error") <> "," <> inspect(reason)
gettext("Error extracting colors") <> ": " <> inspect(reason)
)}
end
end
@@ -146,8 +146,8 @@ defmodule MusicLibraryWeb.WishlistLive.Show do
</.dropdown_link>
<.dropdown_link
id={"actions-#{@record.id}-extract-colors-fast"}
phx-click={JS.push("extract_colors", value: %{id: @record.id, method: :fast})}
id={"actions-#{@record.id}-extract-colors"}
phx-click={JS.push("extract_colors", value: %{id: @record.id})}
>
<.icon
name="hero-paint-brush"
@@ -155,20 +155,7 @@ defmodule MusicLibraryWeb.WishlistLive.Show do
aria-hidden="true"
data-slot="icon"
/>
{gettext("Extract colors (fast)")}
</.dropdown_link>
<.dropdown_link
id={"actions-#{@record.id}-extract-colors-slow"}
phx-click={JS.push("extract_colors", value: %{id: @record.id, method: :slow})}
>
<.icon
name="hero-paint-brush"
class="h-4 w-4 mr-1 phx-click-loading:animate-shake"
aria-hidden="true"
data-slot="icon"
/>
{gettext("Extract colors (slow)")}
{gettext("Extract colors")}
</.dropdown_link>
<.dropdown_separator />
@@ -392,22 +379,22 @@ defmodule MusicLibraryWeb.WishlistLive.Show do
end
end
def handle_event("extract_colors", %{"id" => id, "method" => method}, socket) do
def handle_event("extract_colors", %{"id" => id}, socket) do
record = Records.get_record!(id)
method = String.to_existing_atom(method)
case Records.extract_colors_async(record, method) do
{:ok, _worker} ->
case Records.extract_colors(record) do
{:ok, updated_record} ->
{:noreply,
socket
|> put_toast(:info, gettext("In progress - record will update automatically"))}
|> assign(:record, updated_record)
|> put_toast(:info, gettext("Colors extracted"))}
{:error, reason} ->
{:noreply,
socket
|> put_toast(
:error,
gettext("Error") <> "," <> inspect(reason)
gettext("Error extracting colors") <> ": " <> inspect(reason)
)}
end
end
+14 -13
View File
@@ -815,18 +815,6 @@ msgstr ""
msgid "Error"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Extract colors (fast)"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Extract colors (slow)"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
@@ -2065,6 +2053,19 @@ msgid "Failed to store cover image"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Extract colors (accurate)"
msgid "Colors extracted"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Error extracting colors"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Extract colors"
msgstr ""
+15 -14
View File
@@ -815,18 +815,6 @@ msgstr ""
msgid "Error"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Extract colors (fast)"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Extract colors (slow)"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
@@ -2065,6 +2053,19 @@ msgid "Failed to store cover image"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Extract colors (accurate)"
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Colors extracted"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Error extracting colors"
msgstr ""
#: lib/music_library_web/live/collection_live/show.ex
#: lib/music_library_web/live/wishlist_live/show.ex
#, elixir-autogen, elixir-format
msgid "Extract colors"
msgstr ""
@@ -1,34 +0,0 @@
defmodule MusicLibrary.Colors.ColorFrequencyExtractorTest do
use ExUnit.Case
alias MusicLibrary.Colors.ColorFrequencyExtractor
@image_data MusicLibrary.Fixtures.Records.marbles_cover_data()
describe "extract_dominant_colors/1" do
@describetag :slow
test "extracts colors from an image" do
assert {:ok, colors} = ColorFrequencyExtractor.extract_dominant_colors(@image_data)
assert is_list(colors)
assert colors == ["#000000", "#C08080", "#400000", "#C0C0C0", "#404000"]
end
test "extracts custom number of colors" do
assert {:ok, colors} = ColorFrequencyExtractor.extract_dominant_colors(@image_data, 3)
assert colors == ["#000000", "#C08080", "#400000"]
end
end
describe "extract_dominant_colors!/1" do
@describetag :slow
test "extracts colors or raises" do
colors = ColorFrequencyExtractor.extract_dominant_colors!(@image_data)
assert colors == ["#000000", "#C08080", "#400000", "#C0C0C0", "#404000"]
end
end
end
@@ -1,32 +0,0 @@
defmodule MusicLibrary.Colors.EdgeWeightedExtractorTest do
use ExUnit.Case
alias MusicLibrary.Colors.EdgeWeightedExtractor
@image_data MusicLibrary.Fixtures.Records.marbles_cover_data()
describe "extract_dominant_colors/1" do
@describetag :slow
test "extracts colors from an image" do
assert {:ok, colors} =
EdgeWeightedExtractor.extract_dominant_colors(@image_data)
assert colors == ["#000000", "#A07850", "#785028", "#502828", "#C8A078"]
end
test "extracts custom number of colors" do
assert {:ok, colors} = EdgeWeightedExtractor.extract_dominant_colors(@image_data, 3)
assert colors == ["#000000", "#A07850", "#785028"]
end
end
describe "extract_dominant_colors!/1" do
@describetag :slow
test "extracts colors or raises" do
colors = EdgeWeightedExtractor.extract_dominant_colors!(@image_data)
assert colors == ["#000000", "#A07850", "#785028", "#502828", "#C8A078"]
end
end
end
@@ -0,0 +1,19 @@
defmodule MusicLibrary.Colors.KMeansExtractorTest do
use ExUnit.Case
alias MusicLibrary.Colors.KMeansExtractor
@image_data MusicLibrary.Fixtures.Records.marbles_cover_data()
describe "extract_dominant_colors/1" do
test "extracts 5 colors by default" do
assert {:ok, ["#101111", "#d3b696", "#836246", "#5d412d", "#3c2e22"]} ==
KMeansExtractor.extract_dominant_colors(@image_data)
end
test "extracts custom number of colors" do
assert {:ok, ["#101111", "#d3b696", "#836246"]} ==
KMeansExtractor.extract_dominant_colors(@image_data, 3)
end
end
end
+5 -5
View File
@@ -47,11 +47,11 @@ defmodule MusicLibrary.RecordsTest do
"599407DDF69907D4A60FE13CCAA824D25CF08DC124FD6AA3E8E7ECD98C885FFE"
assert record.dominant_colors == [
"#000000",
"#C0C0C0",
"#C08080",
"#404000",
"#804040"
"#101111",
"#d3b696",
"#836246",
"#5d412d",
"#3c2e22"
]
end
@@ -1,38 +0,0 @@
defmodule MusicLibrary.Worker.ExtractColorsTest do
use MusicLibrary.DataCase
import MusicLibrary.Fixtures.Records
alias MusicLibrary.Records
alias MusicLibrary.Worker.ExtractColors
describe "perform/1" do
@describetag :slow
test "extracts colors using fast method" do
record = record()
assert :ok = perform_job(ExtractColors, %{"id" => record.id, "method" => "fast"})
updated = Records.get_record!(record.id)
assert is_list(updated.dominant_colors)
assert updated.dominant_colors != []
assert Enum.all?(updated.dominant_colors, &String.starts_with?(&1, "#"))
end
test "extracts colors using slow method" do
record = record()
assert :ok = perform_job(ExtractColors, %{"id" => record.id, "method" => "slow"})
updated = Records.get_record!(record.id)
assert is_list(updated.dominant_colors)
assert updated.dominant_colors != []
end
test "raises when record does not exist" do
assert_raise Ecto.NoResultsError, fn ->
perform_job(ExtractColors, %{"id" => Ecto.UUID.generate(), "method" => "fast"})
end
end
end
end
+1 -1
View File
@@ -1,3 +1,3 @@
ExUnit.start(exclude: [:slow])
ExUnit.start()
Ecto.Adapters.SQL.Sandbox.mode(MusicLibrary.Repo, :manual)
Ecto.Adapters.SQL.Sandbox.mode(MusicLibrary.BackgroundRepo, :manual)