From 569f8b5340c30ad153a10577b198d3577d688d0f Mon Sep 17 00:00:00 2001 From: Claudio Ortolina Date: Sun, 8 Jun 2025 07:43:07 +0100 Subject: [PATCH] Distinguish between color extraction strategies By default, use fast color sampling. Run heavy edge weighted extraction only on demand. --- .../colors/color_frequency_extractor.ex | 132 ++++++++++++++++++ .../edge_weighted_extractor.ex} | 4 +- lib/music_library/records.ex | 5 +- lib/music_library/records/record.ex | 7 +- 4 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 lib/music_library/colors/color_frequency_extractor.ex rename lib/music_library/{records/dominant_colors.ex => colors/edge_weighted_extractor.ex} (97%) diff --git a/lib/music_library/colors/color_frequency_extractor.ex b/lib/music_library/colors/color_frequency_extractor.ex new file mode 100644 index 00000000..2a8010d2 --- /dev/null +++ b/lib/music_library/colors/color_frequency_extractor.ex @@ -0,0 +1,132 @@ +defmodule MusicLibrary.Colors.ColorFrequencyExtractor do + @moduledoc """ + Extracts dominant colors from images using Vix. + + Uses a fast but naive approach based on color sampling and histogram + analysis. + + Initially by Claude, using Sonnet 4. + """ + + alias Vix.Vips.{Image, Operation} + + @doc """ + Extracts the n most dominant colors from image data (defaults to 5) + and returns them in hex format, e.g. ["#FF5733", "#33C3FF", "#75FF33"]. + """ + @spec extract_dominant_colors(binary(), pos_integer()) :: {:ok, [String.t()]} | {:error, term()} + def extract_dominant_colors(image_data, num_colors \\ 5) do + with {:ok, image} <- Image.new_from_buffer(image_data), + {:ok, processed_image} <- prepare_image_for_analysis(image), + {:ok, colors} <- extract_colors_via_sampling(processed_image, num_colors) do + hex_colors = Enum.map(colors, &rgb_to_hex/1) + {:ok, hex_colors} + end + end + + @doc """ + Same as `extract-dominant_colors/2`, but raises an error if extraction fails. + """ + @spec extract_dominant_colors!(binary(), pos_integer()) :: [String.t()] | no_return + def extract_dominant_colors!(image_data, num_colors \\ 5) do + case extract_dominant_colors(image_data, num_colors) do + {:ok, colors} -> colors + {:error, reason} -> raise "Failed to extract dominant colors: #{inspect(reason)}" + end + end + + defp prepare_image_for_analysis(image) do + with {:ok, resized} <- Operation.thumbnail_image(image, 150) do + ensure_rgb_channels(resized) + end + end + + defp ensure_rgb_channels(image) do + bands = Image.bands(image) + + cond do + bands >= 3 -> + # Image already has 3+ channels (RGB or RGBA), use as-is + {:ok, image} + + bands == 1 -> + # Grayscale image, convert to 3-channel by copying the single channel + Operation.bandjoin([image, image, image]) + + true -> + {:error, "Unsupported image format with #{bands} bands"} + end + end + + defp extract_colors_via_sampling(image, num_colors) do + width = Image.width(image) + height = Image.height(image) + + # Sample every nth pixel to get a good distribution + sample_step = max(1, div(min(width, height), 10)) + + pixels = + for y <- 0..(height - 1)//sample_step, + x <- 0..(width - 1)//sample_step do + case Operation.getpoint(image, x, y) do + {:ok, [r, g, b | _]} -> + {trunc(r), trunc(g), trunc(b)} + + {:ok, [gray]} -> + gray_val = trunc(gray) + {gray_val, gray_val, gray_val} + + {:ok, [r, g]} -> + # Handle 2-channel images + {trunc(r), trunc(g), 0} + + _ -> + nil + end + end + |> Enum.reject(fn + {r, g, b} -> + # Filter out very dark or very light colors + brightness = (r + g + b) / 3 + brightness < 20 || brightness > 235 + + nil -> + true + end) + + if length(pixels) > 0 do + colors = analyze_color_histogram(pixels, num_colors) + {:ok, colors} + else + {:error, "No valid pixels found for color extraction"} + end + end + + defp analyze_color_histogram(pixels, num_colors) do + # Simple frequency-based approach with color grouping + pixels + |> group_similar_colors() + |> Enum.frequencies() + |> Enum.sort_by(fn {_color, count} -> count end, :desc) + |> Enum.take(num_colors) + |> Enum.map(fn {{r, g, b}, _count} -> {r, g, b} end) + end + + defp group_similar_colors(pixels) do + Enum.map(pixels, fn {r, g, b} -> + # Group colors into buckets to reduce similar colors + bucket_size = 64 + grouped_r = div(r, bucket_size) * bucket_size + grouped_g = div(g, bucket_size) * bucket_size + grouped_b = div(b, bucket_size) * bucket_size + {grouped_r, grouped_g, grouped_b} + end) + end + + defp rgb_to_hex({r, g, b}) do + "#" <> + (Integer.to_string(r, 16) |> String.pad_leading(2, "0") |> String.upcase()) <> + (Integer.to_string(g, 16) |> String.pad_leading(2, "0") |> String.upcase()) <> + (Integer.to_string(b, 16) |> String.pad_leading(2, "0") |> String.upcase()) + end +end diff --git a/lib/music_library/records/dominant_colors.ex b/lib/music_library/colors/edge_weighted_extractor.ex similarity index 97% rename from lib/music_library/records/dominant_colors.ex rename to lib/music_library/colors/edge_weighted_extractor.ex index d2979dfa..e6b11f2a 100644 --- a/lib/music_library/records/dominant_colors.ex +++ b/lib/music_library/colors/edge_weighted_extractor.ex @@ -1,4 +1,4 @@ -defmodule MusicLibrary.Records.DominantColors do +defmodule MusicLibrary.Colors.EdgeWeightedExtractor do @moduledoc """ Simple edge-weighted color extraction using Vix. @@ -6,6 +6,8 @@ defmodule MusicLibrary.Records.DominantColors do giving more importance to colors from visually significant regions without requiring complex algorithms. + Extraction is slow, so it should always be done asynchronously. + Generated by Claude, using Sonnet 4. """ diff --git a/lib/music_library/records.ex b/lib/music_library/records.ex index 8e4b8872..ff93c58f 100644 --- a/lib/music_library/records.ex +++ b/lib/music_library/records.ex @@ -6,7 +6,8 @@ defmodule MusicLibrary.Records do import Ecto.Query, warn: false alias MusicLibrary.Artists - alias MusicLibrary.Records.{ArtistRecord, Cover, DominantColors, Record, SearchParser} + alias MusicLibrary.Colors.EdgeWeightedExtractor + alias MusicLibrary.Records.{ArtistRecord, Cover, Record, SearchParser} alias MusicLibrary.{BackgroundRepo, Repo, Worker} def essential_fields do @@ -255,7 +256,7 @@ defmodule MusicLibrary.Records do end def generate_dominant_colors(record) do - with {:ok, colors} <- DominantColors.extract_dominant_colors(record.cover_data) do + with {:ok, colors} <- EdgeWeightedExtractor.extract_dominant_colors(record.cover_data) do update_record(record, %{"dominant_colors" => colors}) end end diff --git a/lib/music_library/records/record.ex b/lib/music_library/records/record.ex index d8d7997b..9d29de20 100644 --- a/lib/music_library/records/record.ex +++ b/lib/music_library/records/record.ex @@ -5,7 +5,8 @@ defmodule MusicLibrary.Records.Record do alias MusicBrainz.{Release, ReleaseGroup} alias MusicLibrary.Artists.Artist - alias MusicLibrary.Records.{Cover, DominantColors} + alias MusicLibrary.Colors.ColorFrequencyExtractor + alias MusicLibrary.Records.Cover @formats [:cd, :backup, :vinyl, :blu_ray, :dvd, :multi] @types [:album, :ep, :live, :compilation, :single, :other] @@ -170,7 +171,7 @@ defmodule MusicLibrary.Records.Record do end def generate_dominant_colors(%__MODULE__{cover_data: cover_data} = record) do - change(record, dominant_colors: DominantColors.extract_dominant_colors!(cover_data)) + change(record, dominant_colors: ColorFrequencyExtractor.extract_dominant_colors!(cover_data)) end def generate_dominant_colors(changeset) do @@ -182,7 +183,7 @@ defmodule MusicLibrary.Records.Record do put_change( changeset, :dominant_colors, - DominantColors.extract_dominant_colors!(cover_data) + ColorFrequencyExtractor.extract_dominant_colors!(cover_data) ) end end