diff --git a/README.md b/README.md index 69b7f923..a4a0da00 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ mix scrobble.audit --format json ### Understanding the Audit Report The audit report shows: + - Total number of scrobbled tracks - Artists with missing MusicBrainz IDs (grouped by artist name) - Albums with missing MusicBrainz IDs (grouped by album title and artist) @@ -166,6 +167,7 @@ After identifying issues, you can: 1. **Create Scrobble Rules**: Navigate to the Scrobble Rules page in the web interface and add rules to map artist or album names to their correct MusicBrainz IDs. 2. **Apply Rules**: Use the "Apply Rules" button in the Scrobble Rules page to update existing tracks, or run in IEx: + ```elixir MusicLibrary.ScrobbleRules.apply_all_rules() ``` @@ -173,12 +175,12 @@ After identifying issues, you can: 3. **Re-audit**: Run the audit again to verify the fixes worked. The application also provides helper functions in the `MusicLibrary.ScrobbleActivity` context: + - `count_tracks_missing_artist_musicbrainz_id/0` - `count_tracks_missing_album_musicbrainz_id/0` - `get_artists_missing_musicbrainz_id/1` - `get_albums_missing_musicbrainz_id/1` - ## Deployment The application is deployed via Coolify, using a Docker Compose strategy. diff --git a/assets/css/app.css b/assets/css/app.css index fce5c4fd..462a5f09 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -16,7 +16,10 @@ @custom-variant phx-change-loading (.phx-change-loading&, .phx-change-loading &); /* Make LiveView wrapper divs transparent for layout */ -[data-phx-session], [data-phx-teleported-src] { display: contents } +[data-phx-session], +[data-phx-teleported-src] { + display: contents; +} @layer base { a, @@ -37,7 +40,7 @@ @theme { --font-sans: "InterVariable", sans-serif; - --font-sans--font-feature-settings: 'cv02', 'cv03', 'cv04', 'cv11'; + --font-sans--font-feature-settings: "cv02", "cv03", "cv04", "cv11"; --animate-shake: shake 0.82s cubic-bezier(0.36, 0.07, 0.19, 0.97) both; --animate-shine: shine 3s ease infinite; --animate-equalizer-bar: equalizer-bar 1.2s ease-in-out infinite; diff --git a/assets/js/app.js b/assets/js/app.js index d9c46800..f33b4538 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -33,12 +33,12 @@ import { createLiveToastHook } from "live_toast"; import banner from "./banner"; // the duration for each toast to stay on screen in ms -const duration = 4000 +const duration = 4000; // how many toasts to show on screen at once -const maxItems = 3 +const maxItems = 3; -const liveToastHook = createLiveToastHook(duration, maxItems) +const liveToastHook = createLiveToastHook(duration, maxItems); let Hooks = FluxonHooks; Hooks.FormatNumber = FormatNumberHook; @@ -56,8 +56,8 @@ const liveSocket = new LiveSocket("/live", Socket, { params: (view) => { return { _csrf_token: csrfToken, - timezone: Intl.DateTimeFormat().resolvedOptions().timeZone - } + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }; }, hooks: { ...Hooks, ...colocatedHooks }, dom: { @@ -121,32 +121,39 @@ window.liveSocket = liveSocket; // 2. click on elements to jump to their definitions in your code editor // if (process.env.NODE_ENV === "development") { - window.addEventListener("phx:live_reload:attached", ({ detail: reloader }) => { - // Enable server log streaming to client. - // Disable with reloader.disableServerLogs() - reloader.enableServerLogs() + window.addEventListener( + "phx:live_reload:attached", + ({ detail: reloader }) => { + // Enable server log streaming to client. + // Disable with reloader.disableServerLogs() + reloader.enableServerLogs(); - // Open configured PLUG_EDITOR at file:line of the clicked element's HEEx component - // - // * click with "c" key pressed to open at caller location - // * click with "d" key pressed to open at function component definition location - let keyDown - window.addEventListener("keydown", e => keyDown = e.key) - window.addEventListener("keyup", e => keyDown = null) - window.addEventListener("click", e => { - if (keyDown === "c") { - e.preventDefault() - e.stopImmediatePropagation() - reloader.openEditorAtCaller(e.target) - } else if (keyDown === "d") { - e.preventDefault() - e.stopImmediatePropagation() - reloader.openEditorAtDef(e.target) - } - }, true) + // Open configured PLUG_EDITOR at file:line of the clicked element's HEEx component + // + // * click with "c" key pressed to open at caller location + // * click with "d" key pressed to open at function component definition location + let keyDown; + window.addEventListener("keydown", (e) => (keyDown = e.key)); + window.addEventListener("keyup", (e) => (keyDown = null)); + window.addEventListener( + "click", + (e) => { + if (keyDown === "c") { + e.preventDefault(); + e.stopImmediatePropagation(); + reloader.openEditorAtCaller(e.target); + } else if (keyDown === "d") { + e.preventDefault(); + e.stopImmediatePropagation(); + reloader.openEditorAtDef(e.target); + } + }, + true, + ); - window.liveReloader = reloader - }) + window.liveReloader = reloader; + }, + ); } // Credit: https://andrewtimberlake.com/blog/2025/03/see-what-liveview-changes-are-being-made diff --git a/assets/js/banner.js b/assets/js/banner.js index c5d8a116..e72cb0ce 100644 --- a/assets/js/banner.js +++ b/assets/js/banner.js @@ -1,6 +1,7 @@ export default function banner() { const version = document.querySelector('meta[name="version"]').content; - console.log(`%c + console.log( + `%c ███╗ ███╗██╗ ██╗███████╗██╗ ██████╗ ██╗ ██╗██████╗ ██████╗ █████╗ ██████╗ ██╗ ██╗ ████╗ ████║██║ ██║██╔════╝██║██╔════╝ ██║ ██║██╔══██╗██╔══██╗██╔══██╗██╔══██╗╚██╗ ██╔╝ ██╔████╔██║██║ ██║███████╗██║██║ ██║ ██║██████╔╝██████╔╝███████║██████╔╝ ╚████╔╝ @@ -8,5 +9,7 @@ export default function banner() { ██║ ╚═╝ ██║╚██████╔╝███████║██║╚██████╗ ███████╗██║██████╔╝██║ ██║██║ ██║██║ ██║ ██║ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═════╝ ╚══════╝╚═╝╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ Version: ${version} -`, "font-family: IBM Plex Mono,monospace; font-size: 12px"); +`, + "font-family: IBM Plex Mono,monospace; font-size: 12px", + ); } diff --git a/assets/js/hooks/create-navigation-hook.js b/assets/js/hooks/create-navigation-hook.js index 0707138c..4747a3a0 100644 --- a/assets/js/hooks/create-navigation-hook.js +++ b/assets/js/hooks/create-navigation-hook.js @@ -1,4 +1,8 @@ -export default function createNavigationHook({ getContainer, inputId, onSelect }) { +export default function createNavigationHook({ + getContainer, + inputId, + onSelect, +}) { return { mounted() { this.selectedIndex = -1; @@ -89,13 +93,16 @@ export default function createNavigationHook({ getContainer, inputId, onSelect } if (searchInput) { searchInput.focus(); } - } else if (this.selectedIndex >= 0 && this.selectedIndex < results.length) { + } else if ( + this.selectedIndex >= 0 && + this.selectedIndex < results.length + ) { const selectedResult = results[this.selectedIndex]; selectedResult.setAttribute("aria-selected", "true"); selectedResult.scrollIntoView({ block: "nearest", - behavior: "smooth" + behavior: "smooth", }); } }, @@ -105,6 +112,6 @@ export default function createNavigationHook({ getContainer, inputId, onSelect } if (!container) return []; return Array.from(container.querySelectorAll('[role="option"]')); - } + }, }; } diff --git a/assets/js/hooks/format-number.js b/assets/js/hooks/format-number.js index 7ea04499..0147174d 100644 --- a/assets/js/hooks/format-number.js +++ b/assets/js/hooks/format-number.js @@ -4,5 +4,5 @@ export default { }, updated() { this.el.innerText = parseInt(this.el.innerText).toLocaleString(); - } -} + }, +}; diff --git a/assets/js/hooks/record-picker-navigation.js b/assets/js/hooks/record-picker-navigation.js index e139f044..fb81a124 100644 --- a/assets/js/hooks/record-picker-navigation.js +++ b/assets/js/hooks/record-picker-navigation.js @@ -9,5 +9,5 @@ export default createNavigationHook({ } else { hook.navigateUp(); } - } + }, }); diff --git a/assets/js/hooks/rule-picker-navigation.js b/assets/js/hooks/rule-picker-navigation.js index 6540d873..d23d6bd5 100644 --- a/assets/js/hooks/rule-picker-navigation.js +++ b/assets/js/hooks/rule-picker-navigation.js @@ -9,5 +9,5 @@ export default createNavigationHook({ } else { hook.navigateUp(); } - } + }, }); diff --git a/assets/js/hooks/sortable-list.js b/assets/js/hooks/sortable-list.js index 4fa4a6e9..2598f089 100644 --- a/assets/js/hooks/sortable-list.js +++ b/assets/js/hooks/sortable-list.js @@ -28,7 +28,7 @@ export default { onEnd: () => { const items = this.el.querySelectorAll("[data-sortable-item]"); const recordIds = Array.from(items).map( - (item) => item.dataset.recordId + (item) => item.dataset.recordId, ); const payload = { record_ids: recordIds }; diff --git a/assets/js/hooks/universal-search-navigation.js b/assets/js/hooks/universal-search-navigation.js index 45bd48f7..1dc43115 100644 --- a/assets/js/hooks/universal-search-navigation.js +++ b/assets/js/hooks/universal-search-navigation.js @@ -2,5 +2,5 @@ import createNavigationHook from "./create-navigation-hook"; export default createNavigationHook({ getContainer: () => document.getElementById("universal-search-root"), - inputId: "universal-search-input" + inputId: "universal-search-input", }); diff --git a/backlog/docs/nerves-deployment/doc-10 - Deepseek-v4-Pro-xhigh-analysis.md b/backlog/docs/nerves-deployment/doc-10 - Deepseek-v4-Pro-xhigh-analysis.md index 41eb129c..4fd481d7 100644 --- a/backlog/docs/nerves-deployment/doc-10 - Deepseek-v4-Pro-xhigh-analysis.md +++ b/backlog/docs/nerves-deployment/doc-10 - Deepseek-v4-Pro-xhigh-analysis.md @@ -1,9 +1,10 @@ --- id: doc-10 -title: 'Deepseek v4 Pro, xhigh analysis' +title: "Deepseek v4 Pro, xhigh analysis" type: other -created_date: '2026-05-04 15:11' +created_date: "2026-05-04 15:11" --- + # Nerves Deployment Feasibility Analysis for Music Library > Research report — do not implement. May 2026. @@ -29,26 +30,26 @@ steps. ## 1. Native-extensions audit Every dependency that ships a NIF or binary artefact was reviewed for ARM / -Nerves compatibility. Dev‑only tools (esbuild, tailwind, credo, etc.) are +Nerves compatibility. Dev‑only tools (esbuild, tailwind, credo, etc.) are excluded — they never reach the firmware. -| Dependency | Purpose | NIF type | Precompiled ARM? | Nerves feasibility | -|---|---|---|---|---| -| **exqlite** → `ecto_sqlite3` | SQLite driver | C NIF (`cc_precompiler`) | ✅ `aarch64-linux-gnu/musl` (no `armv7-*`) | **🟢 Good** — `force_build: true` compiles from source inside the Nerves toolchain. Explicitly mentioned in Exqlite’s README as an embedded use-case. | -| **vix** → libvips | Image processing (covers, thumbnails) | C NIF (`elixir_make`) | ❌ No ARM binaries advertised | **🔴 High risk** — libvips is ~30 MB with transitive deps (libjpeg, libpng, libwebp, etc.). Each needs Buildroot integration or cross‑compilation. Biggest blocker for a lean firmware. | -| **mdex** → comrak + ammonia | Markdown → HTML | Rust NIF (RustlerPrecompiled) | ⚠️ Standard Rustler targets (`aarch64-unknown-linux-gnu`, possibly `arm-*`) | **🟡 Needs cross‑compilation** — RustlerPrecompiled targets use different triples than Nerves (`aarch64-nerves-linux-gnu`). Precompiled `.so` likely won’t load. Cross‑compile from source with a Nerves‑aware Rust toolchain. | -| **lumis** → tree‑sitter | Syntax highlighting (in mdex) | Rust NIF (RustlerPrecompiled) | Same as mdex | **🟡 Same situation** — part of the mdex stack. Tree‑sitter grammars add per‑language compilation but the NIF itself is manageable. | -| **typst** | PDF generation (tracklists) | Rust NIF (RustlerPrecompiled) | Same as mdex | **🔴 Heavy** — a full typesetting engine. Large binary, plus the same target‑triple mismatch. Likely overkill for embedded PDFs. | -| **dominant_colors** → kmeans_colors | Colour extraction from covers | Rust NIF (`rustler_precompiled ~> 0.7`) | Same as mdex | **🟢 Small** — trivial NIF. Cross‑compilation is straightforward. Low risk. | -| **esbuild** | JS bundling | Go binary | — | **N/A** — dev only, never deployed. | -| **tailwind** | CSS generation | Go binary | — | **N/A** — dev only, never deployed. | +| Dependency | Purpose | NIF type | Precompiled ARM? | Nerves feasibility | +| ----------------------------------- | ------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **exqlite** → `ecto_sqlite3` | SQLite driver | C NIF (`cc_precompiler`) | ✅ `aarch64-linux-gnu/musl` (no `armv7-*`) | **🟢 Good** — `force_build: true` compiles from source inside the Nerves toolchain. Explicitly mentioned in Exqlite’s README as an embedded use-case. | +| **vix** → libvips | Image processing (covers, thumbnails) | C NIF (`elixir_make`) | ❌ No ARM binaries advertised | **🔴 High risk** — libvips is ~30 MB with transitive deps (libjpeg, libpng, libwebp, etc.). Each needs Buildroot integration or cross‑compilation. Biggest blocker for a lean firmware. | +| **mdex** → comrak + ammonia | Markdown → HTML | Rust NIF (RustlerPrecompiled) | ⚠️ Standard Rustler targets (`aarch64-unknown-linux-gnu`, possibly `arm-*`) | **🟡 Needs cross‑compilation** — RustlerPrecompiled targets use different triples than Nerves (`aarch64-nerves-linux-gnu`). Precompiled `.so` likely won’t load. Cross‑compile from source with a Nerves‑aware Rust toolchain. | +| **lumis** → tree‑sitter | Syntax highlighting (in mdex) | Rust NIF (RustlerPrecompiled) | Same as mdex | **🟡 Same situation** — part of the mdex stack. Tree‑sitter grammars add per‑language compilation but the NIF itself is manageable. | +| **typst** | PDF generation (tracklists) | Rust NIF (RustlerPrecompiled) | Same as mdex | **🔴 Heavy** — a full typesetting engine. Large binary, plus the same target‑triple mismatch. Likely overkill for embedded PDFs. | +| **dominant_colors** → kmeans_colors | Colour extraction from covers | Rust NIF (`rustler_precompiled ~> 0.7`) | Same as mdex | **🟢 Small** — trivial NIF. Cross‑compilation is straightforward. Low risk. | +| **esbuild** | JS bundling | Go binary | — | **N/A** — dev only, never deployed. | +| **tailwind** | CSS generation | Go binary | — | **N/A** — dev only, never deployed. | ### Target‑architecture specifics -| Target | CPU | Nerves toolchain triple | Precompiled‑binary match? | -|---|---|---|---| -| RPi 4 / 5 | Cortex‑A72 / A76 (64‑bit) | `aarch64-nerves-linux-gnu` | ⚠️ `aarch64-linux-gnu` *may* be ABI‑compatible (exqlite). Rust NIFs need rebuild. | -| RPi 3 | Cortex‑A53 (32‑bit) | `armv7-nerves-linux-gnueabihf` | ❌ No precompiled binaries match. Everything must be compiled from source. | +| Target | CPU | Nerves toolchain triple | Precompiled‑binary match? | +| --------- | ------------------------- | ------------------------------ | --------------------------------------------------------------------------------- | +| RPi 4 / 5 | Cortex‑A72 / A76 (64‑bit) | `aarch64-nerves-linux-gnu` | ⚠️ `aarch64-linux-gnu` _may_ be ABI‑compatible (exqlite). Rust NIFs need rebuild. | +| RPi 3 | Cortex‑A53 (32‑bit) | `armv7-nerves-linux-gnueabihf` | ❌ No precompiled binaries match. Everything must be compiled from source. | **Recommendation**: Target RPi 4 or 5 (64‑bit) first — the precompiled ecosystem is far better and the 32‑bit path adds unnecessary friction. @@ -60,10 +61,10 @@ far better and the 32‑bit path adds unnecessary friction. The application loads two run‑time extensions via Exqlite’s `load_extensions` config: -| Extension | Source | ARM status | Notes | -|---|---|---|---| -| **vec0** | [sqlite-vec](https://github.com/asg017/sqlite-vec) | ✅ **Good** | “Written in pure C, no dependencies, runs anywhere SQLite runs (… Raspberry Pis, etc.)” — explicitly tested on ARM. Compilable for any Nerves target. | -| **unicode** | likely [sqlean](https://github.com/nalgeon/sqlean) (`text` module or bundle) | ✅ **ARM64 available** | Sqlean ships `sqlean-linux-arm64.zip`. For ARMv7, compile from C source. | +| Extension | Source | ARM status | Notes | +| ----------- | ---------------------------------------------------------------------------- | ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| **vec0** | [sqlite-vec](https://github.com/asg017/sqlite-vec) | ✅ **Good** | “Written in pure C, no dependencies, runs anywhere SQLite runs (… Raspberry Pis, etc.)” — explicitly tested on ARM. Compilable for any Nerves target. | +| **unicode** | likely [sqlean](https://github.com/nalgeon/sqlean) (`text` module or bundle) | ✅ **ARM64 available** | Sqlean ships `sqlean-linux-arm64.zip`. For ARMv7, compile from C source. | ### Loading mechanism @@ -79,7 +80,7 @@ cross‑compiler, place them in the appropriate arch subdirectory, and Exqlite w load them at connection time. **Unknown**: Whether the `unicode` extension is built from the exact same sqlean -release as production, or compiled independently. Sync the build flags and version +release as production, or compiled independently. Sync the build flags and version to avoid subtle behavioural differences. --- @@ -90,13 +91,13 @@ This is the **hardest problem** and the most architecturally open of the three. ### What Litestream does (and doesn’t) -| Capability | Litestream | Needed for device ↔ cloud sync | -|---|---|---| -| Unidirectional backup (app → S3) | ✅ | ❌ | -| Disaster recovery (restore from S3) | ✅ | ❌ | -| Bidirectional sync | ❌ | ✅ | -| Conflict detection / resolution | ❌ | ✅ | -| Multi‑writer support | ❌ | ✅ | +| Capability | Litestream | Needed for device ↔ cloud sync | +| ----------------------------------- | ---------- | ------------------------------ | +| Unidirectional backup (app → S3) | ✅ | ❌ | +| Disaster recovery (restore from S3) | ✅ | ❌ | +| Bidirectional sync | ❌ | ✅ | +| Conflict detection / resolution | ❌ | ✅ | +| Multi‑writer support | ❌ | ✅ | Litestream is purely an asynchronous backup tool. It continuously streams WAL pages to S3. It has **no mechanism** for a second instance to push writes back, no merge @@ -141,7 +142,7 @@ Never write the same record type from both sides: - **Production server**: records, embeddings, notes, record sets, artist info. - **Nerves device**: scrobbles, listening stats, playback logs. -- Each instance has its own SQLite database. They sync via API calls (e.g., device +- Each instance has its own SQLite database. They sync via API calls (e.g., device pushes scrobbles to production, production pushes new records to device). - **Pros**: No database‑level conflicts. Clear ownership boundaries. @@ -184,24 +185,24 @@ Define a custom sync protocol with: Even with a sync strategy in place, Litestream could still run on the Nerves device for **local disaster recovery** — streaming the device’s own WAL to a separate S3 -path. This is independent from the production Litestream instance. +path. This is independent from the production Litestream instance. --- ## Summary of risks -| Risk | Severity | Mitigation | -|---|---|---| -| **vix / libvips on Nerves** | 🔴 High | Replace image processing with a lighter alternative, or offload to the production server via API. | -| **Data‑sync strategy** | 🔴 High | No off‑the‑shelf solution. Start with read‑only replica (option A), iterate toward separate write domains (option B). | -| **typst on embedded** | 🟡 Medium | Switch to server‑side PDF generation; client downloads the result. | -| **Rust NIF cross‑compilation** | 🟡 Medium | Set up Nerves Rust cross‑compilation. Works but adds build complexity. Each library needs individual testing. | -| **ARMv7 (RPi 3) vs AArch64 (RPi 4/5)** | 🟡 Medium | RPi 4/5 (64‑bit) has much better precompiled‑binary support. RPi 3 requires everything compiled from source. | -| **Buildroot package deps (libvips)** | 🟡 Medium | Each transitive C library needs a Buildroot package definition or a custom cross‑compile step. | -| **Firmware size** | 🟡 Medium | libvips + typst + sqlite extensions → image could exceed 200 MB. Typical Nerves firmware is 20–80 MB. | -| **sqlite‑vec on ARM** | 🟢 Low | Explicitly supported. Pure C, no deps. Compiles trivially with the Nerves toolchain. | -| **exqlite on Nerves** | 🟢 Low | Designed for embedded use. Source compilation well‑supported and documented. | -| **dominant\_colors** | 🟢 Low | Tiny Rust NIF. Straightforward cross‑compilation. | +| Risk | Severity | Mitigation | +| -------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------- | +| **vix / libvips on Nerves** | 🔴 High | Replace image processing with a lighter alternative, or offload to the production server via API. | +| **Data‑sync strategy** | 🔴 High | No off‑the‑shelf solution. Start with read‑only replica (option A), iterate toward separate write domains (option B). | +| **typst on embedded** | 🟡 Medium | Switch to server‑side PDF generation; client downloads the result. | +| **Rust NIF cross‑compilation** | 🟡 Medium | Set up Nerves Rust cross‑compilation. Works but adds build complexity. Each library needs individual testing. | +| **ARMv7 (RPi 3) vs AArch64 (RPi 4/5)** | 🟡 Medium | RPi 4/5 (64‑bit) has much better precompiled‑binary support. RPi 3 requires everything compiled from source. | +| **Buildroot package deps (libvips)** | 🟡 Medium | Each transitive C library needs a Buildroot package definition or a custom cross‑compile step. | +| **Firmware size** | 🟡 Medium | libvips + typst + sqlite extensions → image could exceed 200 MB. Typical Nerves firmware is 20–80 MB. | +| **sqlite‑vec on ARM** | 🟢 Low | Explicitly supported. Pure C, no deps. Compiles trivially with the Nerves toolchain. | +| **exqlite on Nerves** | 🟢 Low | Designed for embedded use. Source compilation well‑supported and documented. | +| **dominant_colors** | 🟢 Low | Tiny Rust NIF. Straightforward cross‑compilation. | --- diff --git a/backlog/docs/nerves-deployment/doc-11 - Nerves-Deployment-Research-—-Consolidated-Summary.md b/backlog/docs/nerves-deployment/doc-11 - Nerves-Deployment-Research-—-Consolidated-Summary.md index 46f865ea..fdc7f313 100644 --- a/backlog/docs/nerves-deployment/doc-11 - Nerves-Deployment-Research-—-Consolidated-Summary.md +++ b/backlog/docs/nerves-deployment/doc-11 - Nerves-Deployment-Research-—-Consolidated-Summary.md @@ -2,8 +2,9 @@ id: doc-11 title: Nerves Deployment Research — Consolidated Summary type: other -created_date: '2026-05-04 15:25' +created_date: "2026-05-04 15:25" --- + # Nerves Deployment Research — Consolidated Summary > Synthesized from three parallel research reports (Opus 4.7 xhigh, GPT 5.5 high, Deepseek v4 Pro xhigh) on 2026-05-04. @@ -113,6 +114,7 @@ Litestream is asynchronous backup and disaster recovery. It streams WAL pages to #### D. Offline-Capable Writes (Most Complex) Options if offline writes are required: + - **Application-level outbox**: Local command log + idempotency keys + push to production on reconnect. - **CRDTs** (Automerge, Yjs): True offline-first, no Elixir/SQLite bridge exists. - **Event sourcing**: Append-only mutation log, replay to reconstruct state. Complex. @@ -149,18 +151,18 @@ Options if offline writes are required: ## 5. Risk Summary -| Risk | Severity | Mitigation | -|---|---|---| -| vix / libvips on Nerves | 🔴 High | Offload image processing to production API, or build custom Nerves system | -| Data-sync strategy | 🔴 High | Start read-only, iterate toward write-through API | -| typst on embedded | 🟡 Medium | Server-side PDF generation | -| Rust NIF cross-compilation | 🟡 Medium | Set up Nerves Rust cross-compilation; test each library | -| ARMv7 (RPi 3) vs AArch64 | 🟡 Medium | Target RPi 4/5 (64-bit) only | -| Buildroot deps (libvips, ICU) | 🟡 Medium | Custom Nerves system; ~15-30 min Buildroot rebuild | -| Firmware size | 🟡 Medium | libvips + typst + extensions could exceed 200 MB; set 120 MB budget | -| sqlite-vec / unicode on ARM | 🟢 Low | Pure C, no deps. Cross-compile in Buildroot | -| exqlite on Nerves | 🟢 Low | Designed for embedded; source compilation documented | -| dominant_colors | 🟢 Low | Tiny Rust NIF, straightforward | +| Risk | Severity | Mitigation | +| ----------------------------- | --------- | ------------------------------------------------------------------------- | +| vix / libvips on Nerves | 🔴 High | Offload image processing to production API, or build custom Nerves system | +| Data-sync strategy | 🔴 High | Start read-only, iterate toward write-through API | +| typst on embedded | 🟡 Medium | Server-side PDF generation | +| Rust NIF cross-compilation | 🟡 Medium | Set up Nerves Rust cross-compilation; test each library | +| ARMv7 (RPi 3) vs AArch64 | 🟡 Medium | Target RPi 4/5 (64-bit) only | +| Buildroot deps (libvips, ICU) | 🟡 Medium | Custom Nerves system; ~15-30 min Buildroot rebuild | +| Firmware size | 🟡 Medium | libvips + typst + extensions could exceed 200 MB; set 120 MB budget | +| sqlite-vec / unicode on ARM | 🟢 Low | Pure C, no deps. Cross-compile in Buildroot | +| exqlite on Nerves | 🟢 Low | Designed for embedded; source compilation documented | +| dominant_colors | 🟢 Low | Tiny Rust NIF, straightforward | --- @@ -188,6 +190,7 @@ All three reports collapse into a single question: ## Sources All three reports include extensive source references. See individual documents for full citation lists: + - `doc-8 - Opus-4.7-xhigh-analysis.md` - `doc-9 - GPT-5.5-high-analysis.md` - `doc-10 - Deepseek-v4-Pro-xhigh-analysis.md` diff --git a/backlog/docs/nerves-deployment/doc-8 - Opus-4.7-xhigh-analysis.md b/backlog/docs/nerves-deployment/doc-8 - Opus-4.7-xhigh-analysis.md index 6582a518..b84da31b 100644 --- a/backlog/docs/nerves-deployment/doc-8 - Opus-4.7-xhigh-analysis.md +++ b/backlog/docs/nerves-deployment/doc-8 - Opus-4.7-xhigh-analysis.md @@ -1,29 +1,30 @@ --- id: doc-8 -title: 'Opus 4.7, xhigh analysis' +title: "Opus 4.7, xhigh analysis" type: other -created_date: '2026-05-04 15:08' -updated_date: '2026-05-04 15:10' +created_date: "2026-05-04 15:08" +updated_date: "2026-05-04 15:10" --- + # Nerves Deployment Research — Findings Three parallel research agents covered NIFs, SQLite extensions, and Litestream replication semantics. The user's concerns were well-founded: there are real blockers, but most are solvable with custom Nerves system work. The Litestream question reframes the architecture more than expected. ## 1. Native libraries (NIFs on aarch64-linux-musl) -| Library | aarch64-gnu | aarch64-musl | Status | -|---|---|---|---| -| `mdex` | ✅ | ✅ | Fine | -| `lumis` | ✅ | ✅ | Fine | -| `typst` | ✅ | ✅ | **Nerves-aware** — has `@nerves_rust_target_triple_mapping` in mix.exs | -| `ecto_sqlite3` / `exqlite` | ✅ | ✅ | Driver fine; designed with embedded use in mind. Extensions are the issue (see §2) | -| `dominant_colors` | likely | unknown | Verify GitHub releases; if missing, request musl target from maintainer | -| `vix` | partial | ❌ | **Major blocker** | -| `cloak_ecto` | n/a | n/a | Pure Elixir + `:crypto` | +| Library | aarch64-gnu | aarch64-musl | Status | +| -------------------------- | ----------- | ------------ | ---------------------------------------------------------------------------------- | +| `mdex` | ✅ | ✅ | Fine | +| `lumis` | ✅ | ✅ | Fine | +| `typst` | ✅ | ✅ | **Nerves-aware** — has `@nerves_rust_target_triple_mapping` in mix.exs | +| `ecto_sqlite3` / `exqlite` | ✅ | ✅ | Driver fine; designed with embedded use in mind. Extensions are the issue (see §2) | +| `dominant_colors` | likely | unknown | Verify GitHub releases; if missing, request musl target from maintainer | +| `vix` | partial | ❌ | **Major blocker** | +| `cloak_ecto` | n/a | n/a | Pure Elixir + `:crypto` | **`vix` is the dominant blocker.** Two compounding issues: -- It loads the NIF at *compile time* (macro-driven introspection), which breaks cross-compilation on stock Nerves systems. fhunleth (Nerves core) discussed this in a 2023 ElixirForum thread; the conclusion was "no easy Elixir-level fix." +- It loads the NIF at _compile time_ (macro-driven introspection), which breaks cross-compilation on stock Nerves systems. fhunleth (Nerves core) discussed this in a 2023 ElixirForum thread; the conclusion was "no easy Elixir-level fix." - libvips itself is not in `nerves_system_rpi4` / `nerves_system_rpi5`. You'd need a **custom Nerves system** with `BR2_PACKAGE_LIBVIPS=y` in Buildroot, then `VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS`. This is doable (15–30 min Buildroot rebuild, well-documented) but it commits you to maintaining a custom Nerves system. That single decision affects everything downstream — once you're building custom systems, ICU and sqlite-vec become "while we're at it" additions. @@ -51,7 +52,7 @@ Both extensions are blockers on stock Nerves systems for the same root reason ## 3. Litestream replication — this is the architectural pivot -**Headline:** Litestream is *not* a multi-master replicator. It's a one-way backup tool. If the Nerves device writes locally, those writes are lost on the next restore — no conflict detection, no reconciliation, no merge. +**Headline:** Litestream is _not_ a multi-master replicator. It's a one-way backup tool. If the Nerves device writes locally, those writes are lost on the next restore — no conflict detection, no reconciliation, no merge. **Two viable replica modes today (both newer than the docs you may remember):** @@ -83,6 +84,7 @@ The **biggest non-obvious unknown** is the Litestream architecture decision. Goi ## Sources ### NIF / Phoenix-on-Nerves + - [Underjord: LiveView on Nerves](https://underjord.io/liveview-on-nerves.html) - [ElixirForum: Cross Compiling with NIF as compilation dependency (vix/Nerves thread)](https://elixirforum.com/t/cross-compiling-with-nif-as-compilation-dependency/59821) - [Nerves docs: Compiling Non-BEAM Code](https://hexdocs.pm/nerves/compiling-non-beam-code.html) @@ -96,6 +98,7 @@ The **biggest non-obvious unknown** is the Litestream architecture decision. Goi - [Customizing Your Nerves System](https://hexdocs.pm/nerves/customizing-systems.html) ### SQLite extensions + - [Releases · asg017/sqlite-vec](https://github.com/asg017/sqlite-vec/releases) - [Fix for musl compile PR #199 · asg017/sqlite-vec](https://github.com/asg017/sqlite-vec/pull/199) - [Build and install exqlite with sqlcipher into nerves rpi5 image — ElixirForum](https://elixirforum.com/t/build-and-install-exqlite-with-sqlcipher-into-nerves-rpi5-image/75089) @@ -105,6 +108,7 @@ The **biggest non-obvious unknown** is the Litestream architecture decision. Goi - [Advanced Configuration — Nerves](https://hexdocs.pm/nerves/advanced-configuration.html) ### Litestream + - [Live Read Replication — Litestream (tip)](https://tip.litestream.io/guides/read-replica/) - [VFS Read Replicas — Litestream](https://litestream.io/guides/vfs/) - [How it works — Litestream](https://litestream.io/how-it-works/) diff --git a/backlog/docs/nerves-deployment/doc-9 - GPT-5.5-high-analysis.md b/backlog/docs/nerves-deployment/doc-9 - GPT-5.5-high-analysis.md index 49364950..417c2af3 100644 --- a/backlog/docs/nerves-deployment/doc-9 - GPT-5.5-high-analysis.md +++ b/backlog/docs/nerves-deployment/doc-9 - GPT-5.5-high-analysis.md @@ -1,9 +1,10 @@ --- id: doc-9 -title: 'GPT 5.5, high analysis' +title: "GPT 5.5, high analysis" type: other -created_date: '2026-05-04 15:11' +created_date: "2026-05-04 15:11" --- + # Nerves Deployment Research Report Date: 2026-05-04 diff --git a/docs/architecture.md b/docs/architecture.md index 73e557dd..1797a40c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,6 +11,7 @@ Uses SQLite (via `ecto_sqlite3`) with three databases: one for the app, one for jobs (Oban), and one for telemetry metrics. All schemas use `binary_id` primary keys. Key capabilities: + - Browse/search collected and wishlisted records - Import metadata from MusicBrainz, enrich with Discogs/Wikipedia/Last.fm - Scrobble tracks to Last.fm, import listening history @@ -49,11 +50,11 @@ when `single_line_logging` is `true` — not a supervised process. ## Database & Repos -| Repo | DB file (dev) | Purpose | -|------|---------------|---------| -| `MusicLibrary.Repo` | `data/music_library_dev.db` | All application data | -| `MusicLibrary.BackgroundRepo` | `data/music_library_background_dev.db` | Oban job queue | -| `MusicLibrary.TelemetryRepo` | `data/music_library_telemetry_dev.db` | Telemetry metrics history (persistent across restarts) | +| Repo | DB file (dev) | Purpose | +| ----------------------------- | -------------------------------------- | ------------------------------------------------------ | +| `MusicLibrary.Repo` | `data/music_library_dev.db` | All application data | +| `MusicLibrary.BackgroundRepo` | `data/music_library_background_dev.db` | Oban job queue | +| `MusicLibrary.TelemetryRepo` | `data/music_library_telemetry_dev.db` | Telemetry metrics history (persistent across restarts) | SQLite extensions loaded at runtime: `unicode`, `vec0` (vector search). @@ -64,26 +65,27 @@ write to it directly; insert/update the `records` table instead. ## Schemas -| Schema | Table | PK | Key Fields | -|--------|-------|----|------------| -| `Records.Record` | `records` | `id` (binary_id) | title, type, format, cover_url, cover_hash, musicbrainz_id, genres[], release_date, purchased_at, dominant_colors[], embeds_many :artists | -| `Records.RecordEmbedding` | `record_embeddings` | `id` | embedding (Float32 vector), text_representation, belongs_to :record | -| `Records.SearchIndex` | `records_search_index` | `id` | FTS5 mirror of records (virtual, trigger-synced) | -| `Records.RecordRelease` | `record_releases` | none | record_id, release_id, cover_hash, purchased_at — read-only, no PK | -| `Records.ArtistRecord` | `artist_records` | composite | musicbrainz_id, record_id — DB view joining artists to records | -| `Artists.Artist` | — | — | Embedded schema (name, sort_name, musicbrainz_id, joinphrase) | -| `Artists.ArtistInfo` | `artist_infos` | `id` | musicbrainz_data, discogs_data, wikipedia_data, lastfm_data, image_data_hash | -| `Assets.Asset` | `assets` | `hash` (SHA256) | content (binary), format, properties (map) | -| `Notes.Note` | `notes` | `id` | entity (:record/:artist), content, musicbrainz_id | -| `RecordSets.RecordSet` | `record_sets` | `id` | name, description, has_many :items | -| `RecordSets.RecordSetItem` | `record_set_items` | `id` | position, belongs_to :record_set, belongs_to :record | -| `ScrobbleRules.ScrobbleRule` | `scrobble_rules` | `id` (integer) | type (:album/:artist), match_value, target_musicbrainz_id, enabled | -| `OnlineStoreTemplates.OnlineStoreTemplate` | `online_store_templates` | `id` | name, url_template, enabled | -| `Secrets.Secret` | `secrets` | `name` (string) | value (encrypted binary) | -| `Chats.Chat` | `chats` | `id` (binary_id) | entity (:record/:artist/:collection), musicbrainz_id, topic, has_many :messages | -| `Chats.Message` | `chat_messages` | `id` (binary_id) | role, content, position, belongs_to :chat | +| Schema | Table | PK | Key Fields | +| ------------------------------------------ | ------------------------ | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `Records.Record` | `records` | `id` (binary_id) | title, type, format, cover_url, cover_hash, musicbrainz_id, genres[], release_date, purchased_at, dominant_colors[], embeds_many :artists | +| `Records.RecordEmbedding` | `record_embeddings` | `id` | embedding (Float32 vector), text_representation, belongs_to :record | +| `Records.SearchIndex` | `records_search_index` | `id` | FTS5 mirror of records (virtual, trigger-synced) | +| `Records.RecordRelease` | `record_releases` | none | record_id, release_id, cover_hash, purchased_at — read-only, no PK | +| `Records.ArtistRecord` | `artist_records` | composite | musicbrainz_id, record_id — DB view joining artists to records | +| `Artists.Artist` | — | — | Embedded schema (name, sort_name, musicbrainz_id, joinphrase) | +| `Artists.ArtistInfo` | `artist_infos` | `id` | musicbrainz_data, discogs_data, wikipedia_data, lastfm_data, image_data_hash | +| `Assets.Asset` | `assets` | `hash` (SHA256) | content (binary), format, properties (map) | +| `Notes.Note` | `notes` | `id` | entity (:record/:artist), content, musicbrainz_id | +| `RecordSets.RecordSet` | `record_sets` | `id` | name, description, has_many :items | +| `RecordSets.RecordSetItem` | `record_set_items` | `id` | position, belongs_to :record_set, belongs_to :record | +| `ScrobbleRules.ScrobbleRule` | `scrobble_rules` | `id` (integer) | type (:album/:artist), match_value, target_musicbrainz_id, enabled | +| `OnlineStoreTemplates.OnlineStoreTemplate` | `online_store_templates` | `id` | name, url_template, enabled | +| `Secrets.Secret` | `secrets` | `name` (string) | value (encrypted binary) | +| `Chats.Chat` | `chats` | `id` (binary_id) | entity (:record/:artist/:collection), musicbrainz_id, topic, has_many :messages | +| `Chats.Message` | `chat_messages` | `id` (binary_id) | role, content, position, belongs_to :chat | Last.fm schemas (separate, not Ecto-persisted to main DB): + - `LastFm.Track` — scrobbled tracks from Last.fm API responses - `LastFm.Album`, `LastFm.Artist` — parsed API responses @@ -91,78 +93,78 @@ Last.fm schemas (separate, not Ecto-persisted to main DB): ## Contexts (lib/music_library/) -| Context | Schemas | Responsibility | -|---------|---------|---------------| -| `Records` | Record, RecordEmbedding, SearchIndex | CRUD, search, import from MusicBrainz, cover/genre/color management, PubSub notifications | -| `Collection` | Record (via SearchIndex) | Querying collected records (purchased_at != nil), stats, collected artist IDs, collection summary for AI chat | -| `Wishlist` | Record (via SearchIndex) | Querying wishlisted records (purchased_at is nil) | -| `Artists` | ArtistInfo, ArtistRecord | Artist metadata from MusicBrainz/Discogs/Wikipedia/Last.fm, images, search | -| `Assets` | Asset | Binary asset storage (covers, artist images), cache tracking, pruning unreferenced assets | -| `Notes` | Note | Free-text notes for records and artists | -| `Chats` | Chat, Message, StreamProvider, RecordChat, ArtistChat, CollectionChat | Persistent AI chat conversations for records, artists, and the collection, streaming AI chat behaviour and entity-specific implementations | -| `RecordSets` | RecordSet, RecordSetItem | User-curated record groupings with ordering | -| `ScrobbleRules` | ScrobbleRule | Rules to remap Last.fm scrobble data to correct MusicBrainz IDs; searchable by match_value/target/description, orderable by alphabetical or inserted_at | -| `ScrobbleActivity` | — | Scrobbling releases/media/tracks to Last.fm | -| `ListeningStats` | (LastFm.Track, RecordRelease, ArtistRecord, ArtistInfo) | Scrobble persistence, refresh scheduling, listening analytics, track CRUD, search, listing: scrobble counts, artist play counts (from DB), recent activity, top albums/artists by period | -| `OnlineStoreTemplates` | OnlineStoreTemplate | URL templates for buying records online; searchable by name/description | -| `Errors` | ErrorTracker.Error, ErrorTracker.Occurrence | Read-only queries for production error data tracked by ErrorTracker; filtered listing with pagination, single error with preloaded occurrences and computed counts | -| `Search` | (cross-context) | Universal search dispatcher across collection, wishlist, artists, record sets (delegates to domain contexts) | -| `Secrets` | Secret | Encrypted key-value storage (CRUD + delete) | -| `BarcodeScan` | (Result struct) | Barcode → MusicBrainz lookup workflow, async batch import for multiple new records | -| `Maintenance` | (Oban.Job, LastFm.Track) | Background job monitoring, database vacuum/optimize, scrobble data quality diagnostics | +| Context | Schemas | Responsibility | +| ---------------------- | --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Records` | Record, RecordEmbedding, SearchIndex | CRUD, search, import from MusicBrainz, cover/genre/color management, PubSub notifications | +| `Collection` | Record (via SearchIndex) | Querying collected records (purchased_at != nil), stats, collected artist IDs, collection summary for AI chat | +| `Wishlist` | Record (via SearchIndex) | Querying wishlisted records (purchased_at is nil) | +| `Artists` | ArtistInfo, ArtistRecord | Artist metadata from MusicBrainz/Discogs/Wikipedia/Last.fm, images, search | +| `Assets` | Asset | Binary asset storage (covers, artist images), cache tracking, pruning unreferenced assets | +| `Notes` | Note | Free-text notes for records and artists | +| `Chats` | Chat, Message, StreamProvider, RecordChat, ArtistChat, CollectionChat | Persistent AI chat conversations for records, artists, and the collection, streaming AI chat behaviour and entity-specific implementations | +| `RecordSets` | RecordSet, RecordSetItem | User-curated record groupings with ordering | +| `ScrobbleRules` | ScrobbleRule | Rules to remap Last.fm scrobble data to correct MusicBrainz IDs; searchable by match_value/target/description, orderable by alphabetical or inserted_at | +| `ScrobbleActivity` | — | Scrobbling releases/media/tracks to Last.fm | +| `ListeningStats` | (LastFm.Track, RecordRelease, ArtistRecord, ArtistInfo) | Scrobble persistence, refresh scheduling, listening analytics, track CRUD, search, listing: scrobble counts, artist play counts (from DB), recent activity, top albums/artists by period | +| `OnlineStoreTemplates` | OnlineStoreTemplate | URL templates for buying records online; searchable by name/description | +| `Errors` | ErrorTracker.Error, ErrorTracker.Occurrence | Read-only queries for production error data tracked by ErrorTracker; filtered listing with pagination, single error with preloaded occurrences and computed counts | +| `Search` | (cross-context) | Universal search dispatcher across collection, wishlist, artists, record sets (delegates to domain contexts) | +| `Secrets` | Secret | Encrypted key-value storage (CRUD + delete) | +| `BarcodeScan` | (Result struct) | Barcode → MusicBrainz lookup workflow, async batch import for multiple new records | +| `Maintenance` | (Oban.Job, LastFm.Track) | Background job monitoring, database vacuum/optimize, scrobble data quality diagnostics | --- ## Business Logic Modules -| Module | Purpose | -|--------|---------| -| `Records.SearchParser` | Parses search syntax: `artist:X`, `album:X`, `genre:"Y"`, `format:cd`, `type:album`, `purchase_year:2024`, `release_year:2024`, free text | -| `ListeningStats.SearchParser` | Parses scrobbled tracks search syntax: `record:X`, `album_mbid:X`, `artist_mbid:X`, `artist:X`, `album:X`, `track:X`, free text | -| `Records.Similarity` | Embedding generation and async enqueue (OpenAI, enriched with Last.fm tags, skips API call when text representation unchanged), artist-cascade regeneration when upstream metadata changes, cosine-distance search (sqlite-vec) | -| `Records.TracklistPdf` | Generates 120mm×120mm PDF tracklist from record + release data (Typst) | -| `Batch` | Generic batch runner: stream + transaction + error accumulation | -| `Records.Batch` | Batch operations: refresh all MusicBrainz data, generate all embeddings (uses `Batch`) | -| `Artists.Batch` | Batch refresh: MusicBrainz, Discogs, Wikipedia, Last.fm for all artists (uses `Batch`) | -| `Req.RateLimiter` | ETS-backed Req request step enforcing per-API minimum intervals between requests | -| `Req.RateLimiter.Clock` | Behaviour for time operations (allows test clock injection) | -| `Req.RateLimiter.SystemClock` | Real clock implementation using System.monotonic_time | -| `Assets.Cache` | ETS-based asset cache with TTL (7-day TTL, TTL-only invalidation since assets are content-addressable and immutable) | -| `Assets.Image` / `Assets.Transform` | Image processing via Vix (libvips) | -| `Colors.Extractor` | Behaviour for dominant color extraction (configurable, allows test stubbing) | -| `Colors.KMeansExtractor` | Color extraction via K-Means clustering (dominant_colors library), implements `Colors.Extractor` | -| `Chats.StreamProvider` | Behaviour for streaming AI chat (`stream_response/3` callback) | -| `Chats.RecordChat` | Chat implementation for records (OpenAI streaming, web search enabled) | -| `Chats.ArtistChat` | Chat implementation for artists (OpenAI streaming, uses Wikipedia/artist context) | -| `Chats.CollectionChat` | Chat implementation for the collection (OpenAI streaming via gpt-5.1, uses collection summary as context) | -| `Chats.Prompt` | Builds complete chat prompts by interpolating identity, content, and approach guidelines | -| `Country` | Country code (alpha-2, alpha-3, subdivision, IETF) to flag emoji conversion | -| `ErrorTracker.ErrorNotifier` | GenServer: attaches to ErrorTracker telemetry, skips muted errors, throttles repeated errors, dispatches email notifications | -| `ErrorTracker.ErrorNotifier.Email` | Builds and sends Swoosh error notification emails with stack trace formatting | -| `ErrorIgnorer` | ErrorTracker.Ignorer implementation: filters non-actionable errors (e.g., NoRouteError from bot scanners) | -| `MusicLibrary.ErrorResponse` | Behaviour for structured API error responses (`retryable?/1`, `retry_delay_seconds/1`) — per-API `ErrorResponse` modules implement this | -| `MusicLibrary.HttpError` | Default HTTP status → kind mapping (`:rate_limit`, `:server_error`, `:timeout`, `:auth_error`, `:not_found`, `:client_error`, `:unknown`) used as baseline by per-API `ErrorResponse` modules | -| `MusicLibrary.RetryDelay` | Parses and clamps provider retry/reset headers into Oban snooze delays for structured API errors | -| `MusicLibrary.Worker.ErrorHandler` | Translates per-API `ErrorResponse` structs into Oban tuples — `{:snooze, seconds}` for retryable, `{:cancel, reason}` for permanent | -| `MusicLibraryWeb.RecordsOnThisDayEmail` | Builds and sends daily "records on this day" email with cover images, anniversary styling | -| `MusicLibrary.Mailer` | Swoosh mailer (Mailgun in prod, local adapter in dev) | -| `FormatNumber` | Number formatting utility | -| `QueryReporter` | Dev-only Ecto telemetry reporter: captures executed SQL to a log file with interpolated params, source locations, and timing. Activated at runtime via `start/1` / `stop/0` | -| `MusicLibrary.Logger.SingleLineFormatter` | Production-only Logger.Formatter safety net: replaces embedded newlines (`\n`) with escaped `\\n` in all log messages, ensuring every physical log line is exactly one log event | +| Module | Purpose | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Records.SearchParser` | Parses search syntax: `artist:X`, `album:X`, `genre:"Y"`, `format:cd`, `type:album`, `purchase_year:2024`, `release_year:2024`, free text | +| `ListeningStats.SearchParser` | Parses scrobbled tracks search syntax: `record:X`, `album_mbid:X`, `artist_mbid:X`, `artist:X`, `album:X`, `track:X`, free text | +| `Records.Similarity` | Embedding generation and async enqueue (OpenAI, enriched with Last.fm tags, skips API call when text representation unchanged), artist-cascade regeneration when upstream metadata changes, cosine-distance search (sqlite-vec) | +| `Records.TracklistPdf` | Generates 120mm×120mm PDF tracklist from record + release data (Typst) | +| `Batch` | Generic batch runner: stream + transaction + error accumulation | +| `Records.Batch` | Batch operations: refresh all MusicBrainz data, generate all embeddings (uses `Batch`) | +| `Artists.Batch` | Batch refresh: MusicBrainz, Discogs, Wikipedia, Last.fm for all artists (uses `Batch`) | +| `Req.RateLimiter` | ETS-backed Req request step enforcing per-API minimum intervals between requests | +| `Req.RateLimiter.Clock` | Behaviour for time operations (allows test clock injection) | +| `Req.RateLimiter.SystemClock` | Real clock implementation using System.monotonic_time | +| `Assets.Cache` | ETS-based asset cache with TTL (7-day TTL, TTL-only invalidation since assets are content-addressable and immutable) | +| `Assets.Image` / `Assets.Transform` | Image processing via Vix (libvips) | +| `Colors.Extractor` | Behaviour for dominant color extraction (configurable, allows test stubbing) | +| `Colors.KMeansExtractor` | Color extraction via K-Means clustering (dominant_colors library), implements `Colors.Extractor` | +| `Chats.StreamProvider` | Behaviour for streaming AI chat (`stream_response/3` callback) | +| `Chats.RecordChat` | Chat implementation for records (OpenAI streaming, web search enabled) | +| `Chats.ArtistChat` | Chat implementation for artists (OpenAI streaming, uses Wikipedia/artist context) | +| `Chats.CollectionChat` | Chat implementation for the collection (OpenAI streaming via gpt-5.1, uses collection summary as context) | +| `Chats.Prompt` | Builds complete chat prompts by interpolating identity, content, and approach guidelines | +| `Country` | Country code (alpha-2, alpha-3, subdivision, IETF) to flag emoji conversion | +| `ErrorTracker.ErrorNotifier` | GenServer: attaches to ErrorTracker telemetry, skips muted errors, throttles repeated errors, dispatches email notifications | +| `ErrorTracker.ErrorNotifier.Email` | Builds and sends Swoosh error notification emails with stack trace formatting | +| `ErrorIgnorer` | ErrorTracker.Ignorer implementation: filters non-actionable errors (e.g., NoRouteError from bot scanners) | +| `MusicLibrary.ErrorResponse` | Behaviour for structured API error responses (`retryable?/1`, `retry_delay_seconds/1`) — per-API `ErrorResponse` modules implement this | +| `MusicLibrary.HttpError` | Default HTTP status → kind mapping (`:rate_limit`, `:server_error`, `:timeout`, `:auth_error`, `:not_found`, `:client_error`, `:unknown`) used as baseline by per-API `ErrorResponse` modules | +| `MusicLibrary.RetryDelay` | Parses and clamps provider retry/reset headers into Oban snooze delays for structured API errors | +| `MusicLibrary.Worker.ErrorHandler` | Translates per-API `ErrorResponse` structs into Oban tuples — `{:snooze, seconds}` for retryable, `{:cancel, reason}` for permanent | +| `MusicLibraryWeb.RecordsOnThisDayEmail` | Builds and sends daily "records on this day" email with cover images, anniversary styling | +| `MusicLibrary.Mailer` | Swoosh mailer (Mailgun in prod, local adapter in dev) | +| `FormatNumber` | Number formatting utility | +| `QueryReporter` | Dev-only Ecto telemetry reporter: captures executed SQL to a log file with interpolated params, source locations, and timing. Activated at runtime via `start/1` / `stop/0` | +| `MusicLibrary.Logger.SingleLineFormatter` | Production-only Logger.Formatter safety net: replaces embedded newlines (`\n`) with escaped `\\n` in all log messages, ensuring every physical log line is exactly one log event | --- ## External API Integrations -| Module | API | Rate limit | Purpose | -|--------|-----|-----------|---------| -| `MusicBrainz` / `MusicBrainz.API` | musicbrainz.org | 1000 ms | Release/artist metadata, search | -| `LastFm` / `LastFm.API` | last.fm | 500 ms | Scrobbling, listening history, artist info (tags, similar artists), user profile/session validation | -| `Discogs` / `Discogs.API` | discogs.com | 2000 ms | Artist profiles, images | -| `Wikipedia` / `Wikipedia.API` | wikipedia.org | 1000 ms | Artist biographies | -| `BraveSearch` / `BraveSearch.API` | search.brave.com | 1000 ms | Cover art and artist image search | -| `OpenAI` / `OpenAI.API` | api.openai.com | 250 ms | Text embeddings for similarity, streaming chat via Responses API (gpt-4.1/gpt-5.1 + web search) | -| `MusicLibrary.Mailer` | Mailgun (via Swoosh) | — | Transactional email delivery (error notifications, daily digest) | +| Module | API | Rate limit | Purpose | +| --------------------------------- | -------------------- | ---------- | --------------------------------------------------------------------------------------------------- | +| `MusicBrainz` / `MusicBrainz.API` | musicbrainz.org | 1000 ms | Release/artist metadata, search | +| `LastFm` / `LastFm.API` | last.fm | 500 ms | Scrobbling, listening history, artist info (tags, similar artists), user profile/session validation | +| `Discogs` / `Discogs.API` | discogs.com | 2000 ms | Artist profiles, images | +| `Wikipedia` / `Wikipedia.API` | wikipedia.org | 1000 ms | Artist biographies | +| `BraveSearch` / `BraveSearch.API` | search.brave.com | 1000 ms | Cover art and artist image search | +| `OpenAI` / `OpenAI.API` | api.openai.com | 250 ms | Text embeddings for similarity, streaming chat via Responses API (gpt-4.1/gpt-5.1 + web search) | +| `MusicLibrary.Mailer` | Mailgun (via Swoosh) | — | Transactional email delivery (error notifications, daily digest) | Each has a `Config` module reading from application env. All HTTP clients use `Req` with per-API rate limiting (`Req.RateLimiter`, ETS-backed). In tests, all HTTP calls are @@ -183,73 +185,73 @@ HTTP 429 into `:rate_limit` vs `:auth_error` by reading the body `code` ### Queues -| Queue | Concurrency | Purpose | -|-------|-------------|---------| -| `default` | 10 | General async tasks | -| `heavy_writes` | 1 | DB-intensive or serialized operations | -| `music_brainz` | 1 | MusicBrainz calls (rate-limited at Req layer via `Req.RateLimiter`) | -| `discogs` | 1 | Discogs calls (rate-limited at Req layer via `Req.RateLimiter`) | -| `wikipedia` | 1 | Wikipedia calls | -| `last_fm` | 1 | Last.fm calls (rate-limited at Req layer via `Req.RateLimiter`) | +| Queue | Concurrency | Purpose | +| -------------- | ----------- | ------------------------------------------------------------------- | +| `default` | 10 | General async tasks | +| `heavy_writes` | 1 | DB-intensive or serialized operations | +| `music_brainz` | 1 | MusicBrainz calls (rate-limited at Req layer via `Req.RateLimiter`) | +| `discogs` | 1 | Discogs calls (rate-limited at Req layer via `Req.RateLimiter`) | +| `wikipedia` | 1 | Wikipedia calls | +| `last_fm` | 1 | Last.fm calls (rate-limited at Req layer via `Req.RateLimiter`) | ### Plugins (prod) -| Plugin | Config | Purpose | -|--------|--------|---------| -| `Oban.Plugins.Pruner` | `max_age: 43200` (12h) | Prune completed/cancelled/discarded jobs older than 12 hours | -| `Oban.Plugins.Reindexer` | `schedule: "@weekly"` | Weekly reindex of Oban tables for query performance | -| `Oban.Plugins.Cron` | `timezone: "Europe/London"` | Scheduled recurring workers (see Cron Workers table) | +| Plugin | Config | Purpose | +| ------------------------ | --------------------------- | ------------------------------------------------------------ | +| `Oban.Plugins.Pruner` | `max_age: 43200` (12h) | Prune completed/cancelled/discarded jobs older than 12 hours | +| `Oban.Plugins.Reindexer` | `schedule: "@weekly"` | Weekly reindex of Oban tables for query performance | +| `Oban.Plugins.Cron` | `timezone: "Europe/London"` | Scheduled recurring workers (see Cron Workers table) | ### On-Demand Workers -| Worker | Queue | Trigger | -|--------|-------|---------| -| `FetchArtistInfo` | default | Artist page visit / import (also fetches Last.fm data inline) | -| `FetchArtistLastFmData` | last_fm | Manual / batch | -| `FetchArtistImage` | heavy_writes | Artist info fetched | -| `RefreshCover` | heavy_writes | Manual action / import | -| `ImportFromMusicbrainzRelease` | music_brainz | Barcode scan batch import (2+ new records) | +| Worker | Queue | Trigger | +| ----------------------------------- | ------------ | --------------------------------------------------------------------------- | +| `FetchArtistInfo` | default | Artist page visit / import (also fetches Last.fm data inline) | +| `FetchArtistLastFmData` | last_fm | Manual / batch | +| `FetchArtistImage` | heavy_writes | Artist info fetched | +| `RefreshCover` | heavy_writes | Manual action / import | +| `ImportFromMusicbrainzRelease` | music_brainz | Barcode scan batch import (2+ new records) | | `ImportFromMusicbrainzReleaseGroup` | music_brainz | Cart-style multi-record import in AddRecord component (2+ records selected) | -| `PopulateGenres` | heavy_writes | Manual action (chains → GenerateRecordEmbedding) | -| `GenerateRecordEmbedding` | heavy_writes | Manual / after genre population (delegates to Similarity, skips unchanged) | -| `RecordRefreshMusicBrainzData` | music_brainz | Manual / batch | -| `ArtistRefreshMusicBrainzData` | music_brainz | Manual / batch | -| `ArtistRefreshDiscogsData` | discogs | Manual / batch | -| `ArtistRefreshWikipediaData` | wikipedia | Manual / batch | -| `PruneArtistInfo` | default | Record deleted (cleanup orphaned artist data) | -| `RecordRefreshAllMusicBrainzData` | music_brainz | Manual / cron (bulk refresh via Records.Batch) | -| `RecordGenerateAllEmbeddings` | heavy_writes | Manual / cron (bulk generate via Records.Batch) | -| `ArtistRefreshAllMusicBrainzData` | music_brainz | Manual / cron (bulk refresh via Artists.Batch) | -| `ArtistRefreshAllDiscogsData` | discogs | Manual / cron (bulk refresh via Artists.Batch) | -| `ArtistRefreshAllWikipediaData` | wikipedia | Manual / cron (bulk refresh via Artists.Batch) | -| `RefreshScrobbles` | last_fm | Cron / manual (fetch recent Last.fm scrobbles) | -| `BackfillScrobbledTracks` | heavy_writes | Manual (self-chaining batch import) | -| `SendRecordsOnThisDayEmail` | default | Cron (daily "records on this day" email) | +| `PopulateGenres` | heavy_writes | Manual action (chains → GenerateRecordEmbedding) | +| `GenerateRecordEmbedding` | heavy_writes | Manual / after genre population (delegates to Similarity, skips unchanged) | +| `RecordRefreshMusicBrainzData` | music_brainz | Manual / batch | +| `ArtistRefreshMusicBrainzData` | music_brainz | Manual / batch | +| `ArtistRefreshDiscogsData` | discogs | Manual / batch | +| `ArtistRefreshWikipediaData` | wikipedia | Manual / batch | +| `PruneArtistInfo` | default | Record deleted (cleanup orphaned artist data) | +| `RecordRefreshAllMusicBrainzData` | music_brainz | Manual / cron (bulk refresh via Records.Batch) | +| `RecordGenerateAllEmbeddings` | heavy_writes | Manual / cron (bulk generate via Records.Batch) | +| `ArtistRefreshAllMusicBrainzData` | music_brainz | Manual / cron (bulk refresh via Artists.Batch) | +| `ArtistRefreshAllDiscogsData` | discogs | Manual / cron (bulk refresh via Artists.Batch) | +| `ArtistRefreshAllWikipediaData` | wikipedia | Manual / cron (bulk refresh via Artists.Batch) | +| `RefreshScrobbles` | last_fm | Cron / manual (fetch recent Last.fm scrobbles) | +| `BackfillScrobbledTracks` | heavy_writes | Manual (self-chaining batch import) | +| `SendRecordsOnThisDayEmail` | default | Cron (daily "records on this day" email) | ### Cron Workers -| Schedule | Worker | Queue | -|----------|--------|-------| -| Every 12h | `ApplyScrobbleRules` | heavy_writes | -| Every 12h | `PruneAssetCache` | default | -| Daily 2 AM | `PruneAssets` | default | -| Daily 3 AM | `RepoVacuum` | heavy_writes | -| Daily 4 AM | `RepoOptimize` | heavy_writes | -| Monthly 1st, 6 AM | `RecordRefreshAllMusicBrainzData` | music_brainz | -| Monthly 1st, 7 AM | `RecordGenerateAllEmbeddings` | heavy_writes | -| Monthly 1st, 8 AM | `ArtistRefreshAllMusicBrainzData` | music_brainz | -| Monthly 1st, 9 AM | `ArtistRefreshAllDiscogsData` | discogs | -| Monthly 1st, 10 AM | `ArtistRefreshAllWikipediaData` | wikipedia | -| Daily 7 AM | `SendRecordsOnThisDayEmail` | default | -| Every 5 min | `RefreshScrobbles` | last_fm | +| Schedule | Worker | Queue | +| ------------------ | --------------------------------- | ------------ | +| Every 12h | `ApplyScrobbleRules` | heavy_writes | +| Every 12h | `PruneAssetCache` | default | +| Daily 2 AM | `PruneAssets` | default | +| Daily 3 AM | `RepoVacuum` | heavy_writes | +| Daily 4 AM | `RepoOptimize` | heavy_writes | +| Monthly 1st, 6 AM | `RecordRefreshAllMusicBrainzData` | music_brainz | +| Monthly 1st, 7 AM | `RecordGenerateAllEmbeddings` | heavy_writes | +| Monthly 1st, 8 AM | `ArtistRefreshAllMusicBrainzData` | music_brainz | +| Monthly 1st, 9 AM | `ArtistRefreshAllDiscogsData` | discogs | +| Monthly 1st, 10 AM | `ArtistRefreshAllWikipediaData` | wikipedia | +| Daily 7 AM | `SendRecordsOnThisDayEmail` | default | +| Every 5 min | `RefreshScrobbles` | last_fm | --- ## PubSub Topics -| PubSub | Topic Pattern | Message | Used By | -|--------|---------------|---------|---------| -| `:music_library` | `"records:#{id}"` | `{:update, record}` | CollectionLive.Show, WishlistLive.Show — real-time record updates | +| PubSub | Topic Pattern | Message | Used By | +| ---------------- | -------------------------- | ------------------- | ------------------------------------------------------------------ | +| `:music_library` | `"records:#{id}"` | `{:update, record}` | CollectionLive.Show, WishlistLive.Show — real-time record updates | | `:music_library` | `"listening_stats:update"` | `%{track_count: n}` | StatsLive.Index, ScrobbledTracksLive.Index — new scrobbles arrived | --- @@ -259,91 +261,92 @@ HTTP 429 into `:rate_limit` vs `:auth_error` by reading the body `code` ### Router Structure All authenticated routes live inside a single `live_session` with three `on_mount` hooks: + - `StaticAssets` — detects app updates, shows toast - `GetTimezone` — reads timezone from connect params - `ShowToast` — enables `put_toast!/2` in LiveViews ### LiveViews -| LiveView | Route | Purpose | -|----------|-------|---------| -| `StatsLive.Index` | `/` | Dashboard: counts, recent activity, records on this day | -| `CollectionLive.Index` | `/collection` | Browse/search collected records (grid/list, paginated) | -| `CollectionLive.Show` | `/collection/:id` | Record detail: metadata, scrobbles, similar, colors | -| `WishlistLive.Index` | `/wishlist` | Browse/search wishlisted records | -| `WishlistLive.Show` | `/wishlist/:id` | Wishlist record detail with store links | -| `ArtistLive.Show` | `/artists/:musicbrainz_id` | Artist bio, discography, similar artists | -| `RecordSetLive.Index` | `/record-sets` | Browse/manage curated record sets | -| `RecordSetLive.Show` | `/record-sets/:id` | Set detail with reorderable items | -| `ScrobbleLive.Index` | `/scrobble` | Search MusicBrainz release groups to scrobble | -| `ScrobbleLive.ReleaseGroupShow` | `/scrobble/:rg_id` | List releases within a release group | -| `ScrobbleLive.ReleaseShow` | `/scrobble/:rg_id/releases/:release_id` | Select tracks and scrobble (uses `Release` live_component) | -| `ScrobbledTracksLive.Index` | `/scrobbled-tracks` | Browse/search Last.fm history | -| `ScrobbleRulesLive.Index` | `/scrobble-rules` | Browse/search/sort scrobble remapping rules (paginated, 50 per page) | -| `OnlineStoreTemplateLive.Index` | `/online-store-templates` | Manage store URL templates | -| `MaintenanceLive.Index` | `/maintenance` | Admin: batch jobs, DB maintenance, Last.fm connection | +| LiveView | Route | Purpose | +| ------------------------------- | --------------------------------------- | -------------------------------------------------------------------- | +| `StatsLive.Index` | `/` | Dashboard: counts, recent activity, records on this day | +| `CollectionLive.Index` | `/collection` | Browse/search collected records (grid/list, paginated) | +| `CollectionLive.Show` | `/collection/:id` | Record detail: metadata, scrobbles, similar, colors | +| `WishlistLive.Index` | `/wishlist` | Browse/search wishlisted records | +| `WishlistLive.Show` | `/wishlist/:id` | Wishlist record detail with store links | +| `ArtistLive.Show` | `/artists/:musicbrainz_id` | Artist bio, discography, similar artists | +| `RecordSetLive.Index` | `/record-sets` | Browse/manage curated record sets | +| `RecordSetLive.Show` | `/record-sets/:id` | Set detail with reorderable items | +| `ScrobbleLive.Index` | `/scrobble` | Search MusicBrainz release groups to scrobble | +| `ScrobbleLive.ReleaseGroupShow` | `/scrobble/:rg_id` | List releases within a release group | +| `ScrobbleLive.ReleaseShow` | `/scrobble/:rg_id/releases/:release_id` | Select tracks and scrobble (uses `Release` live_component) | +| `ScrobbledTracksLive.Index` | `/scrobbled-tracks` | Browse/search Last.fm history | +| `ScrobbleRulesLive.Index` | `/scrobble-rules` | Browse/search/sort scrobble remapping rules (paginated, 50 per page) | +| `OnlineStoreTemplateLive.Index` | `/online-store-templates` | Manage store URL templates | +| `MaintenanceLive.Index` | `/maintenance` | Admin: batch jobs, DB maintenance, Last.fm connection | ### LiveComponents -| Component | Used In | Purpose | -|-----------|---------|---------| -| `RecordForm` | Collection/Wishlist (edit) | Record editing: cover search, genre autocomplete, color picker, file upload | -| `ArtistLive.Form` | ArtistLive.Show | Edit artist image (upload + Brave image search) | -| `RecordSetLive.Form` | RecordSetLive.Index | Create/edit record set | -| `RecordSetLive.RecordPicker` | RecordSetLive.Show | Search and add records to set | -| `ScrobbledTracksLive.Form` | ScrobbledTracksLive.Index | Edit scrobbled track | -| `ScrobbleRulesLive.Form` | ScrobbleRulesLive.Index | Create/edit scrobble rule | -| `ScrobbleRulePicker` | ScrobbledTracksLive.Index, StatsLive.Index | Search records and create scrobble rules inline | -| `OnlineStoreTemplateLive.Form` | OnlineStoreTemplateLive.Index | Create/edit store template | -| `StatsLive.TopByPeriod` | StatsLive.TopAlbums, StatsLive.TopArtists | Generic period-tabbed stats display (7d, 30d, 90d, 1y, all-time) | -| `StatsLive.TopAlbums` | StatsLive.Index | Top albums by period (uses TopByPeriod) | -| `StatsLive.TopArtists` | StatsLive.Index | Top artists by period (uses TopByPeriod) | -| `UniversalSearchLive.Index` | Layout (global) | Cmd+K search modal with quick actions (add to wishlist/collection, scrobble, collection chat) | -| `Chat` | CollectionLive.Index, CollectionLive.Show, WishlistLive.Show, ArtistLive.Show | AI chat sheet (OpenAI streaming, configurable per entity) | -| `Notes` | CollectionLive.Show, WishlistLive.Show, ArtistLive.Show | Markdown note rendering and editing | -| `AddRecord` | CollectionLive.Index, WishlistLive.Index | MusicBrainz import interface | -| `BarcodeScanner` | CollectionLive.Index | Barcode scanning UI (uses barcode-detector JS) | -| `Release` | CollectionLive.Show, ScrobbleLive.ReleaseShow | MusicBrainz release display with scrobble (form-based with auto-recovery) | +| Component | Used In | Purpose | +| ------------------------------ | ----------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `RecordForm` | Collection/Wishlist (edit) | Record editing: cover search, genre autocomplete, color picker, file upload | +| `ArtistLive.Form` | ArtistLive.Show | Edit artist image (upload + Brave image search) | +| `RecordSetLive.Form` | RecordSetLive.Index | Create/edit record set | +| `RecordSetLive.RecordPicker` | RecordSetLive.Show | Search and add records to set | +| `ScrobbledTracksLive.Form` | ScrobbledTracksLive.Index | Edit scrobbled track | +| `ScrobbleRulesLive.Form` | ScrobbleRulesLive.Index | Create/edit scrobble rule | +| `ScrobbleRulePicker` | ScrobbledTracksLive.Index, StatsLive.Index | Search records and create scrobble rules inline | +| `OnlineStoreTemplateLive.Form` | OnlineStoreTemplateLive.Index | Create/edit store template | +| `StatsLive.TopByPeriod` | StatsLive.TopAlbums, StatsLive.TopArtists | Generic period-tabbed stats display (7d, 30d, 90d, 1y, all-time) | +| `StatsLive.TopAlbums` | StatsLive.Index | Top albums by period (uses TopByPeriod) | +| `StatsLive.TopArtists` | StatsLive.Index | Top artists by period (uses TopByPeriod) | +| `UniversalSearchLive.Index` | Layout (global) | Cmd+K search modal with quick actions (add to wishlist/collection, scrobble, collection chat) | +| `Chat` | CollectionLive.Index, CollectionLive.Show, WishlistLive.Show, ArtistLive.Show | AI chat sheet (OpenAI streaming, configurable per entity) | +| `Notes` | CollectionLive.Show, WishlistLive.Show, ArtistLive.Show | Markdown note rendering and editing | +| `AddRecord` | CollectionLive.Index, WishlistLive.Index | MusicBrainz import interface | +| `BarcodeScanner` | CollectionLive.Index | Barcode scanning UI (uses barcode-detector JS) | +| `Release` | CollectionLive.Show, ScrobbleLive.ReleaseShow | MusicBrainz release display with scrobble (form-based with auto-recovery) | ### Shared Component Modules (lib/music_library_web/components/) -| Module | Purpose | -|--------|---------| -| `CoreComponents` | Forms, buttons, icons, tables, flash messages | -| `Layouts` | Application layout templates, navigation components (`dropdown_nav/1`) | -| `RecordComponents` | Record cards, cover images, artist images, labels, grids, release status icon badge, shared show-page sections (title, external links, genres, releases, timestamps, debug) | -| `ChartComponents` | Charts for stats dashboard | -| `StatsComponents` | Stats dashboard widgets (`section/1` layout, counters, album preview, records on this day) | -| `ScrobbleComponents` | Scrobble activity displays: status badges, import dropdowns, metadata tooltips, and record-matching UI | -| `SearchComponents` | Search result rendering | -| `CartComponents` | `cart_sidebar/1` — shared cart aside used by `AddRecord` and `BarcodeScanner` | -| `Pagination` | Pagination UI and logic | +| Module | Purpose | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CoreComponents` | Forms, buttons, icons, tables, flash messages | +| `Layouts` | Application layout templates, navigation components (`dropdown_nav/1`) | +| `RecordComponents` | Record cards, cover images, artist images, labels, grids, release status icon badge, shared show-page sections (title, external links, genres, releases, timestamps, debug) | +| `ChartComponents` | Charts for stats dashboard | +| `StatsComponents` | Stats dashboard widgets (`section/1` layout, counters, album preview, records on this day) | +| `ScrobbleComponents` | Scrobble activity displays: status badges, import dropdowns, metadata tooltips, and record-matching UI | +| `SearchComponents` | Search result rendering | +| `CartComponents` | `cart_sidebar/1` — shared cart aside used by `AddRecord` and `BarcodeScanner` | +| `Pagination` | Pagination UI and logic | ### Web Utility Modules (lib/music_library_web/) -| Module | Purpose | -|--------|---------| -| `ErrorMessages` | Maps internal error terms (atoms, structs) to user-friendly gettext strings via `friendly_message/1` | -| `Markdown` | Markdown-to-HTML conversion (MDEx with ammonia sanitization) with `[[double bracket]]` link syntax and streaming document support for chat | -| `Duration` | Milliseconds to human-readable duration formatting | -| `Auth` | Authentication plugs: login password check, API token validation, session enforcement | -| `ArtistLive.Biography` | Artist biography building/rendering from Wikipedia and Last.fm data | -| `LiveHelpers.Params` | URL query param parsing: pagination, search query, sort order, display mode, fallback index | -| `LiveHelpers.IndexActions` | Shared index page logic (search, pagination, import, delete, display mode) for Collection/Wishlist index pages, parameterized by config map | -| `LiveHelpers.RecordActions` | Shared record action handlers (refresh cover, genres, embeddings, MusicBrainz data) for Collection/Wishlist show pages | -| — | Logster v2 handles `[:phoenix, :socket_connected]` telemetry — not a custom module | +| Module | Purpose | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `ErrorMessages` | Maps internal error terms (atoms, structs) to user-friendly gettext strings via `friendly_message/1` | +| `Markdown` | Markdown-to-HTML conversion (MDEx with ammonia sanitization) with `[[double bracket]]` link syntax and streaming document support for chat | +| `Duration` | Milliseconds to human-readable duration formatting | +| `Auth` | Authentication plugs: login password check, API token validation, session enforcement | +| `ArtistLive.Biography` | Artist biography building/rendering from Wikipedia and Last.fm data | +| `LiveHelpers.Params` | URL query param parsing: pagination, search query, sort order, display mode, fallback index | +| `LiveHelpers.IndexActions` | Shared index page logic (search, pagination, import, delete, display mode) for Collection/Wishlist index pages, parameterized by config map | +| `LiveHelpers.RecordActions` | Shared record action handlers (refresh cover, genres, embeddings, MusicBrainz data) for Collection/Wishlist show pages | +| — | Logster v2 handles `[:phoenix, :socket_connected]` telemetry — not a custom module | ### Controllers -| Controller | Routes | Purpose | -|------------|--------|---------| -| `SessionController` | `/login`, `/sessions/create` | Login/logout | -| `HealthController` | `/health` | Health check | -| `LastFmController` | `/auth/last_fm/callback` | Last.fm OAuth | -| `ArchiveController` | `/backup`, `/api/v1/backup` | Database backup download (API route requires token) | -| `AssetController` | `/assets/:transform_payload`, `/public/assets/:transform_payload`, `/api/v1/assets/:transform_payload` | Serve images with transforms (public route for emails, API route requires token) | -| `CollectionController` | `/api/v1/collection/*` | JSON API for collection queries | -| `ErrorController` | `/api/v1/errors`, `/api/v1/errors/:id` | JSON API for production error queries (requires Bearer token) | +| Controller | Routes | Purpose | +| ---------------------- | ------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | +| `SessionController` | `/login`, `/sessions/create` | Login/logout | +| `HealthController` | `/health` | Health check | +| `LastFmController` | `/auth/last_fm/callback` | Last.fm OAuth | +| `ArchiveController` | `/backup`, `/api/v1/backup` | Database backup download (API route requires token) | +| `AssetController` | `/assets/:transform_payload`, `/public/assets/:transform_payload`, `/api/v1/assets/:transform_payload` | Serve images with transforms (public route for emails, API route requires token) | +| `CollectionController` | `/api/v1/collection/*` | JSON API for collection queries | +| `ErrorController` | `/api/v1/errors`, `/api/v1/errors/:id` | JSON API for production error queries (requires Bearer token) | --- @@ -355,26 +358,26 @@ All authenticated routes live inside a single `live_session` with three `on_moun ### JS Hooks -| Hook | Type | Purpose | -|------|------|---------| -| `FormatNumber` | External (`assets/js/hooks/`) | Client-side number formatting | -| `UniversalSearchNavigation` | External | Keyboard navigation in search modal (via `create-navigation-hook` factory) | -| `RecordPickerNavigation` | External | Keyboard navigation in record picker (via `create-navigation-hook` factory) | -| `RulePickerNavigation` | External | Keyboard navigation in scrobble rule picker (via `create-navigation-hook` factory) | -| `SortableList` | External (`assets/js/hooks/`) | Drag-and-drop reordering of record set items (uses sortablejs) | -| `LiveToast` | External (via `createLiveToastHook`) | Toast notification rendering | -| Various `.ColocatedHooks` | Colocated (in .heex) | Inline hooks prefixed with `.` (includes `.ScrollBottom` for Chat) | +| Hook | Type | Purpose | +| --------------------------- | ------------------------------------ | ---------------------------------------------------------------------------------- | +| `FormatNumber` | External (`assets/js/hooks/`) | Client-side number formatting | +| `UniversalSearchNavigation` | External | Keyboard navigation in search modal (via `create-navigation-hook` factory) | +| `RecordPickerNavigation` | External | Keyboard navigation in record picker (via `create-navigation-hook` factory) | +| `RulePickerNavigation` | External | Keyboard navigation in scrobble rule picker (via `create-navigation-hook` factory) | +| `SortableList` | External (`assets/js/hooks/`) | Drag-and-drop reordering of record set items (uses sortablejs) | +| `LiveToast` | External (via `createLiveToastHook`) | Toast notification rendering | +| Various `.ColocatedHooks` | Colocated (in .heex) | Inline hooks prefixed with `.` (includes `.ScrollBottom` for Chat) | ### JS Event Listeners (app.js) All events are namespaced with `music_library:` prefix. -| Event | Action | -|-------|--------| -| `music_library:clipcopy` | Copy text to clipboard | -| `music_library:scroll_top` | Scroll window to top | -| `music_library:confetti` | Trigger canvas-confetti animation | -| `music_library:download` | Decode base64 blob and trigger browser file download (dispatched via `push_event`, prefixed `phx:` on client) | +| Event | Action | +| -------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `music_library:clipcopy` | Copy text to clipboard | +| `music_library:scroll_top` | Scroll window to top | +| `music_library:confetti` | Trigger canvas-confetti animation | +| `music_library:download` | Decode base64 blob and trigger browser file download (dispatched via `push_event`, prefixed `phx:` on client) | ### NPM Dependencies @@ -390,26 +393,26 @@ All events are namespaced with `music_library:` prefix. ### Test Support -| Module | Purpose | -|--------|---------| -| `ConnCase` | HTTP test setup, auto-logged-in session | -| `DataCase` | Database test setup with Ecto sandbox | +| Module | Purpose | +| ----------------- | ------------------------------------------- | +| `ConnCase` | HTTP test setup, auto-logged-in session | +| `DataCase` | Database test setup with Ecto sandbox | | `LiveTestHelpers` | `escape/1` for HTML-escaped text assertions | ### Fixture Modules (test/support/fixtures/) -| Module | Creates | -|--------|---------| -| `MusicLibrary.RecordsFixtures` | Records with MusicBrainz data | -| `MusicLibrary.RecordSetsFixtures` | Record sets with items | -| `MusicLibrary.OnlineStoreTemplatesFixtures` | Store templates | -| `MusicLibrary.ArtistInfoFixtures` | ArtistInfo records | -| `ScrobbleRulesFixtures` | Scrobble rules | -| `ScrobbledTracksFixtures` | Last.fm tracks | -| `Discogs.ArtistFixtures` | Discogs API responses | -| `LastFm.ArtistFixtures` | Last.fm API responses | -| `MusicBrainz.*Fixtures` | MusicBrainz API responses | -| `Wikipedia.Fixtures` | Wikipedia API responses | +| Module | Creates | +| ------------------------------------------- | ----------------------------- | +| `MusicLibrary.RecordsFixtures` | Records with MusicBrainz data | +| `MusicLibrary.RecordSetsFixtures` | Record sets with items | +| `MusicLibrary.OnlineStoreTemplatesFixtures` | Store templates | +| `MusicLibrary.ArtistInfoFixtures` | ArtistInfo records | +| `ScrobbleRulesFixtures` | Scrobble rules | +| `ScrobbledTracksFixtures` | Last.fm tracks | +| `Discogs.ArtistFixtures` | Discogs API responses | +| `LastFm.ArtistFixtures` | Last.fm API responses | +| `MusicBrainz.*Fixtures` | MusicBrainz API responses | +| `Wikipedia.Fixtures` | Wikipedia API responses | ### Test Styles diff --git a/docs/production-infrastructure.md b/docs/production-infrastructure.md index 1353b7c0..e83d5ccf 100644 --- a/docs/production-infrastructure.md +++ b/docs/production-infrastructure.md @@ -19,12 +19,12 @@ push via GitHub Actions. ## Hosting -| Component | Technology | -|-----------|-----------| -| Orchestration | Coolify (self-hosted) | -| Container runtime | Docker | -| SSL termination | Coolify reverse proxy | -| HTTP redirect | HTTP 301 → HTTPS (enforced by app config) | +| Component | Technology | +| ----------------- | ----------------------------------------- | +| Orchestration | Coolify (self-hosted) | +| Container runtime | Docker | +| SSL termination | Coolify reverse proxy | +| HTTP redirect | HTTP 301 → HTTPS (enforced by app config) | The Docker image is a multi-stage build: @@ -42,11 +42,11 @@ Fluxon UI (licensed dependency) is fetched during build via Docker build secrets Three separate SQLite databases, each managed by its own Ecto repo: -| Repo | Purpose | Cache size | Pool size | -|------|---------|------------|-----------| -| `MusicLibrary.Repo` | Application data | 128 MB | `$POOL_SIZE` (default 5) | -| `MusicLibrary.BackgroundRepo` | Oban job queue | 16 MB | `$POOL_SIZE` (default 5) | -| `MusicLibrary.TelemetryRepo` | Persistent telemetry metrics | 4 MB | 2 | +| Repo | Purpose | Cache size | Pool size | +| ----------------------------- | ---------------------------- | ---------- | ------------------------ | +| `MusicLibrary.Repo` | Application data | 128 MB | `$POOL_SIZE` (default 5) | +| `MusicLibrary.BackgroundRepo` | Oban job queue | 16 MB | `$POOL_SIZE` (default 5) | +| `MusicLibrary.TelemetryRepo` | Persistent telemetry metrics | 4 MB | 2 | All databases use incremental auto-vacuum. Paths are configured via environment variables (`DATABASE_PATH`, `BACKGROUND_DATABASE_PATH`, `TELEMETRY_DATABASE_PATH`). @@ -60,13 +60,13 @@ All databases use incremental auto-vacuum. Paths are configured via environment Configured inline in `compose.yaml`. Runs as a separate Docker Compose service (`litestream/litestream:0.5.11-scratch`) sharing the database volume. -| Setting | Value | -|---------|-------| -| S3 endpoint | `https://nbg1.your-objectstorage.com` | -| Bucket | `ffmusiclibrary` | -| Sync interval | 60 minutes | -| Retention | 672 hours (28 days) | -| Healthcheck | `litestream databases` every 30s (timeout 10s, 3 retries) | +| Setting | Value | +| ------------- | --------------------------------------------------------- | +| S3 endpoint | `https://nbg1.your-objectstorage.com` | +| Bucket | `ffmusiclibrary` | +| Sync interval | 60 minutes | +| Retention | 672 hours (28 days) | +| Healthcheck | `litestream databases` every 30s (timeout 10s, 3 retries) | Credentials via environment: `LITESTREAM_ACCESS_KEY_ID`, `LITESTREAM_SECRET_ACCESS_KEY`. @@ -132,10 +132,10 @@ Fluxon (private dependency) is configured with a dedicated registry entry. ### Deployment credentials -| Secret/Variable | Purpose | -|-----------------|---------| -| `COOLIFY_TOKEN` | API authentication (GitHub secret) | -| `COOLIFY_HOST` | Coolify server address (GitHub variable) | +| Secret/Variable | Purpose | +| ------------------ | ---------------------------------------- | +| `COOLIFY_TOKEN` | API authentication (GitHub secret) | +| `COOLIFY_HOST` | Coolify server address (GitHub variable) | | `COOLIFY_APP_UUID` | Application identifier (GitHub variable) | --- @@ -144,32 +144,32 @@ Fluxon (private dependency) is configured with a dedicated registry entry. ### Required -| Variable | Purpose | -|----------|---------| -| `DATABASE_PATH` | Absolute path to main SQLite database | -| `BACKGROUND_DATABASE_PATH` | Absolute path to background jobs database | -| `TELEMETRY_DATABASE_PATH` | Absolute path to telemetry database | -| `SECRET_KEY_BASE` | Phoenix secret (`mix phx.gen.secret`) | -| `CLOAK_ENCRYPTION_KEY` | Base64-encoded 32-byte AES key for encrypted secrets | -| `LOGIN_PASSWORD` | Web login password | -| `API_TOKEN` | Bearer token for API endpoints | -| `MAILGUN_API_KEY` | Mailgun API key | +| Variable | Purpose | +| -------------------------- | ---------------------------------------------------- | +| `DATABASE_PATH` | Absolute path to main SQLite database | +| `BACKGROUND_DATABASE_PATH` | Absolute path to background jobs database | +| `TELEMETRY_DATABASE_PATH` | Absolute path to telemetry database | +| `SECRET_KEY_BASE` | Phoenix secret (`mix phx.gen.secret`) | +| `CLOAK_ENCRYPTION_KEY` | Base64-encoded 32-byte AES key for encrypted secrets | +| `LOGIN_PASSWORD` | Web login password | +| `API_TOKEN` | Bearer token for API endpoints | +| `MAILGUN_API_KEY` | Mailgun API key | ### Optional -| Variable | Default | Purpose | -|----------|---------|---------| -| `SERVICE_FQDN_WEB` | `example.com` | Application domain | -| `PORT` | `4000` | HTTP listen port | -| `POOL_SIZE` | `5` | Database connection pool size | -| `MAILGUN_DOMAIN` | `mailgun.fullyforged.com` | Mailgun sending domain | -| `DEFAULT_TIMEZONE` | `Europe/London` | Application timezone | -| `OPENAI_KEY` | — | OpenAI API (embeddings, chat) | -| `DISCOGS_PERSONAL_ACCESS_TOKEN` | — | Discogs API | -| `BRAVE_SEARCH_API_KEY` | — | Brave Search API | -| `LAST_FM_API_KEY` | — | Last.fm API | -| `LAST_FM_SHARED_SECRET` | — | Last.fm scrobbling auth | -| `LAST_FM_USER` | — | Last.fm username | +| Variable | Default | Purpose | +| ------------------------------- | ------------------------- | ----------------------------- | +| `SERVICE_FQDN_WEB` | `example.com` | Application domain | +| `PORT` | `4000` | HTTP listen port | +| `POOL_SIZE` | `5` | Database connection pool size | +| `MAILGUN_DOMAIN` | `mailgun.fullyforged.com` | Mailgun sending domain | +| `DEFAULT_TIMEZONE` | `Europe/London` | Application timezone | +| `OPENAI_KEY` | — | OpenAI API (embeddings, chat) | +| `DISCOGS_PERSONAL_ACCESS_TOKEN` | — | Discogs API | +| `BRAVE_SEARCH_API_KEY` | — | Brave Search API | +| `LAST_FM_API_KEY` | — | Last.fm API | +| `LAST_FM_SHARED_SECRET` | — | Last.fm scrobbling auth | +| `LAST_FM_USER` | — | Last.fm username | --- @@ -276,22 +276,25 @@ Pi extensions provide additional tools for production observability without manu or browser access. Each extension reads its own environment variables from the pi runtime environment (not server-side config). -| Extension | Tools | Env vars | -|-----------|-------|----------| -| `prod-logs` | `fetch_production_logs` | `PI_COOLIFY_HOST`, `PI_COOLIFY_APP_UUID`, `PI_COOLIFY_TOKEN` | -| `prod-errors` | `fetch_production_errors`, `fetch_production_error`, `/prod-errors` | `PI_API_TOKEN`, `PI_SERVICE_FQDN_WEB` | +| Extension | Tools | Env vars | +| ------------- | ------------------------------------------------------------------- | ------------------------------------------------------------ | +| `prod-logs` | `fetch_production_logs` | `PI_COOLIFY_HOST`, `PI_COOLIFY_APP_UUID`, `PI_COOLIFY_TOKEN` | +| `prod-errors` | `fetch_production_errors`, `fetch_production_error`, `/prod-errors` | `PI_API_TOKEN`, `PI_SERVICE_FQDN_WEB` | **`prod-logs` env vars:** + - `PI_COOLIFY_HOST` — Coolify server base URL (e.g., `https://coolify.example.com`) - `PI_COOLIFY_APP_UUID` — Application UUID in Coolify - `PI_COOLIFY_TOKEN` — Coolify API Bearer token **`prod-errors` tools and command:** + - `fetch_production_errors` — List/filter errors via LLM tool - `fetch_production_error` — Single error detail via LLM tool - `/prod-errors` — Interactive TUI for browsing errors (list, detail, filter toggles) **`prod-errors` env vars:** + - `PI_API_TOKEN` — Must match the `API_TOKEN` env var on the production server (used for Bearer auth on `/api/v1/*`) - `PI_SERVICE_FQDN_WEB` — Production domain with protocol (e.g., `https://musiclibrary.claudio-ortolina.org`, no trailing slash) @@ -301,10 +304,10 @@ runtime environment (not server-side config). Mailgun via Swoosh (`Swoosh.Adapters.Mailgun`). -| Setting | Value | -|---------|-------| -| From | `postmaster@mailgun.fullyforged.com` | -| To | `claudio@fullyforged.com` | +| Setting | Value | +| ------- | ------------------------------------ | +| From | `postmaster@mailgun.fullyforged.com` | +| To | `claudio@fullyforged.com` | Used for error notifications and the daily "records on this day" digest email. diff --git a/scripts/dev/lint b/scripts/dev/lint index cf596bb2..fa97ed97 100755 --- a/scripts/dev/lint +++ b/scripts/dev/lint @@ -22,3 +22,9 @@ mix gettext.extract --merge debug_msg "Running shellcheck..." fd . 'scripts/' --exclude '*.hurl' -t file --exec shellcheck --color + +debug_msg "Running prettier..." +prettier --write '.pi/extensions/**/*.{ts,js,json}' +prettier --write 'assets/css/**/*.css' 'assets/js/**/*.js' +prettier --write 'docs/**/*.md' 'docs/**/*.livemd' 'README.md' +prettier --write 'backlog/archive/**/*.md' 'backlog/completed/**/*.md' 'backlog/tasks/**/*.md' 'backlog/docs/**/*.md'