From 18b3f0974ef594abd914279f5701761d0697c6d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 14:53:29 +0000 Subject: [PATCH] Add genre mixes from clustering recent listens A new cluster_mix_rule clusters a Listenbrainz user's listens from the last few months by MusicBrainz genre (with tracks played in the same session pulled together) and maintains one Subsonic playlist per cluster, filled with the cluster's most played tracks plus unheard library songs from the best matching library genres. - listenbrainz: fetch raw listens since a timestamp and bulk recording genres, handling rate limits - cluster: tag TF-IDF + co-listen smoothing, spherical k-means with k chosen by silhouette score - mix: match cluster profiles to library genres and pick songs, with a per-artist cap; reuse playlists across rebuilds by genre overlap - subsonic: getGenres, getSongsByGenre, rename/delete playlist, genre and MBID fields on tracks, API error detection - daemon, migration, POST/GET /api/playlist/mix and README docs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018opPJMJjLWr4RqdG6vuK6p --- Cargo.lock | 9 +- Cargo.toml | 6 +- README.md | 48 ++ migrations/20261113150000_cluster_mixes.sql | 23 + src/cluster.rs | 539 ++++++++++++++++++++ src/daemon/cluster_mixes.rs | 263 ++++++++++ src/daemon/mod.rs | 2 + src/listenbrainz.rs | 196 +++++++ src/main.rs | 36 +- src/mix.rs | 334 ++++++++++++ src/models.rs | 96 ++++ src/subsonic.rs | 151 ++++++ src/track.rs | 30 ++ 13 files changed, 1728 insertions(+), 5 deletions(-) create mode 100644 migrations/20261113150000_cluster_mixes.sql create mode 100644 src/cluster.rs create mode 100644 src/daemon/cluster_mixes.rs create mode 100644 src/mix.rs diff --git a/Cargo.lock b/Cargo.lock index b958693..bd9a499 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -229,6 +229,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets 0.52.5", ] @@ -1468,9 +1469,11 @@ dependencies = [ "config", "log", "md5", + "rand", "reqwest", "rocket", "serde", + "serde_json", "sqlx", "thiserror 2.0.17", "tokio", @@ -2088,6 +2091,7 @@ checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64 0.22.1", "bytes", + "chrono", "crc", "crossbeam-queue", "either", @@ -2164,6 +2168,7 @@ dependencies = [ "bitflags 2.10.0", "byteorder", "bytes", + "chrono", "crc", "digest", "dotenvy", @@ -2206,6 +2211,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.10.0", "byteorder", + "chrono", "crc", "dotenvy", "etcetera", @@ -2241,6 +2247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ "atoi", + "chrono", "flume", "futures-channel", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index 1b1cfd7..d7bc8d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ description = "Subsonic playlist daemon" [dependencies] rocket = { version = "0.5.0", features = ["json"] } tokio = { version = "1.0", features = ["full"] } -sqlx = { version = "0.8.6", features = ["runtime-tokio", "postgres", "uuid"] } +sqlx = { version = "0.8.6", features = ["runtime-tokio", "postgres", "uuid", "chrono"] } reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } uuid = { version = "1.7", features = ["v4", "serde"] } @@ -17,7 +17,9 @@ md5 = "0.8.0" thiserror = "2.0.17" config = "0.15.19" log = "0.4.21" -chrono = "0.4.38" +chrono = { version = "0.4.38", features = ["serde"] } +rand = "0.8" +serde_json = "1.0" [package.metadata.deb] depends = "$auto" diff --git a/README.md b/README.md index 90ce54c..a8f4b83 100644 --- a/README.md +++ b/README.md @@ -63,3 +63,51 @@ For a yearly top songs playlist, run the following: INSERT INTO create_tracked_playlist (id, name_template, playlist_size, tracking_user, tracking_type) VALUES (gen_random_uuid(), '%Y - Top 100', 100, 'jellywx', 'this_year'); ``` + +### Genre mixes + +A cluster mix rule groups the tracks you've listened to over the last few months by genre, and +keeps one Subsonic playlist ("mix") per group, filled with songs from your library. + +How it works: + +1. Your listens for the period are fetched from Listenbrainz, along with the MusicBrainz genres of + each recording, its release group and its artists. +2. Tracks are clustered on their genres. Tracks you tend to play in the same listening session are + pulled closer together, so untagged tracks and genre-hopping artists land with what you play + them next to. The number of clusters is chosen automatically unless you set one. +3. Each cluster's most characteristic genres are matched against your library's genres + (`getGenres` / `getSongsByGenre`), case and punctuation insensitively, so a library tagged with + MusicBrainz genres by Picard works best. +4. Each mix is made of the cluster's most played tracks (`familiar_ratio` of the mix), plus songs + from the matching genres that you haven't played in the period, picked at random weighted by + how well their genres fit, with at most `playlist_size / 8` songs per artist. + +Mixes are rebuilt every `refresh_hours`. On a rebuild, existing mix playlists are reused for the +new cluster with the most similar genres (and renamed), new playlists are created for new +clusters, and playlists for clusters that no longer appear are deleted. + +Create a rule through the API: + +```sh +curl -X POST localhost:8000/api/playlist/mix -H 'Content-Type: application/json' \ + -d '{"tracking_user": "jellywx", "months": 3, "playlist_size": 50}' +``` + +or with SQL: + +```sql +INSERT INTO cluster_mix_rule (id, tracking_user, months, playlist_size) +VALUES (gen_random_uuid(), 'jellywx', 3, 50); +``` + +| Column | Default | Meaning | +|------------------|-----------------|--------------------------------------------------------------| +| `months` | 3 | Months of listening history to cluster | +| `cluster_count` | automatic | Number of mixes | +| `playlist_size` | 50 | Songs per mix | +| `familiar_ratio` | 0.3 | Share of each mix taken from recently played tracks | +| `name_template` | `Mix: {label}` | Playlist name; `{label}` becomes the mix's top genres | +| `refresh_hours` | 24 | How often to rebuild the mixes | + +Set `last_run` to `NULL` to rebuild a rule's mixes on the next daemon tick. diff --git a/migrations/20261113150000_cluster_mixes.sql b/migrations/20261113150000_cluster_mixes.sql new file mode 100644 index 0000000..403697b --- /dev/null +++ b/migrations/20261113150000_cluster_mixes.sql @@ -0,0 +1,23 @@ +-- A rule that periodically clusters a user's recent listens by genre and +-- maintains one Subsonic "mix" playlist per cluster. +CREATE TABLE "cluster_mix_rule" +( + id UUID NOT NULL PRIMARY KEY, + tracking_user VARCHAR(250) NOT NULL, -- Listenbrainz user to cluster + months INT NOT NULL DEFAULT 3, -- How many months of listens to cluster + cluster_count INT, -- Number of mixes. NULL = choose automatically + playlist_size INT NOT NULL DEFAULT 50, -- Tracks per mix + familiar_ratio DOUBLE PRECISION NOT NULL DEFAULT 0.3, -- Share of each mix taken from recently played tracks + name_template VARCHAR(250) NOT NULL DEFAULT 'Mix: {label}', -- {label} is replaced by the mix's genres + refresh_hours INT NOT NULL DEFAULT 24, -- How often to rebuild the mixes + last_run TIMESTAMPTZ -- When the mixes were last rebuilt +); + +-- The Subsonic playlists currently owned by a cluster mix rule. +CREATE TABLE "cluster_mix_playlist" +( + id UUID NOT NULL PRIMARY KEY, + rule_id UUID NOT NULL REFERENCES "cluster_mix_rule" (id) ON DELETE CASCADE, + playlist_id VARCHAR(250) NOT NULL, -- ID of the playlist in subsonic + tags TEXT[] NOT NULL -- Genres describing the cluster, used to keep playlists stable across rebuilds +); diff --git a/src/cluster.rs b/src/cluster.rs new file mode 100644 index 0000000..5f57c9d --- /dev/null +++ b/src/cluster.rs @@ -0,0 +1,539 @@ +//! Clustering of listened tracks by genre. +//! +//! Every track gets a TF-IDF vector over its MusicBrainz tags (recording, release +//! group and artist tags, as provided by the caller). To take listening habits +//! into account, each vector is then blended with the vectors of the tracks it +//! is usually played next to (within the same listening session), which also +//! gives untagged tracks a position. The vectors are grouped with spherical +//! k-means, weighted by play count, and each cluster is described by the tags +//! that are both common in it and distinctive compared to the rest. + +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use std::collections::HashMap; + +pub struct Play { + /// Unix timestamp of the listen + pub ts: i64, + /// Identifies the track; listens with the same key are the same track + pub key: String, +} + +pub struct ClusterOptions { + /// Ignore tracks with fewer plays than this + pub min_plays: usize, + /// Fixed number of clusters. `None` picks one from `k_range` by silhouette score + pub cluster_count: Option, + pub k_range: (usize, usize), + /// Seconds of silence that end a listening session + pub session_gap: i64, + /// How many following tracks in a session count as played "together" + pub window: usize, + /// How strongly a track is pulled towards the tracks played around it + pub context_weight: f64, + pub seed: u64, +} + +impl Default for ClusterOptions { + fn default() -> Self { + Self { + min_plays: 2, + cluster_count: None, + k_range: (4, 10), + session_gap: 30 * 60, + window: 5, + context_weight: 0.5, + seed: 0, + } + } +} + +#[derive(Debug)] +pub struct Cluster { + /// Track keys and their play counts, most played first + pub tracks: Vec<(String, usize)>, + /// Total plays of the cluster's tracks + pub plays: usize, + /// Tags that characterise the cluster, most characteristic first + pub profile: Vec<(String, f64)>, +} + +const PROFILE_SIZE: usize = 40; +/// Profile tags need at least this fraction of the weight of the cluster's most common tag +const PROFILE_FLOOR: f64 = 0.2; +const N_INIT: usize = 5; +const MAX_ITER: usize = 100; +const SILHOUETTE_SAMPLE: usize = 1500; + +pub fn cluster( + plays: &[Play], + tags: &HashMap>, + opts: &ClusterOptions, +) -> Vec { + // ---- tracks ---- + let mut counts: HashMap<&str, usize> = HashMap::new(); + for p in plays { + *counts.entry(p.key.as_str()).or_default() += 1; + } + let mut keys: Vec<&str> = counts + .iter() + .filter(|(_, &n)| n >= opts.min_plays.max(1)) + .map(|(&k, _)| k) + .collect(); + keys.sort_by(|a, b| counts[b].cmp(&counts[a]).then(a.cmp(b))); + let n = keys.len(); + if n < 2 { + return vec![]; + } + let index: HashMap<&str, usize> = keys.iter().enumerate().map(|(i, &k)| (k, i)).collect(); + + // ---- tag TF-IDF ---- + let mut df: HashMap<&str, usize> = HashMap::new(); + for k in &keys { + for (tag, &w) in tags.get(*k).into_iter().flatten() { + if w > 0.0 { + *df.entry(tag.as_str()).or_default() += 1; + } + } + } + // tags on a single track can't relate tracks to each other + let mut vocab: Vec<&str> = df + .iter() + .filter(|(_, &d)| d >= 2) + .map(|(&t, _)| t) + .collect(); + vocab.sort(); + let dims = vocab.len(); + if dims == 0 { + return vec![]; + } + let vocab_index: HashMap<&str, usize> = + vocab.iter().enumerate().map(|(i, &t)| (t, i)).collect(); + let idf: Vec = vocab + .iter() + .map(|t| ((1 + n) as f64 / (1 + df[t]) as f64).ln() + 1.0) + .collect(); + + let mut tag_vecs = vec![vec![0.0; dims]; n]; + for (i, k) in keys.iter().enumerate() { + for (tag, &w) in tags.get(*k).into_iter().flatten() { + if let Some(&j) = vocab_index.get(tag.as_str()) { + if w > 0.0 { + tag_vecs[i][j] = (1.0 + w).ln() * idf[j]; + } + } + } + normalize(&mut tag_vecs[i]); + } + + // ---- listening context ---- + let mut order: Vec<&Play> = plays.iter().collect(); + order.sort_by_key(|p| p.ts); + let mut co: Vec> = vec![HashMap::new(); n]; + let mut session: Vec = vec![]; + let mut prev_ts: Option = None; + for p in order { + if prev_ts.is_some_and(|t| p.ts - t > opts.session_gap) { + add_cooccurrences(&session, opts.window, &mut co); + session.clear(); + } + if let Some(&i) = index.get(p.key.as_str()) { + session.push(i); + } + prev_ts = Some(p.ts); + } + add_cooccurrences(&session, opts.window, &mut co); + + let mut x: Vec> = Vec::with_capacity(n); + for i in 0..n { + let mut v = tag_vecs[i].clone(); + let total: f64 = co[i].values().sum(); + if total > 0.0 { + for (&j, &c) in &co[i] { + let f = opts.context_weight * c / total; + for (a, b) in v.iter_mut().zip(&tag_vecs[j]) { + *a += f * b; + } + } + } + normalize(&mut v); + x.push(v); + } + + // tracks with no tags and no tagged neighbours can't be placed + let placed: Vec = (0..n).filter(|&i| x[i].iter().any(|&v| v != 0.0)).collect(); + if placed.len() < 2 { + return vec![]; + } + let points: Vec> = placed.iter().map(|&i| x[i].clone()).collect(); + let weights: Vec = placed + .iter() + .map(|&i| (1.0 + counts[keys[i]] as f64).ln()) + .collect(); + + // ---- k-means ---- + let mut rng = StdRng::seed_from_u64(opts.seed); + let k = match opts.cluster_count { + Some(k) => k.clamp(1, points.len()), + None => choose_k(&points, &weights, opts.k_range, &mut rng), + }; + let labels = best_kmeans(&points, &weights, k, &mut rng); + + // ---- describe clusters ---- + let mut overall = vec![0.0; dims]; + let total_weight: f64 = weights.iter().sum(); + for (p, &i) in placed.iter().enumerate() { + for (o, v) in overall.iter_mut().zip(&tag_vecs[i]) { + *o += weights[p] * v / total_weight; + } + } + + let mut clusters = vec![]; + for c in 0..k { + let members: Vec = (0..placed.len()).filter(|&p| labels[p] == c).collect(); + if members.is_empty() { + continue; + } + let member_weight: f64 = members.iter().map(|&p| weights[p]).sum(); + let mut mean = vec![0.0; dims]; + for &p in &members { + for (m, v) in mean.iter_mut().zip(&tag_vecs[placed[p]]) { + *m += weights[p] * v / member_weight; + } + } + // frequent in the cluster AND over-represented compared to everything else, + // ignoring tags only a handful of the cluster's tracks have + let floor = PROFILE_FLOOR * mean.iter().cloned().fold(0.0, f64::max); + let mut profile: Vec<(String, f64)> = (0..dims) + .filter(|&j| mean[j] > 0.0 && mean[j] >= floor) + .map(|j| { + let score = mean[j] * (mean[j] / (overall[j] + 1e-9) + 1.0).ln(); + (vocab[j].to_string(), score) + }) + .collect(); + profile.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + profile.truncate(PROFILE_SIZE); + + let mut tracks: Vec<(String, usize)> = members + .iter() + .map(|&p| (keys[placed[p]].to_string(), counts[keys[placed[p]]])) + .collect(); + tracks.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + + clusters.push(Cluster { + plays: tracks.iter().map(|t| t.1).sum(), + tracks, + profile, + }); + } + clusters.sort_by_key(|c| std::cmp::Reverse(c.plays)); + clusters +} + +fn add_cooccurrences(session: &[usize], window: usize, co: &mut [HashMap]) { + for a in 0..session.len() { + for b in (a + 1)..(a + 1 + window).min(session.len()) { + let (i, j) = (session[a], session[b]); + if i != j { + // closer plays count more + let w = 1.0 / (b - a) as f64; + *co[i].entry(j).or_default() += w; + *co[j].entry(i).or_default() += w; + } + } + } +} + +fn normalize(v: &mut [f64]) { + let norm = dot(v, v).sqrt(); + if norm > 0.0 { + v.iter_mut().for_each(|x| *x /= norm); + } +} + +fn dot(a: &[f64], b: &[f64]) -> f64 { + a.iter().zip(b).map(|(x, y)| x * y).sum() +} + +fn argmax_similarity(point: &[f64], centroids: &[Vec]) -> (usize, f64) { + centroids + .iter() + .enumerate() + .map(|(c, centroid)| (c, dot(point, centroid))) + .fold((0, f64::NEG_INFINITY), |best, cur| { + if cur.1 > best.1 { + cur + } else { + best + } + }) +} + +fn weighted_choice(weights: &[f64], rng: &mut StdRng) -> usize { + let total: f64 = weights.iter().sum(); + if total <= 0.0 { + return rng.gen_range(0..weights.len()); + } + let mut r = rng.gen::() * total; + for (i, &w) in weights.iter().enumerate() { + r -= w; + if r <= 0.0 { + return i; + } + } + weights.len() - 1 +} + +/// Spherical k-means (cosine similarity) with k-means++ seeding. +/// Returns the labels and the objective (weighted sum of similarity to the centroid). +fn kmeans(x: &[Vec], w: &[f64], k: usize, rng: &mut StdRng) -> (Vec, f64) { + let n = x.len(); + let mut centroids = vec![x[weighted_choice(w, rng)].clone()]; + while centroids.len() < k { + let d: Vec = (0..n) + .map(|i| { + let (_, sim) = argmax_similarity(&x[i], ¢roids); + w[i] * (1.0 - sim).max(0.0).powi(2) + }) + .collect(); + centroids.push(x[weighted_choice(&d, rng)].clone()); + } + + let mut labels = vec![usize::MAX; n]; + for _ in 0..MAX_ITER { + let mut changed = false; + for i in 0..n { + let (c, _) = argmax_similarity(&x[i], ¢roids); + if labels[i] != c { + labels[i] = c; + changed = true; + } + } + if !changed { + break; + } + + let dims = x[0].len(); + let mut sums = vec![vec![0.0; dims]; k]; + let mut sizes = vec![0usize; k]; + for i in 0..n { + sizes[labels[i]] += 1; + for (s, v) in sums[labels[i]].iter_mut().zip(&x[i]) { + *s += w[i] * v; + } + } + for c in 0..k { + if sizes[c] == 0 { + // re-seed an empty cluster with the worst-fitting point + let worst = (0..n) + .min_by(|&a, &b| { + dot(&x[a], ¢roids[labels[a]]) + .total_cmp(&dot(&x[b], ¢roids[labels[b]])) + }) + .unwrap(); + sums[c] = x[worst].clone(); + labels[worst] = c; + } + normalize(&mut sums[c]); + } + centroids = sums; + } + + let objective = (0..n) + .map(|i| w[i] * dot(&x[i], ¢roids[labels[i]])) + .sum(); + (labels, objective) +} + +fn best_kmeans(x: &[Vec], w: &[f64], k: usize, rng: &mut StdRng) -> Vec { + (0..N_INIT) + .map(|_| kmeans(x, w, k, rng)) + .max_by(|a, b| a.1.total_cmp(&b.1)) + .unwrap() + .0 +} + +/// Mean silhouette score using cosine distance, over the given sample of points. +fn silhouette(sim: &[Vec], labels: &[usize], k: usize) -> f64 { + let n = labels.len(); + let mut total = 0.0; + for i in 0..n { + let mut dist_sum = vec![0.0; k]; + let mut count = vec![0usize; k]; + for j in 0..n { + if i != j { + dist_sum[labels[j]] += 1.0 - sim[i][j]; + count[labels[j]] += 1; + } + } + if count[labels[i]] == 0 { + continue; // singleton clusters score 0 + } + let a = dist_sum[labels[i]] / count[labels[i]] as f64; + let b = (0..k) + .filter(|&c| c != labels[i] && count[c] > 0) + .map(|c| dist_sum[c] / count[c] as f64) + .fold(f64::INFINITY, f64::min); + if b.is_finite() && a.max(b) > 0.0 { + total += (b - a) / a.max(b); + } + } + total / n as f64 +} + +fn choose_k(x: &[Vec], w: &[f64], range: (usize, usize), rng: &mut StdRng) -> usize { + let n = x.len(); + let (lo, hi) = (range.0.max(2), range.1.min(n - 1)); + if lo > hi { + return range.0.clamp(1, n); + } + + // silhouette is quadratic, so score on a sample + let mut sample: Vec = (0..n).collect(); + for i in (1..n).rev() { + sample.swap(i, rng.gen_range(0..=i)); + } + sample.truncate(SILHOUETTE_SAMPLE); + let sim: Vec> = sample + .iter() + .map(|&i| sample.iter().map(|&j| dot(&x[i], &x[j])).collect()) + .collect(); + + let mut best = (f64::NEG_INFINITY, lo); + for k in lo..=hi { + let labels = best_kmeans(x, w, k, rng); + let sample_labels: Vec = sample.iter().map(|&i| labels[i]).collect(); + let score = silhouette(&sim, &sample_labels, k); + if score > best.0 { + best = (score, k); + } + } + best.1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tags(pairs: &[(&str, &[&str])]) -> HashMap> { + pairs + .iter() + .map(|(k, ts)| { + ( + k.to_string(), + ts.iter().map(|t| (t.to_string(), 1.0)).collect(), + ) + }) + .collect() + } + + fn plays(keys: &[&str], repeats: usize) -> Vec { + let mut out = vec![]; + let mut ts = 0; + for _ in 0..repeats { + for k in keys { + out.push(Play { + ts, + key: k.to_string(), + }); + ts += 200; + } + ts += 10_000; // new session + } + out + } + + #[test] + fn separates_genres() { + let t = tags(&[ + ("m1", &["metal", "death metal"]), + ("m2", &["metal", "death metal"]), + ("m3", &["metal", "black metal"]), + ("f1", &["folk", "indie folk"]), + ("f2", &["folk", "indie folk"]), + ("f3", &["folk", "folk rock"]), + ]); + let p = plays(&["m1", "m2", "m3", "f1", "f2", "f3"], 3); + let opts = ClusterOptions { + cluster_count: Some(2), + context_weight: 0.0, + ..Default::default() + }; + let clusters = cluster(&p, &t, &opts); + + assert_eq!(clusters.len(), 2); + for c in &clusters { + let first = c.tracks[0].0.chars().next().unwrap(); + assert!( + c.tracks.iter().all(|(k, _)| k.starts_with(first)), + "{:?}", + c + ); + let top = &c.profile[0].0; + assert!(if first == 'm' { + top.contains("metal") + } else { + top.contains("folk") + }); + } + } + + #[test] + fn untagged_tracks_follow_context() { + let t = tags(&[ + ("m1", &["metal"]), + ("m2", &["metal"]), + ("f1", &["folk"]), + ("f2", &["folk"]), + ]); + // "x" has no tags but is always played straight after the metal tracks + let mut p = plays(&["m1", "m2", "x"], 3); + let offset = p.last().unwrap().ts + 100_000; + p.extend(plays(&["f1", "f2"], 3).into_iter().map(|mut pl| { + pl.ts += offset; + pl + })); + let opts = ClusterOptions { + cluster_count: Some(2), + ..Default::default() + }; + let clusters = cluster(&p, &t, &opts); + + let metal = clusters + .iter() + .find(|c| c.tracks.iter().any(|(k, _)| k == "m1")) + .unwrap(); + assert!(metal.tracks.iter().any(|(k, _)| k == "x")); + } + + #[test] + fn picks_k_automatically() { + let genres = ["metal", "folk", "hyperpop", "jazz"]; + let mut pairs: Vec<(String, Vec<&str>)> = vec![]; + for g in genres { + for i in 0..6 { + pairs.push((format!("{g}{i}"), vec![g, "music"])); + } + } + let t: HashMap> = pairs + .iter() + .map(|(k, ts)| (k.clone(), ts.iter().map(|t| (t.to_string(), 1.0)).collect())) + .collect(); + let keys: Vec<&str> = pairs.iter().map(|(k, _)| k.as_str()).collect(); + let p = plays(&keys, 2); + let opts = ClusterOptions { + k_range: (2, 8), + context_weight: 0.0, + ..Default::default() + }; + + assert_eq!(cluster(&p, &t, &opts).len(), 4); + } + + #[test] + fn too_little_data() { + let t = tags(&[("a", &["rock"])]); + assert!(cluster(&plays(&["a"], 3), &t, &ClusterOptions::default()).is_empty()); + assert!(cluster(&[], &t, &ClusterOptions::default()).is_empty()); + } +} diff --git a/src/daemon/cluster_mixes.rs b/src/daemon/cluster_mixes.rs new file mode 100644 index 0000000..d97dfc7 --- /dev/null +++ b/src/daemon/cluster_mixes.rs @@ -0,0 +1,263 @@ +use crate::cluster::{self, ClusterOptions, Play}; +use crate::listenbrainz; +use crate::mix::{self, ListenedTrack, MixOptions, RecentlyPlayed}; +use crate::models::{ClusterMixPlaylist, ClusterMixRule}; +use crate::subsonic::Subsonic; +use crate::CONFIG; +use chrono::Utc; +use rand::rngs::StdRng; +use rand::SeedableRng; +use sqlx::postgres::PgPool; +use std::collections::HashMap; +use std::time::Duration; +use tokio::time::{interval, MissedTickBehavior}; +use uuid::Uuid; + +/// Clusters smaller than this don't get a mix +const MIN_CLUSTER_TRACKS: usize = 5; +const SECONDS_PER_MONTH: i64 = 30 * 24 * 60 * 60; + +pub async fn cluster_mixes_daemon( + media_client: impl AsRef + Clone, +) -> Result<(), Box> { + println!("Cluster mix daemon starting..."); + + let database = PgPool::connect( + CONFIG + .get() + .unwrap() + .get::("database.url")? + .as_str(), + ) + .await + .unwrap(); + + let mut ticker = interval(Duration::from_secs( + CONFIG.get().unwrap().get::("daemon.interval")?, + )); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + + loop { + match ClusterMixRule::due(&database).await { + Ok(rules) => { + for rule in rules { + let id = rule.id; + if let Err(e) = build_mixes(media_client.as_ref(), rule, &database).await { + println!("Could not build cluster mixes for rule {}: {:?}", id, e); + } + } + } + + Err(e) => { + println!("Could not fetch cluster_mix_rule: {:?}", e); + } + } + + ticker.tick().await; + } +} + +pub async fn build_mixes( + media_client: &Subsonic, + rule: ClusterMixRule, + database: &PgPool, +) -> Result<(), Box> { + let since = Utc::now().timestamp() - rule.months.max(1) as i64 * SECONDS_PER_MONTH; + let listens = listenbrainz::listens_since(&rule.tracking_user, since).await?; + + // ---- listening history ---- + let mut plays = Vec::with_capacity(listens.len()); + let mut listened: HashMap = HashMap::new(); + let mut recently_played = RecentlyPlayed::default(); + for listen in &listens { + let mbid = listen.recording_mbid(); + let metadata = &listen.track_metadata; + let key = mbid + .clone() + .unwrap_or_else(|| mix::name_key(&metadata.artist_name, &metadata.track_name)); + let track = ListenedTrack { + mbid, + artist: metadata.artist_name.clone(), + title: metadata.track_name.clone(), + }; + recently_played.insert(&track); + listened.entry(key.clone()).or_insert(track); + plays.push(Play { + ts: listen.listened_at, + key, + }); + } + + let mbids: Vec = listened.values().filter_map(|t| t.mbid.clone()).collect(); + let tags = listenbrainz::recording_tags(&mbids).await?; + + // ---- cluster ---- + let clusters = cluster::cluster( + &plays, + &tags, + &ClusterOptions { + cluster_count: rule.cluster_count.map(|k| k.max(1) as usize), + seed: Utc::now().timestamp() as u64, + ..Default::default() + }, + ); + println!( + "{}: {} listens over {} months -> {} clusters", + rule.tracking_user, + listens.len(), + rule.months, + clusters.len() + ); + + // ---- build mixes ---- + let library_genres = mix::genre_index( + media_client + .get_genres() + .await? + .into_iter() + .filter(|g| g.song_count > 0) + .map(|g| g.value), + ); + let options = MixOptions { + size: rule.playlist_size.max(1) as usize, + familiar_ratio: rule.familiar_ratio, + max_genres: 6, + songs_per_genre: 1000, + }; + let mut rng = StdRng::from_entropy(); + + let mut mixes = vec![]; + for cluster in clusters + .iter() + .filter(|c| c.tracks.len() >= MIN_CLUSTER_TRACKS) + { + match mix::build_mix( + media_client, + cluster, + &library_genres, + &listened, + &recently_played, + &options, + &mut rng, + ) + .await? + { + Some(mix) => mixes.push(mix), + None => println!( + "No library genres match cluster {:?}", + cluster + .profile + .iter() + .take(3) + .map(|t| &t.0) + .collect::>() + ), + } + } + + // two clusters can share their top genres + let mut seen_labels: HashMap = HashMap::new(); + for mix in &mut mixes { + let n = seen_labels.entry(mix.label.clone()).or_default(); + *n += 1; + if *n > 1 { + mix.label = format!("{} ({})", mix.label, n); + } + } + + // ---- sync playlists, reusing the previous run's playlists where the genres overlap ---- + let existing = sqlx::query_as!( + ClusterMixPlaylist, + r#"SELECT * FROM "cluster_mix_playlist" WHERE rule_id = $1"#, + rule.id + ) + .fetch_all(database) + .await?; + + let pairs = mix::match_mixes( + &existing.iter().map(|p| p.tags.clone()).collect::>(), + &mixes.iter().map(|m| m.tags.clone()).collect::>(), + ); + let old_for_new: HashMap = pairs.iter().map(|&(o, n)| (n, o)).collect(); + + for (n, mix) in mixes.iter().enumerate() { + let name = rule.name_template.replace("{label}", &mix.label); + + let reused = match old_for_new.get(&n).map(|&o| &existing[o]) { + // the playlist may have been deleted by hand since the last run + Some(old) if media_client.get_playlist(&old.playlist_id).await.is_ok() => { + media_client + .rename_playlist(&old.playlist_id, &name) + .await?; + sqlx::query!( + r#"UPDATE "cluster_mix_playlist" SET tags = $1 WHERE id = $2"#, + &mix.tags, + old.id + ) + .execute(database) + .await?; + Some(old.playlist_id.clone()) + } + Some(old) => { + sqlx::query!( + r#"DELETE FROM "cluster_mix_playlist" WHERE id = $1"#, + old.id + ) + .execute(database) + .await?; + None + } + None => None, + }; + + let playlist_id = match reused { + Some(id) => id, + None => { + let playlist = media_client.create_playlist(&name).await?; + sqlx::query!( + r#" + INSERT INTO "cluster_mix_playlist" (id, rule_id, playlist_id, tags) + VALUES ($1, $2, $3, $4) + "#, + Uuid::new_v4(), + rule.id, + playlist.id, + &mix.tags, + ) + .execute(database) + .await?; + playlist.id + } + }; + + media_client + .replace_playlist_songs(&playlist_id, mix.song_ids.clone()) + .await?; + println!(" {} ({} songs)", name, mix.song_ids.len()); + } + + // mixes for tastes that no longer show up in the listening history + let kept: Vec = pairs.iter().map(|&(o, _)| o).collect(); + for (o, old) in existing.iter().enumerate() { + if !kept.contains(&o) { + if let Err(e) = media_client.delete_playlist(&old.playlist_id).await { + println!("Could not delete playlist {}: {:?}", old.playlist_id, e); + } + sqlx::query!( + r#"DELETE FROM "cluster_mix_playlist" WHERE id = $1"#, + old.id + ) + .execute(database) + .await?; + } + } + + sqlx::query!( + r#"UPDATE "cluster_mix_rule" SET last_run = NOW() WHERE id = $1"#, + rule.id + ) + .execute(database) + .await?; + + Ok(()) +} diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index e5f4dd7..1be21c1 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -1,5 +1,7 @@ +pub mod cluster_mixes; pub mod create_playlists; pub mod update_playlists; +pub use cluster_mixes::cluster_mixes_daemon; pub use create_playlists::create_playlists_daemon; pub use update_playlists::update_playlists_daemon; diff --git a/src/listenbrainz.rs b/src/listenbrainz.rs index 3617d91..0b0a326 100644 --- a/src/listenbrainz.rs +++ b/src/listenbrainz.rs @@ -1,4 +1,5 @@ use serde::Deserialize; +use std::collections::HashMap; use std::fmt::Display; const BASE: &'static str = "https://api.listenbrainz.org/1"; @@ -108,3 +109,198 @@ pub async fn recordings( .await? .payload) } + +const USER_AGENT: &str = concat!("playlistd/", env!("CARGO_PKG_VERSION")); +const LISTENS_PAGE: usize = 1000; +const METADATA_BATCH: usize = 200; + +/// Send a request, waiting out Listenbrainz rate limits and retrying server errors. +async fn send_json( + request: reqwest::RequestBuilder, +) -> Result { + let mut attempt = 0; + loop { + let response = request + .try_clone() + .expect("request bodies are not streamed") + .header(reqwest::header::USER_AGENT, USER_AGENT) + .send() + .await?; + let status = response.status(); + let reset_in = response + .headers() + .get("X-RateLimit-Reset-In") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(1.0); + + if (status == reqwest::StatusCode::TOO_MANY_REQUESTS || status.is_server_error()) + && attempt < 6 + { + attempt += 1; + let wait = if status.is_server_error() { + 2f64.powi(attempt) + } else { + reset_in + 0.5 + }; + tokio::time::sleep(std::time::Duration::from_secs_f64(wait)).await; + continue; + } + + let remaining = response.headers().get("X-RateLimit-Remaining").cloned(); + let body = response.error_for_status()?.json::().await?; + if remaining.as_ref().and_then(|v| v.to_str().ok()) == Some("0") { + tokio::time::sleep(std::time::Duration::from_secs_f64(reset_in + 0.2)).await; + } + return Ok(body); + } +} + +#[derive(Deserialize)] +struct ListensResponse { + payload: ListensPayload, +} + +#[derive(Deserialize)] +struct ListensPayload { + listens: Vec, +} + +#[derive(Deserialize, Clone)] +pub struct Listen { + pub listened_at: i64, + pub track_metadata: TrackMetadata, +} + +#[derive(Deserialize, Clone)] +pub struct TrackMetadata { + pub artist_name: String, + pub track_name: String, + // Free-form and submitted by clients, so don't trust the field types + #[serde(default)] + pub additional_info: Option, + #[serde(default)] + pub mbid_mapping: Option, +} + +#[derive(Deserialize, Clone)] +pub struct MbidMapping { + #[serde(default)] + pub recording_mbid: Option, +} + +impl Listen { + /// The recording MBID, preferring Listenbrainz's own mapping over the submitting client's + pub fn recording_mbid(&self) -> Option { + self.track_metadata + .mbid_mapping + .as_ref() + .and_then(|m| m.recording_mbid.clone()) + .or_else(|| { + self.track_metadata + .additional_info + .as_ref()? + .get("recording_mbid")? + .as_str() + .map(str::to_string) + }) + .filter(|m| !m.is_empty()) + } +} + +/// All of a user's listens at or after `min_ts`, newest first +pub async fn listens_since(user: impl Display, min_ts: i64) -> Result, reqwest::Error> { + let client = reqwest::Client::new(); + let mut listens: Vec = vec![]; + let mut max_ts: Option = None; + + loop { + let mut request = client + .get(format!("{}/user/{}/listens", BASE, user)) + .query(&[("count", LISTENS_PAGE as i64)]); + if let Some(max_ts) = max_ts { + request = request.query(&[("max_ts", max_ts)]); + } + + let page = send_json::(request).await?.payload.listens; + let Some(oldest) = page.last().map(|l| l.listened_at) else { + break; + }; + listens.extend(page.into_iter().filter(|l| l.listened_at >= min_ts)); + if oldest < min_ts || Some(oldest) == max_ts { + break; + } + // max_ts is exclusive + max_ts = Some(oldest); + } + + Ok(listens) +} + +#[derive(Deserialize, Default)] +struct RecordingMetadata { + #[serde(default)] + tag: Option, +} + +#[derive(Deserialize, Default)] +struct RecordingTags { + #[serde(default)] + recording: Vec, + #[serde(default)] + release_group: Vec, + #[serde(default)] + artist: Vec, +} + +#[derive(Deserialize)] +struct Tag { + tag: String, + #[serde(default)] + count: Option, + /// Only set for tags that are MusicBrainz genres (not e.g. "british" or "female vocals") + #[serde(default)] + genre_mbid: Option, +} + +/// Weighted MusicBrainz genres for each recording, combining the recording's own genres (most +/// specific) with those of its release group and artists (broadest). Names are lowercased. +pub async fn recording_tags( + mbids: &[String], +) -> Result>, reqwest::Error> { + let client = reqwest::Client::new(); + let mut out = HashMap::new(); + + for batch in mbids.chunks(METADATA_BATCH) { + let request = + client + .post(format!("{}/metadata/recording/", BASE)) + .json(&serde_json::json!({ + "recording_mbids": batch, + "inc": "artist tag release", + })); + let metadata = send_json::>(request).await?; + + for (mbid, recording) in metadata { + let mut weights: HashMap = HashMap::new(); + if let Some(tags) = recording.tag { + for (source, weight) in [ + (&tags.recording, 3.0), + (&tags.release_group, 2.0), + (&tags.artist, 1.0), + ] { + for tag in source.iter().filter(|t| t.genre_mbid.is_some()) { + let name = tag.tag.trim().to_lowercase(); + if !name.is_empty() { + *weights.entry(name).or_default() += + weight * (1.0 + tag.count.unwrap_or(1).max(1) as f64).ln(); + } + } + } + } + out.insert(mbid, weights); + } + } + + Ok(out) +} diff --git a/src/main.rs b/src/main.rs index e469947..5985b3c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,13 +1,16 @@ +mod cluster; mod daemon; mod listenbrainz; +mod mix; mod models; mod playlist; mod subsonic; mod track; -use crate::daemon::{create_playlists_daemon, update_playlists_daemon}; +use crate::daemon::{cluster_mixes_daemon, create_playlists_daemon, update_playlists_daemon}; use crate::models::{ - CreateTrackedPlaylist, PartialCreateTrackedPlaylist, PartialTrackedPlaylist, TrackedPlaylist, + ClusterMixRule, CreateTrackedPlaylist, PartialClusterMixRule, PartialCreateTrackedPlaylist, + PartialTrackedPlaylist, TrackedPlaylist, }; use crate::subsonic::{Subsonic, SubsonicBuilder}; use config::Config; @@ -61,6 +64,12 @@ async fn main() -> Result<(), Box> { create_playlists_daemon(_media_client).await.unwrap(); }); } + { + let _media_client = media_client.clone(); + tokio::spawn(async move { + cluster_mixes_daemon(_media_client).await.unwrap(); + }); + } rocket::build() .manage(media_client) @@ -72,6 +81,8 @@ async fn main() -> Result<(), Box> { add_tracking_playlist, all_playlists, create_tracking_playlist_rule, + create_cluster_mix_rule, + all_cluster_mix_rules, ], ) .launch() @@ -178,3 +189,24 @@ async fn all_playlists(pool: &State) -> JsonValue { Err(_) => json!([]), } } + +/// Create a rule that maintains genre mixes built from clustering a user's recent listens +#[post("/mix", data = "")] +async fn create_cluster_mix_rule( + pool: &State, + rule: Json, +) -> Json> { + match rule.record(pool.inner()).await { + Ok(rule) => Json(Response::Success(rule)), + Err(e) => Json(Response::Error(e.into())), + } +} + +/// Get all cluster mix rules +#[get("/mix")] +async fn all_cluster_mix_rules(pool: &State) -> JsonValue { + match ClusterMixRule::all(pool.inner()).await { + Ok(rules) => json!(rules), + Err(_) => json!([]), + } +} diff --git a/src/mix.rs b/src/mix.rs new file mode 100644 index 0000000..7b19363 --- /dev/null +++ b/src/mix.rs @@ -0,0 +1,334 @@ +//! Turn a cluster of listened tracks into a playlist of library songs. +//! +//! The cluster's tag profile is matched against the Subsonic library's genres. +//! Songs from the best matching genres are scored by how well their own genres +//! fit the profile, and the mix is made of: +//! * "familiar" tracks: the cluster's most played tracks, if they're in the library +//! * "discovery" tracks: well-fitting songs that weren't played recently, sampled +//! at random (weighted by fit) with a cap on songs per artist + +use crate::cluster::Cluster; +use crate::subsonic::{Subsonic, SubsonicError}; +use crate::track::Track; +use rand::rngs::StdRng; +use rand::seq::SliceRandom; +use rand::Rng; +use std::collections::{HashMap, HashSet}; + +pub struct MixOptions { + pub size: usize, + /// Share of the mix taken from the cluster's recently played tracks + pub familiar_ratio: f64, + /// How many library genres to draw songs from + pub max_genres: usize, + /// Most songs to consider from each genre + pub songs_per_genre: usize, +} + +pub struct Mix { + pub label: String, + /// Normalised genres describing the mix, best first + pub tags: Vec, + pub song_ids: Vec, +} + +/// A track from the listening history +pub struct ListenedTrack { + pub mbid: Option, + pub artist: String, + pub title: String, +} + +/// Lowercase alphanumerics only, so "Post-Hardcore" matches "post hardcore" +pub fn normalize(s: &str) -> String { + s.chars() + .filter(|c| c.is_alphanumeric()) + .flat_map(char::to_lowercase) + .collect() +} + +pub fn name_key(artist: &str, title: &str) -> String { + format!("{}\u{1f}{}", normalize(artist), normalize(title)) +} + +/// Tracks played during the clustered period, for telling "discovery" songs apart +#[derive(Default)] +pub struct RecentlyPlayed { + mbids: HashSet, + names: HashSet, +} + +impl RecentlyPlayed { + pub fn insert(&mut self, track: &ListenedTrack) { + if let Some(mbid) = &track.mbid { + self.mbids.insert(mbid.clone()); + } + self.names.insert(name_key(&track.artist, &track.title)); + } + + fn contains(&self, song: &Track) -> bool { + song.music_brainz_id + .as_ref() + .is_some_and(|m| self.mbids.contains(m)) + || self.names.contains(&name_key(&song.artist, &song.title)) + } +} + +/// Library genres by normalised name +pub fn genre_index(genres: impl IntoIterator) -> HashMap { + let mut index = HashMap::new(); + for genre in genres { + let key = normalize(&genre); + if !key.is_empty() { + index.entry(key).or_insert(genre); + } + } + index +} + +#[derive(Default)] +struct Selection { + songs: Vec, + ids: HashSet, + artists: HashMap, +} + +impl Selection { + fn take(&mut self, song: Track) { + if self.ids.insert(song.id.clone()) { + *self.artists.entry(normalize(&song.artist)).or_default() += 1; + self.songs.push(song); + } + } + + fn artist_count(&self, artist: &str) -> usize { + self.artists.get(&normalize(artist)).copied().unwrap_or(0) + } +} + +pub async fn build_mix( + client: &Subsonic, + cluster: &Cluster, + library_genres: &HashMap, + listened: &HashMap, + recently_played: &RecentlyPlayed, + opts: &MixOptions, + rng: &mut StdRng, +) -> Result, SubsonicError> { + // ---- match the cluster's profile to library genres ---- + let mut weights: HashMap = HashMap::new(); + for (tag, score) in &cluster.profile { + *weights.entry(normalize(tag)).or_default() += score; + } + let mut matched: Vec<(String, f64)> = vec![]; + for (tag, _) in &cluster.profile { + let key = normalize(tag); + if library_genres.contains_key(&key) && !matched.iter().any(|(k, _)| *k == key) { + matched.push((key.clone(), weights[&key])); + } + } + matched.truncate(opts.max_genres); + if matched.is_empty() { + return Ok(None); + } + + // ---- candidate songs ---- + let mut pool: HashMap = HashMap::new(); + for (key, _) in &matched { + for song in client + .songs_by_genre(&library_genres[key], opts.songs_per_genre) + .await? + { + pool.entry(song.id.clone()).or_insert(song); + } + } + + let score = |song: &Track| -> f64 { + let genres = song.genre_names(); + if genres.is_empty() { + return 0.0; + } + let fit: f64 = genres + .iter() + .filter_map(|g| weights.get(&normalize(g))) + .sum(); + // don't let songs tagged with every genre under the sun dominate + fit / (genres.len() as f64).sqrt() + }; + + let by_mbid: HashMap<&str, &Track> = pool + .values() + .filter_map(|s| Some((s.music_brainz_id.as_deref()?, s))) + .filter(|(m, _)| !m.is_empty()) + .collect(); + let by_name: HashMap = pool + .values() + .map(|s| (name_key(&s.artist, &s.title), s)) + .collect(); + + let mut selection = Selection::default(); + let artist_cap = (opts.size / 8).max(2); + + // ---- familiar: the cluster's most played tracks ---- + let familiar_quota = + ((opts.size as f64) * opts.familiar_ratio.clamp(0.0, 1.0)).round() as usize; + for (key, _) in cluster.tracks.iter().take(familiar_quota * 4) { + if selection.songs.len() >= familiar_quota { + break; + } + let Some(track) = listened.get(key) else { + continue; + }; + if selection.artist_count(&track.artist) >= artist_cap { + continue; + } + let found = match track + .mbid + .as_deref() + .and_then(|m| by_mbid.get(m)) + .or_else(|| by_name.get(&name_key(&track.artist, &track.title))) + { + Some(song) => Some((*song).clone()), + // it may be in the library under a genre we didn't pull + None => match &track.mbid { + Some(mbid) => { + let results = client.song_search(mbid.clone()).await?; + if results.len() == 1 { + results.into_iter().next() + } else { + None + } + } + None => None, + }, + }; + if let Some(song) = found { + selection.take(song); + } + } + + // ---- discovery: weighted random sample of well-fitting songs ---- + // Efraimidis-Spirakis: sort by u^(1/w) to sample without replacement, weighted by w + let mut ranked: Vec<(f64, &Track)> = pool + .values() + .filter_map(|song| { + let s = score(song); + (s > 0.0).then(|| (rng.gen::().powf(1.0 / (s * s)), song)) + }) + .collect(); + ranked.sort_by(|a, b| b.0.total_cmp(&a.0).then_with(|| a.1.id.cmp(&b.1.id))); + + // relax the rules in passes if the library is too small to fill the mix + for (allow_recent, cap_artists) in [(false, true), (true, true), (true, false)] { + for (_, song) in &ranked { + if selection.songs.len() >= opts.size { + break; + } + if selection.ids.contains(&song.id) + || (!allow_recent && recently_played.contains(song)) + || (cap_artists && selection.artist_count(&song.artist) >= artist_cap) + { + continue; + } + selection.take((*song).clone()); + } + } + + let mut chosen = selection.songs; + if chosen.is_empty() { + return Ok(None); + } + chosen.shuffle(rng); + + let label = matched + .iter() + .take(3) + .map(|(key, _)| library_genres[key].clone()) + .collect::>() + .join(" / "); + + Ok(Some(Mix { + label, + tags: matched.iter().take(5).map(|(k, _)| k.clone()).collect(), + song_ids: chosen.into_iter().map(|s| s.id).collect(), + })) +} + +/// Pair up previous mixes with new ones by genre overlap, so that playlists keep their identity +/// across rebuilds. Returns `(old index, new index)` pairs. +pub fn match_mixes(old: &[Vec], new: &[Vec]) -> Vec<(usize, usize)> { + let mut pairs: Vec<(f64, usize, usize)> = vec![]; + for (i, a) in old.iter().enumerate() { + for (j, b) in new.iter().enumerate() { + let a: HashSet<_> = a.iter().collect(); + let b: HashSet<_> = b.iter().collect(); + let union = a.union(&b).count(); + let jaccard = if union == 0 { + 0.0 + } else { + a.intersection(&b).count() as f64 / union as f64 + }; + if jaccard > 0.0 { + pairs.push((jaccard, i, j)); + } + } + } + pairs.sort_by(|a, b| b.0.total_cmp(&a.0).then((a.1, a.2).cmp(&(b.1, b.2)))); + + let (mut used_old, mut used_new) = (HashSet::new(), HashSet::new()); + let mut matched = vec![]; + for (_, i, j) in pairs { + if !used_old.contains(&i) && !used_new.contains(&j) { + used_old.insert(i); + used_new.insert(j); + matched.push((i, j)); + } + } + matched +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tags(t: &[&str]) -> Vec { + t.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn normalizes_genre_names() { + assert_eq!(normalize("Post-Hardcore"), normalize("post hardcore")); + assert_eq!(normalize("Drum & Bass"), "drumbass"); + let index = genre_index(["Pop Punk".to_string(), "pop-punk".to_string()]); + assert_eq!(index.len(), 1); + assert_eq!(index["poppunk"], "Pop Punk"); + } + + #[test] + fn matches_mixes_by_overlap() { + let old = vec![ + tags(&["metal", "deathmetal"]), + tags(&["folk"]), + tags(&["jazz"]), + ]; + let new = vec![ + tags(&["folk", "indiefolk"]), + tags(&["metal", "blackmetal"]), + tags(&["pop"]), + ]; + let mut m = match_mixes(&old, &new); + m.sort(); + assert_eq!(m, vec![(0, 1), (1, 0)]); + } + + #[test] + fn genre_names_fall_back_to_genre_field() { + let mut track: Track = serde_json::from_value(serde_json::json!({ + "id": "1", "title": "t", "genre": "Rock; Indie Rock" + })) + .unwrap(); + assert_eq!(track.genre_names(), vec!["Rock", "Indie Rock"]); + track.genres = vec![crate::track::ItemGenre { name: "Emo".into() }]; + assert_eq!(track.genre_names(), vec!["Emo"]); + } +} diff --git a/src/models.rs b/src/models.rs index c0dded5..f306cba 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use sqlx::error::Error; use sqlx::{Executor, Postgres}; @@ -131,3 +132,98 @@ impl CreateTrackedPlaylist { .await } } + +#[derive(Deserialize)] +pub struct PartialClusterMixRule { + pub tracking_user: String, + pub months: Option, + pub cluster_count: Option, + pub playlist_size: Option, + pub familiar_ratio: Option, + pub name_template: Option, + pub refresh_hours: Option, +} + +impl PartialClusterMixRule { + pub async fn record( + &self, + pool: impl Executor<'_, Database = Postgres> + Copy, + ) -> Result { + let uuid = Uuid::new_v4(); + + sqlx::query!( + r#" + INSERT INTO "cluster_mix_rule" + (id, tracking_user, months, cluster_count, playlist_size, familiar_ratio, name_template, refresh_hours) + VALUES ($1, $2, COALESCE($3, 3), $4, COALESCE($5, 50), COALESCE($6::FLOAT8, 0.3), COALESCE($7, 'Mix: {label}'), COALESCE($8, 24)) + "#, + uuid, + self.tracking_user, + self.months, + self.cluster_count, + self.playlist_size, + self.familiar_ratio, + self.name_template, + self.refresh_hours, + ) + .execute(pool) + .await?; + + ClusterMixRule::rule(pool, uuid).await + } +} + +#[derive(Serialize)] +pub struct ClusterMixRule { + pub id: Uuid, + pub tracking_user: String, + pub months: i32, + pub cluster_count: Option, + pub playlist_size: i32, + pub familiar_ratio: f64, + pub name_template: String, + pub refresh_hours: i32, + pub last_run: Option>, +} + +impl ClusterMixRule { + pub async fn all(pool: impl Executor<'_, Database = Postgres>) -> Result, Error> { + sqlx::query_as!(Self, r#"SELECT * FROM "cluster_mix_rule""#) + .fetch_all(pool) + .await + } + + /// Rules whose mixes haven't been built within their refresh interval + pub async fn due(pool: impl Executor<'_, Database = Postgres>) -> Result, Error> { + sqlx::query_as!( + Self, + r#" + SELECT * FROM "cluster_mix_rule" + WHERE last_run IS NULL OR last_run < NOW() - make_interval(hours => refresh_hours) + "# + ) + .fetch_all(pool) + .await + } + + pub async fn rule( + pool: impl Executor<'_, Database = Postgres>, + uuid: Uuid, + ) -> Result { + sqlx::query_as!( + Self, + r#"SELECT * FROM "cluster_mix_rule" WHERE id = $1"#, + uuid + ) + .fetch_one(pool) + .await + } +} + +#[derive(Serialize)] +pub struct ClusterMixPlaylist { + pub id: Uuid, + pub rule_id: Uuid, + pub playlist_id: String, + pub tags: Vec, +} diff --git a/src/subsonic.rs b/src/subsonic.rs index a185511..b1e2a03 100644 --- a/src/subsonic.rs +++ b/src/subsonic.rs @@ -43,6 +43,30 @@ struct GetPlaylistResponse { type CreatePlaylistResponse = GetPlaylistResponse; +#[derive(Deserialize)] +struct GetGenresResponse { + genres: GenreList, +} + +#[derive(Deserialize)] +struct GenreList { + #[serde(default)] + genre: Vec, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Genre { + pub value: String, + #[serde(rename = "songCount", default)] + pub song_count: usize, +} + +#[derive(Deserialize)] +struct SongsByGenreResponse { + #[serde(rename = "songsByGenre")] + songs_by_genre: SearchResponseSongs, +} + struct AuthParams { salt: String, hash: String, @@ -51,6 +75,8 @@ struct AuthParams { #[derive(Debug, Error)] pub enum SubsonicError { Reqwest(reqwest::Error), + Api(String), + Json(serde_json::Error), } impl Display for SubsonicError { @@ -194,6 +220,131 @@ impl Subsonic { self.get_playlist(playlist_id).await } + + /// Authentication and format parameters common to every request + fn base_params(&self) -> Vec<(&'static str, String)> { + let auth_params = self.auth_params(); + vec![ + ("u", self.username.to_string()), + ("s", auth_params.salt), + ("t", auth_params.hash), + ("v", String::from("16")), + ("c", String::from("playlistd")), + ("f", String::from("json")), + ] + } + + async fn request( + &self, + endpoint: &str, + params: &[(&'static str, String)], + ) -> Result { + let mut query = self.base_params(); + query.extend_from_slice(params); + + let response = self + .client + .get(format!("{}/{}", self.base, endpoint)) + .query(&query) + .send() + .await + .map_err(SubsonicError::Reqwest)? + .json::>() + .await + .map_err(SubsonicError::Reqwest)? + .subsonic_response; + + if response.get("status").and_then(|s| s.as_str()) == Some("failed") { + return Err(SubsonicError::Api( + response + .pointer("/error/message") + .and_then(|m| m.as_str()) + .unwrap_or("unknown error") + .to_string(), + )); + } + serde_json::from_value(response).map_err(SubsonicError::Json) + } + + /// All genres in the library + pub async fn get_genres(&self) -> Result, SubsonicError> { + Ok(self + .request::("getGenres", &[]) + .await? + .genres + .genre) + } + + /// Up to `limit` songs with the given genre + pub async fn songs_by_genre( + &self, + genre: &str, + limit: usize, + ) -> Result, SubsonicError> { + // Subsonic caps a single page at 500 + const PAGE: usize = 500; + let mut songs = vec![]; + + while songs.len() < limit { + let count = PAGE.min(limit - songs.len()); + let page = self + .request::( + "getSongsByGenre", + &[ + ("genre", genre.to_string()), + ("count", count.to_string()), + ("offset", songs.len().to_string()), + ], + ) + .await? + .songs_by_genre + .song; + let done = page.len() < count; + songs.extend(page); + if done { + break; + } + } + + Ok(songs) + } + + pub async fn rename_playlist( + &self, + playlist_id: &str, + name: &str, + ) -> Result<(), SubsonicError> { + self.request::( + "updatePlaylist", + &[ + ("playlistId", playlist_id.to_string()), + ("name", name.to_string()), + ], + ) + .await?; + Ok(()) + } + + /// Replace all of a playlist's songs + pub async fn replace_playlist_songs( + &self, + playlist_id: &str, + song_ids: Vec, + ) -> Result { + let playlist = self.get_playlist(playlist_id).await?; + self.update_playlist( + playlist_id.to_string(), + song_ids, + (0..playlist.song_count).collect(), + ) + .await + } + + pub async fn delete_playlist(&self, playlist_id: &str) -> Result<(), SubsonicError> { + self.request::("deletePlaylist", &[("id", playlist_id.to_string())]) + .await?; + Ok(()) + } } #[derive(Default)] diff --git a/src/track.rs b/src/track.rs index 69f9b92..6c6e41c 100644 --- a/src/track.rs +++ b/src/track.rs @@ -4,6 +4,36 @@ use serde::{Deserialize, Serialize}; pub struct Track { pub id: String, pub title: String, + #[serde(default)] pub album: String, + #[serde(default)] pub artist: String, + #[serde(default)] + pub genre: Option, + /// OpenSubsonic: all of the song's genres + #[serde(default)] + pub genres: Vec, + #[serde(rename = "musicBrainzId", default)] + pub music_brainz_id: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ItemGenre { + pub name: String, +} + +impl Track { + /// The song's genres, from the OpenSubsonic `genres` list if the server provides it, + /// otherwise from the plain `genre` field + pub fn genre_names(&self) -> Vec { + if !self.genres.is_empty() { + return self.genres.iter().map(|g| g.name.clone()).collect(); + } + self.genre + .iter() + .flat_map(|g| g.split([';', ',', '/'])) + .map(|g| g.trim().to_string()) + .filter(|g| !g.is_empty()) + .collect() + } }