// Voiceprint: three on-device apps on top of AntSpeaker speaker embeddings. const $ = (id) => document.getElementById(id); const SR = 16000; const MAX_CLIP = 20; // seconds fed to the embedder for a single clip const SPK_VARS = ["--s1", "--s2", "--s3", "--s4", "--s5", "--s6"]; const css = (v) => getComputedStyle(document.documentElement).getPropertyValue(v).trim(); const spkColor = (i) => css(SPK_VARS[i % SPK_VARS.length]); const fmt = (t) => `${Math.floor(t / 60)}:${String(Math.floor(t % 60)).padStart(2, "0")}`; const fmt1 = (t) => `${Math.floor(t / 60)}:${(t % 60).toFixed(1).padStart(4, "0")}`; // ---------------------------------------------------------------- storage const store = { get(k, d) { try { const v = localStorage.getItem("vp:" + k); return v ? JSON.parse(v) : d; } catch (_) { return d; } }, set(k, v) { try { localStorage.setItem("vp:" + k, JSON.stringify(v)); } catch (_) {} }, del(k) { try { localStorage.removeItem("vp:" + k); } catch (_) {} }, }; // ---------------------------------------------------------------- worker const worker = new Worker("worker.js", { type: "module" }); let reqId = 0; const pending = new Map(); let readyResolve; const ready = new Promise((r) => (readyResolve = r)); window.__vp = { ready: null, timings: [] }; worker.onmessage = (ev) => { const m = ev.data; if (m.type === "progress") { $("loadFill").style.width = `${(100 * m.got) / m.total}%`; $("statusText").textContent = `Model ${(m.got / 1e6).toFixed(1)} / ${(m.total / 1e6).toFixed(1)} MB`; } else if (m.type === "status") { $("statusText").textContent = m.text; } else if (m.type === "warn") { console.warn(m.text); } else if (m.type === "ready") { $("status").classList.add("ready"); $("statusText").textContent = `${m.backend === "webgpu" ? "WebGPU" : "WASM"} · ${m.precision} · ready in ${(m.totalMs / 1000).toFixed(1)}s`; window.__vp.ready = m; readyResolve(m); } else if (m.type === "vadProgress") { vadProgressCb && vadProgressCb(m.done / m.total); } else if (m.type === "result" || m.type === "error") { const p = pending.get(m.id); if (!p) { if (m.type === "error") fail(m.text); return; } pending.delete(m.id); m.type === "error" ? p.reject(new Error(m.text)) : p.resolve(m); } }; worker.onerror = (e) => fail(e.message || "worker error"); let vadProgressCb = null; function call(type, data, transfer = []) { return ready.then(() => new Promise((resolve, reject) => { const id = ++reqId; pending.set(id, { resolve, reject }); worker.postMessage({ type, id, ...data }, transfer); })); } async function embed(clips) { const r = await call("embed", { clips: clips.map((c) => c.slice()) }); window.__vp.timings.push({ n: clips.length, ms: r.ms }); return r.embs; } const embedOne = async (pcm) => (await embed([pcm.length > MAX_CLIP * SR ? pcm.subarray(0, MAX_CLIP * SR) : pcm]))[0]; function fail(text) { $("status").classList.add("err"); $("statusText").textContent = "Error: " + text; toast(text); } let toastT; function toast(text) { let el = document.querySelector(".toast"); if (!el) { el = document.createElement("div"); el.className = "toast"; document.body.appendChild(el); } el.textContent = text; el.hidden = false; clearTimeout(toastT); toastT = setTimeout(() => (el.hidden = true), 3500); } worker.postMessage({ type: "load", webgpu: new URLSearchParams(location.search).get("wasm") === null }); // ---------------------------------------------------------------- vector math const dot = (a, b) => { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }; function meanNorm(vs) { const m = new Float32Array(vs[0].length); for (const v of vs) for (let i = 0; i < m.length; i++) m[i] += v[i]; const n = Math.sqrt(dot(m, m)) || 1; for (let i = 0; i < m.length; i++) m[i] /= n; return m; } // ---------------------------------------------------------------- audio io let actx = null; const audioCtx = () => (actx ||= new (window.AudioContext || window.webkitAudioContext)()); async function decodeTo16k(arrayBuf) { // decoding through a 16 kHz offline context resamples straight to the model rate const buf = await new OfflineAudioContext(1, 1, SR).decodeAudioData(arrayBuf); const out = new Float32Array(buf.length); for (let c = 0; c < buf.numberOfChannels; c++) { const d = buf.getChannelData(c); for (let i = 0; i < out.length; i++) out[i] += d[i] / buf.numberOfChannels; } return out; } const fetchSample = async (name) => decodeTo16k(await (await fetch(`samples/${name}.wav`)).arrayBuffer()); function pickFiles(multiple = false) { return new Promise((resolve) => { const inp = $("filePick"); inp.multiple = multiple; inp.value = ""; inp.onchange = () => resolve([...inp.files]); inp.click(); }); } function wavBlob(pcm) { const b = new ArrayBuffer(44 + pcm.length * 2), v = new DataView(b); const w = (o, s) => [...s].forEach((c, i) => v.setUint8(o + i, c.charCodeAt(0))); w(0, "RIFF"); v.setUint32(4, 36 + pcm.length * 2, true); w(8, "WAVE"); w(12, "fmt "); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true); v.setUint32(24, SR, true); v.setUint32(28, SR * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true); w(36, "data"); v.setUint32(40, pcm.length * 2, true); for (let i = 0; i < pcm.length; i++) v.setInt16(44 + i * 2, Math.max(-1, Math.min(1, pcm[i])) * 32767, true); return new Blob([b], { type: "audio/wav" }); } // Recorder: one at a time. Calls onTick(seconds, level). Resolves 16 kHz PCM on stop. let activeRec = null; async function record({ maxSec = 8, onTick = () => {} } = {}) { if (activeRec) { activeRec.stop(); return null; } const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: false, autoGainControl: true } }); const ctx = audioCtx(); await ctx.resume(); const srcNode = ctx.createMediaStreamSource(stream); const an = ctx.createAnalyser(); an.fftSize = 1024; srcNode.connect(an); const tmp = new Float32Array(1024); const mr = new MediaRecorder(stream); const chunks = []; mr.ondataavailable = (e) => e.data.size && chunks.push(e.data); const t0 = performance.now(); let raf; const done = new Promise((resolve) => { mr.onstop = async () => { cancelAnimationFrame(raf); stream.getTracks().forEach((t) => t.stop()); srcNode.disconnect(); activeRec = null; try { resolve(await decodeTo16k(await new Blob(chunks).arrayBuffer())); } catch (e) { toast("Could not decode recording"); resolve(null); } }; }); const tick = () => { const s = (performance.now() - t0) / 1000; an.getFloatTimeDomainData(tmp); let p = 0; for (const x of tmp) p = Math.max(p, Math.abs(x)); onTick(s, Math.min(1, p * 1.4)); if (s >= maxSec) { mr.state === "recording" && mr.stop(); return; } raf = requestAnimationFrame(tick); }; activeRec = { stop: () => mr.state === "recording" && mr.stop() }; mr.start(); tick(); return done; } // Wire a button as a record toggle. function recButton(btn, maxSec, onPcm, label = btn.textContent) { btn.addEventListener("click", async () => { if (activeRec) { activeRec.stop(); return; } btn.classList.add("rec"); let pcm = null; try { pcm = await record({ maxSec, onTick: (s) => (btn.textContent = `Stop · ${s.toFixed(1)}s`) }); } catch (e) { toast("Microphone unavailable: " + e.message); } btn.classList.remove("rec"); btn.textContent = label; if (pcm && pcm.length > SR * 0.5) onPcm(pcm); else if (pcm) toast("That was too short. Speak for a couple of seconds."); }); } // ---------------------------------------------------------------- barcode (a voiceprint drawn as 192 bars) function drawBarcode(canvas, v) { const dpr = window.devicePixelRatio || 1; const w = canvas.clientWidth || 200, h = canvas.clientHeight || 30; canvas.width = w * dpr; canvas.height = h * dpr; const g = canvas.getContext("2d"); g.scale(dpr, dpr); g.clearRect(0, 0, w, h); const ink = css("--ink"), bw = w / v.length; let mx = 0; for (const x of v) mx = Math.max(mx, Math.abs(x)); for (let i = 0; i < v.length; i++) { const a = Math.abs(v[i]) / mx; g.globalAlpha = 0.12 + 0.88 * a; g.fillStyle = ink; const bh = h * (0.35 + 0.65 * a); g.fillRect(i * bw, (h - bh) / 2, Math.max(1, bw - (bw > 2 ? 0.6 : 0)), bh); } g.globalAlpha = 1; } // ---------------------------------------------------------------- tabs document.querySelectorAll(".tabs button").forEach((b) => b.addEventListener("click", () => { document.querySelectorAll(".tabs button").forEach((x) => x.classList.toggle("on", x === b)); document.querySelectorAll(".pane").forEach((p) => p.classList.toggle("on", p.id === "pane-" + b.dataset.tab)); if (b.dataset.tab === "diar" && diar) drawTimeline(); if (b.dataset.tab === "lock" && lockPrint) drawBarcode($("enrollCode"), lockPrint); if (b.dataset.tab === "id") renderRoster(); })); // ================================================================= VOICE LOCK let enroll = [null, null, null]; // {emb, dur, pcm?} let lockPrint = null; const savedLock = store.get("lock", null); if (savedLock) { lockPrint = Float32Array.from(savedLock.print); enroll = savedLock.slots.map((s) => s && { dur: s.dur, emb: null, saved: true }); $("rememberLock").checked = true; } function renderSlots() { const box = $("enrollSlots"); box.innerHTML = ""; enroll.forEach((s, i) => { const el = document.createElement("button"); el.className = "slot" + (s ? " filled" : ""); if (s) { el.innerHTML = `${s.dur.toFixed(1)}sclip ${i + 1}${s.pcm ? ' · play' : ""}`; const a = el.querySelector("[data-play]"); if (a) a.onclick = (e) => { e.stopPropagation(); playPcm(s.pcm); }; } else { el.innerHTML = `Recordclip ${i + 1}`; el.onclick = () => recordSlot(i, el); } box.appendChild(el); }); } async function recordSlot(i, el) { if (activeRec) { activeRec.stop(); return; } el.classList.add("recording"); const t = el.querySelector(".t"), lvl = el.querySelector(".lvl i"); let pcm = null; try { pcm = await record({ maxSec: 5, onTick: (s, l) => { t.textContent = `${s.toFixed(1)}s`; lvl.style.width = `${l * 100}%`; } }); } catch (e) { toast("Microphone unavailable: " + e.message); } if (pcm && pcm.length > SR) await addEnroll(pcm, i); else { if (pcm) toast("Too short. Speak for 2 to 5 seconds."); renderSlots(); } } async function addEnroll(pcm, slot = enroll.findIndex((s) => !s || s.saved)) { if (slot < 0) slot = 2; if (enroll.some((s) => s && s.saved)) enroll = enroll.map((s) => (s && s.saved ? null : s)); // replacing a remembered print enroll[slot] = { pcm, dur: pcm.length / SR, emb: null }; renderSlots(); enroll[slot].emb = await embedOne(pcm); updatePrint(); } function updatePrint() { const embs = enroll.filter((s) => s && s.emb).map((s) => s.emb); if (!embs.length) { if (!enroll.some((s) => s && s.saved)) { lockPrint = null; $("enrollVp").hidden = true; } updateLockIdle(); return; } lockPrint = meanNorm(embs); let cons = ""; if (embs.length > 1) { let s = 0, n = 0; for (let i = 0; i < embs.length; i++) for (let j = i + 1; j < embs.length; j++) { s += dot(embs[i], embs[j]); n++; } cons = ` · clip agreement ${(s / n).toFixed(2)}`; } $("enrollVp").hidden = false; $("enrollMeta").textContent = `Voiceprint from ${embs.length} clip${embs.length > 1 ? "s" : ""}${cons}`; drawBarcode($("enrollCode"), lockPrint); if ($("rememberLock").checked) saveLock(); updateLockIdle(); } function saveLock() { store.set("lock", { print: [...lockPrint], slots: enroll.map((s) => s && { dur: s.dur }) }); } $("rememberLock").onchange = () => ($("rememberLock").checked && lockPrint ? saveLock() : store.del("lock")); function updateLockIdle() { if (lastLock) return; $("lockVerdict").textContent = lockPrint ? "Locked" : "Enroll a voice first"; $("lockSub").textContent = lockPrint ? "Speak to unlock, or try one of the samples." : "Then speak, upload a clip or try a sample."; } $("enrollUpload").onclick = async () => { const files = await pickFiles(true); for (const f of files.slice(0, 3)) { try { await addEnroll(await decodeTo16k(await f.arrayBuffer())); } catch (e) { toast(`Could not read ${f.name}`); } } }; $("enrollSample").onclick = async () => { enroll = [null, null, null]; renderSlots(); const pcms = await Promise.all(["enroll_1", "enroll_2", "enroll_3"].map(fetchSample)); pcms.forEach((p, i) => (enroll[i] = { pcm: p, dur: p.length / SR, emb: null })); renderSlots(); const embs = await Promise.all(pcms.map(embedOne)); embs.forEach((e, i) => (enroll[i].emb = e)); updatePrint(); }; $("enrollClear").onclick = () => { enroll = [null, null, null]; lockPrint = null; lastLock = null; store.del("lock"); $("rememberLock").checked = false; renderSlots(); updatePrint(); setLockState(null); $("lockHist").innerHTML = ""; }; let lastLock = null; // {score, label} const lockTh = () => Number($("lockTh").value); function setLockState(r) { const box = $("lockBox"), th = lockTh(); $("lockThMark").style.left = `${th * 100}%`; box.classList.remove("open", "shut", "idle"); $("lockMeter").classList.remove("ok", "no"); if (!r) { box.classList.add("idle"); $("lockFill").style.width = "0"; updateLockIdle(); return; } const ok = r.score >= th; box.classList.add(ok ? "open" : "shut"); $("lockMeter").classList.add(ok ? "ok" : "no"); $("lockFill").style.width = `${Math.max(0, Math.min(1, r.score)) * 100}%`; $("lockVerdict").textContent = ok ? "Unlocked" : "Access denied"; $("lockSub").textContent = `Similarity ${r.score.toFixed(3)} ${ok ? "≥" : "<"} ${th.toFixed(2)} · ${r.label}`; } async function tryUnlock(pcm, label) { if (!lockPrint) { toast("Enroll a voice first"); return; } $("lockVerdict").textContent = "Listening…"; $("lockSub").textContent = label; const t0 = performance.now(); const e = await embedOne(pcm); const score = dot(lockPrint, e); lastLock = { score, label, ms: performance.now() - t0 }; window.__vp.lastLock = lastLock; setLockState(lastLock); const li = document.createElement("li"); const ok = score >= lockTh(); li.innerHTML = `${ok ? "open" : "denied"}${score.toFixed(3)}`; li.querySelector(".src").textContent = `${label} · ${(pcm.length / SR).toFixed(1)}s`; $("lockHist").prepend(li); while ($("lockHist").children.length > 5) $("lockHist").lastChild.remove(); } $("lockTh").oninput = () => { $("lockThOut").textContent = lockTh().toFixed(2); setLockState(lastLock); }; recButton($("lockRec"), 6, (pcm) => tryUnlock(pcm, "microphone")); $("lockUpload").onclick = async () => { const [f] = await pickFiles(); if (f) tryUnlock(await decodeTo16k(await f.arrayBuffer()), f.name); }; $("lockSame").onclick = async () => tryUnlock(await fetchSample("unlock_same"), "sample: same person, new sentence"); $("lockOther").onclick = async () => tryUnlock(await fetchSample("unlock_impostor"), "sample: a different speaker"); renderSlots(); if (lockPrint) { $("enrollVp").hidden = false; $("enrollMeta").textContent = "Remembered voiceprint"; requestAnimationFrame(() => drawBarcode($("enrollCode"), lockPrint)); } setLockState(null); // ================================================================= VOICE ID let roster = (store.get("roster", []) || []).map((p) => ({ ...p, print: Float32Array.from(p.print), embs: p.embs.map((e) => Float32Array.from(e)) })); const saveRoster = () => store.set("roster", roster.map((p) => ({ name: p.name, sample: !!p.sample, print: [...p.print], embs: p.embs.map((e) => [...e]) }))); function addToRoster(name, emb, sample = false) { name = name.trim() || `Person ${roster.length + 1}`; let p = roster.find((x) => x.name.toLowerCase() === name.toLowerCase()); if (!p) { p = { name, embs: [], sample }; roster.push(p); } p.embs.push(emb); p.print = meanNorm(p.embs); saveRoster(); renderRoster(); return p; } function renderRoster() { const ul = $("roster"); ul.innerHTML = ""; if (!roster.length) { ul.innerHTML = `
  • Nobody yet. Record someone below or load the sample people.
  • `; return; } roster.forEach((p, i) => { const li = document.createElement("li"); li.innerHTML = ``; li.querySelector(".nm").textContent = p.name; const sm = document.createElement("small"); sm.textContent = `${p.embs.length} clip${p.embs.length > 1 ? "s" : ""}${p.sample ? " · sample" : ""}`; li.querySelector(".nm").appendChild(sm); li.querySelector("button").onclick = () => { roster.splice(i, 1); saveRoster(); renderRoster(); }; ul.appendChild(li); requestAnimationFrame(() => drawBarcode(li.querySelector("canvas"), p.print)); }); } recButton($("idRec"), 8, async (pcm) => { addToRoster($("idName").value, await embedOne(pcm)); $("idName").value = ""; }); $("idUpload").onclick = async () => { const files = await pickFiles(true); for (const f of files) { try { const nm = $("idName").value || f.name.replace(/\.[^.]+$/, ""); addToRoster(nm, await embedOne(await decodeTo16k(await f.arrayBuffer()))); } catch (e) { toast(`Could not read ${f.name}`); } } $("idName").value = ""; }; $("idSample").onclick = async () => { const people = ["robin", "sam", "alex", "jordan", "casey"]; const pcms = await Promise.all(people.map((n) => fetchSample("roster_" + n))); const embs = await embed(pcms.map((p) => p.subarray(0, Math.min(p.length, MAX_CLIP * SR)))); roster = roster.filter((p) => !p.sample); people.forEach((n, i) => addToRoster(n[0].toUpperCase() + n.slice(1), embs[i], true)); }; let lastQuery = null; const idTh = () => Number($("idTh").value); function renderRanks() { if (!lastQuery) return; const th = idTh(); const ranked = roster.map((p) => ({ name: p.name, score: dot(p.print, lastQuery.emb) })).sort((a, b) => b.score - a.score); const v = $("idVerdict"); if (!ranked.length) { v.className = "idverdict wait"; v.textContent = "Add some people first."; $("idRanks").innerHTML = ""; return; } const top = ranked[0]; if (top.score >= th) { v.className = "idverdict ok"; v.textContent = `Sounds like ${top.name}`; } else { v.className = "idverdict unk"; v.textContent = "Unknown voice"; } window.__vp.lastId = { label: lastQuery.label, ranked }; $("idRanks").innerHTML = ""; ranked.slice(0, 8).forEach((r, i) => { const li = document.createElement("li"); if (i === 0 && r.score >= th) li.className = "top"; li.innerHTML = `${r.score.toFixed(2)}`; li.querySelector(".nm").textContent = r.name; $("idRanks").appendChild(li); }); } async function identify(pcm, label) { $("idVerdict").className = "idverdict wait"; $("idVerdict").textContent = "Listening…"; lastQuery = { emb: await embedOne(pcm), label }; renderRanks(); } $("idTh").oninput = () => { $("idThOut").textContent = idTh().toFixed(2); renderRanks(); }; recButton($("qRec"), 8, (pcm) => identify(pcm, "microphone")); $("qUpload").onclick = async () => { const [f] = await pickFiles(); if (f) identify(await decodeTo16k(await f.arrayBuffer()), f.name); }; document.querySelectorAll("[data-q]").forEach((b) => (b.onclick = async () => { if (!roster.length) await $("idSample").onclick(); identify(await fetchSample(b.dataset.q), b.textContent); })); renderRoster(); // ================================================================= WHO'S SPEAKING const WIN = 1.5, HOP = 0.75, VAD_HOP = 512 / SR; let diar = null; // {pcm, url, segs, turns, speakers, dur} function vadSegments(probs) { const segs = []; let on = false, s = 0; for (let i = 0; i < probs.length; i++) { if (!on && probs[i] > 0.5) { on = true; s = i; } else if (on && probs[i] < 0.35) { on = false; segs.push([s * VAD_HOP, i * VAD_HOP]); } } if (on) segs.push([s * VAD_HOP, probs.length * VAD_HOP]); const merged = []; for (const [a, b] of segs) { if (merged.length && a - merged[merged.length - 1][1] < 0.15) merged[merged.length - 1][1] = b; else merged.push([a, b]); } return merged.filter(([a, b]) => b - a >= 0.25); } function makeWindows(segs, dur, hop) { const wins = []; for (const [a, b] of segs) { if (b - a <= WIN) { const c = (a + b) / 2; let s = Math.max(0, c - WIN / 2); s = Math.min(s, Math.max(0, dur - WIN)); wins.push({ a: s, b: s + WIN, seg: [a, b] }); continue; } let t = a; for (; t + WIN < b; t += hop) wins.push({ a: t, b: t + WIN, seg: [a, b] }); wins.push({ a: b - WIN, b, seg: [a, b] }); } return wins; } // average-linkage agglomerative clustering on cosine similarity function ahc(E, th, k) { const n = E.length; let S = Array.from({ length: n }, (_, i) => Float64Array.from(E, (e) => dot(E[i], e))); let members = Array.from({ length: n }, (_, i) => [i]); let alive = Array.from({ length: n }, (_, i) => i); while (alive.length > 1) { let best = -Infinity, bi = -1, bj = -1; for (let x = 0; x < alive.length; x++) for (let y = x + 1; y < alive.length; y++) { const v = S[alive[x]][alive[y]]; if (v > best) { best = v; bi = alive[x]; bj = alive[y]; } } if (k ? alive.length <= k : best < th) break; const ni = members[bi].length, nj = members[bj].length; for (const z of alive) if (z !== bi && z !== bj) { const v = (S[bi][z] * ni + S[bj][z] * nj) / (ni + nj); S[bi][z] = S[z][bi] = v; } members[bi] = members[bi].concat(members[bj]); members[bj] = null; alive = alive.filter((z) => z !== bj); } return alive.map((i) => members[i]); } async function runDiar(pcm, label) { if (activeRec) return; const dur = pcm.length / SR; $("diarOut").hidden = true; $("diarProg").hidden = false; const stage = (t, f) => { $("diarStage").textContent = t; $("diarFill").style.width = `${f * 100}%`; }; const t0 = performance.now(); stage("Waiting for model", 0); await ready; stage("Finding speech", 0.02); vadProgressCb = (f) => stage("Finding speech", 0.02 + 0.38 * f); const { probs } = await call("vad", { pcm: pcm.slice() }); const tVad = performance.now(); const segs = vadSegments(probs); if (!segs.length) { stage("No speech found in this recording.", 0); return; } const hop = dur > 900 ? WIN : HOP; const wins = makeWindows(segs, dur, hop); stage(`Fingerprinting ${wins.length} windows`, 0.4); const E = []; const B = 32; for (let i = 0; i < wins.length; i += B) { const clips = wins.slice(i, i + B).map((w) => pcm.subarray(Math.round(w.a * SR), Math.round(w.a * SR) + WIN * SR)); E.push(...(await embed(clips))); stage(`Fingerprinting ${Math.min(i + B, wins.length)} / ${wins.length} windows`, 0.4 + 0.55 * Math.min(1, (i + B) / wins.length)); } const tEmb = performance.now(); stage("Grouping voices", 0.97); await new Promise((r) => setTimeout(r)); const k = Number($("diarK").value); let clusters = ahc(E, 0.35, k || 0); if (!k) { // fold tiny clusters (under ~2 s of evidence) into their nearest neighbour const big = clusters.filter((c) => c.length >= 3); if (big.length) { const cents = big.map((c) => meanNorm(c.map((i) => E[i]))); for (const c of clusters) if (c.length < 3) for (const i of c) { let bj = 0; cents.forEach((m, j) => { if (dot(m, E[i]) > dot(cents[bj], E[i])) bj = j; }); big[bj].push(i); } clusters = big; } } const lab = new Int32Array(wins.length); clusters.forEach((c, ci) => c.forEach((i) => (lab[i] = ci))); // frame-level labels at 50 ms by window vote, inside speech segments only const FR = 0.05, nF = Math.ceil(dur / FR), frame = new Int16Array(nF).fill(-1); const votes = new Float32Array(clusters.length); for (const [a, b] of segs) { for (let f = Math.floor(a / FR); f < Math.min(nF, Math.ceil(b / FR)); f++) { const t = (f + 0.5) * FR; votes.fill(0); for (let w = 0; w < wins.length; w++) { const W = wins[w]; if (t < W.a || t > W.b) continue; votes[lab[w]] += 1 - Math.abs(t - (W.a + W.b) / 2) / WIN; // centre-weighted } let bi = -1, bv = 0; votes.forEach((v, i) => { if (v > bv) { bv = v; bi = i; } }); if (bi < 0) { // nearest window let bd = 1e9; wins.forEach((W, w) => { const d = Math.abs(t - (W.a + W.b) / 2); if (d < bd) { bd = d; bi = lab[w]; } }); } frame[f] = bi; } } // order clusters by first appearance const order = []; for (const f of frame) if (f >= 0 && !order.includes(f)) order.push(f); const remap = new Map(order.map((c, i) => [c, i])); const turns = []; let cur = null; for (let f = 0; f < nF; f++) { const s = frame[f] < 0 ? -1 : remap.get(frame[f]); if (cur && s === cur.spk && f * FR - cur.b < 0.3) { cur.b = (f + 1) * FR; continue; } if (s < 0) continue; cur = { spk: s, a: f * FR, b: (f + 1) * FR }; turns.push(cur); } const speakers = order.map((c, i) => { const print = meanNorm(clusters[c].map((w) => E[w])); const talk = turns.filter((t) => t.spk === i).reduce((s, t) => s + t.b - t.a, 0); let name = `Speaker ${String.fromCharCode(65 + i)}`, match = null; for (const p of roster) { const sc = dot(p.print, print); if (sc >= idTh() && (!match || sc > match.score)) match = { name: p.name, score: sc }; } if (match) name = match.name; return { print, talk, name, match, nWin: clusters[c].length }; }); if (diar && diar.url) URL.revokeObjectURL(diar.url); diar = { pcm, dur, segs, turns, speakers, url: URL.createObjectURL(wavBlob(pcm)), label, timing: { vadMs: tVad - t0, embMs: tEmb - tVad, totalMs: performance.now() - t0, windows: wins.length } }; window.__vp.diar = { speakers: speakers.map((s) => ({ name: s.name, talk: s.talk })), turns: turns.map((t) => [t.spk, +t.a.toFixed(2), +t.b.toFixed(2)]), timing: diar.timing }; $("player").src = diar.url; $("diarProg").hidden = true; $("diarOut").hidden = false; $("diarStats").textContent = `${fmt(dur)} audio · ${wins.length} windows · ${(diar.timing.totalMs / 1000).toFixed(1)}s`; renderDiar(); } function renderDiar() { const box = $("diarSpk"); box.innerHTML = ""; const total = diar.speakers.reduce((s, x) => s + x.talk, 0) || 1; diar.speakers.forEach((s, i) => { const el = document.createElement("div"); el.className = "spk"; el.innerHTML = `${Math.round((100 * s.talk) / total)}% · ${fmt(s.talk)}`; const inp = el.querySelector("input"); inp.value = s.name; if (s.match) inp.title = `Matched ${s.match.name} in Voice ID (${s.match.score.toFixed(2)})`; inp.oninput = () => { s.name = inp.value; renderTurns(); drawTimeline(); }; el.querySelector("button").onclick = () => { addToRoster(s.name, s.print); toast(`Saved ${s.name} to Voice ID`); }; box.appendChild(el); }); renderTurns(); drawTimeline(); } function renderTurns() { const ol = $("diarTurns"); ol.innerHTML = ""; diar.turns.forEach((t, i) => { const li = document.createElement("li"); li.dataset.i = i; const left = (100 * t.a) / diar.dur, w = Math.max(0.4, (100 * (t.b - t.a)) / diar.dur); li.innerHTML = `${fmt1(t.a)}–${fmt1(t.b)}${(t.b - t.a).toFixed(1)}s`; li.querySelector(".who span:last-child").textContent = diar.speakers[t.spk].name; li.onclick = () => playRange(t.a, t.b); ol.appendChild(li); }); } function drawTimeline() { if (!diar || $("diarOut").hidden) return; const dpr = window.devicePixelRatio || 1; const wv = $("diarWave"), ln = $("diarLanes"); const W = wv.clientWidth, H = 72, nS = diar.speakers.length, laneH = 14, LH = nS * (laneH + 4); wv.width = W * dpr; wv.height = H * dpr; ln.style.height = LH + "px"; ln.width = W * dpr; ln.height = LH * dpr; const g = wv.getContext("2d"); g.setTransform(dpr, 0, 0, dpr, 0, 0); g.clearRect(0, 0, W, H); const pcm = diar.pcm, per = pcm.length / W, muted = css("--line"); const spkAt = new Int16Array(W).fill(-1); for (const t of diar.turns) for (let x = Math.floor((t.a / diar.dur) * W); x < Math.min(W, Math.ceil((t.b / diar.dur) * W)); x++) spkAt[x] = t.spk; let peak = 1e-4; for (let i = 0; i < pcm.length; i += 7) peak = Math.max(peak, Math.abs(pcm[i])); for (let x = 0; x < W; x++) { let mx = 0; const s = Math.floor(x * per), e = Math.min(pcm.length, Math.floor((x + 1) * per)); for (let i = s; i < e; i++) { const v = Math.abs(pcm[i]); if (v > mx) mx = v; } const h = Math.max(1, (mx / peak) * (H - 4)); g.fillStyle = spkAt[x] >= 0 ? spkColor(spkAt[x]) : muted; g.fillRect(x, (H - h) / 2, 1, h); } const l = ln.getContext("2d"); l.setTransform(dpr, 0, 0, dpr, 0, 0); l.clearRect(0, 0, W, LH); diar.speakers.forEach((s, i) => { l.fillStyle = css("--soft"); l.fillRect(0, i * (laneH + 4) + 2, W, laneH); }); for (const t of diar.turns) { l.fillStyle = spkColor(t.spk); const x = (t.a / diar.dur) * W, w = Math.max(1.5, ((t.b - t.a) / diar.dur) * W); l.fillRect(x, t.spk * (laneH + 4) + 2, w, laneH); } l.font = `500 10px "IBM Plex Mono", monospace`; l.textBaseline = "middle"; diar.speakers.forEach((s, i) => { l.fillStyle = css("--ink"); l.globalAlpha = 0.75; l.fillText(s.name, 4, i * (laneH + 4) + 2 + laneH / 2 + 0.5); l.globalAlpha = 1; }); // playhead const p = $("player"); if (p.src && !p.paused || p.currentTime > 0) { const x = (p.currentTime / diar.dur) * W; for (const [c, h] of [[g, H], [l, LH]]) { c.fillStyle = css("--ink"); c.fillRect(x, 0, 1.5, h); } } } let stopAt = null; function playRange(a, b) { const p = $("player"); if (!diar) return; if (p.src !== diar.url) p.src = diar.url; p.currentTime = a; stopAt = b; p.play(); } function playPcm(pcm) { const p = $("player"); stopAt = null; p.src = URL.createObjectURL(wavBlob(pcm)); p.play(); } const seek = (ev) => { if (!diar) return; const r = ev.currentTarget.getBoundingClientRect(); const t = ((ev.clientX - r.left) / r.width) * diar.dur; const turn = diar.turns.find((x) => t >= x.a && t <= x.b); playRange(t, null); void turn; }; $("diarWave").onclick = seek; $("diarLanes").onclick = seek; $("diarPlay").onclick = () => { const p = $("player"); if (!diar) return; if (p.src !== diar.url) p.src = diar.url; stopAt = null; p.paused ? p.play() : p.pause(); }; (function loop() { const p = $("player"); if (diar && p.src === diar.url) { if (stopAt !== null && p.currentTime >= stopAt) { p.pause(); stopAt = null; } $("diarTime").textContent = `${fmt1(p.currentTime)} / ${fmt(diar.dur)}`; $("diarPlay").textContent = p.paused ? "Play" : "Pause"; if (!p.paused) { drawTimeline(); const cur = diar.turns.findIndex((t) => p.currentTime >= t.a && p.currentTime <= t.b); document.querySelectorAll("#diarTurns li").forEach((li) => li.classList.toggle("playing", Number(li.dataset.i) === cur)); } } requestAnimationFrame(loop); })(); window.addEventListener("resize", () => { drawTimeline(); if (lockPrint) drawBarcode($("enrollCode"), lockPrint); }); $("diarUpload").onclick = async () => { const [f] = await pickFiles(); if (!f) return; $("diarStage").textContent = "Decoding"; $("diarProg").hidden = false; try { runDiar(await decodeTo16k(await f.arrayBuffer()), f.name); } catch (e) { toast("Could not decode that file"); $("diarProg").hidden = true; } }; recButton($("diarRec"), 300, (pcm) => runDiar(pcm, "microphone")); $("diarSample").onclick = async () => runDiar(await fetchSample("conversation"), "sample conversation"); $("diarK").onchange = () => diar && runDiar(diar.pcm, diar.label); // test hooks window.__vp.embedSample = async (n) => [...(await embedOne(await fetchSample(n)))];