Spaces:
Running
Running
| // Voiceprint: three on-device apps on top of AntSpeaker speaker embeddings. | |
| const $ = (id) => document.getElementById(id); | |
| const SR = 16000; | |
| const MAX_CLIP = 20; // seconds fed to the embedder for a single clip | |
| const SPK_VARS = ["--s1", "--s2", "--s3", "--s4", "--s5", "--s6"]; | |
| const css = (v) => getComputedStyle(document.documentElement).getPropertyValue(v).trim(); | |
| const spkColor = (i) => css(SPK_VARS[i % SPK_VARS.length]); | |
| const fmt = (t) => `${Math.floor(t / 60)}:${String(Math.floor(t % 60)).padStart(2, "0")}`; | |
| const fmt1 = (t) => `${Math.floor(t / 60)}:${(t % 60).toFixed(1).padStart(4, "0")}`; | |
| // ---------------------------------------------------------------- storage | |
| const store = { | |
| get(k, d) { try { const v = localStorage.getItem("vp:" + k); return v ? JSON.parse(v) : d; } catch (_) { return d; } }, | |
| set(k, v) { try { localStorage.setItem("vp:" + k, JSON.stringify(v)); } catch (_) {} }, | |
| del(k) { try { localStorage.removeItem("vp:" + k); } catch (_) {} }, | |
| }; | |
| // ---------------------------------------------------------------- worker | |
| const worker = new Worker("worker.js", { type: "module" }); | |
| let reqId = 0; | |
| const pending = new Map(); | |
| let readyResolve; | |
| const ready = new Promise((r) => (readyResolve = r)); | |
| window.__vp = { ready: null, timings: [] }; | |
| worker.onmessage = (ev) => { | |
| const m = ev.data; | |
| if (m.type === "progress") { | |
| $("loadFill").style.width = `${(100 * m.got) / m.total}%`; | |
| $("statusText").textContent = `Model ${(m.got / 1e6).toFixed(1)} / ${(m.total / 1e6).toFixed(1)} MB`; | |
| } else if (m.type === "status") { | |
| $("statusText").textContent = m.text; | |
| } else if (m.type === "warn") { | |
| console.warn(m.text); | |
| } else if (m.type === "ready") { | |
| $("status").classList.add("ready"); | |
| $("statusText").textContent = `${m.backend === "webgpu" ? "WebGPU" : "WASM"} · ${m.precision} · ready in ${(m.totalMs / 1000).toFixed(1)}s`; | |
| window.__vp.ready = m; | |
| readyResolve(m); | |
| } else if (m.type === "vadProgress") { | |
| vadProgressCb && vadProgressCb(m.done / m.total); | |
| } else if (m.type === "result" || m.type === "error") { | |
| const p = pending.get(m.id); | |
| if (!p) { if (m.type === "error") fail(m.text); return; } | |
| pending.delete(m.id); | |
| m.type === "error" ? p.reject(new Error(m.text)) : p.resolve(m); | |
| } | |
| }; | |
| worker.onerror = (e) => fail(e.message || "worker error"); | |
| let vadProgressCb = null; | |
| function call(type, data, transfer = []) { | |
| return ready.then(() => new Promise((resolve, reject) => { | |
| const id = ++reqId; | |
| pending.set(id, { resolve, reject }); | |
| worker.postMessage({ type, id, ...data }, transfer); | |
| })); | |
| } | |
| async function embed(clips) { | |
| const r = await call("embed", { clips: clips.map((c) => c.slice()) }); | |
| window.__vp.timings.push({ n: clips.length, ms: r.ms }); | |
| return r.embs; | |
| } | |
| const embedOne = async (pcm) => (await embed([pcm.length > MAX_CLIP * SR ? pcm.subarray(0, MAX_CLIP * SR) : pcm]))[0]; | |
| function fail(text) { | |
| $("status").classList.add("err"); | |
| $("statusText").textContent = "Error: " + text; | |
| toast(text); | |
| } | |
| let toastT; | |
| function toast(text) { | |
| let el = document.querySelector(".toast"); | |
| if (!el) { el = document.createElement("div"); el.className = "toast"; document.body.appendChild(el); } | |
| el.textContent = text; el.hidden = false; | |
| clearTimeout(toastT); toastT = setTimeout(() => (el.hidden = true), 3500); | |
| } | |
| worker.postMessage({ type: "load", webgpu: new URLSearchParams(location.search).get("wasm") === null }); | |
| // ---------------------------------------------------------------- vector math | |
| const dot = (a, b) => { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }; | |
| function meanNorm(vs) { | |
| const m = new Float32Array(vs[0].length); | |
| for (const v of vs) for (let i = 0; i < m.length; i++) m[i] += v[i]; | |
| const n = Math.sqrt(dot(m, m)) || 1; | |
| for (let i = 0; i < m.length; i++) m[i] /= n; | |
| return m; | |
| } | |
| // ---------------------------------------------------------------- audio io | |
| let actx = null; | |
| const audioCtx = () => (actx ||= new (window.AudioContext || window.webkitAudioContext)()); | |
| async function decodeTo16k(arrayBuf) { | |
| // decoding through a 16 kHz offline context resamples straight to the model rate | |
| const buf = await new OfflineAudioContext(1, 1, SR).decodeAudioData(arrayBuf); | |
| const out = new Float32Array(buf.length); | |
| for (let c = 0; c < buf.numberOfChannels; c++) { | |
| const d = buf.getChannelData(c); | |
| for (let i = 0; i < out.length; i++) out[i] += d[i] / buf.numberOfChannels; | |
| } | |
| return out; | |
| } | |
| const fetchSample = async (name) => decodeTo16k(await (await fetch(`samples/${name}.wav`)).arrayBuffer()); | |
| function pickFiles(multiple = false) { | |
| return new Promise((resolve) => { | |
| const inp = $("filePick"); | |
| inp.multiple = multiple; inp.value = ""; | |
| inp.onchange = () => resolve([...inp.files]); | |
| inp.click(); | |
| }); | |
| } | |
| function wavBlob(pcm) { | |
| const b = new ArrayBuffer(44 + pcm.length * 2), v = new DataView(b); | |
| const w = (o, s) => [...s].forEach((c, i) => v.setUint8(o + i, c.charCodeAt(0))); | |
| w(0, "RIFF"); v.setUint32(4, 36 + pcm.length * 2, true); w(8, "WAVE"); w(12, "fmt "); | |
| v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true); v.setUint32(24, SR, true); | |
| v.setUint32(28, SR * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true); w(36, "data"); v.setUint32(40, pcm.length * 2, true); | |
| for (let i = 0; i < pcm.length; i++) v.setInt16(44 + i * 2, Math.max(-1, Math.min(1, pcm[i])) * 32767, true); | |
| return new Blob([b], { type: "audio/wav" }); | |
| } | |
| // Recorder: one at a time. Calls onTick(seconds, level). Resolves 16 kHz PCM on stop. | |
| let activeRec = null; | |
| async function record({ maxSec = 8, onTick = () => {} } = {}) { | |
| if (activeRec) { activeRec.stop(); return null; } | |
| const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: false, autoGainControl: true } }); | |
| const ctx = audioCtx(); await ctx.resume(); | |
| const srcNode = ctx.createMediaStreamSource(stream); | |
| const an = ctx.createAnalyser(); an.fftSize = 1024; srcNode.connect(an); | |
| const tmp = new Float32Array(1024); | |
| const mr = new MediaRecorder(stream); | |
| const chunks = []; | |
| mr.ondataavailable = (e) => e.data.size && chunks.push(e.data); | |
| const t0 = performance.now(); | |
| let raf; | |
| const done = new Promise((resolve) => { | |
| mr.onstop = async () => { | |
| cancelAnimationFrame(raf); stream.getTracks().forEach((t) => t.stop()); srcNode.disconnect(); | |
| activeRec = null; | |
| try { resolve(await decodeTo16k(await new Blob(chunks).arrayBuffer())); } catch (e) { toast("Could not decode recording"); resolve(null); } | |
| }; | |
| }); | |
| const tick = () => { | |
| const s = (performance.now() - t0) / 1000; | |
| an.getFloatTimeDomainData(tmp); | |
| let p = 0; for (const x of tmp) p = Math.max(p, Math.abs(x)); | |
| onTick(s, Math.min(1, p * 1.4)); | |
| if (s >= maxSec) { mr.state === "recording" && mr.stop(); return; } | |
| raf = requestAnimationFrame(tick); | |
| }; | |
| activeRec = { stop: () => mr.state === "recording" && mr.stop() }; | |
| mr.start(); tick(); | |
| return done; | |
| } | |
| // Wire a button as a record toggle. | |
| function recButton(btn, maxSec, onPcm, label = btn.textContent) { | |
| btn.addEventListener("click", async () => { | |
| if (activeRec) { activeRec.stop(); return; } | |
| btn.classList.add("rec"); | |
| let pcm = null; | |
| try { | |
| pcm = await record({ maxSec, onTick: (s) => (btn.textContent = `Stop · ${s.toFixed(1)}s`) }); | |
| } catch (e) { toast("Microphone unavailable: " + e.message); } | |
| btn.classList.remove("rec"); btn.textContent = label; | |
| if (pcm && pcm.length > SR * 0.5) onPcm(pcm); | |
| else if (pcm) toast("That was too short. Speak for a couple of seconds."); | |
| }); | |
| } | |
| // ---------------------------------------------------------------- barcode (a voiceprint drawn as 192 bars) | |
| function drawBarcode(canvas, v) { | |
| const dpr = window.devicePixelRatio || 1; | |
| const w = canvas.clientWidth || 200, h = canvas.clientHeight || 30; | |
| canvas.width = w * dpr; canvas.height = h * dpr; | |
| const g = canvas.getContext("2d"); g.scale(dpr, dpr); g.clearRect(0, 0, w, h); | |
| const ink = css("--ink"), bw = w / v.length; | |
| let mx = 0; for (const x of v) mx = Math.max(mx, Math.abs(x)); | |
| for (let i = 0; i < v.length; i++) { | |
| const a = Math.abs(v[i]) / mx; | |
| g.globalAlpha = 0.12 + 0.88 * a; | |
| g.fillStyle = ink; | |
| const bh = h * (0.35 + 0.65 * a); | |
| g.fillRect(i * bw, (h - bh) / 2, Math.max(1, bw - (bw > 2 ? 0.6 : 0)), bh); | |
| } | |
| g.globalAlpha = 1; | |
| } | |
| // ---------------------------------------------------------------- tabs | |
| document.querySelectorAll(".tabs button").forEach((b) => b.addEventListener("click", () => { | |
| document.querySelectorAll(".tabs button").forEach((x) => x.classList.toggle("on", x === b)); | |
| document.querySelectorAll(".pane").forEach((p) => p.classList.toggle("on", p.id === "pane-" + b.dataset.tab)); | |
| if (b.dataset.tab === "diar" && diar) drawTimeline(); | |
| if (b.dataset.tab === "lock" && lockPrint) drawBarcode($("enrollCode"), lockPrint); | |
| if (b.dataset.tab === "id") renderRoster(); | |
| })); | |
| // ================================================================= VOICE LOCK | |
| let enroll = [null, null, null]; // {emb, dur, pcm?} | |
| let lockPrint = null; | |
| const savedLock = store.get("lock", null); | |
| if (savedLock) { lockPrint = Float32Array.from(savedLock.print); enroll = savedLock.slots.map((s) => s && { dur: s.dur, emb: null, saved: true }); $("rememberLock").checked = true; } | |
| function renderSlots() { | |
| const box = $("enrollSlots"); box.innerHTML = ""; | |
| enroll.forEach((s, i) => { | |
| const el = document.createElement("button"); | |
| el.className = "slot" + (s ? " filled" : ""); | |
| if (s) { | |
| el.innerHTML = `<span class="t">${s.dur.toFixed(1)}s</span><span>clip ${i + 1}${s.pcm ? ' · <a data-play>play</a>' : ""}</span>`; | |
| const a = el.querySelector("[data-play]"); | |
| if (a) a.onclick = (e) => { e.stopPropagation(); playPcm(s.pcm); }; | |
| } else { | |
| el.innerHTML = `<span class="t">Record</span><span>clip ${i + 1}</span><span class="lvl"><i></i></span>`; | |
| el.onclick = () => recordSlot(i, el); | |
| } | |
| box.appendChild(el); | |
| }); | |
| } | |
| async function recordSlot(i, el) { | |
| if (activeRec) { activeRec.stop(); return; } | |
| el.classList.add("recording"); | |
| const t = el.querySelector(".t"), lvl = el.querySelector(".lvl i"); | |
| let pcm = null; | |
| try { pcm = await record({ maxSec: 5, onTick: (s, l) => { t.textContent = `${s.toFixed(1)}s`; lvl.style.width = `${l * 100}%`; } }); } | |
| catch (e) { toast("Microphone unavailable: " + e.message); } | |
| if (pcm && pcm.length > SR) await addEnroll(pcm, i); | |
| else { if (pcm) toast("Too short. Speak for 2 to 5 seconds."); renderSlots(); } | |
| } | |
| async function addEnroll(pcm, slot = enroll.findIndex((s) => !s || s.saved)) { | |
| if (slot < 0) slot = 2; | |
| if (enroll.some((s) => s && s.saved)) enroll = enroll.map((s) => (s && s.saved ? null : s)); // replacing a remembered print | |
| enroll[slot] = { pcm, dur: pcm.length / SR, emb: null }; | |
| renderSlots(); | |
| enroll[slot].emb = await embedOne(pcm); | |
| updatePrint(); | |
| } | |
| function updatePrint() { | |
| const embs = enroll.filter((s) => s && s.emb).map((s) => s.emb); | |
| if (!embs.length) { | |
| if (!enroll.some((s) => s && s.saved)) { lockPrint = null; $("enrollVp").hidden = true; } | |
| updateLockIdle(); return; | |
| } | |
| lockPrint = meanNorm(embs); | |
| let cons = ""; | |
| if (embs.length > 1) { | |
| let s = 0, n = 0; | |
| for (let i = 0; i < embs.length; i++) for (let j = i + 1; j < embs.length; j++) { s += dot(embs[i], embs[j]); n++; } | |
| cons = ` · clip agreement ${(s / n).toFixed(2)}`; | |
| } | |
| $("enrollVp").hidden = false; | |
| $("enrollMeta").textContent = `Voiceprint from ${embs.length} clip${embs.length > 1 ? "s" : ""}${cons}`; | |
| drawBarcode($("enrollCode"), lockPrint); | |
| if ($("rememberLock").checked) saveLock(); | |
| updateLockIdle(); | |
| } | |
| function saveLock() { store.set("lock", { print: [...lockPrint], slots: enroll.map((s) => s && { dur: s.dur }) }); } | |
| $("rememberLock").onchange = () => ($("rememberLock").checked && lockPrint ? saveLock() : store.del("lock")); | |
| function updateLockIdle() { | |
| if (lastLock) return; | |
| $("lockVerdict").textContent = lockPrint ? "Locked" : "Enroll a voice first"; | |
| $("lockSub").textContent = lockPrint ? "Speak to unlock, or try one of the samples." : "Then speak, upload a clip or try a sample."; | |
| } | |
| $("enrollUpload").onclick = async () => { | |
| const files = await pickFiles(true); | |
| for (const f of files.slice(0, 3)) { | |
| try { await addEnroll(await decodeTo16k(await f.arrayBuffer())); } catch (e) { toast(`Could not read ${f.name}`); } | |
| } | |
| }; | |
| $("enrollSample").onclick = async () => { | |
| enroll = [null, null, null]; renderSlots(); | |
| const pcms = await Promise.all(["enroll_1", "enroll_2", "enroll_3"].map(fetchSample)); | |
| pcms.forEach((p, i) => (enroll[i] = { pcm: p, dur: p.length / SR, emb: null })); | |
| renderSlots(); | |
| const embs = await Promise.all(pcms.map(embedOne)); | |
| embs.forEach((e, i) => (enroll[i].emb = e)); | |
| updatePrint(); | |
| }; | |
| $("enrollClear").onclick = () => { enroll = [null, null, null]; lockPrint = null; lastLock = null; store.del("lock"); $("rememberLock").checked = false; renderSlots(); updatePrint(); setLockState(null); $("lockHist").innerHTML = ""; }; | |
| let lastLock = null; // {score, label} | |
| const lockTh = () => Number($("lockTh").value); | |
| function setLockState(r) { | |
| const box = $("lockBox"), th = lockTh(); | |
| $("lockThMark").style.left = `${th * 100}%`; | |
| box.classList.remove("open", "shut", "idle"); | |
| $("lockMeter").classList.remove("ok", "no"); | |
| if (!r) { box.classList.add("idle"); $("lockFill").style.width = "0"; updateLockIdle(); return; } | |
| const ok = r.score >= th; | |
| box.classList.add(ok ? "open" : "shut"); | |
| $("lockMeter").classList.add(ok ? "ok" : "no"); | |
| $("lockFill").style.width = `${Math.max(0, Math.min(1, r.score)) * 100}%`; | |
| $("lockVerdict").textContent = ok ? "Unlocked" : "Access denied"; | |
| $("lockSub").textContent = `Similarity ${r.score.toFixed(3)} ${ok ? "≥" : "<"} ${th.toFixed(2)} · ${r.label}`; | |
| } | |
| async function tryUnlock(pcm, label) { | |
| if (!lockPrint) { toast("Enroll a voice first"); return; } | |
| $("lockVerdict").textContent = "Listening…"; $("lockSub").textContent = label; | |
| const t0 = performance.now(); | |
| const e = await embedOne(pcm); | |
| const score = dot(lockPrint, e); | |
| lastLock = { score, label, ms: performance.now() - t0 }; | |
| window.__vp.lastLock = lastLock; | |
| setLockState(lastLock); | |
| const li = document.createElement("li"); | |
| const ok = score >= lockTh(); | |
| li.innerHTML = `<span class="tag ${ok ? "ok" : "no"}">${ok ? "open" : "denied"}</span><span class="src"></span><span class="mono">${score.toFixed(3)}</span>`; | |
| li.querySelector(".src").textContent = `${label} · ${(pcm.length / SR).toFixed(1)}s`; | |
| $("lockHist").prepend(li); | |
| while ($("lockHist").children.length > 5) $("lockHist").lastChild.remove(); | |
| } | |
| $("lockTh").oninput = () => { $("lockThOut").textContent = lockTh().toFixed(2); setLockState(lastLock); }; | |
| recButton($("lockRec"), 6, (pcm) => tryUnlock(pcm, "microphone")); | |
| $("lockUpload").onclick = async () => { const [f] = await pickFiles(); if (f) tryUnlock(await decodeTo16k(await f.arrayBuffer()), f.name); }; | |
| $("lockSame").onclick = async () => tryUnlock(await fetchSample("unlock_same"), "sample: same person, new sentence"); | |
| $("lockOther").onclick = async () => tryUnlock(await fetchSample("unlock_impostor"), "sample: a different speaker"); | |
| renderSlots(); | |
| if (lockPrint) { $("enrollVp").hidden = false; $("enrollMeta").textContent = "Remembered voiceprint"; requestAnimationFrame(() => drawBarcode($("enrollCode"), lockPrint)); } | |
| setLockState(null); | |
| // ================================================================= VOICE ID | |
| let roster = (store.get("roster", []) || []).map((p) => ({ ...p, print: Float32Array.from(p.print), embs: p.embs.map((e) => Float32Array.from(e)) })); | |
| const saveRoster = () => store.set("roster", roster.map((p) => ({ name: p.name, sample: !!p.sample, print: [...p.print], embs: p.embs.map((e) => [...e]) }))); | |
| function addToRoster(name, emb, sample = false) { | |
| name = name.trim() || `Person ${roster.length + 1}`; | |
| let p = roster.find((x) => x.name.toLowerCase() === name.toLowerCase()); | |
| if (!p) { p = { name, embs: [], sample }; roster.push(p); } | |
| p.embs.push(emb); p.print = meanNorm(p.embs); | |
| saveRoster(); renderRoster(); | |
| return p; | |
| } | |
| function renderRoster() { | |
| const ul = $("roster"); ul.innerHTML = ""; | |
| if (!roster.length) { ul.innerHTML = `<li><span class="empty">Nobody yet. Record someone below or load the sample people.</span></li>`; return; } | |
| roster.forEach((p, i) => { | |
| const li = document.createElement("li"); | |
| li.innerHTML = `<span class="nm"></span><canvas></canvas><button class="btn ghost small">Remove</button>`; | |
| li.querySelector(".nm").textContent = p.name; | |
| const sm = document.createElement("small"); sm.textContent = `${p.embs.length} clip${p.embs.length > 1 ? "s" : ""}${p.sample ? " · sample" : ""}`; | |
| li.querySelector(".nm").appendChild(sm); | |
| li.querySelector("button").onclick = () => { roster.splice(i, 1); saveRoster(); renderRoster(); }; | |
| ul.appendChild(li); | |
| requestAnimationFrame(() => drawBarcode(li.querySelector("canvas"), p.print)); | |
| }); | |
| } | |
| recButton($("idRec"), 8, async (pcm) => { addToRoster($("idName").value, await embedOne(pcm)); $("idName").value = ""; }); | |
| $("idUpload").onclick = async () => { | |
| const files = await pickFiles(true); | |
| for (const f of files) { | |
| try { | |
| const nm = $("idName").value || f.name.replace(/\.[^.]+$/, ""); | |
| addToRoster(nm, await embedOne(await decodeTo16k(await f.arrayBuffer()))); | |
| } catch (e) { toast(`Could not read ${f.name}`); } | |
| } | |
| $("idName").value = ""; | |
| }; | |
| $("idSample").onclick = async () => { | |
| const people = ["robin", "sam", "alex", "jordan", "casey"]; | |
| const pcms = await Promise.all(people.map((n) => fetchSample("roster_" + n))); | |
| const embs = await embed(pcms.map((p) => p.subarray(0, Math.min(p.length, MAX_CLIP * SR)))); | |
| roster = roster.filter((p) => !p.sample); | |
| people.forEach((n, i) => addToRoster(n[0].toUpperCase() + n.slice(1), embs[i], true)); | |
| }; | |
| let lastQuery = null; | |
| const idTh = () => Number($("idTh").value); | |
| function renderRanks() { | |
| if (!lastQuery) return; | |
| const th = idTh(); | |
| const ranked = roster.map((p) => ({ name: p.name, score: dot(p.print, lastQuery.emb) })).sort((a, b) => b.score - a.score); | |
| const v = $("idVerdict"); | |
| if (!ranked.length) { v.className = "idverdict wait"; v.textContent = "Add some people first."; $("idRanks").innerHTML = ""; return; } | |
| const top = ranked[0]; | |
| if (top.score >= th) { v.className = "idverdict ok"; v.textContent = `Sounds like ${top.name}`; } | |
| else { v.className = "idverdict unk"; v.textContent = "Unknown voice"; } | |
| window.__vp.lastId = { label: lastQuery.label, ranked }; | |
| $("idRanks").innerHTML = ""; | |
| ranked.slice(0, 8).forEach((r, i) => { | |
| const li = document.createElement("li"); | |
| if (i === 0 && r.score >= th) li.className = "top"; | |
| li.innerHTML = `<span class="nm"></span><span class="track"><i style="width:${Math.max(0, r.score) * 100}%"></i><b style="left:${th * 100}%"></b></span><span class="sc">${r.score.toFixed(2)}</span>`; | |
| li.querySelector(".nm").textContent = r.name; | |
| $("idRanks").appendChild(li); | |
| }); | |
| } | |
| async function identify(pcm, label) { | |
| $("idVerdict").className = "idverdict wait"; $("idVerdict").textContent = "Listening…"; | |
| lastQuery = { emb: await embedOne(pcm), label }; | |
| renderRanks(); | |
| } | |
| $("idTh").oninput = () => { $("idThOut").textContent = idTh().toFixed(2); renderRanks(); }; | |
| recButton($("qRec"), 8, (pcm) => identify(pcm, "microphone")); | |
| $("qUpload").onclick = async () => { const [f] = await pickFiles(); if (f) identify(await decodeTo16k(await f.arrayBuffer()), f.name); }; | |
| document.querySelectorAll("[data-q]").forEach((b) => (b.onclick = async () => { | |
| if (!roster.length) await $("idSample").onclick(); | |
| identify(await fetchSample(b.dataset.q), b.textContent); | |
| })); | |
| renderRoster(); | |
| // ================================================================= WHO'S SPEAKING | |
| const WIN = 1.5, HOP = 0.75, VAD_HOP = 512 / SR; | |
| let diar = null; // {pcm, url, segs, turns, speakers, dur} | |
| function vadSegments(probs) { | |
| const segs = []; let on = false, s = 0; | |
| for (let i = 0; i < probs.length; i++) { | |
| if (!on && probs[i] > 0.5) { on = true; s = i; } | |
| else if (on && probs[i] < 0.35) { on = false; segs.push([s * VAD_HOP, i * VAD_HOP]); } | |
| } | |
| if (on) segs.push([s * VAD_HOP, probs.length * VAD_HOP]); | |
| const merged = []; | |
| for (const [a, b] of segs) { | |
| if (merged.length && a - merged[merged.length - 1][1] < 0.15) merged[merged.length - 1][1] = b; | |
| else merged.push([a, b]); | |
| } | |
| return merged.filter(([a, b]) => b - a >= 0.25); | |
| } | |
| function makeWindows(segs, dur, hop) { | |
| const wins = []; | |
| for (const [a, b] of segs) { | |
| if (b - a <= WIN) { | |
| const c = (a + b) / 2; | |
| let s = Math.max(0, c - WIN / 2); s = Math.min(s, Math.max(0, dur - WIN)); | |
| wins.push({ a: s, b: s + WIN, seg: [a, b] }); | |
| continue; | |
| } | |
| let t = a; | |
| for (; t + WIN < b; t += hop) wins.push({ a: t, b: t + WIN, seg: [a, b] }); | |
| wins.push({ a: b - WIN, b, seg: [a, b] }); | |
| } | |
| return wins; | |
| } | |
| // average-linkage agglomerative clustering on cosine similarity | |
| function ahc(E, th, k) { | |
| const n = E.length; | |
| let S = Array.from({ length: n }, (_, i) => Float64Array.from(E, (e) => dot(E[i], e))); | |
| let members = Array.from({ length: n }, (_, i) => [i]); | |
| let alive = Array.from({ length: n }, (_, i) => i); | |
| while (alive.length > 1) { | |
| let best = -Infinity, bi = -1, bj = -1; | |
| for (let x = 0; x < alive.length; x++) for (let y = x + 1; y < alive.length; y++) { | |
| const v = S[alive[x]][alive[y]]; if (v > best) { best = v; bi = alive[x]; bj = alive[y]; } | |
| } | |
| if (k ? alive.length <= k : best < th) break; | |
| const ni = members[bi].length, nj = members[bj].length; | |
| for (const z of alive) if (z !== bi && z !== bj) { const v = (S[bi][z] * ni + S[bj][z] * nj) / (ni + nj); S[bi][z] = S[z][bi] = v; } | |
| members[bi] = members[bi].concat(members[bj]); members[bj] = null; | |
| alive = alive.filter((z) => z !== bj); | |
| } | |
| return alive.map((i) => members[i]); | |
| } | |
| async function runDiar(pcm, label) { | |
| if (activeRec) return; | |
| const dur = pcm.length / SR; | |
| $("diarOut").hidden = true; $("diarProg").hidden = false; | |
| const stage = (t, f) => { $("diarStage").textContent = t; $("diarFill").style.width = `${f * 100}%`; }; | |
| const t0 = performance.now(); | |
| stage("Waiting for model", 0); await ready; | |
| stage("Finding speech", 0.02); | |
| vadProgressCb = (f) => stage("Finding speech", 0.02 + 0.38 * f); | |
| const { probs } = await call("vad", { pcm: pcm.slice() }); | |
| const tVad = performance.now(); | |
| const segs = vadSegments(probs); | |
| if (!segs.length) { stage("No speech found in this recording.", 0); return; } | |
| const hop = dur > 900 ? WIN : HOP; | |
| const wins = makeWindows(segs, dur, hop); | |
| stage(`Fingerprinting ${wins.length} windows`, 0.4); | |
| const E = []; | |
| const B = 32; | |
| for (let i = 0; i < wins.length; i += B) { | |
| const clips = wins.slice(i, i + B).map((w) => pcm.subarray(Math.round(w.a * SR), Math.round(w.a * SR) + WIN * SR)); | |
| E.push(...(await embed(clips))); | |
| stage(`Fingerprinting ${Math.min(i + B, wins.length)} / ${wins.length} windows`, 0.4 + 0.55 * Math.min(1, (i + B) / wins.length)); | |
| } | |
| const tEmb = performance.now(); | |
| stage("Grouping voices", 0.97); | |
| await new Promise((r) => setTimeout(r)); | |
| const k = Number($("diarK").value); | |
| let clusters = ahc(E, 0.35, k || 0); | |
| if (!k) { // fold tiny clusters (under ~2 s of evidence) into their nearest neighbour | |
| const big = clusters.filter((c) => c.length >= 3); | |
| if (big.length) { | |
| const cents = big.map((c) => meanNorm(c.map((i) => E[i]))); | |
| for (const c of clusters) if (c.length < 3) for (const i of c) { | |
| let bj = 0; cents.forEach((m, j) => { if (dot(m, E[i]) > dot(cents[bj], E[i])) bj = j; }); | |
| big[bj].push(i); | |
| } | |
| clusters = big; | |
| } | |
| } | |
| const lab = new Int32Array(wins.length); | |
| clusters.forEach((c, ci) => c.forEach((i) => (lab[i] = ci))); | |
| // frame-level labels at 50 ms by window vote, inside speech segments only | |
| const FR = 0.05, nF = Math.ceil(dur / FR), frame = new Int16Array(nF).fill(-1); | |
| const votes = new Float32Array(clusters.length); | |
| for (const [a, b] of segs) { | |
| for (let f = Math.floor(a / FR); f < Math.min(nF, Math.ceil(b / FR)); f++) { | |
| const t = (f + 0.5) * FR; votes.fill(0); | |
| for (let w = 0; w < wins.length; w++) { | |
| const W = wins[w]; if (t < W.a || t > W.b) continue; | |
| votes[lab[w]] += 1 - Math.abs(t - (W.a + W.b) / 2) / WIN; // centre-weighted | |
| } | |
| let bi = -1, bv = 0; votes.forEach((v, i) => { if (v > bv) { bv = v; bi = i; } }); | |
| if (bi < 0) { // nearest window | |
| let bd = 1e9; wins.forEach((W, w) => { const d = Math.abs(t - (W.a + W.b) / 2); if (d < bd) { bd = d; bi = lab[w]; } }); | |
| } | |
| frame[f] = bi; | |
| } | |
| } | |
| // order clusters by first appearance | |
| const order = []; for (const f of frame) if (f >= 0 && !order.includes(f)) order.push(f); | |
| const remap = new Map(order.map((c, i) => [c, i])); | |
| const turns = []; let cur = null; | |
| for (let f = 0; f < nF; f++) { | |
| const s = frame[f] < 0 ? -1 : remap.get(frame[f]); | |
| if (cur && s === cur.spk && f * FR - cur.b < 0.3) { cur.b = (f + 1) * FR; continue; } | |
| if (s < 0) continue; | |
| cur = { spk: s, a: f * FR, b: (f + 1) * FR }; turns.push(cur); | |
| } | |
| const speakers = order.map((c, i) => { | |
| const print = meanNorm(clusters[c].map((w) => E[w])); | |
| const talk = turns.filter((t) => t.spk === i).reduce((s, t) => s + t.b - t.a, 0); | |
| let name = `Speaker ${String.fromCharCode(65 + i)}`, match = null; | |
| for (const p of roster) { const sc = dot(p.print, print); if (sc >= idTh() && (!match || sc > match.score)) match = { name: p.name, score: sc }; } | |
| if (match) name = match.name; | |
| return { print, talk, name, match, nWin: clusters[c].length }; | |
| }); | |
| if (diar && diar.url) URL.revokeObjectURL(diar.url); | |
| diar = { pcm, dur, segs, turns, speakers, url: URL.createObjectURL(wavBlob(pcm)), label, | |
| timing: { vadMs: tVad - t0, embMs: tEmb - tVad, totalMs: performance.now() - t0, windows: wins.length } }; | |
| window.__vp.diar = { speakers: speakers.map((s) => ({ name: s.name, talk: s.talk })), turns: turns.map((t) => [t.spk, +t.a.toFixed(2), +t.b.toFixed(2)]), timing: diar.timing }; | |
| $("player").src = diar.url; | |
| $("diarProg").hidden = true; $("diarOut").hidden = false; | |
| $("diarStats").textContent = `${fmt(dur)} audio · ${wins.length} windows · ${(diar.timing.totalMs / 1000).toFixed(1)}s`; | |
| renderDiar(); | |
| } | |
| function renderDiar() { | |
| const box = $("diarSpk"); box.innerHTML = ""; | |
| const total = diar.speakers.reduce((s, x) => s + x.talk, 0) || 1; | |
| diar.speakers.forEach((s, i) => { | |
| const el = document.createElement("div"); el.className = "spk"; | |
| el.innerHTML = `<span class="sw" style="background:${spkColor(i)}"></span><input value=""><span class="pct">${Math.round((100 * s.talk) / total)}% · ${fmt(s.talk)}</span><button class="btn ghost">Save voice</button>`; | |
| const inp = el.querySelector("input"); inp.value = s.name; | |
| if (s.match) inp.title = `Matched ${s.match.name} in Voice ID (${s.match.score.toFixed(2)})`; | |
| inp.oninput = () => { s.name = inp.value; renderTurns(); drawTimeline(); }; | |
| el.querySelector("button").onclick = () => { addToRoster(s.name, s.print); toast(`Saved ${s.name} to Voice ID`); }; | |
| box.appendChild(el); | |
| }); | |
| renderTurns(); drawTimeline(); | |
| } | |
| function renderTurns() { | |
| const ol = $("diarTurns"); ol.innerHTML = ""; | |
| diar.turns.forEach((t, i) => { | |
| const li = document.createElement("li"); | |
| li.dataset.i = i; | |
| const left = (100 * t.a) / diar.dur, w = Math.max(0.4, (100 * (t.b - t.a)) / diar.dur); | |
| li.innerHTML = `<span class="tm">${fmt1(t.a)}–${fmt1(t.b)}</span><span class="who"><span class="sw" style="background:${spkColor(t.spk)}"></span><span></span></span><span class="segw" style="position:relative;height:6px"><span class="seg" style="position:absolute;left:${left}%;width:${w}%;top:0;bottom:0;background:${spkColor(t.spk)}"></span></span><span class="tm">${(t.b - t.a).toFixed(1)}s</span>`; | |
| li.querySelector(".who span:last-child").textContent = diar.speakers[t.spk].name; | |
| li.onclick = () => playRange(t.a, t.b); | |
| ol.appendChild(li); | |
| }); | |
| } | |
| function drawTimeline() { | |
| if (!diar || $("diarOut").hidden) return; | |
| const dpr = window.devicePixelRatio || 1; | |
| const wv = $("diarWave"), ln = $("diarLanes"); | |
| const W = wv.clientWidth, H = 72, nS = diar.speakers.length, laneH = 14, LH = nS * (laneH + 4); | |
| wv.width = W * dpr; wv.height = H * dpr; ln.style.height = LH + "px"; ln.width = W * dpr; ln.height = LH * dpr; | |
| const g = wv.getContext("2d"); g.setTransform(dpr, 0, 0, dpr, 0, 0); g.clearRect(0, 0, W, H); | |
| const pcm = diar.pcm, per = pcm.length / W, muted = css("--line"); | |
| const spkAt = new Int16Array(W).fill(-1); | |
| for (const t of diar.turns) for (let x = Math.floor((t.a / diar.dur) * W); x < Math.min(W, Math.ceil((t.b / diar.dur) * W)); x++) spkAt[x] = t.spk; | |
| let peak = 1e-4; for (let i = 0; i < pcm.length; i += 7) peak = Math.max(peak, Math.abs(pcm[i])); | |
| for (let x = 0; x < W; x++) { | |
| let mx = 0; const s = Math.floor(x * per), e = Math.min(pcm.length, Math.floor((x + 1) * per)); | |
| for (let i = s; i < e; i++) { const v = Math.abs(pcm[i]); if (v > mx) mx = v; } | |
| const h = Math.max(1, (mx / peak) * (H - 4)); | |
| g.fillStyle = spkAt[x] >= 0 ? spkColor(spkAt[x]) : muted; | |
| g.fillRect(x, (H - h) / 2, 1, h); | |
| } | |
| const l = ln.getContext("2d"); l.setTransform(dpr, 0, 0, dpr, 0, 0); l.clearRect(0, 0, W, LH); | |
| diar.speakers.forEach((s, i) => { l.fillStyle = css("--soft"); l.fillRect(0, i * (laneH + 4) + 2, W, laneH); }); | |
| for (const t of diar.turns) { | |
| l.fillStyle = spkColor(t.spk); | |
| const x = (t.a / diar.dur) * W, w = Math.max(1.5, ((t.b - t.a) / diar.dur) * W); | |
| l.fillRect(x, t.spk * (laneH + 4) + 2, w, laneH); | |
| } | |
| l.font = `500 10px "IBM Plex Mono", monospace`; l.textBaseline = "middle"; | |
| diar.speakers.forEach((s, i) => { l.fillStyle = css("--ink"); l.globalAlpha = 0.75; l.fillText(s.name, 4, i * (laneH + 4) + 2 + laneH / 2 + 0.5); l.globalAlpha = 1; }); | |
| // playhead | |
| const p = $("player"); | |
| if (p.src && !p.paused || p.currentTime > 0) { | |
| const x = (p.currentTime / diar.dur) * W; | |
| for (const [c, h] of [[g, H], [l, LH]]) { c.fillStyle = css("--ink"); c.fillRect(x, 0, 1.5, h); } | |
| } | |
| } | |
| let stopAt = null; | |
| function playRange(a, b) { | |
| const p = $("player"); | |
| if (!diar) return; | |
| if (p.src !== diar.url) p.src = diar.url; | |
| p.currentTime = a; stopAt = b; p.play(); | |
| } | |
| function playPcm(pcm) { | |
| const p = $("player"); stopAt = null; | |
| p.src = URL.createObjectURL(wavBlob(pcm)); p.play(); | |
| } | |
| const seek = (ev) => { | |
| if (!diar) return; | |
| const r = ev.currentTarget.getBoundingClientRect(); | |
| const t = ((ev.clientX - r.left) / r.width) * diar.dur; | |
| const turn = diar.turns.find((x) => t >= x.a && t <= x.b); | |
| playRange(t, null); | |
| void turn; | |
| }; | |
| $("diarWave").onclick = seek; $("diarLanes").onclick = seek; | |
| $("diarPlay").onclick = () => { const p = $("player"); if (!diar) return; if (p.src !== diar.url) p.src = diar.url; stopAt = null; p.paused ? p.play() : p.pause(); }; | |
| (function loop() { | |
| const p = $("player"); | |
| if (diar && p.src === diar.url) { | |
| if (stopAt !== null && p.currentTime >= stopAt) { p.pause(); stopAt = null; } | |
| $("diarTime").textContent = `${fmt1(p.currentTime)} / ${fmt(diar.dur)}`; | |
| $("diarPlay").textContent = p.paused ? "Play" : "Pause"; | |
| if (!p.paused) { | |
| drawTimeline(); | |
| const cur = diar.turns.findIndex((t) => p.currentTime >= t.a && p.currentTime <= t.b); | |
| document.querySelectorAll("#diarTurns li").forEach((li) => li.classList.toggle("playing", Number(li.dataset.i) === cur)); | |
| } | |
| } | |
| requestAnimationFrame(loop); | |
| })(); | |
| window.addEventListener("resize", () => { drawTimeline(); if (lockPrint) drawBarcode($("enrollCode"), lockPrint); }); | |
| $("diarUpload").onclick = async () => { | |
| const [f] = await pickFiles(); if (!f) return; | |
| $("diarStage").textContent = "Decoding"; $("diarProg").hidden = false; | |
| try { runDiar(await decodeTo16k(await f.arrayBuffer()), f.name); } catch (e) { toast("Could not decode that file"); $("diarProg").hidden = true; } | |
| }; | |
| recButton($("diarRec"), 300, (pcm) => runDiar(pcm, "microphone")); | |
| $("diarSample").onclick = async () => runDiar(await fetchSample("conversation"), "sample conversation"); | |
| $("diarK").onchange = () => diar && runDiar(diar.pcm, diar.label); | |
| // test hooks | |
| window.__vp.embedSample = async (n) => [...(await embedOne(await fetchSample(n)))]; | |