diff --git a/desktop/src-tauri/locales/en-US/common.json b/desktop/src-tauri/locales/en-US/common.json index e337933..6ee8579 100644 --- a/desktop/src-tauri/locales/en-US/common.json +++ b/desktop/src-tauri/locales/en-US/common.json @@ -37,14 +37,20 @@ "customize-info": "Download any supported model in ggml or gguf format (with the file extension ending in '.bin'). Transfer it to the models directory, then select from the dropdown. No need to restart 🌟", "dark": "Dark", "diarize-threshold": "Speaker recognition threshold", + "diarize-max-speakers-note": "Supports up to 4 speakers detection.", + "diarization": "Diarization", "discord-community": "Discord Community", "download-file": "Download and transcribe", "download-model": "Download model", "download-models-link": "Download Models", + "download-complete": "Download complete", + "download-diarize-model": "Speaker diarization requires a model (~25MB). Download it now?", "downloading": "Downloading... {{progress}}%", + "downloading-diarize-model": "Downloading diarization model...", "downloading-ai-models": "Downloading AI models...", "downloading-model": "Downloading {{company}} Model...", "downloading-ytdlp": "Downloading ytdlp", + "enable-diarization": "Enable speaker diarization", "enable-logs": "Enable Logs", "error": "Error", "error-title": "Error", @@ -55,10 +61,13 @@ "format": "Format", "formats": "Formats", "general": "General", + "global-dictation": "Global Dictation", + "global-dictation-enable": "Enable", + "global-dictation-promo": "Press a shortcut anywhere to instantly transcribe. Works while Vibe runs in the background.", "global-hotkey": "Global Hotkey", "global-hotkey-enabled": "Enable global hotkey", "global-hotkey-shortcut": "Shortcut", - "global-hotkey-description": "Press once to start recording, press again to stop. Transcription is copied to clipboard", + "global-hotkey-description": "Hold the shortcut to record, release to transcribe.", "gpu-device": "GPU Device number", "high-gpu-performance": "Set Graphics performance to high", "hotkey-output-mode": "Output mode", @@ -69,6 +78,7 @@ "include-sub-folders": "Include sub folders", "info-cancel-download": "You can cancel and download the model manually later.", "info-diarize-threshold": "Threshold for speaker recognition or consider as not detected", + "info-diarization": "Detect and label different speakers in the audio. Requires downloading a separate model.", "info-enable-logs": "Write logs to file. Please restart after enabling it.", "info-gpu-device": "Select the GPU device for transcription. Enter the GPU device number, starting from 0. If you have 2 GPUs, choose either 0 or 1.", "info-high-gpu-performance": "Enhances graphics performance for Vibe, but will consume more system resources.", diff --git a/desktop/src-tauri/src/cli.rs b/desktop/src-tauri/src/cli.rs index 9880265..1295af3 100644 --- a/desktop/src-tauri/src/cli.rs +++ b/desktop/src-tauri/src/cli.rs @@ -4,7 +4,7 @@ use std::process; use tauri::AppHandle; use tauri_plugin_aptabase::EventTracker; -use crate::cmd::{resolve_ffmpeg_path, resolve_sona_binary}; +use crate::cmd::{resolve_diarize_path, resolve_ffmpeg_path, resolve_sona_binary}; /// Attach to console if cli detected in Windows #[cfg(all(windows, not(debug_assertions)))] @@ -40,6 +40,7 @@ pub async fn run(app_handle: &AppHandle) -> Result<()> { let sona_binary = resolve_sona_binary(app_handle)?; let ffmpeg_path = resolve_ffmpeg_path(app_handle); + let diarize_path = resolve_diarize_path(app_handle); // Forward all args after the executable name to sona let args: Vec = std::env::args().skip(1).collect(); @@ -52,6 +53,9 @@ pub async fn run(app_handle: &AppHandle) -> Result<()> { if let Some(ref ffmpeg) = ffmpeg_path { cmd.env("SONA_FFMPEG_PATH", ffmpeg); } + if let Some(ref diarize) = diarize_path { + cmd.env("SONA_DIARIZE_PATH", diarize); + } let mut child = cmd.spawn().map_err(|e| eyre::eyre!("failed to spawn sona: {}", e))?; diff --git a/desktop/src-tauri/src/cmd/mod.rs b/desktop/src-tauri/src/cmd/mod.rs index 6314f38..b4d459b 100644 --- a/desktop/src-tauri/src/cmd/mod.rs +++ b/desktop/src-tauri/src/cmd/mod.rs @@ -214,6 +214,7 @@ pub struct TranscribeOptions { pub sampling_strategy: Option, pub best_of: Option, pub beam_size: Option, + pub diarize_model: Option, } #[tauri::command] @@ -313,6 +314,22 @@ pub fn resolve_ffmpeg_path(app_handle: &tauri::AppHandle) -> Option { None } +pub fn resolve_diarize_path(app_handle: &tauri::AppHandle) -> Option { + let resource_dir = app_handle.path().resource_dir().ok()?; + + #[cfg(target_os = "windows")] + let binary_name = "sona-diarize.exe"; + #[cfg(not(target_os = "windows"))] + let binary_name = "sona-diarize"; + + let sidecar_path = resource_dir.join(binary_name); + if sidecar_path.exists() { + return Some(sidecar_path); + } + + None +} + #[tauri::command] pub async fn transcribe( app_handle: tauri::AppHandle, @@ -354,11 +371,12 @@ pub async fn transcribe( SonaEvent::Progress { progress } => { let _ = set_progress_bar(&app_handle, Some(progress.into())); } - SonaEvent::Segment { start, end, text } => { + SonaEvent::Segment { start, end, text, speaker } => { let segment = Segment { start: (start * 100.0) as i64, stop: (end * 100.0) as i64, text, + speaker, }; app_handle.emit_to("main", "new_segment", segment.clone()).log_error(); segments.push(segment); @@ -463,7 +481,8 @@ pub async fn load_model(app_handle: tauri::AppHandle, model_path: String) -> Res if state_guard.process.is_none() { let binary_path = resolve_sona_binary(&app_handle)?; let ffmpeg_path = resolve_ffmpeg_path(&app_handle); - match crate::sona::SonaProcess::spawn(&binary_path, ffmpeg_path.as_deref()) { + let diarize_path = resolve_diarize_path(&app_handle); + match crate::sona::SonaProcess::spawn(&binary_path, ffmpeg_path.as_deref(), diarize_path.as_deref()) { Ok(process) => state_guard.process = Some(process), Err(e) => { let error_msg = format!("{:#}", e); @@ -497,7 +516,8 @@ pub async fn start_api_server(app_handle: tauri::AppHandle, sona_state: State<'_ if state_guard.process.is_none() { let binary_path = resolve_sona_binary(&app_handle)?; let ffmpeg_path = resolve_ffmpeg_path(&app_handle); - let process = crate::sona::SonaProcess::spawn(&binary_path, ffmpeg_path.as_deref())?; + let diarize_path = resolve_diarize_path(&app_handle); + let process = crate::sona::SonaProcess::spawn(&binary_path, ffmpeg_path.as_deref(), diarize_path.as_deref())?; state_guard.process = Some(process); } let process = state_guard.process.as_ref().context("API server process missing")?; diff --git a/desktop/src-tauri/src/sona.rs b/desktop/src-tauri/src/sona.rs index e9c6801..2d96bfc 100644 --- a/desktop/src-tauri/src/sona.rs +++ b/desktop/src-tauri/src/sona.rs @@ -25,13 +25,13 @@ struct ReadySignal { #[allow(dead_code)] pub enum SonaEvent { Progress { progress: i32 }, - Segment { start: f64, end: f64, text: String }, + Segment { start: f64, end: f64, text: String, speaker: Option }, Result { text: String }, Error { message: String }, } impl SonaProcess { - pub fn spawn(binary_path: &Path, ffmpeg_path: Option<&Path>) -> Result { + pub fn spawn(binary_path: &Path, ffmpeg_path: Option<&Path>, diarize_path: Option<&Path>) -> Result { tracing::debug!("spawning sona at {}", binary_path.display()); let mut cmd = Command::new(binary_path); @@ -44,6 +44,11 @@ impl SonaProcess { cmd.env("SONA_FFMPEG_PATH", ffmpeg); } + if let Some(diarize) = diarize_path { + tracing::debug!("setting SONA_DIARIZE_PATH={}", diarize.display()); + cmd.env("SONA_DIARIZE_PATH", diarize); + } + #[cfg(target_os = "windows")] { use std::os::windows::process::CommandExt; @@ -220,6 +225,11 @@ impl SonaProcess { form = form.text("beam_size", n.to_string()); } } + if let Some(ref model) = options.diarize_model { + if !model.is_empty() { + form = form.text("diarize_model", model.clone()); + } + } let resp = self .client diff --git a/desktop/src-tauri/src/types.rs b/desktop/src-tauri/src/types.rs index 2147d7a..a2dde4b 100644 --- a/desktop/src-tauri/src/types.rs +++ b/desktop/src-tauri/src/types.rs @@ -36,6 +36,8 @@ pub struct Segment { pub start: i64, pub stop: i64, pub text: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub speaker: Option, } impl Segment { diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index dc1215d..16c5bd6 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -44,7 +44,8 @@ "icons/icon.ico" ], "externalBin": [ - "binaries/sona" + "binaries/sona", + "binaries/sona-diarize" ], "resources": { "locales": "locales" diff --git a/desktop/src-tauri/tauri.macos.conf.json b/desktop/src-tauri/tauri.macos.conf.json index 8b9b677..9d880e2 100644 --- a/desktop/src-tauri/tauri.macos.conf.json +++ b/desktop/src-tauri/tauri.macos.conf.json @@ -1,6 +1,6 @@ { "bundle": { - "externalBin": ["binaries/sona", "binaries/ffmpeg"], + "externalBin": ["binaries/sona", "binaries/ffmpeg", "binaries/sona-diarize"], "macOS": { "entitlements": "entitlements.plist", "signingIdentity": "-", diff --git a/desktop/src/components/HtmlView.tsx b/desktop/src/components/HtmlView.tsx index 540b363..c050ad0 100644 --- a/desktop/src/components/HtmlView.tsx +++ b/desktop/src/components/HtmlView.tsx @@ -1,3 +1,4 @@ +import { useTranslation } from 'react-i18next' import { Segment, formatTimestamp } from '~/lib/transcript' import { NamedPath } from '~/lib/utils' import { Preference } from '~/providers/Preference' @@ -20,6 +21,7 @@ export function formatDuration(start: number, stop: number, direction: 'rtl' | ' } export default function HTMLView({ segments, file, preference }: HTMLViewProps) { + const { t } = useTranslation() return (
{formatDuration(segment.start, segment.stop)} + {segment.speaker != null && ( + + {t('common.speaker-prefix')} {segment.speaker + 1} + + )}
{segment.text}
diff --git a/desktop/src/components/Params.tsx b/desktop/src/components/Params.tsx index 5dd1353..8231d3a 100644 --- a/desktop/src/components/Params.tsx +++ b/desktop/src/components/Params.tsx @@ -7,7 +7,10 @@ import { ModelOptions as IModelOptions, usePreferenceProvider } from '~/provider import { useToastProvider } from '~/providers/Toast' import { listen } from '@tauri-apps/api/event' import * as config from '~/lib/config' +import * as fs from '@tauri-apps/plugin-fs' +import { invoke } from '@tauri-apps/api/core' import { open as shellOpen } from '@tauri-apps/plugin-shell' +import { join } from '@tauri-apps/api/path' import { toast as hotToast } from 'sonner' import * as dialog from '@tauri-apps/plugin-dialog' import { Claude, defaultClaudeConfig, defaultOllamaConfig, defaultOpenAIConfig, Llm, Ollama, OpenAICompatible } from '~/lib/llm' @@ -282,6 +285,60 @@ export default function ModelOptions({ options, setOptions }: ParamsProps) {
+ {/* Diarization Section */} +
+

{t('common.diarization')}

+
+ + + {t('common.enable-diarization')} + + { + if (!checked) { + preference.setDiarizeEnabled(false) + return + } + try { + const modelsFolder = await invoke('get_models_folder') + const modelPath = await join(modelsFolder, config.diarizeModelFilename) + const exists = await fs.exists(modelPath) + if (exists) { + preference.setDiarizeEnabled(true) + } else { + const confirmed = await dialog.ask( + t('common.download-diarize-model'), + { title: t('common.diarization'), kind: 'info' } + ) + if (confirmed) { + toast.setMessage(t('common.downloading-diarize-model') as string) + toast.setOpen(true) + toast.setProgress(0) + try { + await invoke('download_model', { url: config.diarizeModelUrl, path: modelPath }) + preference.setDiarizeEnabled(true) + hotToast.success(t('common.download-complete')) + } finally { + toast.setOpen(false) + toast.setProgress(null) + } + } + } + } catch (e) { + console.error('diarization setup failed:', e) + hotToast.error(String(e)) + } + }} + /> +
+ {preference.diarizeEnabled && ( +

{t('common.diarize-max-speakers-note')}

+ )} +
+ +
+ {/* Model Options Section */}

{t('common.model-options')}

diff --git a/desktop/src/components/TextArea.tsx b/desktop/src/components/TextArea.tsx index ed2a2ee..fd8dc2b 100644 --- a/desktop/src/components/TextArea.tsx +++ b/desktop/src/components/TextArea.tsx @@ -72,18 +72,19 @@ export default function TextArea({ const preference = usePreferenceProvider() const [text, setText] = useState('') + const speakerLabel = t('common.speaker-prefix') useEffect(() => { if (segments) { setText( preference.textFormat === 'vtt' - ? asVtt(segments) + ? asVtt(segments, speakerLabel) : preference.textFormat === 'srt' - ? asSrt(segments) + ? asSrt(segments, speakerLabel) : preference.textFormat === 'json' ? asJson(segments) : preference.textFormat === 'csv' ? asCsv(segments) - : asText(segments), + : asText(segments, speakerLabel), ) } else { setText('') @@ -111,7 +112,7 @@ export default function TextArea({ if (format === 'docx') { const fileName = await path.basename(filePath) - const doc = await toDocx(fileName, segments!, preference.textAreaDirection) + const doc = await toDocx(fileName, segments!, preference.textAreaDirection, speakerLabel) const arrayBuffer = await doc.arrayBuffer() await fs.writeFile(filePath, new Uint8Array(arrayBuffer)) } else { diff --git a/desktop/src/lib/config.ts b/desktop/src/lib/config.ts index ed07bbd..2ea6204 100644 --- a/desktop/src/lib/config.ts +++ b/desktop/src/lib/config.ts @@ -21,6 +21,9 @@ export const segmentModelFilename = 'segmentation-3.0.onnx' export const embeddingModelUrl = 'https://github.com/thewh1teagle/vibe/releases/download/v0.0.1/wespeaker_en_voxceleb_CAM++.onnx' export const segmentModelUrl = 'https://github.com/thewh1teagle/vibe/releases/download/v0.0.1/segmentation-3.0.onnx' +export const diarizeModelFilename = 'diar_streaming_sortformer_4spk-v2.1.onnx' +export const diarizeModelUrl = 'https://huggingface.co/altunenes/parakeet-rs/resolve/main/diar_streaming_sortformer_4spk-v2.1.onnx' + export const llmApiKeyUrl = 'https://console.anthropic.com/settings/keys' export const llmDefaultMaxTokens = 8192 // https://docs.anthropic.com/en/docs/about-claude/models export const llmLimitsUrl = 'https://console.anthropic.com/settings/limits' diff --git a/desktop/src/lib/docx.ts b/desktop/src/lib/docx.ts index 6e3c604..542446a 100644 --- a/desktop/src/lib/docx.ts +++ b/desktop/src/lib/docx.ts @@ -2,7 +2,7 @@ import { Document, Packer, Paragraph, TextRun, AlignmentType } from 'docx' import { Segment } from './transcript' import { formatDuration } from '~/components/HtmlView' -export async function toDocx(title: string, segments: Segment[], direction: 'rtl' | 'ltr') { +export async function toDocx(title: string, segments: Segment[], direction: 'rtl' | 'ltr', speakerLabel: string = 'Speaker') { const isRtl = direction === 'rtl' const doc = new Document({ sections: [ @@ -26,11 +26,13 @@ export async function toDocx(title: string, segments: Segment[], direction: 'rtl new Paragraph({}), ...segments.map((segment) => { const duration = formatDuration(segment.start, segment.stop, direction) + const speakerText = segment.speaker != null ? ` ${speakerLabel} ${segment.speaker + 1}` : '' return new Paragraph({ alignment: isRtl ? AlignmentType.RIGHT : AlignmentType.LEFT, bidirectional: isRtl, children: [ new TextRun({ text: duration, bold: true, rightToLeft: isRtl }), + ...(speakerText ? [new TextRun({ text: speakerText, bold: true, rightToLeft: isRtl })] : []), new TextRun({ text: `\n${segment.text}`, break: 1, rightToLeft: isRtl }), ], }) diff --git a/desktop/src/lib/transcript.ts b/desktop/src/lib/transcript.ts index 81482de..e822372 100644 --- a/desktop/src/lib/transcript.ts +++ b/desktop/src/lib/transcript.ts @@ -13,6 +13,7 @@ export interface Segment { start: number stop: number text: string + speaker?: number } export function formatTimestamp(seconds: number, alwaysIncludeHours: boolean, decimalMarker: string, includeMilliseconds: boolean = true): string { @@ -42,30 +43,34 @@ export function formatTimestamp(seconds: number, alwaysIncludeHours: boolean, de return result } -export function asSrt(segments: Segment[]) { +function speakerPrefix(segment: Segment, label: string): string { + return segment.speaker != null ? `[${label} ${segment.speaker + 1}] ` : '' +} + +export function asSrt(segments: Segment[], speakerLabel: string = 'Speaker') { return segments.reduce((transcript, segment, i) => { return ( transcript + `${i > 0 ? '\n' : ''}${i + 1}\n` + `${formatTimestamp(segment.start, true, ',')} --> ${formatTimestamp(segment.stop, true, ',')}\n` + - `${segment.text.trim().replace('-->', '->')}\n` + `${speakerPrefix(segment, speakerLabel)}${segment.text.trim().replace('-->', '->')}\n` ) }, '') } -export function asVtt(segments: Segment[]) { +export function asVtt(segments: Segment[], speakerLabel: string = 'Speaker') { return segments.reduce((transcript, segment) => { return ( transcript + `${formatTimestamp(segment.start, false, '.')} --> ${formatTimestamp(segment.stop, false, '.')}\n` + - `${segment.text.trim().replace('-->', '->')}\n` + `${speakerPrefix(segment, speakerLabel)}${segment.text.trim().replace('-->', '->')}\n` ) }, '') } -export function asText(segments: Segment[]) { +export function asText(segments: Segment[], speakerLabel: string = 'Speaker') { return segments.reduce((transcript, segment) => { - return transcript + `${segment.text.trim()}\n` + return transcript + `${speakerPrefix(segment, speakerLabel)}${segment.text.trim()}\n` }, '') } @@ -78,11 +83,16 @@ function escapeCsv(value: string) { } export function asCsv(segments: Segment[]) { - const header = 'start,end,text' + const hasSpeakers = segments.some((s) => s.speaker != null) + const header = hasSpeakers ? 'start,end,speaker,text' : 'start,end,text' const rows = segments.map((segment) => { const start = formatTimestamp(segment.start, true, '.') const end = formatTimestamp(segment.stop, true, '.') const text = segment.text.trim() + if (hasSpeakers) { + const speaker = segment.speaker != null ? String(segment.speaker + 1) : '' + return `${escapeCsv(start)},${escapeCsv(end)},${escapeCsv(speaker)},${escapeCsv(text)}` + } return `${escapeCsv(start)},${escapeCsv(end)},${escapeCsv(text)}` }) return [header, ...rows].join('\n') diff --git a/desktop/src/pages/batch/viewModel.tsx b/desktop/src/pages/batch/viewModel.tsx index ae2ca52..a9924d0 100644 --- a/desktop/src/pages/batch/viewModel.tsx +++ b/desktop/src/pages/batch/viewModel.tsx @@ -1,5 +1,6 @@ import { invoke } from '@tauri-apps/api/core' import { useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' import { useLocation, useNavigate } from 'react-router-dom' import { TextFormat, formatExtensions } from '~/components/FormatSelect' import { Segment, Transcript, asCsv, asJson, asSrt, asText, asVtt } from '~/lib/transcript' @@ -31,6 +32,7 @@ export function viewModel() { const isAbortingRef = useRef(false) const preference = usePreferenceProvider() const navigate = useNavigate() + const { t } = useTranslation() const [llm, setLlm] = useState(null) const location = useLocation() const [outputFolder, setOutputFolder] = useState('') @@ -45,12 +47,13 @@ export function viewModel() { } }, [preference.llmConfig]) + const speakerLabel = t('common.speaker-prefix') function getText(segments: Segment[], format: TextFormat) { if (format === 'srt') { - return asSrt(segments) + return asSrt(segments, speakerLabel) } if (format === 'vtt') { - return asVtt(segments) + return asVtt(segments, speakerLabel) } if (format === 'json') { return asJson(segments) @@ -58,7 +61,7 @@ export function viewModel() { if (format === 'csv') { return asCsv(segments) } - return asText(segments) + return asText(segments, speakerLabel) } async function checkFilesState() { @@ -149,6 +152,11 @@ export function viewModel() { throw new Error('No model selected. Please download or select a model first.') } await invoke('load_model', { modelPath: preference.modelPath }) + let diarize_model: string | undefined + if (preference.diarizeEnabled) { + const modelsFolder = await invoke('get_models_folder') + diarize_model = modelsFolder + '/' + config.diarizeModelFilename + } setCurrentIndex(localIndex) const loopStartTime = performance.now() for (const file of files) { @@ -160,6 +168,7 @@ export function viewModel() { const options = { path: file.path, ...preference.modelOptions, + ...(diarize_model ? { diarize_model } : {}), } const startTime = performance.now() @@ -199,7 +208,7 @@ export function viewModel() { let llmSegments: Segment[] | null = null if (llm && preference.llmConfig?.enabled) { try { - const question = `${preference.llmConfig.prompt.replace('%s', transcript.asText(res.segments))}` + const question = `${preference.llmConfig.prompt.replace('%s', transcript.asText(res.segments, speakerLabel))}` const answer = await llm.ask(question) if (answer) { llmSegments = [{ start: 0, stop: res.segments?.[res.segments?.length - 1].stop ?? 0, text: answer }] @@ -215,7 +224,7 @@ export function viewModel() { // Write file if (format === 'docx') { const fileName = await path.basename(dst) - const doc = await toDocx(fileName, res.segments, preference.textAreaDirection) + const doc = await toDocx(fileName, res.segments, preference.textAreaDirection, speakerLabel) const arrayBuffer = await doc.arrayBuffer() const buffer = new Uint8Array(arrayBuffer) fs.writeFile(dst, buffer) diff --git a/desktop/src/pages/home/Page.tsx b/desktop/src/pages/home/Page.tsx index 899eb07..bb1b5ea 100644 --- a/desktop/src/pages/home/Page.tsx +++ b/desktop/src/pages/home/Page.tsx @@ -6,12 +6,13 @@ import TextArea from '~/components/TextArea' import AudioInput from '~/pages/home/AudioInput' import AudioPlayer from './AudioPlayer' import ProgressPanel from './ProgressPanel' +import { useHotkeyProvider, type HotkeyOutputMode } from '~/providers/Hotkey' import { viewModel } from './viewModel' import AudioDeviceInput from '~/components/AudioDeviceInput' import { ReactComponent as FileIcon } from '~/icons/file.svg' import { ReactComponent as MicrphoneIcon } from '~/icons/microphone.svg' import { ReactComponent as LinkIcon } from '~/icons/link.svg' -import { useEffect } from 'react' +import { useEffect, useMemo } from 'react' import { webviewWindow } from '@tauri-apps/api' import * as keepAwake from 'tauri-plugin-keepawake-api' import { Button } from '~/components/ui/button' @@ -23,6 +24,20 @@ import { Tabs, TabsList, TabsTrigger } from '~/components/ui/tabs' export default function Home() { const { t } = useTranslation() const vm = viewModel() + const hotkey = useHotkeyProvider() + const isMac = navigator.platform.toUpperCase().includes('MAC') + + const shortcutKeys = useMemo(() => { + const keyMap: Record = { + CmdOrCtrl: isMac ? '⌘' : 'Ctrl', + Cmd: '⌘', + Ctrl: isMac ? '⌃' : 'Ctrl', + Shift: isMac ? '⇧' : 'Shift', + Alt: isMac ? '⌥' : 'Alt', + Option: '⌥', + } + return hotkey.hotkeyShortcut.split('+').map((k) => keyMap[k] ?? k) + }, [hotkey.hotkeyShortcut, isMac]) async function showWindow() { const currentWindow = webviewWindow.getCurrentWebviewWindow() @@ -80,6 +95,68 @@ export default function Home() { )} +
+
+ + {t('common.global-dictation')} + +
+
+ + {!hotkey.hotkeyEnabled ? ( +
+
+

{t('common.global-dictation-promo')}

+ +
+
+ ) : ( +
+
+ {t('common.global-hotkey-enabled')} + +
+
+
+ {t('common.global-hotkey-shortcut')}: +
+ {shortcutKeys.map((key, i) => ( + + {key} + + ))} +
+
+ hotkey.setHotkeyShortcut(e.target.value)} + className="h-7 text-xs text-muted-foreground" + /> +
+
+ {(['clipboard', 'type'] as HotkeyOutputMode[]).map((mode) => ( + + ))} +
+

{t('common.global-hotkey-description')}

+
+ )} +
)} diff --git a/desktop/src/pages/home/viewModel.ts b/desktop/src/pages/home/viewModel.ts index 634c1a7..4bfdf5e 100644 --- a/desktop/src/pages/home/viewModel.ts +++ b/desktop/src/pages/home/viewModel.ts @@ -448,9 +448,15 @@ export function viewModel() { throw new Error('No model selected. Please download or select a model first.') } await invoke('load_model', { modelPath }) + let diarize_model: string | undefined + if (preferenceRef.current.diarizeEnabled) { + const modelsFolder = await invoke('get_models_folder') + diarize_model = modelsFolder + '/' + config.diarizeModelFilename + } const options = { path, ...preferenceRef.current.modelOptions, + ...(diarize_model ? { diarize_model } : {}), } const startTime = performance.now() const res: transcript.Transcript = await invoke('transcribe', { @@ -499,7 +505,7 @@ export function viewModel() { if (newSegments && llm && preferenceRef.current.llmConfig?.enabled) { try { - const question = `${preferenceRef.current.llmConfig.prompt.replace('%s', transcript.asText(newSegments))}` + const question = `${preferenceRef.current.llmConfig.prompt.replace('%s', transcript.asText(newSegments, t('common.speaker-prefix')))}` const answerPromise = llm.ask(question) hotToast.promise(answerPromise, { loading: t('common.summarize-loading'), diff --git a/desktop/src/pages/settings/Page.tsx b/desktop/src/pages/settings/Page.tsx index 179e71d..3804ca1 100644 --- a/desktop/src/pages/settings/Page.tsx +++ b/desktop/src/pages/settings/Page.tsx @@ -15,8 +15,6 @@ import { ReactComponent as CopyIcon } from '~/icons/copy.svg' import * as config from '~/lib/config' import { supportedLanguages } from '~/lib/i18n' import { ModifyState } from '~/lib/utils' -import ModelOptions from '~/components/Params' -import { useHotkeyProvider, HotkeyOutputMode } from '~/providers/Hotkey' import { viewModel } from './viewModel' import { Button } from '~/components/ui/button' import { Input } from '~/components/ui/input' @@ -44,7 +42,6 @@ function SectionCard({ children }: { children: ReactNode }) { export default function SettingsPage({ setVisible }: SettingsPageProps) { const { t, i18n } = useTranslation() const vm = viewModel() - const hotkey = useHotkeyProvider() const apiDocsUrl = vm.apiBaseUrl ? `${vm.apiBaseUrl}/docs` : null const serverActionBusy = vm.isStartingApiServer || vm.isStoppingApiServer @@ -267,7 +264,7 @@ export default function SettingsPage({ setVisible }: SettingsPageProps) { variant="ghost" onMouseDown={() => shell.open(config.supportVibeURL)} className="h-12 w-full justify-between rounded-none px-4 font-medium first:rounded-t-lg last:rounded-b-lg hover:bg-accent/55"> - {t('common.support-the-project')} + {t('common.support-the-project')}
-
- - -
-
- {t('common.global-hotkey-enabled')} - -
- {hotkey.hotkeyEnabled && ( -
- - hotkey.setHotkeyShortcut(e.target.value)} - /> -
- )} -
- -
- {(['clipboard', 'type'] as HotkeyOutputMode[]).map((mode) => ( - - ))} -
-
-

{t('common.global-hotkey-description')}

- -
- -
-
diff --git a/desktop/src/providers/Hotkey.tsx b/desktop/src/providers/Hotkey.tsx index 6c4255a..323080e 100644 --- a/desktop/src/providers/Hotkey.tsx +++ b/desktop/src/providers/Hotkey.tsx @@ -14,6 +14,8 @@ import { useTranslation } from 'react-i18next' // when hotkey-triggered recording finishes export let hotkeyRecordingActive = false +export const DEFAULT_HOTKEY_SHORTCUT = 'CmdOrCtrl+Shift+V' + export type HotkeyOutputMode = 'clipboard' | 'type' interface HotkeyContextType { @@ -55,7 +57,7 @@ export function HotkeyProvider({ children }: { children: ReactNode }) { const preferenceRef = useRef(preference) const [hotkeyEnabled, setHotkeyEnabled] = useLocalStorage('prefs_hotkey_enabled', false) - const [hotkeyShortcut, setHotkeyShortcut] = useLocalStorage('prefs_hotkey_shortcut', 'CmdOrCtrl+Shift+V') + const [hotkeyShortcut, setHotkeyShortcut] = useLocalStorage('prefs_hotkey_shortcut', DEFAULT_HOTKEY_SHORTCUT) const [hotkeyOutputMode, setHotkeyOutputMode] = useLocalStorage('prefs_hotkey_output_mode', 'clipboard') const [isHotkeyRecording, setIsHotkeyRecording] = useState(false) @@ -79,37 +81,37 @@ export function HotkeyProvider({ children }: { children: ReactNode }) { return new Claude(config) }, []) - const handleHotkeyPress = useCallback(async () => { - if (isHotkeyRecordingRef.current) { - // Stop recording - await emit('stop_record') - } else { - // Start recording - try { - const devices = await invoke('get_audio_devices') - const defaultInput = devices.find((d) => d.isDefault && d.isInput) - if (!defaultInput) { - console.error('No default input device found') - return - } - - isHotkeyRecordingRef.current = true - hotkeyRecordingActive = true - setIsHotkeyRecording(true) - - await invoke('start_record', { - devices: [defaultInput], - storeInDocuments: false, - }) - } catch (error) { - console.error('Hotkey start_record error:', error) - isHotkeyRecordingRef.current = false - hotkeyRecordingActive = false - setIsHotkeyRecording(false) + const handleHotkeyDown = useCallback(async () => { + if (isHotkeyRecordingRef.current) return + try { + const devices = await invoke('get_audio_devices') + const defaultInput = devices.find((d) => d.isDefault && d.isInput) + if (!defaultInput) { + console.error('No default input device found') + return } + + isHotkeyRecordingRef.current = true + hotkeyRecordingActive = true + setIsHotkeyRecording(true) + + await invoke('start_record', { + devices: [defaultInput], + storeInDocuments: false, + }) + } catch (error) { + console.error('Hotkey start_record error:', error) + isHotkeyRecordingRef.current = false + hotkeyRecordingActive = false + setIsHotkeyRecording(false) } }, []) + const handleHotkeyUp = useCallback(async () => { + if (!isHotkeyRecordingRef.current) return + await emit('stop_record') + }, []) + // Listen for record_finish and process when hotkey-triggered useEffect(() => { const unlisten = listen<{ path: string; name: string }>('record_finish', async (event) => { @@ -129,7 +131,7 @@ export function HotkeyProvider({ children }: { children: ReactNode }) { ...preferenceRef.current.modelOptions, } const res: transcript.Transcript = await invoke('transcribe', { options }) - let resultText = transcript.asText(res.segments) + let resultText = transcript.asText(res.segments, t('common.speaker-prefix')) // Optional LLM summarization const llm = createLlm() @@ -188,7 +190,9 @@ export function HotkeyProvider({ children }: { children: ReactNode }) { try { await register(hotkeyShortcut, (event) => { if (event.state === 'Pressed') { - handleHotkeyPress() + handleHotkeyDown() + } else if (event.state === 'Released') { + handleHotkeyUp() } }) registeredShortcutRef.current = hotkeyShortcut @@ -206,7 +210,7 @@ export function HotkeyProvider({ children }: { children: ReactNode }) { registeredShortcutRef.current = null } } - }, [hotkeyEnabled, hotkeyShortcut, handleHotkeyPress]) + }, [hotkeyEnabled, hotkeyShortcut, handleHotkeyDown, handleHotkeyUp]) const value: HotkeyContextType = { hotkeyEnabled, diff --git a/desktop/src/providers/Preference.tsx b/desktop/src/providers/Preference.tsx index 46edb2c..2c884f3 100644 --- a/desktop/src/providers/Preference.tsx +++ b/desktop/src/providers/Preference.tsx @@ -58,6 +58,9 @@ export interface Preference { advancedTranscribeOptions: AdvancedTranscribeOptions setAdvancedTranscribeOptions: ModifyState + diarizeEnabled: boolean + setDiarizeEnabled: ModifyState + analyticsEnabled: boolean setAnalyticsEnabled: (value: boolean) => void } @@ -149,6 +152,8 @@ export function PreferenceProvider({ children }: { children: ReactNode }) { skipIfExists: true, }) + const [diarizeEnabled, setDiarizeEnabled] = useLocalStorage('prefs_diarize_enabled', false) + const [analyticsEnabled, setAnalyticsEnabledLocal] = useState(true) useEffect(() => { load(config.storeFilename).then((store) => { @@ -258,6 +263,8 @@ export function PreferenceProvider({ children }: { children: ReactNode }) { setShouldCheckYtDlpVersion, advancedTranscribeOptions, setAdvancedTranscribeOptions, + diarizeEnabled, + setDiarizeEnabled, analyticsEnabled, setAnalyticsEnabled, } diff --git a/plans/sona-diarize-integration/sona-diarize_001.md b/plans/sona-diarize-integration/sona-diarize_001.md new file mode 100644 index 0000000..57b46f1 --- /dev/null +++ b/plans/sona-diarize-integration/sona-diarize_001.md @@ -0,0 +1,124 @@ +# Integrate Diarization Support into Vibe + +## Context +Sona now supports optional speaker diarization via `sona-diarize` binary + Sortformer model. We need to integrate this into Vibe so users can enable diarization from the UI, download the model, and get speaker-attributed transcription segments. + +## 1. Packaging — `scripts/pre_build.py` + +**Add `sona-diarize` to `SONA_ASSET_MAP` and download it alongside sona.** + +The `sona-diarize` binaries are raw binaries (not archives), same as Linux sona. They live in the same GitHub release (`v0.1.1`). + +Add a new asset map for diarize: +```python +DIARIZE_ASSET_MAP = { + "aarch64-apple-darwin": "sona-diarize-darwin-arm64", + "x86_64-unknown-linux-gnu": "sona-diarize-linux-amd64", + "aarch64-unknown-linux-gnu": "sona-diarize-linux-arm64", + "x86_64-pc-windows-msvc": "sona-diarize-windows-amd64.exe", + # x86_64-apple-darwin: not available (ort lacks prebuilt binaries) +} +``` + +Add `download_diarize()` function mirroring `download_sona()` but simpler (always raw binary, no archive extraction). Downloads to `desktop/src-tauri/binaries/sona-diarize-{target-triple}[.exe]`. + +Call it from `main()` after `download_sona()`. + +**File:** `/Users/yqbqwlny/Documents/audio/vibe/scripts/pre_build.py` + +## 2. Tauri Config — Bundle `sona-diarize` + +Add to `externalBin`: +```json +"externalBin": ["binaries/sona", "binaries/sona-diarize"] +``` + +**Files:** +- `/Users/yqbqwlny/Documents/audio/vibe/desktop/src-tauri/tauri.conf.json` +- `/Users/yqbqwlny/Documents/audio/vibe/desktop/src-tauri/tauri.macos.conf.json` + +## 3. Rust — Resolve `sona-diarize` binary & set env var + +**Add `resolve_diarize_path()`** in `cmd/mod.rs`, same pattern as `resolve_ffmpeg_path()`: check resource dir, return `Option`. + +**In `SonaProcess::spawn()`** (`sona.rs`): accept optional `diarize_path` parameter, set `SONA_DIARIZE_PATH` env var (same as `SONA_FFMPEG_PATH` pattern). + +**In `load_model()`** (`cmd/mod.rs`): pass `resolve_diarize_path()` result into `SonaProcess::spawn()`. + +**In `TranscribeOptions`**: add `diarize_model: Option` field. + +**In `transcribe_stream()`** (`sona.rs`): if `diarize_model` is set, add `diarize_model` text part to the multipart form. Also add `response_format=verbose_json` (needed for speaker fields). + +**In `SonaEvent`**: add optional `speaker: Option` to `Segment` variant. + +**In `transcribe()`** (`cmd/mod.rs`): pass speaker from event into `Segment` type. + +**Files:** +- `/Users/yqbqwlny/Documents/audio/vibe/desktop/src-tauri/src/cmd/mod.rs` +- `/Users/yqbqwlny/Documents/audio/vibe/desktop/src-tauri/src/sona.rs` +- `/Users/yqbqwlny/Documents/audio/vibe/desktop/src-tauri/src/types.rs` (add `speaker` to `Segment`) + +## 4. Frontend — Config constants + +Add to `config.ts`: +```typescript +export const diarizeModelFilename = 'diar_streaming_sortformer_4spk-v2.1.onnx' +export const diarizeModelUrl = 'https://huggingface.co/altunenes/parakeet-rs/resolve/main/diar_streaming_sortformer_4spk-v2.1.onnx' +``` + +**File:** `/Users/yqbqwlny/Documents/audio/vibe/desktop/src/lib/config.ts` + +## 5. Frontend — Preference state + +Add to `Preference` interface and provider: +```typescript +diarizeEnabled: boolean +setDiarizeEnabled: ModifyState +``` + +Store in localStorage as `prefs_diarize_enabled`, default `false`. + +**File:** `/Users/yqbqwlny/Documents/audio/vibe/desktop/src/providers/Preference.tsx` + +## 6. Frontend — More Options UI (Params.tsx) + +Add a new **Diarization** section between FFmpeg Options and Presets: + +``` +── Diarization ── +[Switch] Enable speaker diarization +``` + +When toggling ON: +1. Check if diarize model exists in models folder (`get_models_folder` + `diarizeModelFilename`) +2. If not: show dialog asking to download (~25MB), if yes → navigate to download flow or download inline with progress +3. If yes: enable the switch + +The switch state is `preference.diarizeEnabled`. + +**File:** `/Users/yqbqwlny/Documents/audio/vibe/desktop/src/components/Params.tsx` + +## 7. Frontend — Pass `diarize_model` in transcription + +In `viewModel.ts` (home and batch), when building `TranscribeOptions`: +- If `preference.diarizeEnabled`, set `diarize_model` to `{models_folder}/{diarizeModelFilename}` +- Otherwise omit it + +**Files:** +- `/Users/yqbqwlny/Documents/audio/vibe/desktop/src/pages/home/viewModel.ts` +- `/Users/yqbqwlny/Documents/audio/vibe/desktop/src/pages/batch/viewModel.tsx` + +## 8. Frontend — Display speaker in segments + +Update segment rendering to show speaker label when present. + +**Files:** Segment display components (wherever segments are rendered in the UI). + +## Verification + +1. `uv run scripts/pre_build.py` — should download both `sona` and `sona-diarize` to `desktop/src-tauri/binaries/` +2. `pnpm exec tauri dev` — app builds and runs +3. More Options → Diarization → enable toggle → prompts to download model → downloads +4. Transcribe audio → segments show speaker labels +5. Disable diarization → transcribe without speaker labels (normal behavior) +6. If `sona-diarize` binary missing → diarization silently skipped, transcription works normally diff --git a/samples/6_speakers.wav b/samples/6_speakers.wav new file mode 100644 index 0000000..03f02d0 Binary files /dev/null and b/samples/6_speakers.wav differ diff --git a/scripts/pre_build.py b/scripts/pre_build.py index 2846fa8..5abfba5 100644 --- a/scripts/pre_build.py +++ b/scripts/pre_build.py @@ -27,6 +27,15 @@ SONA_ASSET_MAP = { "x86_64-pc-windows-msvc": ("sona-windows-amd64-with-ffmpeg.zip", "sona.exe", "ffmpeg.exe"), } +# Raw sona-diarize binaries (no archive). Not available for x86_64-apple-darwin +# because ort lacks prebuilt ONNX Runtime binaries for that target. +DIARIZE_ASSET_MAP = { + "aarch64-apple-darwin": "sona-diarize-darwin-arm64", + "x86_64-unknown-linux-gnu": "sona-diarize-linux-amd64", + "aarch64-unknown-linux-gnu": "sona-diarize-linux-arm64", + "x86_64-pc-windows-msvc": "sona-diarize-windows-amd64.exe", +} + HOST_TRIPLE_MAP = { ("Darwin", "arm64"): "aarch64-apple-darwin", ("Darwin", "x86_64"): "x86_64-apple-darwin", @@ -55,6 +64,23 @@ def detect_host_target() -> str | None: return HOST_TRIPLE_MAP.get((platform.system(), platform.machine())) +def download_with_progress(client: httpx.Client, url: str, label: str) -> bytes: + with client.stream("GET", url) as response: + response.raise_for_status() + total = int(response.headers.get("content-length", 0)) + downloaded = 0 + chunks: list[bytes] = [] + for chunk in response.iter_bytes(): + chunks.append(chunk) + downloaded += len(chunk) + if total: + pct = downloaded * 100 // total + print(f"\r {label}: {pct}% ({downloaded // 1024 // 1024}MB / {total // 1024 // 1024}MB)", end="", flush=True) + if total: + print() + return b"".join(chunks) + + def download_sona(script_root: Path, target_triple: str | None) -> None: resolved_target = target_triple or detect_host_target() if not resolved_target: @@ -106,9 +132,7 @@ def download_sona(script_root: Path, target_triple: str | None) -> None: try: with httpx.Client(follow_redirects=True, timeout=120) as client: - response = client.get(url) - response.raise_for_status() - data = response.content + data = download_with_progress(client, url, "sona") except Exception as exc: print(f"Warning: Failed to download sona from {url}: {exc}") print(f"Warning: You can manually place the sidecar at {sona_dest}") @@ -150,6 +174,62 @@ def download_sona(script_root: Path, target_triple: str | None) -> None: print(f"Extracted ffmpeg sidecar to {ffmpeg_dest}") +def download_diarize(script_root: Path, target_triple: str | None) -> None: + resolved_target = target_triple or detect_host_target() + if not resolved_target: + return + + asset_name = DIARIZE_ASSET_MAP.get(resolved_target) + if not asset_name: + # Create a stub so Tauri can bundle externalBin without error + is_windows = resolved_target.endswith("windows-msvc") + sidecar = f"sona-diarize-{resolved_target}" + (".exe" if is_windows else "") + binaries_dir = script_root.parent / "desktop" / "src-tauri" / "binaries" + dest = binaries_dir / sidecar + if not dest.exists(): + binaries_dir.mkdir(parents=True, exist_ok=True) + if is_windows: + dest.write_text("@echo off\necho sona-diarize is not supported on this platform\nexit /b 1\n") + else: + dest.write_text("#!/bin/sh\necho 'sona-diarize is not supported on this platform'\nexit 1\n") + dest.chmod(dest.stat().st_mode | 0o111) + print(f"Created sona-diarize stub at {dest} (not available for '{resolved_target}')") + return + + repo_root = script_root.parent + version_file = repo_root / ".sona-version" + try: + tag = version_file.read_text(encoding="utf-8").strip() + except OSError: + return + + if not tag: + return + + is_windows = resolved_target.endswith("windows-msvc") + sidecar = f"sona-diarize-{resolved_target}" + (".exe" if is_windows else "") + binaries_dir = repo_root / "desktop" / "src-tauri" / "binaries" + dest = binaries_dir / sidecar + + if dest.exists(): + print(f"sona-diarize sidecar already exists at {dest}; skipping download.") + return + + binaries_dir.mkdir(parents=True, exist_ok=True) + url = f"https://github.com/thewh1teagle/sona/releases/download/{tag}/{asset_name}" + + try: + with httpx.Client(follow_redirects=True, timeout=120) as client: + data = download_with_progress(client, url, "sona-diarize") + dest.write_bytes(data) + if not is_windows: + dest.chmod(dest.stat().st_mode | 0o111) + print(f"Downloaded sona-diarize sidecar to {dest}") + except Exception as exc: + print(f"Warning: Failed to download sona-diarize from {url}: {exc}") + print("Diarization support will not be available in this build.") + + def main() -> int: original_cwd = Path.cwd() script_root = Path(__file__).resolve().parent @@ -159,6 +239,7 @@ def main() -> int: argv = sys.argv[1:] target_triple = parse_target_arg(argv) download_sona(script_root, target_triple) + download_diarize(script_root, target_triple) platform_map = { "Windows": "windows", diff --git a/sona b/sona index 5dfda24..5e235fd 160000 --- a/sona +++ b/sona @@ -1 +1 @@ -Subproject commit 5dfda245e9eafe31c6b1d91729cd6fedcdef5365 +Subproject commit 5e235fdd877b51c1c071e94260e42df3f43788f6 diff --git a/website/src/components/SupportButton.tsx b/website/src/components/SupportButton.tsx index 9a00e05..d93df7e 100644 --- a/website/src/components/SupportButton.tsx +++ b/website/src/components/SupportButton.tsx @@ -10,9 +10,11 @@ export default function SupportButton({ onOpenKofi }: SupportButtonProps) { const { t } = useTranslation() return ( - ) } diff --git a/website/src/icons/Heart.tsx b/website/src/icons/Heart.tsx index ad70adb..744a247 100644 --- a/website/src/icons/Heart.tsx +++ b/website/src/icons/Heart.tsx @@ -1,6 +1,10 @@ -export default function Heart() { +interface HeartProps { + className?: string +} + +export default function Heart({ className }: HeartProps) { return ( -