Speech

import { ControlSize } from "theme.slint";
import { Speech, SpeechButton, SpeechStatus, SpeechWaveform } from "speech.slint";

Input {
    placeholder: "Type or dictate";

    // One bar per level, newest against the trailing edge. An empty history
    // draws nothing, so no condition is needed to keep it out of the way.
    SpeechWaveform {
        levels: root.levels;
        capturing: Speech.is-capturing(root.status);
        interval: root.level-interval;
        size: ControlSize.xsmall;
        width: 48px;
    }

    SpeechButton {
        status: root.status;
        has-recognizer: root.has-recognizer;
        available: root.available;
        size: ControlSize.xsmall;
        // The application starts or stops its own session here.
        toggled(active) => { root.request-dictation(active); }
    }
}

[dependencies]
gpui-kit = { version = "{{gpui_kit_version}}", features = ["speech"] }

use gpui_kit::component::speech::{
    SpeechButton, SpeechEvent, SpeechState, SpeechStatus, SpeechWaveform,
};

// Installed in ui/components. That directory is on the Slint include path, so import by file name:
import { SpeechStatus, Speech, SpeechButton, SpeechWaveform } from "speech.slint";

use crate::components::button::Button;
use crate::components::group_box::{self, GroupBox};
use crate::components::input::Input;
use crate::components::speech::{SpeechButton, SpeechStatus, SpeechWaveform};
use crate::components::theme::{self, ControlSize, Tokens};
use iced::time::Instant;
use iced::widget::{column, container, mouse_area, row, scrollable, svg, text};
use iced::{Alignment as Axis, Color, Element, Length, Subscription, Task, Theme};
use std::time::Duration;

// The waveform and the microphone are the field's suffix. The waveform is
// always there — an empty history draws nothing, so no condition is needed
// to keep it out of the way — and the application starts and stops its own
// session on the button's callback.
let dictate = column![
    Input::new("speech-field")
        .value(&app.field)
        .placeholder("Type or dictate")
        .on_input(Message::FieldChanged)
        .suffix(
            row![
                SpeechWaveform::new()
                    .levels(app.levels.iter().copied())
                    .capturing(app.status.is_capturing())
                    .size(ControlSize::XSmall)
                    .width(48.0),
                SpeechButton::new()
                    .status(app.status)
                    .size(ControlSize::XSmall)
                    .on_toggle(Message::Toggle),
            ]
            .spacing(theme::SPACING_SM)
            .align_y(Axis::Center),
        ),
    line(&app.transcript, theme::FONT_SIZE_SM, |tokens| tokens
        .muted_foreground,),
]
.spacing(theme::SPACING_SM);

use gpui_kit::component::{
    WindowExt as _, h_flex,
    input::{Input, InputState},
    notification::Notification,
    speech::{SpeechButton, SpeechEvent, SpeechState, SpeechWaveform},
};

let input = cx.new(|cx| InputState::new(window, cx));
let speech = cx.new(SpeechState::new);

let subscription = cx.subscribe_in(&speech, window, {
    let input = input.clone();
    move |_, _, event, window, cx| match event {
        SpeechEvent::Final(text) if !text.is_empty() => {
            input.update(cx, |input, cx| input.insert(text.clone(), window, cx));
        }
        SpeechEvent::Error(error) => {
            window.push_notification(Notification::error(error.to_string()), cx);
        }
        _ => {}
    }
});

VerticalLayout {
    spacing: Theme.spacing-sm;
    field := Input {
        placeholder: "Type or dictate";
        min-width: 280px;
        // No `if` around it: an empty history draws nothing, and a
        // conditional here would move both controls off the trailing
        // edge.
        SpeechWaveform {
            levels: root.levels;
            capturing: Speech.is-capturing(root.status);
            interval: root.level-interval;
            size: ControlSize.xsmall;
            width: 48px;
        }
        SpeechButton {
            status: root.status;
            size: ControlSize.xsmall;
            toggled(active) => {
                if active {
                    root.start-session();
                }
                else {
                    root.stop-session();
                }
            }
        }
    }
    if Speech.is-active(root.status): Text {
        text: root.transcript;
        color: Theme.muted-foreground;
        font-size: Theme.font-size-sm;
        wrap: word-wrap;
    }
}

let capturing = self.speech.read(cx).status().is_capturing();

Input::new(&self.input).suffix(
    h_flex()
        .gap_2()
        .when(capturing, |this| {
            this.child(SpeechWaveform::new(&self.speech).w(px(48.)).xsmall())
        })
        .child(SpeechButton::new(&self.speech).xsmall()),
)

// Every event carries the whole transcript, so the latest one is the preview.
// The events themselves are the application's: the component reports nothing,
// and this demo shows what the host would have carried.
let events = column![
    line(
        "Every event carries the whole transcript, so the latest one is the preview.",
        theme::FONT_SIZE_SM,
        |tokens| tokens.muted_foreground,
    ),
    row![
        SpeechWaveform::new()
            .levels(app.levels.iter().copied())
            .capturing(app.status.is_capturing())
            .size(ControlSize::Small),
        if app.last_event.is_empty() {
            line("Nothing yet", theme::FONT_SIZE_SM, |tokens| {
                tokens.muted_foreground
            })
        } else {
            line(&app.last_event, theme::FONT_SIZE_SM, |tokens| {
                tokens.foreground
            })
        },
    ]
    .spacing(theme::SPACING_MD)
    .align_y(Axis::Center),
]
.spacing(theme::SPACING_SM);

let speech = self.speech.read(cx);

v_flex()
    .gap_2()
    .child(Input::new(&self.input))
    .when(speech.status().is_active(), |this| {
        this.child(
            div()
                .text_sm()
                .text_color(cx.theme().muted_foreground)
                .child(speech.transcript()),
        )
    })

VerticalLayout {
    spacing: Theme.spacing-sm;
    Text {
        text: "Every event carries the whole transcript, so the latest one is the preview.";
        color: Theme.muted-foreground;
        font-size: Theme.font-size-sm;
        wrap: word-wrap;
    }
    HorizontalLayout {
        spacing: Theme.spacing-md;
        alignment: start;
        cross-axis-alignment: center;
        SpeechWaveform {
            levels: root.levels;
            capturing: Speech.is-capturing(root.status);
            interval: root.level-interval;
            size: ControlSize.small;
            width: 96px;
        }
        Text {
            text: root.transcript == "" ? "Nothing yet" : root.transcript;
            color: root.transcript == "" ? Theme.muted-foreground : Theme.foreground;
            font-size: Theme.font-size-sm;
            wrap: word-wrap;
            vertical-alignment: center;
        }
    }
}

// The state the component reads, and the three calls an application makes to
// it: start, stop, cancel. iced's element cannot own the session, so the
// buttons move the one this demo holds.
let session_status = format!("status: {}", status_text(app.status));
let session = column![
    row![
        control("Start", !app.status.is_active(), Message::Start),
        control("Stop", app.status.is_capturing(), Message::Stop),
        control("Cancel", app.status.is_active(), Message::Cancel),
    ]
    .spacing(theme::SPACING_SM)
    .align_y(Axis::Center),
    owned_muted(session_status, theme::FONT_SIZE_SM),
]
.spacing(theme::SPACING_SM);

speech.update(cx, |speech, cx| speech.start(cx));  // Does nothing while running.
speech.update(cx, |speech, cx| speech.stop(cx));   // Emits `Final` once the result is in.
speech.update(cx, |speech, cx| speech.cancel(cx)); // Emits `Cancelled` at once.
speech.update(cx, |speech, cx| speech.toggle(cx)); // `start` when idle, otherwise `stop`.

VerticalLayout {
    spacing: Theme.spacing-sm;
    HorizontalLayout {
        spacing: Theme.spacing-sm;
        alignment: start;
        Button {
            text: "Start";
            size: ControlSize.xsmall;
            enabled: !Speech.is-active(root.status);
            clicked => {
                root.start-session();
            }
        }
        Button {
            text: "Stop";
            size: ControlSize.xsmall;
            enabled: Speech.is-capturing(root.status);
            clicked => {
                root.stop-session();
            }
        }
        Button {
            text: "Cancel";
            size: ControlSize.xsmall;
            enabled: Speech.is-active(root.status);
            clicked => {
                root.cancel-session();
            }
        }
    }
    Text {
        text: "status: " + root.status-text();
        color: Theme.muted-foreground;
        font-size: Theme.font-size-sm;
    }
}

let speech = cx.new(|cx| SpeechState::new(cx).stop_timeout(Duration::from_secs(5)));

// The same control in its six states: idle, capturing, stopping, disabled,
// unavailable, and hidden when the host has no recognizer at all.
let states = row![
    state_cell(
        SpeechButton::new()
            .status(SpeechStatus::Idle)
            .size(ControlSize::XSmall),
        "Idle",
    ),
    state_cell(
        SpeechButton::new()
            .status(SpeechStatus::Recording)
            .size(ControlSize::XSmall),
        "Capturing",
    ),
    state_cell(
        SpeechButton::new()
            .status(SpeechStatus::Stopping)
            .size(ControlSize::XSmall),
        "Stopping",
    ),
    state_cell(
        SpeechButton::new().disabled(true).size(ControlSize::XSmall),
        "Disabled",
    ),
    state_cell(
        SpeechButton::new()
            .available(false)
            .size(ControlSize::XSmall),
        "Unavailable",
    ),
    state_cell(
        SpeechButton::new()
            .has_recognizer(false)
            .size(ControlSize::XSmall),
        "Hidden",
    ),
]
.spacing(theme::SPACING_LG)
.align_y(Axis::Start);

SpeechButton::new(&speech)
    .small()
    .show_when_unsupported(true)
    .disabled(readonly)

GridLayout {
    horizontal-stretch: 1;
    spacing: 16px;
    VerticalLayout {
        spacing: Theme.spacing-xs;
        alignment: center;
        // The cell is as wide as its caption, so the button needs
        // centring across it.
        cross-axis-alignment: center;
        SpeechButton {
            status: SpeechStatus.idle;
            size: ControlSize.xsmall;
        }
        Text {
            text: "Idle";
            font-size: Theme.font-size-xs;
            color: Theme.muted-foreground;
            horizontal-alignment: center;
        }
    }
    VerticalLayout {
        spacing: Theme.spacing-xs;
        alignment: center;
        // The cell is as wide as its caption, so the button needs
        // centring across it.
        cross-axis-alignment: center;
        SpeechButton {
            status: SpeechStatus.recording;
            size: ControlSize.xsmall;
        }
        Text {
            text: "Capturing";
            font-size: Theme.font-size-xs;
            color: Theme.muted-foreground;
            horizontal-alignment: center;
        }
    }
    VerticalLayout {
        spacing: Theme.spacing-xs;
        alignment: center;
        // The cell is as wide as its caption, so the button needs
        // centring across it.
        cross-axis-alignment: center;
        SpeechButton {
            status: SpeechStatus.stopping;
            size: ControlSize.xsmall;
        }
        Text {
            text: "Stopping";
            font-size: Theme.font-size-xs;
            color: Theme.muted-foreground;
            horizontal-alignment: center;
        }
    }
    VerticalLayout {
        spacing: Theme.spacing-xs;
        alignment: center;
        // The cell is as wide as its caption, so the button needs
        // centring across it.
        cross-axis-alignment: center;
        SpeechButton {
            status: SpeechStatus.idle;
            size: ControlSize.xsmall;
            enabled: false;
        }
        Text {
            text: "Disabled";
            font-size: Theme.font-size-xs;
            color: Theme.muted-foreground;
            horizontal-alignment: center;
        }
    }
    VerticalLayout {
        spacing: Theme.spacing-xs;
        alignment: center;
        // The cell is as wide as its caption, so the button needs
        // centring across it.
        cross-axis-alignment: center;
        SpeechButton {
            available: false;
            size: ControlSize.xsmall;
        }
        Text {
            text: "Unavailable";
            font-size: Theme.font-size-xs;
            color: Theme.muted-foreground;
            horizontal-alignment: center;
        }
    }
    VerticalLayout {
        spacing: Theme.spacing-xs;
        alignment: center;
        // The cell is as wide as its caption, so the button needs
        // centring across it.
        cross-axis-alignment: center;
        SpeechButton {
            has-recognizer: false;
            size: ControlSize.xsmall;
        }
        Text {
            text: "Hidden";
            font-size: Theme.font-size-xs;
            color: Theme.muted-foreground;
            horizontal-alignment: center;
        }
    }
}

use gpui_kit::component::speech::SystemRecognizer;

// Dictate Chinese regardless of the system language.
let speech = cx.new(|cx| {
    SpeechState::new(cx).recognizer(SystemRecognizer::new().locale("zh-CN"))
});

// Use the application's own recognizer instead of the system's.
let speech = cx.new(|cx| SpeechState::new(cx).recognizer(CloudRecognizer::new(client)));

// Never fall back to the system recognizer. Without a recognizer of its own,
// `has_recognizer()` is false and the button renders nothing.
let speech = cx.new(|cx| SpeechState::new(cx).system_fallback(false));

use gpui_kit::App;
use gpui_kit::component::speech::{
    RecognitionSession, SpeechError, SpeechRecognizer, SpeechSink,
};

/// Reports how much audio it has heard, as a stand-in for a real service.
struct DurationRecognizer;

impl SpeechRecognizer for DurationRecognizer {
    fn start(
        &self,
        sink: SpeechSink,
        cx: &mut App,
    ) -> Result<Box<dyn RecognitionSession>, SpeechError> {
        // A service would connect here and call `ready` once connected.
        sink.ready(cx);
        Ok(Box::new(DurationSession { sink, samples: 0 }))
    }
}

struct DurationSession {
    sink: SpeechSink,
    samples: usize,
}

impl RecognitionSession for DurationSession {
    fn push_audio(&mut self, samples: &[i16], cx: &mut App) {
        // 16 kHz mono, the default `audio_format`.
        self.samples += samples.len();
        let seconds = self.samples / 16_000;
        self.sink.hypothesis(format!("{seconds} s of audio"), cx);
    }

    fn finish(&mut self, cx: &mut App) {
        let seconds = self.samples / 16_000;
        self.sink.phrase(format!("{seconds} s of audio"), cx);
        self.sink.finish(cx);
    }
}

use std::time::Duration;

use gpui_kit::{App, Subscription};
use gpui_kit::component::speech::{AudioFormat, AudioInput, AudioSink, SpeechError};

/// Pushes 100 ms of silence at a time.
struct Silence;

impl AudioInput for Silence {
    fn start(
        &self,
        format: AudioFormat,
        sink: AudioSink,
        cx: &mut App,
    ) -> Result<Subscription, SpeechError> {
        let len = format.sample_rate() as usize / 10 * format.channels() as usize;
        let task = cx.spawn(async move |cx| {
            loop {
                cx.background_executor().timer(Duration::from_millis(100)).await;
                cx.update(|cx| sink.push(vec![0; len], cx));
            }
        });
        // Capture runs until the state drops this subscription.
        Ok(Subscription::new(move || drop(task)))
    }
}

let speech = cx.new(|cx| SpeechState::new(cx).recognizer(recognizer).input(Silence));

<key>NSMicrophoneUsageDescription</key>
<string>Dictate text into messages.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>Turn your speech into text.</string>