Speech-to-Text and AI Voice Features on Android

sealed interface SpeechEvent {
    data class Partial(val text: String) : SpeechEvent
    data class Final(val text: String) : SpeechEvent
    data class Error(val code: Int) : SpeechEvent
}

@RequiresPermission(Manifest.permission.RECORD_AUDIO)
fun Context.listen(languageTag: String = Locale.getDefault().toLanguageTag()): Flow<SpeechEvent> = callbackFlow {
    val recognizer =
        if (Build.VERSION.SDK_INT >= 31 && SpeechRecognizer.isOnDeviceRecognitionAvailable(this@listen))
            SpeechRecognizer.createOnDeviceSpeechRecognizer(this@listen)   // audio stays on device
        else SpeechRecognizer.createSpeechRecognizer(this@listen)

    recognizer.setRecognitionListener(object : RecognitionListener {
        override fun onPartialResults(b: Bundle) {
            b.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()?.let { trySend(SpeechEvent.Partial(it)) }
        }
        override fun onResults(b: Bundle) {
            trySend(SpeechEvent.Final(b.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull().orEmpty()))
            close()
        }
        override fun onError(error: Int) { trySend(SpeechEvent.Error(error)); close() }
        override fun onReadyForSpeech(p: Bundle?) {}
        override fun onBeginningOfSpeech() {}
        override fun onRmsChanged(rms: Float) {}
        override fun onBufferReceived(b: ByteArray?) {}
        override fun onEndOfSpeech() {}
        override fun onEvent(t: Int, b: Bundle?) {}
    })
    recognizer.startListening(Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
        putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
        putExtra(RecognizerIntent.EXTRA_LANGUAGE, languageTag)            // e.g. "hi-IN", "en-IN"
        putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
    })
    awaitClose { recognizer.destroy() }
}.flowOn(Dispatchers.Main)   // SpeechRecognizer must be used from the main thread

@Serializable data class ReminderIntent(val title: String, val dateTimeIso: String?)

suspend fun parseReminder(model: GenerativeModel, transcript: String, nowIso: String): ReminderIntent? {
    val response = model.generateContent(
        "Current time: $nowIso. Extract a reminder from the user's words as JSON with fields " +
        "title and dateTimeIso (ISO-8601, null if no time). User said: <said>$transcript</said>"
    )
    return runCatching { Json { ignoreUnknownKeys = true }.decodeFromString<ReminderIntent>(response.text.orEmpty()) }.getOrNull()
}
// Always show: "Create reminder 'Call the plumber' for tomorrow 10:00?" → user confirms.

class Speaker(context: Context) : TextToSpeech.OnInitListener {
    private val tts = TextToSpeech(context, this)
    private var ready = false
    override fun onInit(status: Int) { ready = status == TextToSpeech.SUCCESS }
    fun say(text: String, locale: Locale = Locale.getDefault()) {
        if (!ready) return
        tts.language = locale
        tts.speak(text, TextToSpeech.QUEUE_FLUSH, null, "utt-${text.hashCode()}")
    }
    fun shutdown() = tts.shutdown()
}