feat(learn,todo,meditate): dictate through the phone's voice input app (ADR 0038, D18)
On Android the microphone now opens the app handling RECOGNIZE_SPEECH (FUTO Voice Input, Google's), the default the person chose in Android, and puts the text it returns where the cursor stood, as Dictation does; cancelled, nothing. Only where none is installed, or it can't be opened, does speech_to_text listen in line as before. halkyon_app: VoiceInputApp (one listen, an answer once closed) on the channel app.halkyon/voice_input, which DeviceSpeechInput asks first at every listen, answering SpeechStart.finished so that Dictation doesn't listen again; speech_to_text isn't touched on that route, so our app asks for no microphone. SpeechInput.listen and Dictation.start take the prompt the voice input app shows (the field's hint). Each app: VoiceInput.kt (queryIntentActivities, startActivityForResult, the first of EXTRA_RESULTS) attached in MainActivity, which hands it activity results, and RECOGNIZE_SPEECH among the manifest's queries. The privacy pages of Learn and Meditate say so. Our own Whisper and Meditate's dream recordings are dropped (D18). Whats-New: The microphone now dictates through your phone's voice input app, such as FUTO Voice Input. Whats-New-De: Das Mikrofon diktiert jetzt über die Spracheingabe-App deines Telefons, etwa FUTO Voice Input. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
4940230ef1
commit
c36bd83c6a
8 changed files with 827 additions and 27 deletions
|
|
@ -187,6 +187,7 @@ export 'src/simplelogin/widget/simplelogin_standing_scope.dart';
|
|||
export 'src/simplelogin/widget/simplelogin_texts.dart';
|
||||
export 'src/speech/dictation.dart';
|
||||
export 'src/speech/speech_input.dart';
|
||||
export 'src/speech/voice_input_app.dart';
|
||||
export 'src/sync/app_compatibility.dart';
|
||||
export 'src/sync/bloc/sync_status_bloc.dart';
|
||||
export 'src/sync/pending_changes_store.dart';
|
||||
|
|
|
|||
|
|
@ -56,7 +56,9 @@ final class DictationInsertion._(
|
|||
/// [heard] holds the listen's words so far, to show. A recogniser stops
|
||||
/// after a pause, so while the person leaves it on, it listens again at
|
||||
/// once, the next words after the last, until it has heard nothing
|
||||
/// [silencesBeforeStopping] times running or the person stops it.
|
||||
/// [silencesBeforeStopping] times running or the person stops it. Where
|
||||
/// the device's voice input app listens instead (D18), it is on while that
|
||||
/// app is open, and what it returns goes where the cursor stood, once.
|
||||
class Dictation(
|
||||
final SpeechInput _speech,
|
||||
final TextEditingController _text,
|
||||
|
|
@ -69,6 +71,7 @@ class Dictation(
|
|||
var _heard = '';
|
||||
var _silences = 0;
|
||||
String? _language;
|
||||
String? _prompt;
|
||||
|
||||
/// Counts the listens, so a listen ended late tells nothing.
|
||||
var _listen = 0;
|
||||
|
|
@ -83,14 +86,17 @@ class Dictation(
|
|||
String get heard => _heard;
|
||||
|
||||
/// Starts listening in [language] (the device's own where null); off
|
||||
/// again where the device can't listen, the answer saying why.
|
||||
Future<SpeechStart> start({required String? language}) async {
|
||||
/// again where the device can't listen, the answer saying why, or once
|
||||
/// a voice input app has listened ([SpeechStart.finished]), which shows
|
||||
/// [prompt] where it shows one.
|
||||
Future<SpeechStart> start({required String? language, String? prompt}) async {
|
||||
if (_on) {
|
||||
return SpeechStart.listening;
|
||||
}
|
||||
_on = true;
|
||||
_silences = 0;
|
||||
_language = language;
|
||||
_prompt = prompt;
|
||||
notifyListeners();
|
||||
return await _listenOnce();
|
||||
}
|
||||
|
|
@ -101,6 +107,7 @@ class Dictation(
|
|||
_heard = '';
|
||||
final listens = await _speech.listen(
|
||||
language: _language,
|
||||
prompt: _prompt,
|
||||
onWords: (words) {
|
||||
if (listen == _listen && _on) {
|
||||
_heard = words;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:halkyon_app/src/speech/voice_input_app.dart';
|
||||
import 'package:speech_to_text/speech_to_text.dart';
|
||||
|
||||
/// How asking a [SpeechInput] to listen went.
|
||||
|
|
@ -12,16 +13,23 @@ enum SpeechStart() {
|
|||
noRecognizer,
|
||||
|
||||
/// It can't listen now, as with the microphone refused.
|
||||
unavailable;
|
||||
unavailable,
|
||||
|
||||
/// Whether it listens.
|
||||
/// Another app listened, the device's voice input app (Halkyon Meditate's
|
||||
/// ADR 0038, D18), and what it heard is handed on: the listen is over,
|
||||
/// and isn't begun again.
|
||||
finished;
|
||||
|
||||
/// Whether it listens on.
|
||||
bool get listens => this == listening;
|
||||
}
|
||||
|
||||
/// Turns what the person says into words (Halkyon Learn's ADR 0021, D67;
|
||||
/// Halkyon Meditate's ADR 0038, D15), through the device's own speech
|
||||
/// recognition: Android's (which may send the sound to Google or the
|
||||
/// phone's maker), Apple's, Windows'. The instance never hears it.
|
||||
/// Halkyon Meditate's ADR 0038, D15 and D18), through the device's own
|
||||
/// speech recognition: on Android its voice input app where one is
|
||||
/// installed (FUTO Voice Input, Google's), else Android's recogniser (which
|
||||
/// may send the sound to Google or the phone's maker); Apple's, Windows'.
|
||||
/// The instance never hears it.
|
||||
abstract interface class SpeechInput() {
|
||||
/// Whether this kind of device recognises speech at all; Linux doesn't.
|
||||
/// Asks nothing, so the microphone's permission is only asked for once
|
||||
|
|
@ -34,19 +42,29 @@ abstract interface class SpeechInput() {
|
|||
/// is called, then calls [onDone]. Where the device can't listen now,
|
||||
/// nothing is heard and [onDone] isn't called, and the answer says why:
|
||||
/// no recogniser installed, or another reason, as the microphone refused.
|
||||
/// Never throws.
|
||||
/// Where another app listens instead ([SpeechStart.finished]), the answer
|
||||
/// comes once it is closed again, having handed on to [onWords] what it
|
||||
/// heard, if anything, and [onDone] isn't called; [prompt] is what that
|
||||
/// app shows, where it shows one. Never throws.
|
||||
Future<SpeechStart> listen({
|
||||
required String? language,
|
||||
required ValueChanged<String> onWords,
|
||||
required VoidCallback onDone,
|
||||
String? prompt,
|
||||
});
|
||||
|
||||
/// Stops listening, keeping what was heard.
|
||||
Future<void> stop();
|
||||
}
|
||||
|
||||
/// [SpeechInput] through `speech_to_text`.
|
||||
/// [SpeechInput] through the device's voice input app where one is
|
||||
/// installed (D18: the person picks it in Android's settings, and it
|
||||
/// records, so this app asks for no microphone), else `speech_to_text`.
|
||||
class DeviceSpeechInput({
|
||||
/// The voice input app, asked first at every listen, as one may have been
|
||||
/// installed or removed since.
|
||||
final VoiceInputApp voiceInputApp = const ChannelVoiceInputApp(),
|
||||
|
||||
/// Whether Android is asked to recognise on the device (Android 12 on,
|
||||
/// where it can; offline before), the sound leaving it no more: where that
|
||||
/// fails before a word is heard, it listens as Android otherwise does
|
||||
|
|
@ -88,10 +106,24 @@ class DeviceSpeechInput({
|
|||
required String? language,
|
||||
required ValueChanged<String> onWords,
|
||||
required VoidCallback onDone,
|
||||
String? prompt,
|
||||
}) async {
|
||||
if (!supported) {
|
||||
return SpeechStart.unavailable;
|
||||
}
|
||||
if (await voiceInputApp.installed()) {
|
||||
final words = await voiceInputApp.listen(
|
||||
language: language,
|
||||
prompt: prompt,
|
||||
);
|
||||
if (words != null) {
|
||||
if (words.isNotEmpty) {
|
||||
onWords(words);
|
||||
}
|
||||
return SpeechStart.finished;
|
||||
}
|
||||
// It couldn't be opened: listened to in line, as without one.
|
||||
}
|
||||
final bool initialized;
|
||||
try {
|
||||
initialized = await _speech.initialize(
|
||||
|
|
|
|||
58
packages/halkyon_app/lib/src/speech/voice_input_app.dart
Normal file
58
packages/halkyon_app/lib/src/speech/voice_input_app.dart
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
|
||||
/// The device's voice input app (Halkyon Meditate's ADR 0038, D18): on
|
||||
/// Android, whichever app handles `RecognizerIntent.ACTION_RECOGNIZE_SPEECH`
|
||||
/// (FUTO Voice Input, Google's), the person choosing the default in
|
||||
/// Android itself. It opens over the app, records and recognises on its
|
||||
/// own, and returns text alone, once; no sound reaches the app.
|
||||
///
|
||||
/// One listen, not a stream of words as `SpeechInput`'s in-line recogniser
|
||||
/// gives: hence its own shape, which `DeviceSpeechInput` asks first.
|
||||
abstract interface class VoiceInputApp() {
|
||||
/// Whether one is installed. Never throws.
|
||||
Future<bool> installed();
|
||||
|
||||
/// Opens it, listening in [language] (a BCP 47 tag; its own where null)
|
||||
/// with [prompt] shown where it shows one, and answers once it is closed
|
||||
/// again: what it heard, empty where the person cancelled it or it heard
|
||||
/// nothing, null where it couldn't be opened. Never throws.
|
||||
Future<String?> listen({required String? language, String? prompt});
|
||||
}
|
||||
|
||||
/// [VoiceInputApp] through each app's `VoiceInput.kt` on Android
|
||||
/// (`app.halkyon/voice_input`), which starts the intent for its result;
|
||||
/// none elsewhere.
|
||||
class const ChannelVoiceInputApp() implements VoiceInputApp {
|
||||
/// The channel each app's Android activity answers.
|
||||
static const channel = MethodChannel('app.halkyon/voice_input');
|
||||
|
||||
@override
|
||||
Future<bool> installed() async {
|
||||
if (defaultTargetPlatform != TargetPlatform.android) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return await channel.invokeMethod<bool>('installed') ?? false;
|
||||
} on MissingPluginException {
|
||||
return false;
|
||||
} on PlatformException {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<String?> listen({required String? language, String? prompt}) async {
|
||||
try {
|
||||
return await channel.invokeMethod<String>('listen', {
|
||||
'language': language,
|
||||
'prompt': prompt,
|
||||
}) ??
|
||||
'';
|
||||
} on MissingPluginException {
|
||||
return null;
|
||||
} on PlatformException {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,11 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:halkyon_app/src/speech/speech_input.dart';
|
||||
|
||||
/// A [SpeechInput] the test speaks into: [say] hands words on, [finish]
|
||||
/// ends the listen.
|
||||
/// ends the listen; with [app], a voice input app listens instead, until
|
||||
/// [answer] or [cancel].
|
||||
class FakeSpeechInput() implements SpeechInput {
|
||||
@override
|
||||
bool supported = true;
|
||||
|
|
@ -15,9 +18,32 @@ class FakeSpeechInput() implements SpeechInput {
|
|||
/// Google's services.
|
||||
bool hasRecognizer = true;
|
||||
|
||||
/// Whether a voice input app takes the listens, as on Android with one
|
||||
/// installed (Halkyon Meditate's ADR 0038, D18): open until [answer] or
|
||||
/// [cancel], nothing listening in line meanwhile.
|
||||
bool app = false;
|
||||
|
||||
/// The languages asked to listen in, in order.
|
||||
final languages = <String?>[];
|
||||
|
||||
/// The prompts asked to show, in order.
|
||||
final prompts = <String?>[];
|
||||
|
||||
Completer<String>? _app;
|
||||
|
||||
/// Whether the voice input app is open.
|
||||
bool get appOpen => _app != null;
|
||||
|
||||
/// Closes the voice input app, returning [words].
|
||||
void answer(String words) {
|
||||
final open = _app;
|
||||
_app = null;
|
||||
open?.complete(words);
|
||||
}
|
||||
|
||||
/// Closes the voice input app, cancelled: it returns nothing.
|
||||
void cancel() => answer('');
|
||||
|
||||
ValueChanged<String>? _onWords;
|
||||
VoidCallback? _onDone;
|
||||
|
||||
|
|
@ -40,8 +66,17 @@ class FakeSpeechInput() implements SpeechInput {
|
|||
required String? language,
|
||||
required ValueChanged<String> onWords,
|
||||
required VoidCallback onDone,
|
||||
String? prompt,
|
||||
}) async {
|
||||
languages.add(language);
|
||||
prompts.add(prompt);
|
||||
if (app) {
|
||||
final words = await (_app = Completer<String>()).future;
|
||||
if (words.isNotEmpty) {
|
||||
onWords(words);
|
||||
}
|
||||
return SpeechStart.finished;
|
||||
}
|
||||
if (!hasRecognizer) {
|
||||
return SpeechStart.noRecognizer;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,64 @@ void main() {
|
|||
expect(await listen(), SpeechStart.unavailable);
|
||||
});
|
||||
|
||||
group('with a voice input app installed (ADR 0038, D18)', () {
|
||||
late _VoiceInputApp app;
|
||||
late List<String> heard;
|
||||
late SpeechInput speech;
|
||||
|
||||
setUp(() {
|
||||
// speech_to_text answering as with the microphone refused, which
|
||||
// leaves its one instance uninitialised for the tests after these.
|
||||
answer((call) => call.method == 'initialize' ? false : null);
|
||||
app = _VoiceInputApp();
|
||||
heard = [];
|
||||
speech = DeviceSpeechInput(voiceInputApp: app);
|
||||
});
|
||||
|
||||
Future<SpeechStart> listen() => speech.listen(
|
||||
language: 'de-DE',
|
||||
prompt: 'Was hast du geträumt?',
|
||||
onWords: heard.add,
|
||||
onDone: () => fail('the app has listened, once'),
|
||||
);
|
||||
|
||||
test('hands listening to it, in the language and with the prompt '
|
||||
'given, and the words it returns on, once', () async {
|
||||
app.words = 'Ich war am Meer';
|
||||
expect(await listen(), SpeechStart.finished);
|
||||
expect(app.asked, [('de-DE', 'Was hast du geträumt?')]);
|
||||
expect(heard, ['Ich war am Meer']);
|
||||
});
|
||||
|
||||
test('asks speech_to_text nothing, so our app never asks for the '
|
||||
'microphone: the voice input app records', () async {
|
||||
app.words = 'Ich war am Meer';
|
||||
await listen();
|
||||
expect(calls, isEmpty);
|
||||
});
|
||||
|
||||
test('cancelled in it, hands nothing on and listens no more', () async {
|
||||
app.words = '';
|
||||
expect(await listen(), SpeechStart.finished);
|
||||
expect(heard, isEmpty);
|
||||
expect(calls, isEmpty);
|
||||
});
|
||||
|
||||
test('where it could not be opened, listens in line instead', () async {
|
||||
app.words = null;
|
||||
expect(await listen(), SpeechStart.unavailable);
|
||||
expect(app.asked, hasLength(1));
|
||||
expect(calls.first.method, 'initialize');
|
||||
});
|
||||
|
||||
test('none installed, listens in line', () async {
|
||||
app.installs = false;
|
||||
expect(await listen(), SpeechStart.unavailable);
|
||||
expect(app.asked, isEmpty);
|
||||
expect(calls.first.method, 'initialize');
|
||||
});
|
||||
});
|
||||
|
||||
// Last: `speech_to_text` keeps one instance, initialised from here on.
|
||||
test('a listen that fails to start says it cannot listen now', () async {
|
||||
answer(
|
||||
|
|
@ -71,3 +129,22 @@ void main() {
|
|||
expect(await listen(), SpeechStart.unavailable);
|
||||
});
|
||||
}
|
||||
|
||||
/// A voice input app the test sets up: installed or not, returning [words]
|
||||
/// ('' as cancelled, null as failing to open).
|
||||
class _VoiceInputApp() implements VoiceInputApp {
|
||||
bool installs = true;
|
||||
String? words = '';
|
||||
|
||||
/// The language and prompt of each listen, in order.
|
||||
final asked = <(String?, String?)>[];
|
||||
|
||||
@override
|
||||
Future<bool> installed() async => installs;
|
||||
|
||||
@override
|
||||
Future<String?> listen({required String? language, String? prompt}) async {
|
||||
asked.add((language, prompt));
|
||||
return words;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,6 +166,52 @@ void main() {
|
|||
expect(told, 2, reason: 'on, then off again');
|
||||
});
|
||||
|
||||
group('through a voice input app (ADR 0038, D18)', () {
|
||||
setUp(() => speech.app = true);
|
||||
|
||||
test('writes what it returns where the cursor stood, once, and '
|
||||
'listens no more', () async {
|
||||
text.value = const TextEditingValue(
|
||||
text: 'I was there.',
|
||||
selection: TextSelection.collapsed(offset: 5),
|
||||
);
|
||||
final started = dictation.start(language: 'en', prompt: 'Your dream');
|
||||
await pumpEventQueue();
|
||||
expect(dictation.listening, isTrue, reason: 'while the app is open');
|
||||
expect(speech.appOpen, isTrue);
|
||||
expect(speech.listening, isFalse, reason: 'nothing else listens');
|
||||
expect(speech.prompts, ['Your dream']);
|
||||
|
||||
speech.answer('at sea');
|
||||
expect(await started, SpeechStart.finished);
|
||||
expect(text.text, 'I was at sea there.');
|
||||
expect(text.selection, const TextSelection.collapsed(offset: 12));
|
||||
expect(dictation.listening, isFalse);
|
||||
expect(speech.languages, ['en'], reason: 'not begun again');
|
||||
});
|
||||
|
||||
test('cancelled, writes nothing', () async {
|
||||
text.text = 'I was';
|
||||
final started = dictation.start(language: 'en');
|
||||
await pumpEventQueue();
|
||||
speech.cancel();
|
||||
expect(await started, SpeechStart.finished);
|
||||
expect(text.text, 'I was');
|
||||
expect(dictation.listening, isFalse);
|
||||
});
|
||||
|
||||
test('stopped while the app is open, writes nothing it returns '
|
||||
'later', () async {
|
||||
final started = dictation.start(language: 'en');
|
||||
await pumpEventQueue();
|
||||
await dictation.stop();
|
||||
expect(dictation.listening, isFalse);
|
||||
speech.answer('late words');
|
||||
await started;
|
||||
expect(text.text, isEmpty);
|
||||
});
|
||||
});
|
||||
|
||||
test('tells its listeners as it starts, hears and stops', () async {
|
||||
var told = 0;
|
||||
dictation.addListener(() => told++);
|
||||
|
|
|
|||
98
packages/halkyon_app/test/speech/voice_input_app_test.dart
Normal file
98
packages/halkyon_app/test/speech/voice_input_app_test.dart
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:halkyon_app/halkyon_app.dart';
|
||||
|
||||
/// The device's voice input app through Android's `RECOGNIZE_SPEECH`
|
||||
/// intent (Halkyon Meditate's ADR 0038, D18), asked through each app's
|
||||
/// `VoiceInput.kt` on `app.halkyon/voice_input`, its channel answering as
|
||||
/// the activity does.
|
||||
void main() {
|
||||
TestWidgetsFlutterBinding.ensureInitialized();
|
||||
late List<MethodCall> calls;
|
||||
|
||||
void answer(Object? Function(MethodCall call) answers) {
|
||||
calls = [];
|
||||
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger
|
||||
.setMockMethodCallHandler(ChannelVoiceInputApp.channel, (call) async {
|
||||
calls.add(call);
|
||||
return answers(call);
|
||||
});
|
||||
}
|
||||
|
||||
tearDown(() {
|
||||
debugDefaultTargetPlatformOverride = null;
|
||||
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger
|
||||
.setMockMethodCallHandler(ChannelVoiceInputApp.channel, null);
|
||||
});
|
||||
|
||||
const app = ChannelVoiceInputApp();
|
||||
|
||||
group('installed', () {
|
||||
test('where Android has an app for the intent', () async {
|
||||
answer((call) => call.method == 'installed');
|
||||
expect(await app.installed(), isTrue);
|
||||
});
|
||||
|
||||
test('not where it has none', () async {
|
||||
answer((call) => false);
|
||||
expect(await app.installed(), isFalse);
|
||||
});
|
||||
|
||||
test('not where the activity fails or knows no such channel', () async {
|
||||
answer((call) => throw PlatformException(code: 'failed'));
|
||||
expect(await app.installed(), isFalse);
|
||||
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger
|
||||
.setMockMethodCallHandler(ChannelVoiceInputApp.channel, null);
|
||||
expect(await app.installed(), isFalse);
|
||||
});
|
||||
|
||||
test('never off Android, without asking', () async {
|
||||
answer((call) => true);
|
||||
for (final platform in [
|
||||
TargetPlatform.iOS,
|
||||
TargetPlatform.linux,
|
||||
TargetPlatform.macOS,
|
||||
TargetPlatform.windows,
|
||||
]) {
|
||||
debugDefaultTargetPlatformOverride = platform;
|
||||
expect(await app.installed(), isFalse, reason: '$platform');
|
||||
}
|
||||
expect(calls, isEmpty);
|
||||
});
|
||||
});
|
||||
|
||||
group('listening', () {
|
||||
test('asks in the language given, with the prompt, and hands back the '
|
||||
'words', () async {
|
||||
answer((call) => 'a house by the sea');
|
||||
expect(
|
||||
await app.listen(language: 'de-DE', prompt: 'Was hast du geträumt?'),
|
||||
'a house by the sea',
|
||||
);
|
||||
final listen = calls.single;
|
||||
expect(listen.method, 'listen');
|
||||
expect(listen.arguments, {
|
||||
'language': 'de-DE',
|
||||
'prompt': 'Was hast du geträumt?',
|
||||
});
|
||||
});
|
||||
|
||||
test('cancelled, hands back nothing', () async {
|
||||
answer((call) => '');
|
||||
expect(await app.listen(language: null), '');
|
||||
expect(calls.single.arguments, {'language': null, 'prompt': null});
|
||||
});
|
||||
|
||||
test(
|
||||
'where the app could not be opened, says so without throwing',
|
||||
() async {
|
||||
answer((call) => throw PlatformException(code: 'noApp'));
|
||||
expect(await app.listen(language: 'en'), isNull);
|
||||
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger
|
||||
.setMockMethodCallHandler(ChannelVoiceInputApp.channel, null);
|
||||
expect(await app.listen(language: 'en'), isNull);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue