feat(learn): voice mode, a conversation spoken as Claude's "Let's talk" (ADR 0021, D68)
Beside the field's microphone, a button opens voice mode over the whole screen: a glow rising from below, a big microphone, and +, who answers, the speaker and End in one row. It listens with the device's recogniser, asks what was heard in one kept conversation, reads the answer aloud with the device's speech (flutter_tts, its Markdown left out) and listens again. On Linux there is none, as there is no microphone; where nothing reads aloud, answers are shown as text. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
7be31dcbfc
commit
c984678c5b
GPG key ID:
07F6FCF405488B64
22 changed files with 1180 additions and 6 deletions
|
|
@ -96,5 +96,10 @@
|
|||
<intent>
|
||||
<action android:name="android.speech.RecognitionService"/>
|
||||
</intent>
|
||||
<!-- The device's text-to-speech engines, which read an assistant's
|
||||
answers aloud in voice mode (ADR 0021, D68). -->
|
||||
<intent>
|
||||
<action android:name="android.intent.action.TTS_SERVICE"/>
|
||||
</intent>
|
||||
</queries>
|
||||
</manifest>
|
||||
|
|
|
|||
|
|
@ -2135,5 +2135,15 @@
|
|||
"assistantAddTitle": "Zu deiner Frage hinzufügen",
|
||||
"assistantAddCamera": "Kamera",
|
||||
"assistantAddPhotos": "Fotos",
|
||||
"assistantAddFiles": "Dateien"
|
||||
"assistantAddFiles": "Dateien",
|
||||
"assistantTalk": "Sprachmodus",
|
||||
"voiceTitle": "Lass uns reden",
|
||||
"voiceListening": "Ich höre zu …",
|
||||
"voiceThinking": "Ich denke nach …",
|
||||
"voiceTapToTalk": "Tippe aufs Mikrofon, um zu sprechen",
|
||||
"voiceTalk": "Sprechen",
|
||||
"voiceDone": "Senden",
|
||||
"voiceSpeakerOff": "Antworten nicht vorlesen",
|
||||
"voiceSpeakerOn": "Antworten vorlesen",
|
||||
"voiceEnd": "Sprachmodus beenden"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2557,5 +2557,25 @@
|
|||
"assistantAddPhotos": "Photos",
|
||||
"@assistantAddPhotos": {"description": "Tile in +'s sheet: picks photos from the device."},
|
||||
"assistantAddFiles": "Files",
|
||||
"@assistantAddFiles": {"description": "Tile in +'s sheet: picks files (PDFs, texts, images)."}
|
||||
"@assistantAddFiles": {"description": "Tile in +'s sheet: picks files (PDFs, texts, images)."},
|
||||
"assistantTalk": "Voice mode",
|
||||
"@assistantTalk": {"description": "The field's button opening voice mode, a conversation spoken rather than typed (ADR 0021, D68)"},
|
||||
"voiceTitle": "Let's talk",
|
||||
"@voiceTitle": {"description": "Voice mode's heading"},
|
||||
"voiceListening": "Listening…",
|
||||
"@voiceListening": {"description": "Voice mode while it hears the person"},
|
||||
"voiceThinking": "Thinking…",
|
||||
"@voiceThinking": {"description": "Voice mode while the answer is awaited"},
|
||||
"voiceTapToTalk": "Tap the microphone to talk",
|
||||
"@voiceTapToTalk": {"description": "Voice mode waiting for the microphone"},
|
||||
"voiceTalk": "Talk",
|
||||
"@voiceTalk": {"description": "The microphone's tooltip: listens, stopping any reading"},
|
||||
"voiceDone": "Send",
|
||||
"@voiceDone": {"description": "The microphone's tooltip while listening: stops and asks what was heard"},
|
||||
"voiceSpeakerOff": "Don't read answers aloud",
|
||||
"@voiceSpeakerOff": {"description": "Switches reading answers aloud off"},
|
||||
"voiceSpeakerOn": "Read answers aloud",
|
||||
"@voiceSpeakerOn": {"description": "Switches reading answers aloud on"},
|
||||
"voiceEnd": "End voice mode",
|
||||
"@voiceEnd": {"description": "Closes voice mode"}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12601,6 +12601,66 @@ abstract class AppLocalizations {
|
|||
/// In en, this message translates to:
|
||||
/// **'Files'**
|
||||
String get assistantAddFiles;
|
||||
|
||||
/// The field's button opening voice mode, a conversation spoken rather than typed (ADR 0021, D68)
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Voice mode'**
|
||||
String get assistantTalk;
|
||||
|
||||
/// Voice mode's heading
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Let\'s talk'**
|
||||
String get voiceTitle;
|
||||
|
||||
/// Voice mode while it hears the person
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Listening…'**
|
||||
String get voiceListening;
|
||||
|
||||
/// Voice mode while the answer is awaited
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Thinking…'**
|
||||
String get voiceThinking;
|
||||
|
||||
/// Voice mode waiting for the microphone
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Tap the microphone to talk'**
|
||||
String get voiceTapToTalk;
|
||||
|
||||
/// The microphone's tooltip: listens, stopping any reading
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Talk'**
|
||||
String get voiceTalk;
|
||||
|
||||
/// The microphone's tooltip while listening: stops and asks what was heard
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Send'**
|
||||
String get voiceDone;
|
||||
|
||||
/// Switches reading answers aloud off
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Don\'t read answers aloud'**
|
||||
String get voiceSpeakerOff;
|
||||
|
||||
/// Switches reading answers aloud on
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Read answers aloud'**
|
||||
String get voiceSpeakerOn;
|
||||
|
||||
/// Closes voice mode
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'End voice mode'**
|
||||
String get voiceEnd;
|
||||
}
|
||||
|
||||
class _AppLocalizationsDelegate
|
||||
|
|
|
|||
|
|
@ -8079,4 +8079,34 @@ class AppLocalizationsDe extends AppLocalizations {
|
|||
|
||||
@override
|
||||
String get assistantAddFiles => 'Dateien';
|
||||
|
||||
@override
|
||||
String get assistantTalk => 'Sprachmodus';
|
||||
|
||||
@override
|
||||
String get voiceTitle => 'Lass uns reden';
|
||||
|
||||
@override
|
||||
String get voiceListening => 'Ich höre zu …';
|
||||
|
||||
@override
|
||||
String get voiceThinking => 'Ich denke nach …';
|
||||
|
||||
@override
|
||||
String get voiceTapToTalk => 'Tippe aufs Mikrofon, um zu sprechen';
|
||||
|
||||
@override
|
||||
String get voiceTalk => 'Sprechen';
|
||||
|
||||
@override
|
||||
String get voiceDone => 'Senden';
|
||||
|
||||
@override
|
||||
String get voiceSpeakerOff => 'Antworten nicht vorlesen';
|
||||
|
||||
@override
|
||||
String get voiceSpeakerOn => 'Antworten vorlesen';
|
||||
|
||||
@override
|
||||
String get voiceEnd => 'Sprachmodus beenden';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8006,6 +8006,36 @@ class AppLocalizationsEn extends AppLocalizations {
|
|||
|
||||
@override
|
||||
String get assistantAddFiles => 'Files';
|
||||
|
||||
@override
|
||||
String get assistantTalk => 'Voice mode';
|
||||
|
||||
@override
|
||||
String get voiceTitle => 'Let\'s talk';
|
||||
|
||||
@override
|
||||
String get voiceListening => 'Listening…';
|
||||
|
||||
@override
|
||||
String get voiceThinking => 'Thinking…';
|
||||
|
||||
@override
|
||||
String get voiceTapToTalk => 'Tap the microphone to talk';
|
||||
|
||||
@override
|
||||
String get voiceTalk => 'Talk';
|
||||
|
||||
@override
|
||||
String get voiceDone => 'Send';
|
||||
|
||||
@override
|
||||
String get voiceSpeakerOff => 'Don\'t read answers aloud';
|
||||
|
||||
@override
|
||||
String get voiceSpeakerOn => 'Read answers aloud';
|
||||
|
||||
@override
|
||||
String get voiceEnd => 'End voice mode';
|
||||
}
|
||||
|
||||
/// The translations for English, as used in the United Kingdom (`en_GB`).
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_files
|
|||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_lookups.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/photo_picker.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/speech_output.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/sync_conversation_gateway.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/web_search.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/card/data/card_deletions.dart';
|
||||
|
|
@ -201,6 +202,9 @@ class Dependencies({
|
|||
/// Photos taken or picked to go with a question (D67).
|
||||
required final PhotoPicker photos,
|
||||
|
||||
/// Reads an assistant's answers aloud in voice mode, on the device (D68).
|
||||
required final SpeechOutput speechOutput,
|
||||
|
||||
/// The person's own Jellyfin server, to watch and learn from: the
|
||||
/// device's, kept across instances.
|
||||
required final JellyfinRepository jellyfin,
|
||||
|
|
@ -336,6 +340,7 @@ class Dependencies({
|
|||
webSearch: WebSearch(),
|
||||
speech: DeviceSpeechInput(),
|
||||
photos: ImagePickerPhotoPicker(),
|
||||
speechOutput: DeviceSpeechOutput(),
|
||||
jellyfin: device.jellyfin,
|
||||
jellyfinSuggestion: JellyfinSuggestion(
|
||||
() async => (await client.instance.info()).jellyfinUrl,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter_tts/flutter_tts.dart';
|
||||
|
||||
/// Reads text aloud with the device's own text-to-speech (ADR 0021, D68):
|
||||
/// Android's engine, Apple's, Windows'. Nothing leaves the device.
|
||||
abstract interface class SpeechOutput() {
|
||||
/// Whether this kind of device reads aloud at all; Linux doesn't.
|
||||
bool get supported;
|
||||
|
||||
/// Reads [text] aloud in [language] (a BCP 47 tag; the device's own
|
||||
/// voice where it has none for it), completing once it has been read,
|
||||
/// stopped, or couldn't be.
|
||||
Future<void> speak(String text, {String? language});
|
||||
|
||||
/// Stops reading.
|
||||
Future<void> stop();
|
||||
}
|
||||
|
||||
/// [SpeechOutput] through `flutter_tts`.
|
||||
class DeviceSpeechOutput() implements SpeechOutput {
|
||||
final _tts = FlutterTts();
|
||||
|
||||
/// Completes the reading under way.
|
||||
Completer<void>? _reading;
|
||||
|
||||
var _ready = false;
|
||||
|
||||
@override
|
||||
bool get supported => switch (defaultTargetPlatform) {
|
||||
TargetPlatform.android ||
|
||||
TargetPlatform.iOS ||
|
||||
TargetPlatform.macOS ||
|
||||
TargetPlatform.windows => true,
|
||||
TargetPlatform.linux || TargetPlatform.fuchsia => false,
|
||||
};
|
||||
|
||||
void _done() {
|
||||
final reading = _reading;
|
||||
_reading = null;
|
||||
if (reading != null && !reading.isCompleted) {
|
||||
reading.complete();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> speak(String text, {String? language}) async {
|
||||
if (!supported || text.trim().isEmpty) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (!_ready) {
|
||||
_ready = true;
|
||||
_tts
|
||||
..setCompletionHandler(_done)
|
||||
..setCancelHandler(_done)
|
||||
..setErrorHandler((_) => _done());
|
||||
}
|
||||
if (language != null &&
|
||||
await _tts.isLanguageAvailable(language) == true) {
|
||||
await _tts.setLanguage(language);
|
||||
}
|
||||
_done();
|
||||
final reading = _reading = Completer<void>();
|
||||
await _tts.speak(text);
|
||||
await reading.future;
|
||||
} on Exception {
|
||||
// No engine, or none that speaks: the answer stays to be read.
|
||||
_done();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> stop() async {
|
||||
try {
|
||||
await _tts.stop();
|
||||
} on Exception {
|
||||
// Nothing was being read.
|
||||
}
|
||||
_done();
|
||||
}
|
||||
}
|
||||
|
||||
/// [markdown], an assistant's answer, as it is read aloud: without its
|
||||
/// marks (emphasis, headings, code, a link's address, a list's bullets, a
|
||||
/// table's bars), which a voice would otherwise spell out.
|
||||
String speakableText(String markdown) {
|
||||
final lines = <String>[];
|
||||
for (final line in markdown.split('\n')) {
|
||||
final trimmed = line.trim();
|
||||
// Code fences and a table's rule go; a table's row reads as its cells.
|
||||
if (trimmed.startsWith('```') ||
|
||||
RegExp(r'^\|?[ \t:|-]+\|[ \t:|-]*$').hasMatch(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
if (trimmed.startsWith('|')) {
|
||||
lines.add(
|
||||
[
|
||||
for (final cell
|
||||
in trimmed.replaceAll(RegExp(r'^\||\|$'), '').split('|'))
|
||||
cell.trim(),
|
||||
].join(', '),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
lines.add(
|
||||
line
|
||||
// Headings, quotes and bullets at a line's start.
|
||||
.replaceFirst(RegExp(r'^[ \t]{0,3}(#{1,6}|>|[-*+]|\d+\.)[ \t]+'), ''),
|
||||
);
|
||||
}
|
||||
final text = lines
|
||||
.join('\n')
|
||||
// [a link](address) reads as its words.
|
||||
.replaceAllMapped(
|
||||
RegExp(r'!?\[([^\]]*)\]\([^)]*\)'),
|
||||
(match) => match.group(1)!,
|
||||
)
|
||||
// Emphasis, inline code and formulas' dollars.
|
||||
.replaceAll(RegExp(r'(\*\*|__|\*|_|`|~~|\$)'), '')
|
||||
.replaceAll(RegExp(r'\n{3,}'), '\n\n');
|
||||
return text.trim();
|
||||
}
|
||||
|
|
@ -18,6 +18,7 @@ import 'package:halkyon_learn_flutter/src/feature/assistant/widget/assistant_wor
|
|||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/composer.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/conversation_history.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/conversation_sheet.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/voice_mode.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/settings/model/person_settings.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/studyset/data/studyset_repository.dart';
|
||||
import 'package:halkyon_llm/halkyon_llm.dart';
|
||||
|
|
@ -273,6 +274,43 @@ class _AssistantFieldState() extends State<AssistantField> {
|
|||
widget.onAsked?.call();
|
||||
}
|
||||
|
||||
/// Opens voice mode with [choice] (ADR 0021, D68), within the field's
|
||||
/// scope, the parts chosen and attached going with the first question;
|
||||
/// first agreed to as any question to [choice] (D40, D9, D17).
|
||||
Future<void> _talk(AssistantChoice choice, List<Answerer> answerers) async {
|
||||
final answerer = AssistantAnswerer(choice);
|
||||
final chosen = [
|
||||
for (final part in _offered(answerer))
|
||||
if (_chosen.contains(part)) part,
|
||||
];
|
||||
final attached = _attachedFor(answerer);
|
||||
final scope = AssistantScope(
|
||||
reach: _reach,
|
||||
studysetId: widget.topic.studysetId,
|
||||
includePrivate: _reach != AssistantReach.ticked && _includePrivate,
|
||||
partsFrom: widget.topic.partsFrom(chosen),
|
||||
);
|
||||
if (!await readyToAsk(context, choice: choice, scope: scope) || !mounted) {
|
||||
return;
|
||||
}
|
||||
final effort = effortFor(
|
||||
choice,
|
||||
context.read<DeviceSettingsBloc>().state.askEfforts,
|
||||
);
|
||||
_clear();
|
||||
await showVoiceMode(
|
||||
context,
|
||||
choice: choice,
|
||||
assistants: [
|
||||
for (final each in answerers)
|
||||
if (each case AssistantAnswerer(:final choice)) choice,
|
||||
],
|
||||
scope: scope,
|
||||
chosen: [...chosen, ...attached],
|
||||
effort: effort,
|
||||
);
|
||||
}
|
||||
|
||||
/// Opens the sheet choosing what goes along with [answerer]'s question.
|
||||
Future<void> _openContext(Answerer answerer) => showAdaptiveSheet<void>(
|
||||
context,
|
||||
|
|
@ -407,6 +445,12 @@ class _AssistantFieldState() extends State<AssistantField> {
|
|||
? l10n.assistantSendAsk
|
||||
: l10n.assistantSendWeb,
|
||||
sendIcon: asksAssistant ? LucideIcons.arrowUp : LucideIcons.search,
|
||||
onTalk: switch (answerer) {
|
||||
AssistantAnswerer(:final choice) => () => unawaited(
|
||||
_talk(choice, answerers),
|
||||
),
|
||||
WebAnswerer() => null,
|
||||
},
|
||||
),
|
||||
],
|
||||
);
|
||||
|
|
|
|||
|
|
@ -355,6 +355,10 @@ class const SpeakOrSend({
|
|||
required final VoidCallback onSend,
|
||||
required final String sendTooltip,
|
||||
final IconData sendIcon = LucideIcons.arrowUp,
|
||||
|
||||
/// Opens voice mode (ADR 0021, D68), offered beside the microphone while
|
||||
/// nothing could be sent, as Claude's field does; none where null.
|
||||
final VoidCallback? onTalk,
|
||||
super.key,
|
||||
}) extends StatefulWidget {
|
||||
@override
|
||||
|
|
@ -439,7 +443,17 @@ class _SpeakOrSendState() extends State<SpeakOrSend> {
|
|||
icon: const Icon(LucideIcons.mic, size: 20),
|
||||
onPressed: () => unawaited(_listen()),
|
||||
),
|
||||
if (canSend) ...[const SizedBox(width: 6), send],
|
||||
if (canSend) ...[
|
||||
const SizedBox(width: 6),
|
||||
send,
|
||||
] else if (widget.onTalk case final talk?) ...[
|
||||
const SizedBox(width: 6),
|
||||
IconButton.filled(
|
||||
tooltip: l10n.assistantTalk,
|
||||
icon: const Icon(LucideIcons.audioLines, size: 20),
|
||||
onPressed: talk,
|
||||
),
|
||||
],
|
||||
],
|
||||
);
|
||||
},
|
||||
|
|
|
|||
|
|
@ -288,7 +288,7 @@ class const _ConversationView({
|
|||
view: TurnView.of(asking.messages),
|
||||
streaming: asking.text,
|
||||
approving: switch (asking.approving) {
|
||||
final image? => _ImageApproval(
|
||||
final image? => ImageApproval(
|
||||
name: name,
|
||||
image: image,
|
||||
),
|
||||
|
|
@ -507,9 +507,10 @@ class const _Turn({
|
|||
|
||||
/// The image [name] asks to see (D12), shown only once the person
|
||||
/// approves it: Show it, or Don't show.
|
||||
class const _ImageApproval({
|
||||
class const ImageApproval({
|
||||
required final String name,
|
||||
required final AskedImage image,
|
||||
super.key,
|
||||
}) extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,580 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_bloc/flutter_bloc.dart';
|
||||
import 'package:halkyon_app/halkyon_app.dart';
|
||||
import 'package:halkyon_learn_flutter/l10n/app_localizations.dart';
|
||||
import 'package:halkyon_learn_flutter/src/core/dependencies/dependencies.dart';
|
||||
import 'package:halkyon_learn_flutter/src/core/dependencies/person_dependencies.dart';
|
||||
import 'package:halkyon_learn_flutter/src/core/device_settings/bloc/device_settings_bloc.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/bloc/conversation_cubit.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/speech_output.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_topic.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/turn_view.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/answerers.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/asking.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/composer.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/conversation_sheet.dart';
|
||||
import 'package:halkyon_llm/halkyon_llm.dart';
|
||||
import 'package:lucide_icons_flutter/lucide_icons.dart';
|
||||
import 'package:material_ui/material_ui.dart';
|
||||
|
||||
/// Opens voice mode (ADR 0021, D68), as Claude's "Let's talk": over the
|
||||
/// whole screen, a conversation with [choice] within [scope] spoken rather
|
||||
/// than typed, the first question going with the parts [chosen], at
|
||||
/// [effort]. Any other assistant among [assistants] can be picked, which
|
||||
/// starts a conversation with it. Completes once it ends.
|
||||
Future<void> showVoiceMode(
|
||||
BuildContext context, {
|
||||
required AssistantChoice choice,
|
||||
required List<AssistantChoice> assistants,
|
||||
required AssistantScope scope,
|
||||
List<AssistantPart> chosen = const [],
|
||||
LlmEffort? effort,
|
||||
}) => Navigator.of(context, rootNavigator: true).push(
|
||||
PageRouteBuilder<void>(
|
||||
pageBuilder: (context, _, _) => _VoiceMode(
|
||||
choice: choice,
|
||||
assistants: assistants,
|
||||
scope: scope,
|
||||
chosen: chosen,
|
||||
effort: effort,
|
||||
),
|
||||
transitionsBuilder: (context, animation, _, child) =>
|
||||
FadeTransition(opacity: animation, child: child),
|
||||
),
|
||||
);
|
||||
|
||||
/// Where voice mode stands.
|
||||
enum _Phase() {
|
||||
/// Waiting for the microphone to be tapped.
|
||||
idle,
|
||||
|
||||
/// Hearing the person.
|
||||
listening,
|
||||
|
||||
/// Waiting for the answer.
|
||||
thinking,
|
||||
|
||||
/// Reading the answer aloud.
|
||||
speaking,
|
||||
}
|
||||
|
||||
class const _VoiceMode({
|
||||
required final AssistantChoice choice,
|
||||
required final List<AssistantChoice> assistants,
|
||||
required final AssistantScope scope,
|
||||
required final List<AssistantPart> chosen,
|
||||
required final LlmEffort? effort,
|
||||
}) extends StatefulWidget {
|
||||
@override
|
||||
State<_VoiceMode> createState() => _VoiceModeState();
|
||||
}
|
||||
|
||||
class _VoiceModeState() extends State<_VoiceMode> {
|
||||
late final _speech = Dependencies.of(context).speech;
|
||||
late final _output = Dependencies.of(context).speechOutput;
|
||||
late var _choice = widget.choice;
|
||||
late ConversationCubit _cubit = _conversationWith(_choice);
|
||||
|
||||
/// The parts chosen in the field, which go with the first question only.
|
||||
late var _chosen = widget.chosen;
|
||||
|
||||
/// The files and photos + attached, for the next question.
|
||||
final _attached = <AssistantPart>[];
|
||||
|
||||
var _phase = _Phase.idle;
|
||||
|
||||
/// What was heard of the question being spoken.
|
||||
var _heard = '';
|
||||
|
||||
/// The last answer, as written.
|
||||
var _answer = '';
|
||||
|
||||
/// Whether answers are read aloud.
|
||||
var _speaker = true;
|
||||
|
||||
/// Whether voice mode is ending, after which nothing listens or reads.
|
||||
var _ending = false;
|
||||
|
||||
String? get _language => Localizations.localeOf(context).toLanguageTag();
|
||||
|
||||
ConversationCubit _conversationWith(AssistantChoice choice) {
|
||||
final person = PersonDependencies.of(context);
|
||||
return ConversationCubit(
|
||||
assistants: Dependencies.of(context).assistants,
|
||||
lookups: person.assistantLookups,
|
||||
conversations: person.conversations,
|
||||
choice: choice,
|
||||
assistant: assistantRefOf(choice),
|
||||
scope: widget.scope,
|
||||
answerIn: Localizations.localeOf(context).toLanguageTag(),
|
||||
effort: choice == widget.choice ? widget.effort : null,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
void didChangeDependencies() {
|
||||
super.didChangeDependencies();
|
||||
if (_phase == _Phase.idle && !_started) {
|
||||
_started = true;
|
||||
unawaited(_start());
|
||||
}
|
||||
}
|
||||
|
||||
var _started = false;
|
||||
|
||||
Future<void> _start() async {
|
||||
await _cubit.open();
|
||||
if (mounted) {
|
||||
await _listen();
|
||||
}
|
||||
}
|
||||
|
||||
/// Listens for a question, stopping any reading first.
|
||||
Future<void> _listen() async {
|
||||
if (_ending) {
|
||||
return;
|
||||
}
|
||||
final messenger = ScaffoldMessenger.of(context);
|
||||
final l10n = AppLocalizations.of(context);
|
||||
// Listening before the reading stops, so the reading ending doesn't
|
||||
// listen a second time.
|
||||
setState(() {
|
||||
_phase = _Phase.listening;
|
||||
_heard = '';
|
||||
});
|
||||
await _output.stop();
|
||||
if (!mounted || _ending) {
|
||||
return;
|
||||
}
|
||||
final listens = await _speech.listen(
|
||||
language: _language,
|
||||
onWords: (words) {
|
||||
if (mounted && _phase == _Phase.listening) {
|
||||
setState(() => _heard = words);
|
||||
}
|
||||
},
|
||||
onDone: () => unawaited(_heardAll()),
|
||||
);
|
||||
if (!listens && mounted) {
|
||||
setState(() => _phase = _Phase.idle);
|
||||
messenger.showSnackBar(
|
||||
SnackBar(content: Text(l10n.assistantSpeechUnavailable)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Asks what was heard, then reads the answer aloud and listens again;
|
||||
/// with nothing heard, waits for the microphone.
|
||||
Future<void> _heardAll() async {
|
||||
if (!mounted || _ending || _phase != _Phase.listening) {
|
||||
return;
|
||||
}
|
||||
final question = _heard.trim();
|
||||
if (question.isEmpty && _attached.isEmpty) {
|
||||
setState(() => _phase = _Phase.idle);
|
||||
return;
|
||||
}
|
||||
final sent = [..._chosen, ..._attached];
|
||||
setState(() {
|
||||
_phase = _Phase.thinking;
|
||||
_answer = '';
|
||||
_chosen = const [];
|
||||
_attached.clear();
|
||||
});
|
||||
await _cubit.ask(question, sent);
|
||||
if (!mounted || _ending) {
|
||||
return;
|
||||
}
|
||||
final state = _cubit.state;
|
||||
if (state.unanswered != null || state.turns.isEmpty) {
|
||||
setState(() => _phase = _Phase.idle);
|
||||
return;
|
||||
}
|
||||
final answer = TurnView.of(state.turns.last.messages).answer;
|
||||
setState(() => _answer = answer);
|
||||
if (_speaker && _output.supported) {
|
||||
setState(() => _phase = _Phase.speaking);
|
||||
await _output.speak(speakableText(answer), language: _language);
|
||||
// Tapping the microphone meanwhile listens already.
|
||||
if (!mounted || _phase != _Phase.speaking) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
await _listen();
|
||||
}
|
||||
|
||||
/// The microphone: stops listening, which asks what was heard; while the
|
||||
/// answer is read or awaited, or while waiting, listens at once.
|
||||
Future<void> _microphone() async {
|
||||
switch (_phase) {
|
||||
case _Phase.listening:
|
||||
await _speech.stop();
|
||||
case _Phase.speaking || _Phase.idle:
|
||||
await _listen();
|
||||
case _Phase.thinking:
|
||||
await _cubit.stop();
|
||||
await _listen();
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _toggleSpeaker() async {
|
||||
setState(() => _speaker = !_speaker);
|
||||
if (!_speaker && _phase == _Phase.speaking) {
|
||||
await _output.stop();
|
||||
await _listen();
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts a conversation with [choice] instead, once it may be asked.
|
||||
Future<void> _pick(AssistantChoice choice) async {
|
||||
if (choice == _choice ||
|
||||
!await readyToAsk(context, choice: choice, scope: widget.scope) ||
|
||||
!mounted) {
|
||||
return;
|
||||
}
|
||||
context.read<DeviceSettingsBloc>().add(
|
||||
DeviceSettingsEventAskedWith(AssistantAnswerer(choice).key),
|
||||
);
|
||||
// Idle first, so stopping asks nothing of the one before.
|
||||
setState(() => _phase = _Phase.idle);
|
||||
await _speech.stop();
|
||||
await _output.stop();
|
||||
if (!mounted) {
|
||||
return;
|
||||
}
|
||||
final previous = _cubit;
|
||||
setState(() {
|
||||
_choice = choice;
|
||||
_cubit = _conversationWith(choice);
|
||||
_answer = '';
|
||||
_heard = '';
|
||||
});
|
||||
await previous.close();
|
||||
await _start();
|
||||
}
|
||||
|
||||
Future<void> _end() async {
|
||||
_ending = true;
|
||||
await _speech.stop();
|
||||
await _output.stop();
|
||||
if (mounted) {
|
||||
Navigator.of(context).pop();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
_ending = true;
|
||||
unawaited(_speech.stop());
|
||||
unawaited(_output.stop());
|
||||
unawaited(_cubit.close());
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final l10n = AppLocalizations.of(context);
|
||||
final theme = Theme.of(context);
|
||||
// What was heard while it's said, the answer otherwise, written as it
|
||||
// is read; beneath it, where voice mode stands.
|
||||
final said = switch (_phase) {
|
||||
_Phase.listening when _heard.isNotEmpty => _heard,
|
||||
_Phase.thinking => _heard,
|
||||
_ => speakableText(_answer),
|
||||
};
|
||||
final hint = switch (_phase) {
|
||||
_Phase.listening when _heard.isEmpty => l10n.voiceListening,
|
||||
_Phase.thinking => l10n.voiceThinking,
|
||||
_Phase.idle => l10n.voiceTapToTalk,
|
||||
_ => null,
|
||||
};
|
||||
return PopScope(
|
||||
onPopInvokedWithResult: (didPop, _) {
|
||||
if (didPop) {
|
||||
_ending = true;
|
||||
}
|
||||
},
|
||||
child: BlocProvider.value(
|
||||
value: _cubit,
|
||||
child: Scaffold(
|
||||
backgroundColor: theme.colorScheme.surface,
|
||||
body: Stack(
|
||||
children: [
|
||||
Positioned.fill(child: _Glow(phase: _phase)),
|
||||
SafeArea(
|
||||
child: Column(
|
||||
children: [
|
||||
Expanded(
|
||||
child: Center(
|
||||
child: SingleChildScrollView(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 32),
|
||||
child: Column(
|
||||
children: [
|
||||
ShaderMask(
|
||||
shaderCallback: voiceGradient.createShader,
|
||||
child: const Icon(
|
||||
LucideIcons.sparkles,
|
||||
size: 48,
|
||||
color: Colors.white,
|
||||
),
|
||||
),
|
||||
const SizedBox(height: 16),
|
||||
Text(
|
||||
l10n.voiceTitle,
|
||||
style: theme.textTheme.headlineMedium,
|
||||
textAlign: TextAlign.center,
|
||||
),
|
||||
const SizedBox(height: 24),
|
||||
if (said.isNotEmpty)
|
||||
Text(
|
||||
said,
|
||||
style: theme.textTheme.bodyLarge,
|
||||
textAlign: TextAlign.center,
|
||||
),
|
||||
if (hint != null) ...[
|
||||
const SizedBox(height: 12),
|
||||
Text(
|
||||
hint,
|
||||
style: theme.textTheme.bodyMedium?.copyWith(
|
||||
color: theme.colorScheme.onSurfaceVariant,
|
||||
),
|
||||
textAlign: TextAlign.center,
|
||||
),
|
||||
],
|
||||
BlocBuilder<ConversationCubit, ConversationState>(
|
||||
builder: (context, state) =>
|
||||
switch (state.asking?.approving) {
|
||||
final image? => Padding(
|
||||
padding: const EdgeInsets.only(top: 16),
|
||||
child: ImageApproval(
|
||||
name: _choice.name,
|
||||
image: image,
|
||||
),
|
||||
),
|
||||
null => const SizedBox.shrink(),
|
||||
},
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
_Microphone(phase: _phase, onPressed: _microphone),
|
||||
const SizedBox(height: 24),
|
||||
Padding(
|
||||
padding: const EdgeInsets.fromLTRB(16, 0, 16, 16),
|
||||
child: Row(
|
||||
children: [
|
||||
Badge(
|
||||
isLabelVisible: _attached.isNotEmpty,
|
||||
label: Text('${_attached.length}'),
|
||||
child: AttachButton(
|
||||
images: _choice.images,
|
||||
assistant: _choice.name,
|
||||
onAttached: (part) =>
|
||||
setState(() => _attached.add(part)),
|
||||
),
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
Flexible(
|
||||
child: _Picker(
|
||||
current: _choice,
|
||||
assistants: widget.assistants,
|
||||
onPicked: (choice) => unawaited(_pick(choice)),
|
||||
),
|
||||
),
|
||||
const Spacer(),
|
||||
IconButton(
|
||||
tooltip: _speaker
|
||||
? l10n.voiceSpeakerOff
|
||||
: l10n.voiceSpeakerOn,
|
||||
style: _round(context),
|
||||
icon: Icon(
|
||||
_speaker
|
||||
? LucideIcons.volume2
|
||||
: LucideIcons.volumeX,
|
||||
),
|
||||
onPressed: () => unawaited(_toggleSpeaker()),
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
IconButton(
|
||||
tooltip: l10n.voiceEnd,
|
||||
style: IconButton.styleFrom(
|
||||
backgroundColor: theme.colorScheme.inverseSurface,
|
||||
foregroundColor:
|
||||
theme.colorScheme.onInverseSurface,
|
||||
fixedSize: const Size.square(48),
|
||||
),
|
||||
icon: const Icon(LucideIcons.x),
|
||||
onPressed: () => unawaited(_end()),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The colours of voice mode's sign and glow: the assistant's ring's.
|
||||
const voiceGradient = LinearGradient(
|
||||
colors: [Color(0xFF179FD9), Color(0xFF415DF0), Color(0xFF8080FF)],
|
||||
);
|
||||
|
||||
/// A round button of voice mode's row.
|
||||
ButtonStyle _round(BuildContext context) => IconButton.styleFrom(
|
||||
backgroundColor: Theme.of(context).colorScheme.surfaceContainerHigh,
|
||||
fixedSize: const Size.square(48),
|
||||
);
|
||||
|
||||
/// The glow rising from the foot of the screen, brighter while listening
|
||||
/// or reading aloud.
|
||||
class const _Glow({required final _Phase phase}) extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final strength = switch (phase) {
|
||||
_Phase.listening => 0.55,
|
||||
_Phase.speaking => 0.45,
|
||||
_Phase.thinking => 0.3,
|
||||
_Phase.idle => 0.15,
|
||||
};
|
||||
return IgnorePointer(
|
||||
child: TweenAnimationBuilder<double>(
|
||||
tween: Tween(end: strength),
|
||||
duration: const Duration(milliseconds: 600),
|
||||
curve: Curves.easeInOut,
|
||||
builder: (context, strength, _) => DecoratedBox(
|
||||
decoration: BoxDecoration(
|
||||
gradient: RadialGradient(
|
||||
center: const Alignment(0, 1.35),
|
||||
radius: 1.1,
|
||||
colors: [
|
||||
const Color(0xFF415DF0).withValues(alpha: strength),
|
||||
const Color(0xFF8080FF).withValues(alpha: strength / 3),
|
||||
Colors.transparent,
|
||||
],
|
||||
stops: const [0, 0.45, 1],
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The big microphone: listening, tapped, it stops and the question goes;
|
||||
/// otherwise it listens, stopping the reading or the answer on its way.
|
||||
class const _Microphone({
|
||||
required final _Phase phase,
|
||||
required final VoidCallback onPressed,
|
||||
}) extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final l10n = AppLocalizations.of(context);
|
||||
final theme = Theme.of(context);
|
||||
final listening = phase == _Phase.listening;
|
||||
return Tooltip(
|
||||
message: listening ? l10n.voiceDone : l10n.voiceTalk,
|
||||
child: AnimatedContainer(
|
||||
duration: const Duration(milliseconds: 200),
|
||||
width: 72,
|
||||
height: 72,
|
||||
decoration: ShapeDecoration(
|
||||
shape: const CircleBorder(),
|
||||
color: listening
|
||||
? theme.colorScheme.primary
|
||||
: theme.colorScheme.surfaceContainerHigh,
|
||||
),
|
||||
child: Material(
|
||||
type: MaterialType.transparency,
|
||||
shape: const CircleBorder(),
|
||||
clipBehavior: Clip.antiAlias,
|
||||
child: InkWell(
|
||||
onTap: onPressed,
|
||||
child: Icon(
|
||||
listening ? LucideIcons.audioLines : LucideIcons.mic,
|
||||
size: 28,
|
||||
color: listening
|
||||
? theme.colorScheme.onPrimary
|
||||
: theme.colorScheme.onSurface,
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Who answers, as a round chip with its name, its menu the other
|
||||
/// assistants.
|
||||
class const _Picker({
|
||||
required final AssistantChoice current,
|
||||
required final List<AssistantChoice> assistants,
|
||||
required final ValueChanged<AssistantChoice> onPicked,
|
||||
}) extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final l10n = AppLocalizations.of(context);
|
||||
final theme = Theme.of(context);
|
||||
final label = Row(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
children: [
|
||||
Flexible(
|
||||
child: Text(
|
||||
current.name,
|
||||
style: theme.textTheme.labelLarge,
|
||||
maxLines: 1,
|
||||
overflow: TextOverflow.ellipsis,
|
||||
),
|
||||
),
|
||||
if (assistants.length > 1) ...[
|
||||
const SizedBox(width: 4),
|
||||
Icon(
|
||||
LucideIcons.chevronsUpDown,
|
||||
size: 16,
|
||||
color: theme.colorScheme.onSurfaceVariant,
|
||||
),
|
||||
],
|
||||
],
|
||||
);
|
||||
return MenuAnchor(
|
||||
menuChildren: [
|
||||
for (final choice in assistants)
|
||||
MenuItemButton(
|
||||
trailingIcon: choice == current
|
||||
? const Icon(LucideIcons.check)
|
||||
: null,
|
||||
onPressed: () => onPicked(choice),
|
||||
child: Text(choice.name),
|
||||
),
|
||||
],
|
||||
builder: (context, controller, _) => Tooltip(
|
||||
message: l10n.assistantChoose,
|
||||
child: Material(
|
||||
color: theme.colorScheme.surfaceContainerHigh,
|
||||
shape: const StadiumBorder(),
|
||||
clipBehavior: Clip.antiAlias,
|
||||
child: InkWell(
|
||||
onTap: assistants.length < 2
|
||||
? null
|
||||
: () => controller.isOpen
|
||||
? controller.close()
|
||||
: controller.open(),
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 18, vertical: 14),
|
||||
child: label,
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ import device_info_plus
|
|||
import file_picker_darwin
|
||||
import file_selector_macos
|
||||
import flutter_secure_storage_darwin
|
||||
import flutter_tts
|
||||
import flutter_web_auth_2
|
||||
import google_sign_in_ios
|
||||
import media_kit_video
|
||||
|
|
@ -33,6 +34,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) {
|
|||
FilePickerPlugin.register(with: registry.registrar(forPlugin: "FilePickerPlugin"))
|
||||
FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin"))
|
||||
FlutterSecureStorageDarwinPlugin.register(with: registry.registrar(forPlugin: "FlutterSecureStorageDarwinPlugin"))
|
||||
FlutterTtsPlugin.register(with: registry.registrar(forPlugin: "FlutterTtsPlugin"))
|
||||
FlutterWebAuth2Plugin.register(with: registry.registrar(forPlugin: "FlutterWebAuth2Plugin"))
|
||||
GoogleSignInPlugin.register(with: registry.registrar(forPlugin: "GoogleSignInPlugin"))
|
||||
MediaKitVideoPlugin.register(with: registry.registrar(forPlugin: "MediaKitVideoPlugin"))
|
||||
|
|
|
|||
|
|
@ -61,6 +61,9 @@ dependencies:
|
|||
# Keeps the token of a Jellyfin server in the system's keyring; Serverpod's
|
||||
# sign-in keeps its session there with it already.
|
||||
flutter_secure_storage: ^10.0.0
|
||||
# Reads an assistant's answers aloud in voice mode with the device's own
|
||||
# text-to-speech (Android's, Apple's, Windows'; ADR 0021, D68).
|
||||
flutter_tts: ^4.2.5
|
||||
fractional_indexing_dart: ^1.0.7
|
||||
# Pinned exactly: every device on one app version must compute the same
|
||||
# schedules.
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import '../../support/fake_conversation_gateway.dart';
|
|||
import '../../support/fake_grammar_gateway.dart';
|
||||
import '../../support/fake_photo_picker.dart';
|
||||
import '../../support/fake_settings_gateway.dart';
|
||||
import '../../support/fake_speech_output.dart';
|
||||
import '../../support/fake_studyset_gateway.dart';
|
||||
import '../../support/memory_asset_file_cache.dart';
|
||||
import '../../support/memory_device_settings_store.dart';
|
||||
|
|
@ -119,6 +120,8 @@ void main() {
|
|||
late FakeLlmClient llm;
|
||||
late FakeFilePicker picker;
|
||||
late FakePhotoPicker photos;
|
||||
late FakeSpeechInput speech;
|
||||
late FakeSpeechOutput voice;
|
||||
late MemoryDeviceSettingsStore store;
|
||||
late List<Uri> searched;
|
||||
late FakeAssetGateway assetGateway;
|
||||
|
|
@ -146,6 +149,8 @@ void main() {
|
|||
llm = FakeLlmClient();
|
||||
picker = FakeFilePicker();
|
||||
photos = FakePhotoPicker();
|
||||
speech = FakeSpeechInput();
|
||||
voice = FakeSpeechOutput();
|
||||
searched = [];
|
||||
assetGateway = FakeAssetGateway();
|
||||
assetFiles = MemoryAssetFileCache();
|
||||
|
|
@ -208,6 +213,8 @@ void main() {
|
|||
grammarLanguages: grammarLanguages,
|
||||
files: picker,
|
||||
photos: photos,
|
||||
speech: speech,
|
||||
speechOutput: voice,
|
||||
),
|
||||
location: location,
|
||||
);
|
||||
|
|
@ -361,8 +368,10 @@ void main() {
|
|||
);
|
||||
final history = at('Conversations');
|
||||
final speak = at('Speak your question');
|
||||
final talk = at('Voice mode');
|
||||
final field = tester.getRect(find.byType(AssistantField));
|
||||
expect(field.right - speak.right, lessThan(16), reason: 'at the right');
|
||||
expect(field.right - talk.right, lessThan(16), reason: 'at the right');
|
||||
expect(speak.right, lessThan(talk.left), reason: 'the microphone before');
|
||||
|
||||
await openPicker(tester);
|
||||
await tester.tap(find.text('Claude Opus, thinking longer').last);
|
||||
|
|
@ -370,6 +379,7 @@ void main() {
|
|||
expect(find.text('Ask Claude Opus, thinking longer…'), findsOneWidget);
|
||||
expect(at('Conversations'), history);
|
||||
expect(at('Speak your question'), speak);
|
||||
expect(at('Voice mode'), talk);
|
||||
|
||||
for (final icon in [
|
||||
LucideIcons.plus,
|
||||
|
|
@ -1549,4 +1559,129 @@ void main() {
|
|||
expect(find.text('Was not shown an image'), findsOneWidget);
|
||||
});
|
||||
});
|
||||
|
||||
group("voice mode, as Claude's (D68)", () {
|
||||
/// Opens voice mode from the field.
|
||||
Future<void> talk(WidgetTester tester) async {
|
||||
await tester.tap(
|
||||
find.descendant(
|
||||
of: find.byType(AssistantField),
|
||||
matching: find.byTooltip('Voice mode'),
|
||||
),
|
||||
);
|
||||
await tester.pumpAndSettle();
|
||||
}
|
||||
|
||||
/// Says [words] and stops, as the recogniser does after a pause, and
|
||||
/// lets the answer come.
|
||||
Future<void> say(WidgetTester tester, String words) async {
|
||||
speech.say(words);
|
||||
await tester.pump();
|
||||
speech.finish();
|
||||
await tester.pump(const Duration(seconds: 1));
|
||||
await tester.pumpAndSettle();
|
||||
}
|
||||
|
||||
testWidgets('the field opens it over the whole screen, no bar beneath; '
|
||||
'it listens, asks what was said, reads the answer aloud without its '
|
||||
'marks, and listens again; ending it keeps the conversation', (
|
||||
tester,
|
||||
) async {
|
||||
await showApp(tester);
|
||||
await talk(tester);
|
||||
|
||||
expect(find.text("Let's talk"), findsOneWidget);
|
||||
expect(find.text('Home'), findsNothing, reason: 'no navigation bar');
|
||||
expect(speech.listening, isTrue);
|
||||
expect(speech.languages.single, 'en-US');
|
||||
|
||||
assistants.answers.add(_says('Because it **joins** clauses.'));
|
||||
speech.say('Why て-form?');
|
||||
await tester.pump();
|
||||
expect(find.text('Why て-form?'), findsOneWidget, reason: 'as heard');
|
||||
speech.finish();
|
||||
await tester.pump(const Duration(seconds: 1));
|
||||
await tester.pumpAndSettle();
|
||||
|
||||
expect(_asked(assistants.asked.single.$2), startsWith('Why て-form?'));
|
||||
expect(voice.spoken.single, ('Because it joins clauses.', 'en-US'));
|
||||
expect(
|
||||
find.text('Because it **joins** clauses.', findRichText: true),
|
||||
findsNothing,
|
||||
);
|
||||
expect(speech.listening, isTrue, reason: 'listening again');
|
||||
|
||||
await tester.tap(find.byTooltip('End voice mode'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(find.text("Let's talk"), findsNothing);
|
||||
expect(speech.listening, isFalse);
|
||||
expect(kept.written, isNotEmpty, reason: 'kept, as any conversation');
|
||||
});
|
||||
|
||||
testWidgets('a second question goes on the same conversation', (
|
||||
tester,
|
||||
) async {
|
||||
await showApp(tester);
|
||||
await talk(tester);
|
||||
assistants.answers
|
||||
..add(_says('It joins.'))
|
||||
..add(_says('As in 食べて.'));
|
||||
|
||||
await say(tester, 'Why て-form?');
|
||||
await say(tester, 'An example?');
|
||||
|
||||
final second = assistants.asked.last.$2;
|
||||
expect(_asked(second), startsWith('An example?'));
|
||||
expect(second.messages, hasLength(3), reason: 'the first turn before');
|
||||
});
|
||||
|
||||
testWidgets('with the speaker off, answers are shown, not read, and it '
|
||||
'listens again at once', (tester) async {
|
||||
await showApp(tester);
|
||||
await talk(tester);
|
||||
await tester.tap(find.byTooltip("Don't read answers aloud"));
|
||||
await tester.pumpAndSettle();
|
||||
assistants.answers.add(_says('Because it joins clauses.'));
|
||||
|
||||
await say(tester, 'Why て-form?');
|
||||
|
||||
expect(voice.spoken, isEmpty);
|
||||
expect(
|
||||
find.text('Because it joins clauses.', findRichText: true),
|
||||
findsOneWidget,
|
||||
);
|
||||
expect(speech.listening, isTrue);
|
||||
expect(find.byTooltip('Read answers aloud'), findsOneWidget);
|
||||
});
|
||||
|
||||
testWidgets('the microphone stops the reading and listens at once; '
|
||||
'nothing heard, it waits for the microphone', (tester) async {
|
||||
voice.holds = true;
|
||||
await showApp(tester);
|
||||
await talk(tester);
|
||||
assistants.answers.add(_says('A long answer.'));
|
||||
await say(tester, 'Why?');
|
||||
expect(voice.speaking, isTrue);
|
||||
expect(speech.listening, isFalse);
|
||||
|
||||
await tester.tap(find.byTooltip('Talk'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(voice.speaking, isFalse);
|
||||
expect(speech.listening, isTrue);
|
||||
|
||||
speech.finish();
|
||||
await tester.pumpAndSettle();
|
||||
expect(speech.listening, isFalse, reason: 'nothing heard');
|
||||
expect(find.text('Tap the microphone to talk'), findsOneWidget);
|
||||
expect(assistants.asked, hasLength(1));
|
||||
});
|
||||
|
||||
testWidgets('only for an assistant, and only where the device listens', (
|
||||
tester,
|
||||
) async {
|
||||
speech.supported = false;
|
||||
await showApp(tester);
|
||||
expect(find.byTooltip('Voice mode'), findsNothing);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/speech_output.dart';
|
||||
|
||||
void main() {
|
||||
test('an answer is read without its marks, a link as its words, a '
|
||||
"list's items as lines (ADR 0021, D68)", () {
|
||||
expect(
|
||||
speakableText(
|
||||
'## Why て-form?\n\n'
|
||||
'It **joins** clauses, as `食べて` does.\n\n'
|
||||
'- First, *this*.\n'
|
||||
'1. Then [that one](https://example.org).\n'
|
||||
'> A quote.',
|
||||
),
|
||||
'Why て-form?\n\n'
|
||||
'It joins clauses, as 食べて does.\n\n'
|
||||
'First, this.\n'
|
||||
'Then that one.\n'
|
||||
'A quote.',
|
||||
);
|
||||
});
|
||||
|
||||
test('a table reads as its cells, a code block as its code', () {
|
||||
expect(
|
||||
speakableText('| a | b |\n|---|---|\n| 1 | 2 |'),
|
||||
contains('1, 2'),
|
||||
);
|
||||
expect(speakableText('```dart\nprint(1);\n```'), 'print(1);');
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/speech_output.dart';
|
||||
|
||||
/// A [SpeechOutput] that notes what it reads; while [holds], a reading
|
||||
/// lasts until [finish].
|
||||
class FakeSpeechOutput() implements SpeechOutput {
|
||||
@override
|
||||
bool supported = true;
|
||||
|
||||
/// Whether a reading lasts until [finish] rather than ending at once.
|
||||
bool holds = false;
|
||||
|
||||
/// What was read aloud, in order, with the language asked for.
|
||||
final spoken = <(String, String?)>[];
|
||||
|
||||
Completer<void>? _reading;
|
||||
|
||||
/// Whether a reading is under way.
|
||||
bool get speaking => _reading != null;
|
||||
|
||||
/// Ends the reading under way.
|
||||
void finish() {
|
||||
final reading = _reading;
|
||||
_reading = null;
|
||||
reading?.complete();
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> speak(String text, {String? language}) async {
|
||||
spoken.add((text, language));
|
||||
if (holds) {
|
||||
final reading = _reading = Completer<void>();
|
||||
await reading.future;
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> stop() async => finish();
|
||||
}
|
||||
|
|
@ -25,6 +25,7 @@ import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_files
|
|||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_lookups.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/photo_picker.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/speech_output.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/web_search.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/card/data/card_deletions.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/card/data/card_gateway.dart';
|
||||
|
|
@ -102,6 +103,7 @@ import 'fake_publish_gateway.dart';
|
|||
import 'fake_review_gateway.dart';
|
||||
import 'fake_settings_gateway.dart';
|
||||
import 'fake_sound_player.dart';
|
||||
import 'fake_speech_output.dart';
|
||||
import 'fake_studyset_gateway.dart';
|
||||
import 'fake_tag_gateway.dart';
|
||||
import 'fake_youtube_videos.dart';
|
||||
|
|
@ -175,6 +177,7 @@ Dependencies testDependencies({
|
|||
WebSearch? webSearch,
|
||||
SpeechInput? speech,
|
||||
PhotoPicker? photos,
|
||||
SpeechOutput? speechOutput,
|
||||
JellyfinGateway? jellyfinGateway,
|
||||
JellyfinStore? jellyfinStore,
|
||||
JellyfinSuggestion? jellyfinSuggestion,
|
||||
|
|
@ -268,6 +271,7 @@ Dependencies testDependencies({
|
|||
),
|
||||
speech: speech ?? FakeSpeechInput(),
|
||||
photos: photos ?? FakePhotoPicker(),
|
||||
speechOutput: speechOutput ?? FakeSpeechOutput(),
|
||||
jellyfin: JellyfinRepository(
|
||||
(device) => jellyfinGateway ?? FakeJellyfinGateway(),
|
||||
jellyfinStore ?? MemoryJellyfinStore(),
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
#include <desktop_webview_window/desktop_webview_window_plugin.h>
|
||||
#include <file_selector_windows/file_selector_windows.h>
|
||||
#include <flutter_secure_storage_windows/flutter_secure_storage_windows_plugin.h>
|
||||
#include <flutter_tts/flutter_tts_plugin.h>
|
||||
#include <media_kit_libs_windows_video/media_kit_libs_windows_video_plugin_c_api.h>
|
||||
#include <media_kit_video/media_kit_video_plugin_c_api.h>
|
||||
#include <speech_to_text_windows/speech_to_text_windows.h>
|
||||
|
|
@ -28,6 +29,8 @@ void RegisterPlugins(flutter::PluginRegistry* registry) {
|
|||
registry->GetRegistrarForPlugin("FileSelectorWindows"));
|
||||
FlutterSecureStorageWindowsPluginRegisterWithRegistrar(
|
||||
registry->GetRegistrarForPlugin("FlutterSecureStorageWindowsPlugin"));
|
||||
FlutterTtsPluginRegisterWithRegistrar(
|
||||
registry->GetRegistrarForPlugin("FlutterTtsPlugin"));
|
||||
MediaKitLibsWindowsVideoPluginCApiRegisterWithRegistrar(
|
||||
registry->GetRegistrarForPlugin("MediaKitLibsWindowsVideoPluginCApi"));
|
||||
MediaKitVideoPluginCApiRegisterWithRegistrar(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
|
|||
desktop_webview_window
|
||||
file_selector_windows
|
||||
flutter_secure_storage_windows
|
||||
flutter_tts
|
||||
media_kit_libs_windows_video
|
||||
media_kit_video
|
||||
speech_to_text_windows
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue