halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

feat(learn): an assistant reads PDFs' pages, and sees images the person chooses or approves

Browse source 
ADR 0021, D12: text first. Lookups find a studyset's PDFs and read a page's
text as PDFium extracts it. An assistant that reads images is offered a
card's images as chips, and may ask to see a card's image or a PDF's page
drawn whole; the conversation asks the person each time (Show it, Don't
show) and sends nothing meanwhile. Images go only as png, jpeg, gif or webp
of at most 5 MB, are never kept in a turn, and a follow-up goes without them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-25 21:54:56 +01:00
commit 2b7d04fd70
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
26 changed files with 2030 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

13
apps/learn/halkyon_learn_flutter/lib/l10n/app_de.arb
View file

@ -1814,6 +1814,7 @@
"assistantLineExplanation": "Erklärung: {text}",
"assistantLineDescription": "Beschreibung: {text}",
"assistantChipStudyset": "Lernset: {title}",
"assistantChipImage": "Bild {number}",
"assistantHintWeb": "Im Web suchen…",
"assistantSendWeb": "Suchen",
"assistantSearchFailed": "Die Suche ließ sich nicht öffnen.",
@ -1858,6 +1859,18 @@
"assistantLookedUpNotes": "Notizen nach „{query}“ durchsucht: {count, plural, =0{keine gefunden} =1{1 gefunden} other{{count} gefunden}}",
"assistantLookedUpNote": "Eine Notiz gelesen",
"assistantLookedUpNoNote": "Eine Notiz gesucht, keine gefunden",
"assistantListedPdfs": "{count, plural, =0{Keine PDFs} =1{1 PDF} other{{count} PDFs}} aufgelistet",
"assistantLookedUpPdfs": "PDFs nach „{query}“ durchsucht: {count, plural, =0{keine gefunden} =1{1 gefunden} other{{count} gefunden}}",
"assistantLookedUpPage": "Eine Seite einer PDF gelesen",
"assistantLookedUpNoPage": "Eine Seite einer PDF gesucht, nicht lesbar",
"assistantShownImage": "Hat ein Bild gesehen, das du freigegeben hast",
"assistantNotShownImage": "Hat ein Bild nicht gesehen",
"assistantShownPage": "Hat eine Seite gesehen, die du freigegeben hast",
"assistantNotShownPage": "Hat eine Seite nicht gesehen",
"assistantImageAskCard": "{name} möchte Bild {number} der Karte „{title}“ sehen.",
"assistantImageAskPage": "{name} möchte Seite {page} von „{title}“ als Bild sehen.",
"assistantImageShow": "Zeigen",
"assistantImageDontShow": "Nicht zeigen",
"assistantPrivateTitle": "{count, plural, =1{Ein privates Lernset an {name} senden?} other{Private Lernsets an {name} senden?}}",
"assistantPrivateGranted": "{count, plural, =1{Es ist Ende-zu-Ende-verschlüsselt, sodass nicht einmal {instance} es lesen kann. Was {name} darin nachschlägt oder was du daraus auswählst, geht durch {instance}, das nichts davon aufbewahrt, an {host}, wo es nach den Bedingungen des Anbieters gelesen wird.} other{Sie sind Ende-zu-Ende-verschlüsselt, sodass nicht einmal {instance} sie lesen kann. Was {name} darin nachschlägt oder was du daraus auswählst, geht durch {instance}, das nichts davon aufbewahrt, an {host}, wo es nach den Bedingungen des Anbieters gelesen wird.}}",
"assistantPrivateOwn": "{count, plural, =1{Es ist Ende-zu-Ende-verschlüsselt, sodass nicht einmal die Instanz es lesen kann. Was {name} darin nachschlägt oder was du daraus auswählst, geht direkt an {host}, wo es nach den Bedingungen des Anbieters gelesen wird.} other{Sie sind Ende-zu-Ende-verschlüsselt, sodass nicht einmal die Instanz sie lesen kann. Was {name} darin nachschlägt oder was du daraus auswählst, geht direkt an {host}, wo es nach den Bedingungen des Anbieters gelesen wird.}}",

18
apps/learn/halkyon_learn_flutter/lib/l10n/app_en.arb
View file

@ -2128,6 +2128,8 @@
"@assistantLineDescription": {"placeholders": {"text": {"type": "String"}}},
"assistantChipStudyset": "Studyset: {title}",
"@assistantChipStudyset": {"placeholders": {"title": {"type": "String"}}},
"assistantChipImage": "Image {number}",
"@assistantChipImage": {"description": "A chip sending an image on the card (ADR 0021, D12).", "placeholders": {"number": {"type": "int"}}},
"assistantHintWeb": "Search the web…",
"assistantSendWeb": "Search",
"assistantSearchFailed": "The search couldn't be opened.",
@ -2182,6 +2184,22 @@
"@assistantLookedUpNotes": {"placeholders": {"query": {"type": "String"}, "count": {"type": "int"}}},
"assistantLookedUpNote": "Read a note",
"assistantLookedUpNoNote": "Looked for a note, found none",
"assistantListedPdfs": "Listed {count, plural, =0{no PDFs} =1{1 PDF} other{{count} PDFs}}",
"@assistantListedPdfs": {"placeholders": {"count": {"type": "int"}}},
"assistantLookedUpPdfs": "Looked for “{query}” in PDFs: {count, plural, =0{none found} =1{1 found} other{{count} found}}",
"@assistantLookedUpPdfs": {"placeholders": {"query": {"type": "String"}, "count": {"type": "int"}}},
"assistantLookedUpPage": "Read a page of a PDF",
"assistantLookedUpNoPage": "Looked for a page of a PDF, couldn't read it",
"assistantShownImage": "Was shown an image you approved",
"assistantNotShownImage": "Was not shown an image",
"assistantShownPage": "Was shown a page you approved",
"assistantNotShownPage": "Was not shown a page",
"assistantImageAskCard": "{name} asks to see image {number} of the card “{title}”.",
"@assistantImageAskCard": {"description": "The model asks for an image; the person approves each (D12).", "placeholders": {"name": {"type": "String"}, "number": {"type": "int"}, "title": {"type": "String"}}},
"assistantImageAskPage": "{name} asks to see page {page} of “{title}” as a picture.",
"@assistantImageAskPage": {"placeholders": {"name": {"type": "String"}, "page": {"type": "int"}, "title": {"type": "String"}}},
"assistantImageShow": "Show it",
"assistantImageDontShow": "Don't show",
"assistantPrivateTitle": "{count, plural, =1{Send a private studyset to {name}?} other{Send private studysets to {name}?}}",
"@assistantPrivateTitle": {"description": "The warning before a private studyset's content goes to a remote model (ADR 0021, D9).", "placeholders": {"count": {"type": "int"}, "name": {"type": "String"}}},
"assistantPrivateGranted": "{count, plural, =1{It is end-to-end encrypted, so that not even {instance} can read it. What {name} looks up in it, or what you chose of it, passes through {instance}, which keeps none of it, to {host}, which reads it under the provider's terms.} other{They are end-to-end encrypted, so that not even {instance} can read them. What {name} looks up in them, or what you chose of them, passes through {instance}, which keeps none of it, to {host}, which reads it under the provider's terms.}}",

78
apps/learn/halkyon_learn_flutter/lib/l10n/app_localizations.dart
View file

@ -10701,6 +10701,12 @@ abstract class AppLocalizations {
/// **'Studyset: {title}'**
String assistantChipStudyset(String title);
/// A chip sending an image on the card (ADR 0021, D12).
///
/// In en, this message translates to:
/// **'Image {number}'**
String assistantChipImage(int number);
/// No description provided for @assistantHintWeb.
///
/// In en, this message translates to:
@ -10965,6 +10971,78 @@ abstract class AppLocalizations {
/// **'Looked for a note, found none'**
String get assistantLookedUpNoNote;
/// No description provided for @assistantListedPdfs.
///
/// In en, this message translates to:
/// **'Listed {count, plural, =0{no PDFs} =1{1 PDF} other{{count} PDFs}}'**
String assistantListedPdfs(int count);
/// No description provided for @assistantLookedUpPdfs.
///
/// In en, this message translates to:
/// **'Looked for “{query}” in PDFs: {count, plural, =0{none found} =1{1 found} other{{count} found}}'**
String assistantLookedUpPdfs(String query, int count);
/// No description provided for @assistantLookedUpPage.
///
/// In en, this message translates to:
/// **'Read a page of a PDF'**
String get assistantLookedUpPage;
/// No description provided for @assistantLookedUpNoPage.
///
/// In en, this message translates to:
/// **'Looked for a page of a PDF, couldn\'t read it'**
String get assistantLookedUpNoPage;
/// No description provided for @assistantShownImage.
///
/// In en, this message translates to:
/// **'Was shown an image you approved'**
String get assistantShownImage;
/// No description provided for @assistantNotShownImage.
///
/// In en, this message translates to:
/// **'Was not shown an image'**
String get assistantNotShownImage;
/// No description provided for @assistantShownPage.
///
/// In en, this message translates to:
/// **'Was shown a page you approved'**
String get assistantShownPage;
/// No description provided for @assistantNotShownPage.
///
/// In en, this message translates to:
/// **'Was not shown a page'**
String get assistantNotShownPage;
/// The model asks for an image; the person approves each (D12).
///
/// In en, this message translates to:
/// **'{name} asks to see image {number} of the card “{title}”.'**
String assistantImageAskCard(String name, int number, String title);
/// No description provided for @assistantImageAskPage.
///
/// In en, this message translates to:
/// **'{name} asks to see page {page} of “{title}” as a picture.'**
String assistantImageAskPage(String name, int page, String title);
/// No description provided for @assistantImageShow.
///
/// In en, this message translates to:
/// **'Show it'**
String get assistantImageShow;
/// No description provided for @assistantImageDontShow.
///
/// In en, this message translates to:
/// **'Don\'t show'**
String get assistantImageDontShow;
/// The warning before a private studyset's content goes to a remote model (ADR 0021, D9).
///
/// In en, this message translates to:

66
apps/learn/halkyon_learn_flutter/lib/l10n/app_localizations_de.dart
View file

@ -6707,6 +6707,11 @@ class AppLocalizationsDe extends AppLocalizations {
return 'Lernset: $title';
}
@override
String assistantChipImage(int number) {
return 'Bild $number';
}
@override
String get assistantHintWeb => 'Im Web suchen…';
@ -6876,6 +6881,67 @@ class AppLocalizationsDe extends AppLocalizations {
@override
String get assistantLookedUpNoNote => 'Eine Notiz gesucht, keine gefunden';
@override
String assistantListedPdfs(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count PDFs',
one: '1 PDF',
zero: 'Keine PDFs',
);
return '$_temp0 aufgelistet';
}
@override
String assistantLookedUpPdfs(String query, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count gefunden',
one: '1 gefunden',
zero: 'keine gefunden',
);
return 'PDFs nach „$query“ durchsucht: $_temp0';
}
@override
String get assistantLookedUpPage => 'Eine Seite einer PDF gelesen';
@override
String get assistantLookedUpNoPage =>
'Eine Seite einer PDF gesucht, nicht lesbar';
@override
String get assistantShownImage =>
'Hat ein Bild gesehen, das du freigegeben hast';
@override
String get assistantNotShownImage => 'Hat ein Bild nicht gesehen';
@override
String get assistantShownPage =>
'Hat eine Seite gesehen, die du freigegeben hast';
@override
String get assistantNotShownPage => 'Hat eine Seite nicht gesehen';
@override
String assistantImageAskCard(String name, int number, String title) {
return '$name möchte Bild $number der Karte „$title“ sehen.';
}
@override
String assistantImageAskPage(String name, int page, String title) {
return '$name möchte Seite $page von „$title“ als Bild sehen.';
}
@override
String get assistantImageShow => 'Zeigen';
@override
String get assistantImageDontShow => 'Nicht zeigen';
@override
String assistantPrivateTitle(int count, String name) {
String _temp0 = intl.Intl.pluralLogic(

64
apps/learn/halkyon_learn_flutter/lib/l10n/app_localizations_en.dart
View file

@ -6643,6 +6643,11 @@ class AppLocalizationsEn extends AppLocalizations {
return 'Studyset: $title';
}
@override
String assistantChipImage(int number) {
return 'Image $number';
}
@override
String get assistantHintWeb => 'Search the web…';
@ -6812,6 +6817,65 @@ class AppLocalizationsEn extends AppLocalizations {
@override
String get assistantLookedUpNoNote => 'Looked for a note, found none';
@override
String assistantListedPdfs(int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count PDFs',
one: '1 PDF',
zero: 'no PDFs',
);
return 'Listed $_temp0';
}
@override
String assistantLookedUpPdfs(String query, int count) {
String _temp0 = intl.Intl.pluralLogic(
count,
locale: localeName,
other: '$count found',
one: '1 found',
zero: 'none found',
);
return 'Looked for “$query” in PDFs: $_temp0';
}
@override
String get assistantLookedUpPage => 'Read a page of a PDF';
@override
String get assistantLookedUpNoPage =>
'Looked for a page of a PDF, couldn\'t read it';
@override
String get assistantShownImage => 'Was shown an image you approved';
@override
String get assistantNotShownImage => 'Was not shown an image';
@override
String get assistantShownPage => 'Was shown a page you approved';
@override
String get assistantNotShownPage => 'Was not shown a page';
@override
String assistantImageAskCard(String name, int number, String title) {
return '$name asks to see image $number of the card “$title”.';
}
@override
String assistantImageAskPage(String name, int page, String title) {
return '$name asks to see page $page of “$title” as a picture.';
}
@override
String get assistantImageShow => 'Show it';
@override
String get assistantImageDontShow => 'Don\'t show';
@override
String assistantPrivateTitle(int count, String name) {
String _temp0 = intl.Intl.pluralLogic(

9
apps/learn/halkyon_learn_flutter/lib/src/core/content/rich_content.dart
View file

@ -117,6 +117,15 @@ final class RichContent(
if (node case {'data': {'url': final url}}) ?assetIdOf(url),
};
/// The person's images it shows, by their asset rows' ids, in order and
/// each once: what an assistant that reads images may be sent of it (ADR
/// 0021, D12).
List<String> get imageIds => {
for (final node in _nodes)
if (node case {'type': 'image', 'data': {'url': final url}})
?assetIdOf(url),
}.toList();
bool _hasBlock(String type) => _nodes.any((node) => node['type'] == type);
/// This content without its first block when that is a heading reading

8
apps/learn/halkyon_learn_flutter/lib/src/core/dependencies/dependencies.dart
View file

@ -23,6 +23,7 @@ import 'package:halkyon_learn_flutter/src/core/video/media_players.dart';
import 'package:halkyon_learn_flutter/src/core/video/youtube_videos.dart';
import 'package:halkyon_learn_flutter/src/feature/anki/data/anki_package_reader.dart';
import 'package:halkyon_learn_flutter/src/feature/archive/data/archive_reader.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_files.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_lookups.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/sync_conversation_gateway.dart';
@ -271,6 +272,7 @@ class Dependencies({
final instanceOperators = InstanceOperatorRepository(
ServerpodInstanceOperatorGateway(client.modules.base),
);
final pdfEngine = PdfrxEngine();
final assistants = AssistantRepository(
ServerpodAssistantGateway(client.modules.base),
);
@ -309,7 +311,7 @@ class Dependencies({
files: FileSelectorFilePicker(),
fileSaver: PlatformFileSaver(),
videos: IframeYoutubeVideos(),
pdfEngine: PdfrxEngine(),
pdfEngine: pdfEngine,
ankiPackages: IsolateAnkiPackageReader(),
archives: IsolateArchiveReader(),
exportArchiver: IsolateExportArchiver(),
@ -338,6 +340,7 @@ class Dependencies({
person,
await folders.openOnly(person.authUserId),
accountKeys: () => accountKeys.read(person.authUserId),
pdfEngine: pdfEngine,
onClosed: closed.complete,
);
} on Object {
@ -433,6 +436,7 @@ Future<PersonDependencies> _openPersonData(
SignedIn person,
Directory folder, {
required Future<UnlockedAccountKeys?> Function() accountKeys,
required PdfEngine pdfEngine,
required void Function() onClosed,
}) async {
final db = await client.createSyncSession(
@ -599,6 +603,8 @@ Future<PersonDependencies> _openPersonData(
studysets: studysets.withDeleted,
cardsOf: cards.cardsOf,
notesOf: notes.notesOf,
pdfsOf: pdfSources.sourcesOf,
files: DeviceAssistantFiles(assets, pdfEngine),
),
assets: assets,
deletions: deletions,

137
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/bloc/conversation_cubit.dart
View file

@ -8,18 +8,21 @@ import 'package:halkyon_app/halkyon_app.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_lookups.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_lookup.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_prompt.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_topic.dart';
import 'package:halkyon_llm/halkyon_llm.dart';
/// A question on its way: the turn's [messages] so far, the question first
/// and the lookups made, and the [text] of the answer being written.
/// and the lookups made, the [text] of the answer being written, and the
/// image the model asks to see while the person is [approving] it (D12).
final class const Asking({
required final List<LlmMessage> messages,
final String text = '',
final AskedImage? approving,
}) with Equatable {
@override
List<Object?> get props => [messages, text];
List<Object?> get props => [messages, text, approving];
}
/// A question that got no answer: [failure] says why, or null where the
@ -88,10 +91,12 @@ AssistantRef assistantRefOf(AssistantChoice choice) => switch (choice) {
/// A conversation with an assistant (ADR 0021, D7, D47, D51, D52): asks
/// [_choice] each question with the conversation so far, answers the
/// model's lookups on the device within [_scope], and keeps each turn,
/// sealed, once it is answered: nothing of a question stopped or failed,
/// and nothing at all where the device can't seal it. Continues the kept
/// conversation whose id it is given, if any.
/// model's lookups on the device within [_scope], shows it an image only
/// where it reads them and the person chose or approves it (D12), and
/// keeps each turn, sealed, once it is answered, without its images:
/// nothing of a question stopped or failed, and nothing at all where the
/// device can't seal it. Continues the kept conversation whose id it is
/// given, if any.
class ConversationCubit({
required final AssistantRepository _assistants,
required final AssistantLookups _lookups,
@ -136,8 +141,12 @@ class ConversationCubit({
StreamSubscription<LlmEvent>? _answering;
Completer<LlmFinished>? _answered;
Completer<bool>? _approval;
var _stopping = false;
/// Whether the assistant answering reads images.
bool get _readsImages => _choice?.images ?? false;
/// Reads whether this device keeps the conversation and, where it
/// continues one, its turns.
Future<void> open() async {
@ -156,12 +165,38 @@ class ConversationCubit({
}
/// Asks [question] with the parts [chosen], one of which holds something;
/// nothing while a question is on its way or the turns are read.
/// nothing while a question is on its way or the turns are read. An
/// image chosen goes only to an assistant that reads images, which is
/// told where one can't be had.
Future<void> ask(String question, List<AssistantPart> chosen) async {
if (question.trim().isEmpty && chosen.isEmpty) {
final sent = [
for (final part in chosen)
if (part.kind != AssistantPartKind.image || _readsImages) part,
];
if (question.trim().isEmpty && sent.isEmpty) {
return;
}
await _ask(questionMessage(question, chosen));
final images = <String, LlmImage>{};
for (final part in sent) {
if (part.kind == AssistantPartKind.image) {
try {
if (await _lookups.chosenImage(part.text) case final image?) {
images[part.text] = image;
}
} on Exception catch (error) {
log('Reading an image failed', name: _log, error: error);
}
}
}
await _ask(questionMessage(question, sent, images: images));
}
/// Shows the model the image it asks to see, where [approved], or tells
/// it the person chose not to.
void approveImage({required bool approved}) {
if (_approval case final approval? when !approval.isCompleted) {
approval.complete(approved);
}
}
/// Asks the question that got no answer again.
@ -177,6 +212,7 @@ class ConversationCubit({
return;
}
_stopping = true;
approveImage(approved: false);
await _answering?.cancel();
if (_answered case final answered? when !answered.isCompleted) {
answered.completeError(const _Stopped());
@ -203,7 +239,7 @@ class ConversationCubit({
: null,
answerIn: _answerIn,
);
final tools = _lookups.toolsFor(_scope);
final tools = _lookups.toolsFor(_scope, images: _readsImages);
for (var round = 0; ; round++) {
if (_stopping) {
throw const _Stopped();
@ -228,12 +264,16 @@ class ConversationCubit({
break;
}
// Every call gets its result, those of the last round too, or the
// conversation could no longer be sent.
turn.add(
LlmMessage.user([
for (final call in calls) await _resultOf(call, round),
]),
);
// conversation could no longer be sent; the images approved come
// after them, as a tool's result holds only text.
final results = <LlmPart>[];
final shown = <LlmPart>[];
for (final call in calls) {
final (result, image) = await _resultOf(call, round, turn);
results.add(result);
shown.addAll(image);
}
turn.add(LlmMessage.user([...results, ...shown]));
if (round > maxLookupRounds) {
break;
}
@ -287,6 +327,7 @@ class ConversationCubit({
}
},
onError: (Object error) {
_answering = null;
if (!answered.isCompleted) {
answered.completeError(switch (error) {
final AssistantFailed failure => failure,
@ -295,6 +336,7 @@ class ConversationCubit({
}
},
onDone: () {
_answering = null;
if (answered.isCompleted) {
return;
}
@ -312,11 +354,19 @@ class ConversationCubit({
return answered.future;
}
/// [call]'s result in [round]: the lookup's within the scope for the
/// first [maxLookupRounds], then an error telling the model to answer.
Future<LlmToolResult> _resultOf(LlmToolCall call, int round) async {
LlmToolResult refused(String why) =>
LlmToolResult(callId: call.id, content: why, isError: true);
/// [call]'s result in [round], with the image it shows: the lookup's
/// within the scope for the first [maxLookupRounds], then an error
/// telling the model to answer. An image the model asks to see goes
/// once the person approves it, the [turn] so far shown meanwhile.
Future<(LlmToolResult, List<LlmPart>)> _resultOf(
LlmToolCall call,
int round,
List<LlmMessage> turn,
) async {
(LlmToolResult, List<LlmPart>) refused(String why) => (
LlmToolResult(callId: call.id, content: why, isError: true),
const [],
);
if (round == maxLookupRounds) {
return refused('No more lookups: answer with what you found.');
}
@ -324,13 +374,53 @@ class ConversationCubit({
return refused('The question ends here.');
}
try {
return await _lookups.answer(call, _scope);
if (_readsImages && (LookupKind.ofTool(call.name)?.showsImage ?? false)) {
final asked = await _lookups.askedImage(call, _scope);
if (asked == null) {
return refused('There is no such image within reach.');
}
if (!await _approve(asked, turn)) {
return refused('The person chose not to show it.');
}
final image = await _lookups.imageOf(asked, _scope);
if (image == null) {
return refused('That image cannot be had now.');
}
return (
LlmToolResult(callId: call.id, content: 'The image follows.'),
[LlmText(asked.label), image],
);
}
return (
await _lookups.answer(call, _scope, images: _readsImages),
const <LlmPart>[],
);
} on Exception catch (error) {
log('A lookup failed', name: _log, error: error);
return refused('The lookup failed.');
}
}
/// Whether the person approves showing the model [asked], which the
/// sheet asks after the [turn] so far; never once they stopped.
Future<bool> _approve(AskedImage asked, List<LlmMessage> turn) async {
if (_stopping || isClosed) {
return false;
}
final approval = _approval = Completer<bool>();
emit(
state.next(
asking: Asking(messages: [...turn], approving: asked),
),
);
final approved = await approval.future;
_approval = null;
if (!isClosed) {
emit(state.next(asking: Asking(messages: [...turn])));
}
return approved;
}
/// Keeps the turn [messages] answered, starting the conversation with
/// its first [question]; where this device keeps none, in this sheet
/// alone.
@ -368,7 +458,7 @@ class ConversationCubit({
...state.turns,
AssistantTurn(
id: turnId ?? 'unkept-${state.turns.length + 1}',
messages: messages,
messages: withoutImages(messages),
usage: usage,
writtenAt: clock.now().toUtc(),
),
@ -389,6 +479,7 @@ class ConversationCubit({
@override
Future<void> close() async {
approveImage(approved: false);
await _answering?.cancel();
await super.close();
}

93
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/data/assistant_files.dart Normal file
View file

@ -0,0 +1,93 @@
import 'dart:ui' show Rect;
import 'package:halkyon_learn_flutter/src/core/asset/data/asset_repository.dart';
import 'package:halkyon_learn_flutter/src/feature/pdf_source/data/pdf_engine.dart';
import 'package:halkyon_learn_flutter/src/feature/pdf_source/model/pdf_source.dart';
import 'package:halkyon_llm/halkyon_llm.dart';
/// What an assistant's lookups read of the person's files (ADR 0021, D12):
/// a PDF's pages, as text and as pictures, and images, each as a model is
/// sent them. Null wherever the file can't be had now, or won't do.
abstract interface class AssistantFiles() {
/// The text of [pdf]'s [page] as PDFium reads it; empty for a page
/// without any.
Future<String?> pageText(PdfSource pdf, int page);
/// [pdf]'s [page] drawn whole, as a PNG.
Future<LlmImage?> pageImage(PdfSource pdf, int page);
/// The image of the asset row [assetId], where it is one a model reads.
Future<LlmImage?> image(String assetId);
}
/// [AssistantFiles] on this device: files through [_assets], from the
/// device or downloaded, PDFs opened with [_engine] for each page read.
class DeviceAssistantFiles(
final AssetRepository _assets,
final PdfEngine _engine,
) implements AssistantFiles {
/// The images every provider reads.
static const imageTypes = {
'image/png',
'image/jpeg',
'image/gif',
'image/webp',
};
/// How large an image may be: the smallest of the providers' limits for
/// one (Anthropic's).
static const maxImageBytes = 5 * 1024 * 1024;
@override
Future<String?> pageText(PdfSource pdf, int page) =>
_withPage(pdf, page, (document) => document.pageText(page));
@override
Future<LlmImage?> pageImage(PdfSource pdf, int page) async {
final png = await _withPage(
pdf,
page,
(document) =>
document.renderRegion(page, const Rect.fromLTWH(0, 0, 1, 1)),
);
return png == null || png.lengthInBytes > maxImageBytes
? null
: LlmImage(mediaType: 'image/png', bytes: png);
}
@override
Future<LlmImage?> image(String assetId) async {
final file = await _assets.fileOf(assetId);
if (file == null ||
!imageTypes.contains(file.mimeType) ||
file.bytes.lengthInBytes > maxImageBytes) {
return null;
}
return LlmImage(mediaType: file.mimeType, bytes: file.bytes);
}
/// What [read] reads of [pdf] opened, where it has [page].
Future<T?> _withPage<T extends Object>(
PdfSource pdf,
int page,
Future<T?> Function(PdfDocumentView document) read,
) async {
final file = await _assets.fileOf(pdf.content.assetId);
if (file == null) {
return null;
}
final PdfDocumentView document;
try {
document = await _engine.open(file.bytes);
} on FormatException {
return null;
}
try {
return page < 1 || page > document.pageCount
? null
: await read(document);
} finally {
await document.dispose();
}
}
}

249
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/data/assistant_lookups.dart
View file

@ -1,10 +1,12 @@
import 'dart:convert';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_files.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_lookup.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_topic.dart';
import 'package:halkyon_learn_flutter/src/feature/card/model/card.dart';
import 'package:halkyon_learn_flutter/src/feature/note/model/note.dart';
import 'package:halkyon_learn_flutter/src/feature/pdf_source/model/pdf_source.dart';
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset.dart';
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset_tree.dart';
import 'package:halkyon_llm/halkyon_llm.dart';
@ -13,7 +15,9 @@ import 'package:halkyon_llm/halkyon_llm.dart';
/// answered on this device from the person's own studysets, and only within
/// the scope they chose; private studysets only where they switched them on
/// (D8), never what is locked here, never progress (D11), and nothing at
/// all where only what is ticked goes (D50).
/// all where only what is ticked goes (D50). Text first: a PDF as PDFium
/// reads it; an image only for an assistant that reads them, each once the
/// person approves it (D12).
class AssistantLookups({
/// Every studyset of the person's on this device, deleted ones too.
required final Future<List<Studyset>> Function() studysets,
@ -23,6 +27,12 @@ class AssistantLookups({
/// The notes of a studyset, in order.
required final Future<List<Note>> Function(String studysetId) notesOf,
/// The PDFs of a studyset, in order.
required final Future<List<PdfSource>> Function(String studysetId) pdfsOf,
/// What is read of the person's files: PDFs' pages, images.
required final AssistantFiles files,
}) {
/// How many cards or notes a search returns unless the model says.
static const defaultLimit = 10;
@ -34,9 +44,15 @@ class AssistantLookups({
/// a long note, not a book.
static const maxNoteLength = 6000;
/// The tools offered within [scope]; none where only what is ticked goes.
List<LlmTool> toolsFor(AssistantScope scope) =>
scope.reach == AssistantReach.ticked ? const [] : _tools;
/// How much of a PDF's page reading one ([LookupKind.pdfPage]) sends.
static const maxPageLength = maxNoteLength;
/// The tools offered within [scope], those showing images only where
/// the assistant reads [images]; none where only what is ticked goes.
List<LlmTool> toolsFor(AssistantScope scope, {bool images = false}) =>
scope.reach == AssistantReach.ticked
? const []
: [..._tools, if (images) ..._imageTools];
static const _query = {
'type': 'string',
@ -91,6 +107,61 @@ class AssistantLookups({
'required': ['id'],
},
),
LlmTool(
name: LookupKind.pdfs.tool,
description:
"Finds the person's PDFs within reach whose title contains the "
'query, or lists them all without one: each with its id, title, '
"page count and studyset's title. read_pdf_page reads a page.",
inputSchema: {
'type': 'object',
'properties': {'query': _query, 'limit': _limit},
},
),
LlmTool(
name: LookupKind.pdfPage.tool,
description:
'Reads the text of a page of a PDF find_pdfs found, as it is '
'extracted: figures and graphs drawn on the page are not in it.',
inputSchema: _pageInput,
),
];
static const _pageInput = {
'type': 'object',
'properties': {
'id': {'type': 'string', 'description': "The PDF's id."},
'page': {'type': 'integer', 'description': 'The page, from 1.'},
},
'required': ['id', 'page'],
};
static final _imageTools = [
LlmTool(
name: LookupKind.cardImage.tool,
description:
'Shows you an image on a card find_cards found with images, once '
'the person approves it; ask only where the question needs it.',
inputSchema: {
'type': 'object',
'properties': {
'id': {'type': 'string', 'description': "The card's id."},
'number': {
'type': 'integer',
'description': 'Which of its images, from 1.',
},
},
'required': ['id', 'number'],
},
),
LlmTool(
name: LookupKind.pageImage.tool,
description:
'Shows you a page of a PDF as a picture, once the person approves '
'it: for figures, graphs or formulas its text leaves out; ask only '
'where the question needs it.',
inputSchema: _pageInput,
),
];
/// The studysets [scope] reaches, in the library's order: the studyset
@ -211,11 +282,21 @@ class AssistantLookups({
/// [call]'s answer within [scope]: what it found as JSON, or an error the
/// model reads, for a tool not offered there, input it doesn't take, or
/// something outside the scope, which reads as if there were none.
Future<LlmToolResult> answer(LlmToolCall call, AssistantScope scope) async {
/// something outside the scope, which reads as if there were none. A
/// card says how many images it has where the assistant reads [images].
/// An image is never answered here: the person approves each first
/// ([askedImage]).
Future<LlmToolResult> answer(
LlmToolCall call,
AssistantScope scope, {
bool images = false,
}) async {
LlmToolResult error(String why) =>
LlmToolResult(callId: call.id, content: why, isError: true);
if (!toolsFor(scope).any((tool) => tool.name == call.name)) {
if (!toolsFor(
scope,
images: images,
).any((tool) => tool.name == call.name)) {
return error('There is no tool ${call.name} in this conversation.');
}
final reached = await reach(scope);
@ -253,7 +334,11 @@ class AssistantLookups({
for (final studyset in reached) {
if (call.name == LookupKind.cards.tool) {
for (final card in await cardsOf(studyset.id)) {
final json = _cardJson(card, studyset.content.title);
final json = _cardJson(
card,
studyset.content.title,
images: images,
);
if (_mentions(json, needle)) {
found.add(json);
}
@ -308,15 +393,158 @@ class AssistantLookups({
}
}
return error('No note $id within reach.');
case LookupKind.pdfs:
final query = call.input['query'];
if (query != null && query is! String) {
return error('"query" must be text.');
}
final limit = switch (call.input['limit']) {
final int limit when limit > 0 => limit.clamp(1, maxLimit),
_ => defaultLimit,
};
final needle = (query as String? ?? '').trim().toLowerCase();
final found = <Map<String, Object?>>[];
for (final studyset in reached) {
for (final pdf in await pdfsOf(studyset.id)) {
if (found.length < limit &&
pdf.content.title.toLowerCase().contains(needle)) {
found.add({
'id': pdf.id,
'studyset': studyset.content.title,
'title': pdf.content.title,
'pages': ?pdf.content.pageCount,
});
}
}
}
return LlmToolResult(callId: call.id, content: result(found));
case LookupKind.pdfPage:
final (id, page) = (call.input['id'], call.input['page']);
if (id is! String || page is! int) {
return error('"id" must be text and "page" a number.');
}
final (pdf, studyset) = await _pdfWithin(id, reached) ?? (null, null);
if (pdf == null || studyset == null) {
return error('No PDF $id within reach.');
}
final text = await files.pageText(pdf, page);
if (text == null) {
return error(
'Page $page of that PDF cannot be read now: it has no such page, '
'or its file is not on this device.',
);
}
return LlmToolResult(
callId: call.id,
content: result({
'id': pdf.id,
'studyset': studyset.content.title,
'title': pdf.content.title,
'page': page,
'pages': ?pdf.content.pageCount,
'text': _cut(text.trim(), maxPageLength),
}),
);
case LookupKind.cardImage || LookupKind.pageImage:
return error('An image is shown only once the person approves it.');
case null:
return error('There is no tool ${call.name} in this conversation.');
}
}
/// The image [call] asks to see within [scope], for the person to
/// approve; null for one not within reach, or input the tool doesn't
/// take.
Future<AskedImage?> askedImage(LlmToolCall call, AssistantScope scope) async {
final reached = await reach(scope);
final id = call.input['id'];
if (id is! String) {
return null;
}
switch (LookupKind.ofTool(call.name)) {
case LookupKind.cardImage:
final number = call.input['number'];
if (number is! int) {
return null;
}
for (final studyset in reached) {
for (final card in await cardsOf(studyset.id)) {
if (card.id == id) {
final images = cardImageIds(card.content);
return number < 1 || number > images.length
? null
: AskedImage(
kind: LookupKind.cardImage,
title: _cut(cardFrontText(card.content), 80),
number: number,
assetId: images[number - 1],
);
}
}
}
return null;
case LookupKind.pageImage:
final page = call.input['page'];
final (pdf, _) = await _pdfWithin(id, reached) ?? (null, null);
if (pdf == null || page is! int || page < 1) {
return null;
}
if (pdf.content.pageCount case final pages? when page > pages) {
return null;
}
return AskedImage(
kind: LookupKind.pageImage,
title: pdf.content.title,
number: page,
pdfId: pdf.id,
);
case _:
return null;
}
}
/// The image [asked] names, approved, within [scope]; null where it can't
/// be had now, or no longer is within reach.
Future<LlmImage?> imageOf(AskedImage asked, AssistantScope scope) async {
if (asked.assetId case final assetId?) {
return await files.image(assetId);
}
if (asked.pdfId case final pdfId?) {
final (pdf, _) =
await _pdfWithin(pdfId, await reach(scope)) ?? (null, null);
return pdf == null ? null : await files.pageImage(pdf, asked.number);
}
return null;
}
/// The image of the asset row [assetId], chosen to go along (D12).
Future<LlmImage?> chosenImage(String assetId) => files.image(assetId);
/// The PDF [id] within the studysets [reached], with its studyset.
Future<(PdfSource, Studyset)?> _pdfWithin(
String id,
List<Studyset> reached,
) async {
for (final studyset in reached) {
for (final pdf in await pdfsOf(studyset.id)) {
if (pdf.id == id) {
return (pdf, studyset);
}
}
}
return null;
}
/// What a card says, as a lookup sends it: its parts, each cut as a
/// ticked part is; never its progress.
static Map<String, Object?> _cardJson(Card card, String studyset) {
/// ticked part is, and how many images it has where the assistant reads
/// [images]; never its progress.
static Map<String, Object?> _cardJson(
Card card,
String studyset, {
bool images = false,
}) {
final content = card.content;
final imageCount = images ? cardImageIds(content).length : 0;
final hints = [
for (final hint in content.hints)
if (hint.plainText.trim() case final text when text.isNotEmpty)
@ -331,6 +559,7 @@ class AssistantLookups({
if (hints.isNotEmpty) 'hints': hints,
if (explanation.isNotEmpty)
'explanation': _cut(explanation, AssistantPart.maxLength),
if (imageCount > 0) 'images': imageCount,
};
}

5
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/data/conversation_payload.dart
View file

@ -2,6 +2,7 @@ import 'dart:typed_data';
import 'package:halkyon_learn_flutter/src/core/content/payload.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_prompt.dart';
import 'package:halkyon_llm/halkyon_llm.dart';
// A conversation's and its turns' content as their synced rows store it (see
@ -17,10 +18,6 @@ const conversationSchemaVersion = 1;
/// The schema this app writes a turn in.
const turnSchemaVersion = 1;
/// What a kept turn holds in an image's place: images aren't kept (they
/// would make a turn megabytes), so a follow-up goes without them.
const imageNotKept = '[An image was sent here.]';
/// Encodes a conversation's [title], [assistant], [scope] and [task].
ByteData encodeConversationPayload({
required String title,

51
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/model/assistant_lookup.dart
View file

@ -18,7 +18,23 @@ enum LookupKind(
notes('find_notes'),
/// One note, read whole.
note('read_note');
note('read_note'),
/// PDFs, by their titles (D12).
pdfs('find_pdfs'),
/// A page of a PDF, its text as PDFium reads it (D12).
pdfPage('read_pdf_page'),
/// An image on a card, shown once the person approves (D12).
cardImage('view_card_image'),
/// A page of a PDF as a picture, shown once the person approves (D12).
pageImage('view_pdf_page');
/// Whether it shows the model an image, which the person approves each
/// time (D12).
bool get showsImage => this == cardImage || this == pageImage;
/// The kind [tool] looks through; null for a tool this app doesn't
/// offer.
@ -35,13 +51,42 @@ final class const AssistantLookup({
/// What it looked for; null where it looked for nothing in particular.
final String? query,
/// How many it found: studysets, cards, notes, or 1 for a note read.
/// How many it found: studysets, cards, notes or PDFs, or 1 for a note
/// or page read and an image shown.
required final int found,
}) with Equatable {
@override
List<Object?> get props => [kind, query, found];
}
/// An image the model asks to see (D12), within reach, as the person is
/// asked to approve it: an image on a card, or a page of a PDF.
final class const AskedImage({
/// What shows it: a card's image, or a PDF's page.
required final LookupKind kind,
/// The card's front, or the PDF's title.
required final String title,
/// The image's number on the card, from 1, or the page.
required final int number,
/// The asset row of a card's image; null for a page.
final String? assetId,
/// The PDF of a page; null for a card's image.
final String? pdfId,
}) with Equatable {
/// What the model is told it is looking at, before the image.
String get label => switch (kind) {
LookupKind.pageImage => 'Page $number of the PDF “$title”:',
_ => 'Image $number of the card “$title”:',
};
@override
List<Object?> get props => [kind, title, number, assetId, pdfId];
}
/// [call] and its [result] as the conversation shows them; null for a tool
/// this app doesn't offer.
AssistantLookup? describeLookup(LlmToolCall call, LlmToolResult result) {
@ -51,6 +96,8 @@ AssistantLookup? describeLookup(LlmToolCall call, LlmToolResult result) {
}
final found = result.isError
? 0
: kind.showsImage
? 1
: switch (_decoded(result.content)) {
final List<Object?> items => items.length,
Map<Object?, Object?>() => 1,

75
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/model/assistant_prompt.dart
View file

@ -6,23 +6,58 @@ import 'package:halkyon_llm/halkyon_llm.dart';
// What the model reads besides the conversation: written in English, which
// every model reads best, and told to answer in the person's language.
/// What a kept turn holds in an image's place: images aren't kept (they
/// would make a turn megabytes), so a follow-up goes without them.
const imageNotKept = '[An image was sent here.]';
/// What goes in the place of an image chosen that can't be had now.
const imageNotHad = '[The image could not be had on this device.]';
/// The person's [question] as the model reads it, then each part they
/// [chosen] to go along, labelled (ADR 0021, D7): each a text of its own,
/// so [askedIn] reads them back, those after the first starting on a blank
/// line, as a provider may join them. [question] or [chosen] holds
/// something.
LlmMessage questionMessage(String question, List<AssistantPart> chosen) {
final texts = [
if (question.trim() case final text when text.isNotEmpty) text,
for (final part in chosen)
'${_label(part.kind, part.number)}$_labelEnd${part.sent}',
];
return LlmMessage.user([
for (final (index, text) in texts.indexed)
LlmText(index == 0 ? text : '\n\n$text'),
]);
/// line, as a provider may join them; an image after its label, from
/// [images] by its asset row (D12). [question] or [chosen] holds something.
LlmMessage questionMessage(
String question,
List<AssistantPart> chosen, {
Map<String, LlmImage> images = const {},
}) {
final parts = <LlmPart>[];
void add(String text) =>
parts.add(LlmText(parts.isEmpty ? text : '\n\n$text'));
if (question.trim() case final text when text.isNotEmpty) {
add(text);
}
for (final part in chosen) {
final label = '${_label(part.kind, part.number)}$_labelEnd';
if (part.kind != AssistantPartKind.image) {
add('$label${part.sent}');
} else if (images[part.text] case final image?) {
add(label);
parts.add(image);
} else {
add('$label$imageNotHad');
}
}
return LlmMessage.user(parts);
}
/// [messages] with [imageNotKept] in place of each image: what a turn
/// holds once answered, kept and sent again with a follow-up.
List<LlmMessage> withoutImages(List<LlmMessage> messages) => [
for (final message in messages)
if (message.role == LlmRole.user && message.parts.any(_isImage))
LlmMessage.user([
for (final part in message.parts)
if (_isImage(part)) const LlmText(imageNotKept) else part,
])
else
message,
];
bool _isImage(LlmPart part) => part is LlmImage;
/// What [message], made by [questionMessage], asked: the question as typed
/// (empty where only parts went), and the parts that went along.
({String question, List<AssistantPart> parts}) askedIn(LlmMessage message) {
@ -31,7 +66,7 @@ LlmMessage questionMessage(String question, List<AssistantPart> chosen) {
for (final LlmText(:text) in message.parts.whereType<LlmText>()) {
if (_partOf(text.trimLeft()) case final part?) {
parts.add(part);
} else if (question.isEmpty) {
} else if (question.isEmpty && text.trim() != imageNotKept) {
question = text.trim();
}
}
@ -47,9 +82,11 @@ String _label(AssistantPartKind kind, int? number) => switch (kind) {
AssistantPartKind.explanation => "The card's explanation",
AssistantPartKind.studyset => 'The studyset',
AssistantPartKind.description => "The studyset's description",
AssistantPartKind.image => "The card's image ${number ?? 1}",
};
final _hintLabel = RegExp(r"^The card's hint (\d+)$");
final _imageLabel = RegExp(r"^The card's image (\d+)$");
AssistantPart? _partOf(String text) {
final end = text.indexOf(_labelEnd);
@ -65,8 +102,18 @@ AssistantPart? _partOf(String text) {
number: int.parse(match.group(1)!),
);
}
// An image's text isn't kept: what the part showed is gone with it.
if (_imageLabel.firstMatch(label) case final match?) {
return AssistantPart(
kind: AssistantPartKind.image,
text: '',
number: int.parse(match.group(1)!),
);
}
for (final kind in AssistantPartKind.values) {
if (kind != AssistantPartKind.hint && _label(kind, null) == label) {
if (kind != AssistantPartKind.hint &&
kind != AssistantPartKind.image &&
_label(kind, null) == label) {
return AssistantPart(kind: kind, text: rest);
}
}

20
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/model/assistant_topic.dart
View file

@ -101,6 +101,8 @@ final class const AssistantTopicCard({
if (card.explanation?.plainText.trim() case final text?
when text.isNotEmpty)
AssistantPart(kind: AssistantPartKind.explanation, text: text),
for (final (index, id) in cardImageIds(card).indexed)
AssistantPart(kind: AssistantPartKind.image, text: id, number: index + 1),
AssistantPart(kind: AssistantPartKind.studyset, text: studysetTitle),
];
}
@ -113,6 +115,10 @@ enum AssistantPartKind() {
explanation,
studyset,
description,
/// An image on a card, by its asset row (D12): offered only to an
/// assistant that reads images, and sent as the image itself.
image,
}
/// A part of a topic that can go along with a question: its [kind] and its
@ -124,7 +130,7 @@ final class const AssistantPart({
/// Its text, as sent.
required final String text,
/// A hint's number, from 1.
/// A hint's or an image's number, from 1.
final int? number,
}) with Equatable {
/// How long a part's text may be when sent: enough for a sentence or
@ -135,7 +141,8 @@ final class const AssistantPart({
/// and description don't (they would only narrow the results wrongly).
bool get searchable =>
kind != AssistantPartKind.studyset &&
kind != AssistantPartKind.description;
kind != AssistantPartKind.description &&
kind != AssistantPartKind.image;
/// [text] as sent, cut to [maxLength].
String get sent =>
@ -153,6 +160,15 @@ String searchQuery(String question, List<AssistantPart> chosen) => [
if (part.searchable) part.sent,
].where((text) => text.isNotEmpty).join(' ');
/// The images [card] shows, by their asset rows, in order and each once:
/// on its front, its back, its hints and its explanation.
List<String> cardImageIds(CardContent card) => {
...card.front.imageIds,
...card.back.imageIds,
for (final hint in card.hints) ...hint.imageIds,
...?card.explanation?.imageIds,
}.toList();
/// What [card] asks, as plain text: its front, question, expression, or
/// what to sort.
String cardFrontText(CardContent card) => card.front.plainText.trim();

33
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/widget/assistant_field.dart
View file

@ -4,6 +4,7 @@ import 'package:flutter/foundation.dart' show setEquals;
import 'package:flutter_bloc/flutter_bloc.dart';
import 'package:halkyon_app/halkyon_app.dart';
import 'package:halkyon_learn_flutter/l10n/app_localizations.dart';
import 'package:halkyon_learn_flutter/src/core/asset/widget/stored_file_image.dart';
import 'package:halkyon_learn_flutter/src/core/dependencies/dependencies.dart';
import 'package:halkyon_learn_flutter/src/core/dependencies/person_dependencies.dart';
import 'package:halkyon_learn_flutter/src/core/device_settings/bloc/device_settings_bloc.dart';
@ -160,10 +161,16 @@ class _AssistantFieldState() extends State<AssistantField> {
}
/// The parts [answerer] can take, in the topic's order: a web search
/// only those a search takes, an assistant all of them.
/// only those a search takes, an assistant all of them, the card's
/// images only where it reads images (D12).
List<AssistantPart> _offered(Answerer? answerer) => [
for (final part in widget.topic.parts)
if (answerer is AssistantAnswerer || part.searchable) part,
if (switch (answerer) {
AssistantAnswerer(:final choice) =>
part.kind != AssistantPartKind.image || choice.images,
_ => part.searchable,
})
part,
];
void _clear() {
@ -578,7 +585,10 @@ class const _Parts({
?leading,
for (final part in parts)
Tooltip(
message: part.sent,
// An image's text is its asset row's id.
message: part.kind == AssistantPartKind.image
? partText(l10n, part)
: part.sent,
waitDuration: const Duration(milliseconds: 400),
child: ConstrainedBox(
constraints: BoxConstraints(
@ -586,6 +596,9 @@ class const _Parts({
),
child: FilterChip(
visualDensity: VisualDensity.compact,
avatar: part.kind == AssistantPartKind.image
? _Thumbnail(assetId: part.text)
: null,
label: Text(
partText(l10n, part),
maxLines: 1,
@ -605,6 +618,20 @@ class const _Parts({
}
}
/// A card's image on its chip, small.
class const _Thumbnail({required final String assetId})
extends StatelessWidget {
@override
Widget build(BuildContext context) => ClipRRect(
borderRadius: BorderRadius.circular(4),
child: Image(
image: StoredFileImage(PersonDependencies.of(context).assets, assetId),
fit: BoxFit.cover,
errorBuilder: (context, _, _) => const Icon(LucideIcons.image),
),
);
}
/// Home's field at its foot, above the navigation: asks about the person's
/// learning. Nothing while neither the web search nor an assistant is
/// there.

18
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/widget/assistant_words.dart
View file

@ -47,6 +47,23 @@ String lookupText(AppLocalizations l10n, AssistantLookup lookup) =>
lookup.found > 0
? l10n.assistantLookedUpNote
: l10n.assistantLookedUpNoNote,
LookupKind.pdfs => switch (lookup.query) {
final query? when query.trim().isNotEmpty => l10n.assistantLookedUpPdfs(
query,
lookup.found,
),
_ => l10n.assistantListedPdfs(lookup.found),
},
LookupKind.pdfPage =>
lookup.found > 0
? l10n.assistantLookedUpPage
: l10n.assistantLookedUpNoPage,
LookupKind.cardImage =>
lookup.found > 0
? l10n.assistantShownImage
: l10n.assistantNotShownImage,
LookupKind.pageImage =>
lookup.found > 0 ? l10n.assistantShownPage : l10n.assistantNotShownPage,
};
/// [part] on its chip: what it is and its text, on one line.
@ -59,5 +76,6 @@ String partText(AppLocalizations l10n, AssistantPart part) {
AssistantPartKind.explanation => l10n.assistantLineExplanation(text),
AssistantPartKind.studyset => l10n.assistantChipStudyset(text),
AssistantPartKind.description => l10n.assistantLineDescription(text),
AssistantPartKind.image => l10n.assistantChipImage(part.number ?? 1),
};
}

119
apps/learn/halkyon_learn_flutter/lib/src/feature/assistant/widget/conversation_sheet.dart
View file

@ -12,6 +12,7 @@ import 'package:halkyon_learn_flutter/src/core/editor/rich_content_document.dart
import 'package:halkyon_learn_flutter/src/core/editor/rich_content_view.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/bloc/conversation_cubit.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_lookup.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_topic.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/turn_view.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/widget/assistant_words.dart';
@ -159,6 +160,13 @@ class const _ConversationView({
_Turn(
view: TurnView.of(asking.messages),
streaming: asking.text,
approving: switch (asking.approving) {
final image? => _ImageApproval(
name: name,
image: image,
),
null => null,
},
),
if (state.unanswered case final unanswered?)
_Unanswered(
@ -291,6 +299,10 @@ class const _Note({
class const _Turn({
required final TurnView view,
final String? streaming,
/// What the person is asked to approve, in place of the answer being
/// written.
final Widget? approving,
}) extends StatelessWidget {
@override
Widget build(BuildContext context) {
@ -315,7 +327,13 @@ class const _Turn({
padding: const EdgeInsets.only(top: 8),
child: Row(
children: [
Icon(LucideIcons.search, size: 14, color: quiet?.color),
Icon(
lookup.kind.showsImage
? LucideIcons.image
: LucideIcons.search,
size: 14,
color: quiet?.color,
),
const SizedBox(width: 8),
Expanded(child: Text(lookupText(l10n, lookup), style: quiet)),
],
@ -324,19 +342,22 @@ class const _Turn({
const SizedBox(height: 12),
if (streaming != null) ...[
if (written.isNotEmpty) Text(written),
Padding(
padding: const EdgeInsets.only(top: 8),
child: Row(
children: [
const SizedBox.square(
dimension: 16,
child: ButtonLoadingIndicator(),
),
const SizedBox(width: 8),
Text(l10n.assistantAnswering, style: quiet),
],
if (approving case final approving?)
approving
else
Padding(
padding: const EdgeInsets.only(top: 8),
child: Row(
children: [
const SizedBox.square(
dimension: 16,
child: ButtonLoadingIndicator(),
),
const SizedBox(width: 8),
Text(l10n.assistantAnswering, style: quiet),
],
),
),
),
] else ...[
if (written.isEmpty)
Text(l10n.assistantNoAnswer, style: quiet)
@ -354,6 +375,78 @@ class const _Turn({
}
}
/// The image [name] asks to see (D12), shown only once the person
/// approves it: Show it, or Don't show.
class const _ImageApproval({
required final String name,
required final AskedImage image,
}) extends StatelessWidget {
@override
Widget build(BuildContext context) {
final l10n = AppLocalizations.of(context);
final theme = Theme.of(context);
final conversation = context.read<ConversationCubit>();
return Padding(
padding: const EdgeInsets.only(top: 8),
child: DecoratedBox(
decoration: BoxDecoration(
color: theme.colorScheme.surfaceContainerHigh,
borderRadius: BorderRadius.circular(16),
),
child: Padding(
padding: const EdgeInsets.fromLTRB(14, 12, 8, 4),
child: Column(
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
Row(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Icon(
image.kind == LookupKind.pageImage
? LucideIcons.fileImage
: LucideIcons.image,
size: 20,
color: theme.colorScheme.primary,
),
const SizedBox(width: 12),
Expanded(
child: Text(switch (image.kind) {
LookupKind.pageImage => l10n.assistantImageAskPage(
name,
image.number,
image.title,
),
_ => l10n.assistantImageAskCard(
name,
image.number,
image.title,
),
}),
),
],
),
OverflowBar(
alignment: MainAxisAlignment.end,
spacing: 8,
children: [
TextButton(
onPressed: () => conversation.approveImage(approved: false),
child: Text(l10n.assistantImageDontShow),
),
FilledButton.tonal(
onPressed: () => conversation.approveImage(approved: true),
child: Text(l10n.assistantImageShow),
),
],
),
],
),
),
),
);
}
}
/// What was asked, on the end side as the person's own words.
class const _Question({required final TurnView view}) extends StatelessWidget {
@override

22
apps/learn/halkyon_learn_flutter/test/core/content/rich_content_test.dart
View file

@ -164,6 +164,28 @@ void main() {
});
});
test("RichContent.imageIds lists the person's images it shows, in order "
'and once each; no sound, nor an image from elsewhere', () {
Map<String, Object?> block(String type, String url) => {
'type': type,
'data': {'url': url},
};
final content = RichContent({
'document': {
'type': 'page',
'children': [
block('image', 'asset:b'),
block('audio', 'asset:sound'),
block('image', 'https://example.org/cat.png'),
block('image', 'asset:a'),
block('image', 'asset:b'),
],
},
});
expect(content.imageIds, ['b', 'a']);
});
group('RichContent.plainText', () {
test('is the text without formatting, one line per block', () {
expect(

298
apps/learn/halkyon_learn_flutter/test/feature/assistant/bloc/conversation_cubit_test.dart
View file

@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:typed_data';
import 'package:flutter_test/flutter_test.dart';
import 'package:halkyon_app/halkyon_app.dart';
@ -9,14 +10,17 @@ import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_looku
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_lookup.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_prompt.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_topic.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/turn_view.dart';
import 'package:halkyon_learn_flutter/src/feature/card/model/card.dart';
import 'package:halkyon_learn_flutter/src/feature/card/model/card_content.dart';
import 'package:halkyon_learn_flutter/src/feature/pdf_source/model/pdf_source.dart';
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset.dart';
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset_content.dart';
import 'package:halkyon_llm/halkyon_llm.dart';
import '../../../support/fake_assistant_files.dart';
import '../../../support/fake_conversation_gateway.dart';
// Asking an assistant in a conversation (ADR 0021, D7, D47, D51, D52): the
@ -33,6 +37,16 @@ const _claude = OfferedAssistant(
model: 'claude-sonnet',
);
/// An assistant that reads images (D12).
const _seeing = OfferedAssistant(
id: 'seeing',
name: 'Seeing',
protocol: 'openai',
host: 'api.openai.com',
model: 'gpt',
images: true,
);
const _usage = LlmUsage(inputTokens: 100, outputTokens: 20);
/// The model answering [text], streamed in two pieces.
@ -99,6 +113,8 @@ void main() {
_card('c-kuru', '来る', 'irregular'),
],
notesOf: (_) async => const [],
pdfsOf: (_) async => const [],
files: FakeAssistantFiles(),
);
setUp(() {
@ -241,7 +257,14 @@ void main() {
final (_, first) = gateway.asked.first;
expect(
[for (final tool in first.tools) tool.name],
['list_studysets', 'find_cards', 'find_notes', 'read_note'],
[
'list_studysets',
'find_cards',
'find_notes',
'read_note',
'find_pdfs',
'read_pdf_page',
],
);
expect(first.system, contains('the studyset “Verbs”'));
final (_, second) = gateway.asked.last;
@ -267,6 +290,8 @@ void main() {
studysets: () async => [_verbs],
cardsOf: (_) async => throw const FormatException('unreadable'),
notesOf: (_) async => const [],
pdfsOf: (_) async => const [],
files: FakeAssistantFiles(),
);
addTearDown(() => lookups = working);
final conversation = cubit(
@ -506,6 +531,8 @@ void main() {
],
cardsOf: (_) async => const [],
notesOf: (_) async => const [],
pdfsOf: (_) async => const [],
files: FakeAssistantFiles(),
);
final conversation = cubit(
task: AssistantTask.check,
@ -523,4 +550,273 @@ void main() {
await conversation.close();
},
);
group('images (D12)', () {
final files = FakeAssistantFiles()
..images['img-1'] = LlmImage(
mediaType: 'image/png',
bytes: Uint8List.fromList([1, 2, 3]),
)
..pages['p-genki'] = {2: 'A table of verbs.'};
final pictured = AssistantLookups(
studysets: () async => [_verbs],
cardsOf: (_) async => const [],
notesOf: (_) async => const [],
pdfsOf: (_) async => [
const PdfSource(
id: 'p-genki',
studysetId: 'verbs',
content: PdfSourceContent(
title: 'Genki I',
fileName: 'genki.pdf',
assetId: 'asset-genki',
pageCount: 12,
),
),
],
files: files,
);
const library = AssistantScope(reach: AssistantReach.library);
const viewPage = LlmToolCall(
id: 'call-page',
name: 'view_pdf_page',
input: {'id': 'p-genki', 'page': 2},
);
const asked = AskedImage(
kind: LookupKind.pageImage,
title: 'Genki I',
number: 2,
pdfId: 'p-genki',
);
const chosenImage = AssistantPart(
kind: AssistantPartKind.image,
text: 'img-1',
number: 1,
);
setUp(() {
final working = lookups;
lookups = pictured;
addTearDown(() => lookups = working);
});
List<LlmImage> imagesIn(List<LlmMessage> messages) => [
for (final message in messages) ...message.parts.whereType<LlmImage>(),
];
/// Asks [conversation] [question], answering the image the model asks
/// to see with [answer] once it is asked; the image asked about.
Future<AskedImage?> askApproving(
ConversationCubit conversation,
String question,
void Function() answer,
) async {
AskedImage? approving;
final listening = conversation.stream.listen((state) {
if (state.asking?.approving case final image? when approving == null) {
approving = image;
answer();
}
});
await conversation.ask(question, const []);
await listening.cancel();
return approving;
}
test('an image chosen goes along to an assistant reading images, and '
'is kept as a mark, so a follow-up goes without it', () async {
final conversation = cubit(choice: const GrantedAssistant(_seeing));
await conversation.open();
gateway.answers
..add(_says('A cat.'))
..add(_says('Grey.'));
await conversation.ask('What is it?', const [chosenImage]);
final question = gateway.asked.single.$2.messages.single;
expect(imagesIn([question]).single.bytes, [1, 2, 3]);
expect(imagesIn(conversation.state.turns.single.messages), isEmpty);
final saved = (await conversations.all()).single;
expect(
imagesIn((await conversations.turnsOf(saved.id)).single.messages),
isEmpty,
);
await conversation.ask('Which colour?', const []);
expect(imagesIn(gateway.asked.last.$2.messages), isEmpty);
expect(
TurnView.of(conversation.state.turns.first.messages).sent,
[
const AssistantPart(
kind: AssistantPartKind.image,
text: '',
number: 1,
),
],
);
await conversation.close();
});
test('an image chosen never goes to an assistant not reading them, nor '
'is it said to', () async {
final conversation = cubit();
await conversation.open();
gateway.answers.add(_says('Hello.'));
await conversation.ask('Hi', const [chosenImage]);
final question = gateway.asked.single.$2.messages.single;
expect(imagesIn([question]), isEmpty);
expect(question.text, isNot(contains('image')));
// Alone, it is no question at all.
await conversation.ask('', const [chosenImage]);
expect(gateway.asked, hasLength(1));
await conversation.close();
});
test('an image chosen that cannot be had is said to be missing', () async {
final conversation = cubit(choice: const GrantedAssistant(_seeing));
await conversation.open();
gateway.answers.add(_says('I see nothing.'));
await conversation.ask('What is it?', const [
AssistantPart(kind: AssistantPartKind.image, text: 'gone', number: 1),
]);
final question = gateway.asked.single.$2.messages.single;
expect(imagesIn([question]), isEmpty);
expect(question.text, contains(imageNotHad));
await conversation.close();
});
test(
'what shows images is offered only to an assistant reading them',
() async {
for (final (choice, offered) in [
(const GrantedAssistant(_seeing), true),
(const GrantedAssistant(_claude), false),
]) {
final conversation = cubit(scope: library, choice: choice);
await conversation.open();
gateway.answers.add(_says('Yes.'));
await conversation.ask('Any figures?', const []);
expect(
[for (final tool in gateway.asked.last.$2.tools) tool.name],
offered
? contains('view_pdf_page')
: isNot(contains('view_pdf_page')),
);
await conversation.close();
}
},
);
test('a page the model asks to see goes once the person approves it, '
'after the results', () async {
final conversation = cubit(
scope: library,
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('The table lists godan verbs.'));
final approving = await askApproving(
conversation,
'What does page 2 show?',
() => conversation.approveImage(approved: true),
);
expect(approving, asked);
final shown = gateway.asked.last.$2.messages.last.parts;
expect(shown, hasLength(3));
expect((shown[0] as LlmToolResult).isError, isFalse);
expect((shown[1] as LlmText).text, asked.label);
expect((shown[2] as LlmImage).mediaType, 'image/png');
expect(files.drawn, ['p-genki:2']);
final turn = conversation.state.turns.single;
expect(imagesIn(turn.messages), isEmpty);
expect(
TurnView.of(turn.messages).lookups,
[const AssistantLookup(kind: LookupKind.pageImage, found: 1)],
);
await conversation.close();
});
test('refused, the model is told so and sees nothing', () async {
files.drawn.clear();
final conversation = cubit(
scope: library,
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('Without it, I cannot say.'));
await askApproving(
conversation,
'What does page 2 show?',
() => conversation.approveImage(approved: false),
);
final results = gateway.asked.last.$2.messages.last.parts;
final result = results.single as LlmToolResult;
expect(result.isError, isTrue);
expect(result.content, contains('chose not to show it'));
expect(imagesIn(gateway.asked.last.$2.messages), isEmpty);
expect(files.drawn, isEmpty);
await conversation.close();
});
test('stopped while asked, the question goes no further', () async {
final conversation = cubit(
scope: library,
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('Never asked.'));
await askApproving(
conversation,
'What does page 2 show?',
() => unawaited(conversation.stop()),
);
expect(gateway.asked, hasLength(1));
expect(conversation.state.asking, isNull);
expect(conversation.state.unanswered?.failure, isNull);
expect(conversation.state.turns, isEmpty);
await conversation.close();
});
test('an image out of reach is never asked about', () async {
final conversation = cubit(
scope: const AssistantScope(
reach: AssistantReach.studyset,
studysetId: 'elsewhere',
),
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('Nothing there.'));
final approving = await askApproving(
conversation,
'What does page 2 show?',
() => fail('asked to approve an image out of reach'),
);
expect(approving, isNull);
final result =
gateway.asked.last.$2.messages.last.parts.single as LlmToolResult;
expect(result.isError, isTrue);
await conversation.close();
});
});
}

201
apps/learn/halkyon_learn_flutter/test/feature/assistant/conversation_test.dart
View file

@ -1,8 +1,10 @@
import 'dart:convert';
import 'dart:typed_data';
import 'package:flutter_test/flutter_test.dart';
import 'package:halkyon_app/halkyon_app.dart';
import 'package:halkyon_app/testing.dart';
import 'package:halkyon_learn_flutter/src/core/asset/data/asset_repository.dart';
import 'package:halkyon_learn_flutter/src/core/content/rich_content.dart';
import 'package:halkyon_learn_flutter/src/core/device_settings/model/device_settings.dart';
import 'package:halkyon_learn_flutter/src/core/router/route_paths.dart';
@ -24,10 +26,14 @@ import 'package:halkyon_llm/halkyon_llm.dart';
import 'package:lucide_icons_flutter/lucide_icons.dart';
import 'package:material_ui/material_ui.dart';
import '../../support/fake_asset_gateway.dart';
import '../../support/fake_asset_sealing.dart';
import '../../support/fake_asset_server_gateway.dart';
import '../../support/fake_card_gateway.dart';
import '../../support/fake_conversation_gateway.dart';
import '../../support/fake_settings_gateway.dart';
import '../../support/fake_studyset_gateway.dart';
import '../../support/memory_asset_file_cache.dart';
import '../../support/memory_device_settings_store.dart';
import '../../support/pump_app.dart';
import '../../support/test_dependencies.dart';
@ -49,6 +55,17 @@ OfferedAssistant _claude({bool agreed = true}) => OfferedAssistant(
agreedAt: agreed ? _agreed : null,
);
/// An assistant that reads images (D12).
final _seeing = OfferedAssistant(
id: 'seeing',
name: 'Seeing',
protocol: 'openai',
host: 'api.openai.com',
model: 'gpt',
images: true,
agreedAt: _agreed,
);
List<LlmEvent> _says(String text) => [
LlmTextDelta(text),
LlmFinished(
@ -99,6 +116,8 @@ void main() {
late FakeLlmClient llm;
late MemoryDeviceSettingsStore store;
late List<Uri> searched;
late FakeAssetGateway assetGateway;
late MemoryAssetFileCache assetFiles;
setUp(() {
assistants = FakeAssistantGateway()
@ -120,6 +139,8 @@ void main() {
settings = FakeSettingsGateway();
llm = FakeLlmClient();
searched = [];
assetGateway = FakeAssetGateway();
assetFiles = MemoryAssetFileCache();
});
Future<void> showApp(
@ -140,6 +161,8 @@ void main() {
studysetGateway: studysets,
cardGateway: cards,
settingsGateway: settings,
assetGateway: assetGateway,
assetFiles: assetFiles,
llm: llm,
webSearch: WebSearch(
open: (url) async {
@ -282,7 +305,15 @@ void main() {
final request = assistants.asked.single.$2;
expect(
[for (final tool in request.tools) tool.name],
['list_studysets', 'find_cards', 'find_notes', 'read_note'],
[
'list_studysets',
'find_cards',
'find_notes',
'read_note',
'find_pdfs',
'read_pdf_page',
],
reason: 'nothing showing images to an assistant not reading them',
);
expect(request.system, contains('the studyset “Japanese”'));
});
@ -795,4 +826,172 @@ void main() {
expect(find.byTooltip('Check for mistakes'), findsNothing);
});
});
group('images (D12)', () {
/// Writes a card into Verbs with a picture of a bowl on its front, as
/// another device would have; its id.
Future<String> picturedCard() async {
final assets = AssetRepository(
assetGateway,
FakeAssetServerGateway(),
assetFiles,
FakeAssetSealing(),
);
addTearDown(assets.dispose);
final image = await assets.add(
Uint8List.fromList(utf8.encode('a bowl of rice')),
fileName: 'bowl.png',
mimeType: 'image/png',
studysetId: null,
);
final repository = CardRepository(cards);
addTearDown(repository.dispose);
return await repository.create(
'verbs',
CardContent(
front: RichContent({
'document': {
'type': 'page',
'children': [
{
'type': 'paragraph',
'data': {
'delta': [
{'insert': '飯'},
],
},
},
{
'type': 'image',
'data': {'url': 'asset:$image'},
},
],
},
}),
back: RichContent.plain('rice, a meal'),
),
);
}
List<LlmImage> imagesIn(LlmRequest request) => [
for (final message in request.messages)
...message.parts.whereType<LlmImage>(),
];
testWidgets("a card's image is offered to an assistant reading images, "
'and goes along once chosen', (tester) async {
assistants.offer = AssistantsOffer(assistants: [_seeing]);
await picturedCard();
await showApp(tester, location: RoutePaths.study('japanese'));
await unfold(tester);
// The chips wrap to four lines on a phone's study screen.
await tester.ensureVisible(find.text('Image 1'));
await tester.pumpAndSettle();
await tester.tap(find.text('Image 1'));
await tester.pump();
assistants.answers.add(_says('A bowl of rice.'));
await ask(tester, 'What is this?');
final request = assistants.asked.single.$2;
expect(imagesIn(request).single.bytes, utf8.encode('a bowl of rice'));
expect(find.text('A bowl of rice.', findRichText: true), findsOneWidget);
});
testWidgets('an assistant not reading images is offered none', (
tester,
) async {
await picturedCard();
await showApp(tester, location: RoutePaths.study('japanese'));
await unfold(tester);
expect(find.text('Image 1'), findsNothing);
expect(find.textContaining('Front'), findsOneWidget);
});
testWidgets('an image the model asks to see is shown to it only once '
'the person approves it in the conversation', (tester) async {
assistants.offer = AssistantsOffer(assistants: [_seeing]);
final card = await picturedCard();
await showApp(tester, location: RoutePaths.studyset('japanese'));
await unfold(tester);
await tester.tap(find.text('Only what I choose'));
await tester.pumpAndSettle();
await tester.tap(find.text('This studyset with its topics'));
await tester.pumpAndSettle();
assistants.answers
..add([
LlmFinished(
message: LlmMessage.assistant([
LlmToolCall(
id: 'call-1',
name: 'view_card_image',
input: {'id': card, 'number': 1},
),
]),
stop: LlmStop.toolCalls,
usage: const LlmUsage(inputTokens: 10, outputTokens: 5),
),
])
..add(_says('Your card shows a bowl of rice.'));
await ask(tester, 'What does my card about 飯 show?');
expect(
find.text('Seeing asks to see image 1 of the card “飯”.'),
findsOneWidget,
);
expect(assistants.asked, hasLength(1), reason: 'nothing sent meanwhile');
await tester.tap(find.text('Show it'));
await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();
expect(find.text('Show it'), findsNothing);
expect(
imagesIn(assistants.asked.last.$2).single.bytes,
utf8.encode('a bowl of rice'),
);
expect(find.text('Was shown an image you approved'), findsOneWidget);
expect(
find.text('Your card shows a bowl of rice.', findRichText: true),
findsOneWidget,
);
});
testWidgets("Don't show tells the model so, and shows it nothing", (
tester,
) async {
assistants.offer = AssistantsOffer(assistants: [_seeing]);
final card = await picturedCard();
await showApp(tester, location: RoutePaths.studyset('japanese'));
await unfold(tester);
await tester.tap(find.text('Only what I choose'));
await tester.pumpAndSettle();
await tester.tap(find.text('This studyset with its topics'));
await tester.pumpAndSettle();
assistants.answers
..add([
LlmFinished(
message: LlmMessage.assistant([
LlmToolCall(
id: 'call-1',
name: 'view_card_image',
input: {'id': card, 'number': 1},
),
]),
stop: LlmStop.toolCalls,
usage: const LlmUsage(inputTokens: 10, outputTokens: 5),
),
])
..add(_says('I cannot see it.'));
await ask(tester, 'What does my card show?');
await tester.tap(find.text("Don't show"));
await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();
expect(imagesIn(assistants.asked.last.$2), isEmpty);
expect(find.text('Was not shown an image'), findsOneWidget);
});
});
}

Some files were not shown because too many files have changed in this diff Show more