halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

feat(learn): an assistant reads PDFs' pages, and sees images the person chooses or approves

Browse source 
ADR 0021, D12: text first. Lookups find a studyset's PDFs and read a page's
text as PDFium extracts it. An assistant that reads images is offered a
card's images as chips, and may ask to see a card's image or a PDF's page
drawn whole; the conversation asks the person each time (Show it, Don't
show) and sends nothing meanwhile. Images go only as png, jpeg, gif or webp
of at most 5 MB, are never kept in a turn, and a follow-up goes without them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-25 21:54:56 +01:00
commit 2b7d04fd70
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
1 changed files with 2030 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

298
apps/learn/halkyon_learn_flutter/test/feature/assistant/bloc/conversation_cubit_test.dart
View file

@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:typed_data';
import 'package:flutter_test/flutter_test.dart';
import 'package:halkyon_app/halkyon_app.dart';
@ -9,14 +10,17 @@ import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_looku
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_lookup.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_prompt.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_topic.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/model/turn_view.dart';
import 'package:halkyon_learn_flutter/src/feature/card/model/card.dart';
import 'package:halkyon_learn_flutter/src/feature/card/model/card_content.dart';
import 'package:halkyon_learn_flutter/src/feature/pdf_source/model/pdf_source.dart';
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset.dart';
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset_content.dart';
import 'package:halkyon_llm/halkyon_llm.dart';
import '../../../support/fake_assistant_files.dart';
import '../../../support/fake_conversation_gateway.dart';
// Asking an assistant in a conversation (ADR 0021, D7, D47, D51, D52): the
@ -33,6 +37,16 @@ const _claude = OfferedAssistant(
model: 'claude-sonnet',
);
/// An assistant that reads images (D12).
const _seeing = OfferedAssistant(
id: 'seeing',
name: 'Seeing',
protocol: 'openai',
host: 'api.openai.com',
model: 'gpt',
images: true,
);
const _usage = LlmUsage(inputTokens: 100, outputTokens: 20);
/// The model answering [text], streamed in two pieces.
@ -99,6 +113,8 @@ void main() {
_card('c-kuru', '来る', 'irregular'),
],
notesOf: (_) async => const [],
pdfsOf: (_) async => const [],
files: FakeAssistantFiles(),
);
setUp(() {
@ -241,7 +257,14 @@ void main() {
final (_, first) = gateway.asked.first;
expect(
[for (final tool in first.tools) tool.name],
['list_studysets', 'find_cards', 'find_notes', 'read_note'],
[
'list_studysets',
'find_cards',
'find_notes',
'read_note',
'find_pdfs',
'read_pdf_page',
],
);
expect(first.system, contains('the studyset “Verbs”'));
final (_, second) = gateway.asked.last;
@ -267,6 +290,8 @@ void main() {
studysets: () async => [_verbs],
cardsOf: (_) async => throw const FormatException('unreadable'),
notesOf: (_) async => const [],
pdfsOf: (_) async => const [],
files: FakeAssistantFiles(),
);
addTearDown(() => lookups = working);
final conversation = cubit(
@ -506,6 +531,8 @@ void main() {
],
cardsOf: (_) async => const [],
notesOf: (_) async => const [],
pdfsOf: (_) async => const [],
files: FakeAssistantFiles(),
);
final conversation = cubit(
task: AssistantTask.check,
@ -523,4 +550,273 @@ void main() {
await conversation.close();
},
);
group('images (D12)', () {
final files = FakeAssistantFiles()
..images['img-1'] = LlmImage(
mediaType: 'image/png',
bytes: Uint8List.fromList([1, 2, 3]),
)
..pages['p-genki'] = {2: 'A table of verbs.'};
final pictured = AssistantLookups(
studysets: () async => [_verbs],
cardsOf: (_) async => const [],
notesOf: (_) async => const [],
pdfsOf: (_) async => [
const PdfSource(
id: 'p-genki',
studysetId: 'verbs',
content: PdfSourceContent(
title: 'Genki I',
fileName: 'genki.pdf',
assetId: 'asset-genki',
pageCount: 12,
),
),
],
files: files,
);
const library = AssistantScope(reach: AssistantReach.library);
const viewPage = LlmToolCall(
id: 'call-page',
name: 'view_pdf_page',
input: {'id': 'p-genki', 'page': 2},
);
const asked = AskedImage(
kind: LookupKind.pageImage,
title: 'Genki I',
number: 2,
pdfId: 'p-genki',
);
const chosenImage = AssistantPart(
kind: AssistantPartKind.image,
text: 'img-1',
number: 1,
);
setUp(() {
final working = lookups;
lookups = pictured;
addTearDown(() => lookups = working);
});
List<LlmImage> imagesIn(List<LlmMessage> messages) => [
for (final message in messages) ...message.parts.whereType<LlmImage>(),
];
/// Asks [conversation] [question], answering the image the model asks
/// to see with [answer] once it is asked; the image asked about.
Future<AskedImage?> askApproving(
ConversationCubit conversation,
String question,
void Function() answer,
) async {
AskedImage? approving;
final listening = conversation.stream.listen((state) {
if (state.asking?.approving case final image? when approving == null) {
approving = image;
answer();
}
});
await conversation.ask(question, const []);
await listening.cancel();
return approving;
}
test('an image chosen goes along to an assistant reading images, and '
'is kept as a mark, so a follow-up goes without it', () async {
final conversation = cubit(choice: const GrantedAssistant(_seeing));
await conversation.open();
gateway.answers
..add(_says('A cat.'))
..add(_says('Grey.'));
await conversation.ask('What is it?', const [chosenImage]);
final question = gateway.asked.single.$2.messages.single;
expect(imagesIn([question]).single.bytes, [1, 2, 3]);
expect(imagesIn(conversation.state.turns.single.messages), isEmpty);
final saved = (await conversations.all()).single;
expect(
imagesIn((await conversations.turnsOf(saved.id)).single.messages),
isEmpty,
);
await conversation.ask('Which colour?', const []);
expect(imagesIn(gateway.asked.last.$2.messages), isEmpty);
expect(
TurnView.of(conversation.state.turns.first.messages).sent,
[
const AssistantPart(
kind: AssistantPartKind.image,
text: '',
number: 1,
),
],
);
await conversation.close();
});
test('an image chosen never goes to an assistant not reading them, nor '
'is it said to', () async {
final conversation = cubit();
await conversation.open();
gateway.answers.add(_says('Hello.'));
await conversation.ask('Hi', const [chosenImage]);
final question = gateway.asked.single.$2.messages.single;
expect(imagesIn([question]), isEmpty);
expect(question.text, isNot(contains('image')));
// Alone, it is no question at all.
await conversation.ask('', const [chosenImage]);
expect(gateway.asked, hasLength(1));
await conversation.close();
});
test('an image chosen that cannot be had is said to be missing', () async {
final conversation = cubit(choice: const GrantedAssistant(_seeing));
await conversation.open();
gateway.answers.add(_says('I see nothing.'));
await conversation.ask('What is it?', const [
AssistantPart(kind: AssistantPartKind.image, text: 'gone', number: 1),
]);
final question = gateway.asked.single.$2.messages.single;
expect(imagesIn([question]), isEmpty);
expect(question.text, contains(imageNotHad));
await conversation.close();
});
test(
'what shows images is offered only to an assistant reading them',
() async {
for (final (choice, offered) in [
(const GrantedAssistant(_seeing), true),
(const GrantedAssistant(_claude), false),
]) {
final conversation = cubit(scope: library, choice: choice);
await conversation.open();
gateway.answers.add(_says('Yes.'));
await conversation.ask('Any figures?', const []);
expect(
[for (final tool in gateway.asked.last.$2.tools) tool.name],
offered
? contains('view_pdf_page')
: isNot(contains('view_pdf_page')),
);
await conversation.close();
}
},
);
test('a page the model asks to see goes once the person approves it, '
'after the results', () async {
final conversation = cubit(
scope: library,
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('The table lists godan verbs.'));
final approving = await askApproving(
conversation,
'What does page 2 show?',
() => conversation.approveImage(approved: true),
);
expect(approving, asked);
final shown = gateway.asked.last.$2.messages.last.parts;
expect(shown, hasLength(3));
expect((shown[0] as LlmToolResult).isError, isFalse);
expect((shown[1] as LlmText).text, asked.label);
expect((shown[2] as LlmImage).mediaType, 'image/png');
expect(files.drawn, ['p-genki:2']);
final turn = conversation.state.turns.single;
expect(imagesIn(turn.messages), isEmpty);
expect(
TurnView.of(turn.messages).lookups,
[const AssistantLookup(kind: LookupKind.pageImage, found: 1)],
);
await conversation.close();
});
test('refused, the model is told so and sees nothing', () async {
files.drawn.clear();
final conversation = cubit(
scope: library,
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('Without it, I cannot say.'));
await askApproving(
conversation,
'What does page 2 show?',
() => conversation.approveImage(approved: false),
);
final results = gateway.asked.last.$2.messages.last.parts;
final result = results.single as LlmToolResult;
expect(result.isError, isTrue);
expect(result.content, contains('chose not to show it'));
expect(imagesIn(gateway.asked.last.$2.messages), isEmpty);
expect(files.drawn, isEmpty);
await conversation.close();
});
test('stopped while asked, the question goes no further', () async {
final conversation = cubit(
scope: library,
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('Never asked.'));
await askApproving(
conversation,
'What does page 2 show?',
() => unawaited(conversation.stop()),
);
expect(gateway.asked, hasLength(1));
expect(conversation.state.asking, isNull);
expect(conversation.state.unanswered?.failure, isNull);
expect(conversation.state.turns, isEmpty);
await conversation.close();
});
test('an image out of reach is never asked about', () async {
final conversation = cubit(
scope: const AssistantScope(
reach: AssistantReach.studyset,
studysetId: 'elsewhere',
),
choice: const GrantedAssistant(_seeing),
);
await conversation.open();
gateway.answers
..add(_looksUp([viewPage]))
..add(_says('Nothing there.'));
final approving = await askApproving(
conversation,
'What does page 2 show?',
() => fail('asked to approve an image out of reach'),
);
expect(approving, isNull);
final result =
gateway.asked.last.$2.messages.last.parts.single as LlmToolResult;
expect(result.isError, isTrue);
await conversation.close();
});
});
}