feat(learn): an assistant reads PDFs' pages, and sees images the person chooses or approves

ADR 0021, D12: text first. Lookups find a studyset's PDFs and read a page's
text as PDFium extracts it. An assistant that reads images is offered a
card's images as chips, and may ask to see a card's image or a PDF's page
drawn whole; the conversation asks the person each time (Show it, Don't
show) and sends nothing meanwhile. Images go only as png, jpeg, gif or webp
of at most 5 MB, are never kept in a turn, and a follow-up goes without them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

apps/learn/halkyon_learn_flutter/test/feature/assistant/data/assistant_files_test.dart

@@ -0,0 +1,107 @@
+import 'dart:convert';
+import 'dart:typed_data';
+import 'dart:ui' show Rect;
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:halkyon_learn_flutter/src/core/asset/data/asset_repository.dart';
+import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_files.dart';
+import 'package:halkyon_learn_flutter/src/feature/pdf_source/model/pdf_source.dart';
+
+import '../../../support/fake_asset_gateway.dart';
+import '../../../support/fake_asset_sealing.dart';
+import '../../../support/fake_asset_server_gateway.dart';
+import '../../../support/fake_pdf_engine.dart';
+import '../../../support/memory_asset_file_cache.dart';
+
+// What a model is sent of the person's files (ADR 0021, D12): images every
+// provider reads, none larger than the smallest limit, and a PDF's pages
+// as text and drawn whole.
+
+void main() {
+  late AssetRepository assets;
+  late FakePdfEngine engine;
+  late DeviceAssistantFiles files;
+
+  setUp(() {
+    assets = AssetRepository(
+      FakeAssetGateway(),
+      FakeAssetServerGateway(),
+      MemoryAssetFileCache(),
+      FakeAssetSealing(),
+    );
+    engine = FakePdfEngine();
+    files = DeviceAssistantFiles(assets, engine);
+  });
+
+  Future<String> add(List<int> bytes, String mimeType) => assets.add(
+    Uint8List.fromList(bytes),
+    fileName: 'file',
+    mimeType: mimeType,
+    studysetId: null,
+  );
+
+  Future<PdfSource> pdf({int pages = 3}) async => PdfSource(
+    id: 'p',
+    studysetId: 's',
+    content: PdfSourceContent(
+      title: 'Genki I',
+      fileName: 'genki.pdf',
+      assetId: await add(utf8.encode('%PDF- pages:$pages'), 'application/pdf'),
+    ),
+  );
+
+  test('gives an image a model reads', () async {
+    final types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp'];
+    for (final (index, type) in types.indexed) {
+      // Bytes of its own, or it would be the same file.
+      final id = await add([1, 2, 3, index], type);
+      final image = await files.image(id);
+      expect(image?.mediaType, type);
+      expect(image?.bytes, [1, 2, 3, index]);
+    }
+  });
+
+  test('gives no other file, nor one larger than a provider takes', () async {
+    expect(await files.image(await add([1], 'image/svg+xml')), isNull);
+    expect(await files.image(await add([2], 'audio/mpeg')), isNull);
+    expect(
+      await files.image(
+        await add(
+          List.filled(DeviceAssistantFiles.maxImageBytes + 1, 7),
+          'image/png',
+        ),
+      ),
+      isNull,
+    );
+    expect(await files.image('no-such-asset'), isNull);
+  });
+
+  test("reads a page's text, and none of a page it hasn't", () async {
+    engine.texts[2] = 'Verbs ending in -ru.';
+    final genki = await pdf();
+
+    expect(await files.pageText(genki, 2), 'Verbs ending in -ru.');
+    expect(await files.pageText(genki, 4), isNull);
+    expect(await files.pageText(genki, 0), isNull);
+  });
+
+  test('draws a page whole, as a PNG', () async {
+    final genki = await pdf();
+
+    final image = await files.pageImage(genki, 3);
+
+    expect(image?.mediaType, 'image/png');
+    expect(image?.bytes, engine.image);
+    expect(engine.rendered.single.page, 3);
+    expect(engine.rendered.single.region, const Rect.fromLTWH(0, 0, 1, 1));
+  });
+
+  test('gives nothing of a PDF it cannot open, nor a page too large', () async {
+    final genki = await pdf();
+    engine.failNext = true;
+    expect(await files.pageText(genki, 1), isNull);
+
+    engine.image = Uint8List(DeviceAssistantFiles.maxImageBytes + 1);
+    expect(await files.pageImage(genki, 1), isNull);
+  });
+}

apps/learn/halkyon_learn_flutter/test/feature/assistant/data/assistant_lookups_test.dart

@@ -1,4 +1,5 @@
 import 'dart:convert';
+import 'dart:typed_data';
 
 import 'package:flutter_test/flutter_test.dart';
 import 'package:halkyon_learn_flutter/src/core/content/rich_content.dart';
@@ -9,10 +10,13 @@ import 'package:halkyon_learn_flutter/src/feature/card/model/card.dart';
 import 'package:halkyon_learn_flutter/src/feature/card/model/card_content.dart';
 import 'package:halkyon_learn_flutter/src/feature/note/model/note.dart';
 import 'package:halkyon_learn_flutter/src/feature/note/model/note_content.dart';
+import 'package:halkyon_learn_flutter/src/feature/pdf_source/model/pdf_source.dart';
 import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset.dart';
 import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset_content.dart';
 import 'package:halkyon_llm/halkyon_llm.dart';
 
+import '../../../support/fake_assistant_files.dart';
+
 // What a model may look up, and only within the scope the person chose
 // (ADR 0021, D7, D8, D11): read-only, on the device, never progress.
 
@@ -50,6 +54,24 @@ Card _card(
   heads: const {},
 );
 
+PdfSource _pdf(String id, String studysetId, String title, {int? pages}) =>
+    PdfSource(
+      id: id,
+      studysetId: studysetId,
+      content: PdfSourceContent(
+        title: title,
+        fileName: '$title.pdf',
+        assetId: 'asset-$id',
+        pageCount: pages,
+      ),
+    );
+
+/// A block showing the person's image [assetId].
+Map<String, Object?> _image(String assetId) => {
+  'type': 'image',
+  'data': {'url': 'asset:$assetId'},
+};
+
 Note _note(String id, String title, String body) => Note(
   id: id,
   position: 'a0',
@@ -86,6 +108,46 @@ final _cards = {
   'locked': [_card('c-locked', 'secret', 'irregular')],
 };
 
+// A card of Verbs with two images, one on each side.
+final _pictured = Card(
+  id: 'c-pictured',
+  position: 'a1',
+  content: CardContent(
+    front: RichContent({
+      'document': {
+        'type': 'page',
+        'children': [
+          {
+            'type': 'paragraph',
+            'data': {
+              'delta': [
+                {'insert': '食べる'},
+              ],
+            },
+          },
+          _image('img-eat'),
+        ],
+      },
+    }),
+    back: RichContent({
+      'document': {
+        'type': 'page',
+        'children': [_image('img-bowl')],
+      },
+    }),
+  ),
+  heads: const {},
+);
+
+final _pdfs = {
+  'verbs': [
+    _pdf('p-genki', 'verbs', 'Genki I', pages: 12),
+    _pdf('p-drill', 'verbs', 'Verb drills'),
+  ],
+  'kanji': [_pdf('p-kanji', 'kanji', 'Kanji in context', pages: 3)],
+  'es': [_pdf('p-es', 'es', 'Gramática', pages: 5)],
+};
+
 final _notes = {
   'verbs': [
     _note(
@@ -97,10 +159,12 @@ final _notes = {
   'es': [_note('n-es', 'Irregular verbs', 'ser, ir, hacer')],
 };
 
-AssistantLookups _lookups() => AssistantLookups(
+AssistantLookups _lookups({FakeAssistantFiles? files}) => AssistantLookups(
   studysets: () async => _studysets,
   cardsOf: (id) async => _cards[id] ?? const [],
   notesOf: (id) async => _notes[id] ?? const [],
+  pdfsOf: (id) async => _pdfs[id] ?? const [],
+  files: files ?? FakeAssistantFiles(),
 );
 
 Future<Object?> _answer(
@@ -141,11 +205,36 @@ void main() {
           ))
             tool.name,
         ],
-        ['list_studysets', 'find_cards', 'find_notes', 'read_note'],
+        [
+          'list_studysets',
+          'find_cards',
+          'find_notes',
+          'read_note',
+          'find_pdfs',
+          'read_pdf_page',
+        ],
       );
     }
   });
 
+  test('offers what shows images only to an assistant reading them, and '
+      'only with its lookups (D12)', () {
+    const scope = AssistantScope(
+      reach: AssistantReach.studyset,
+      studysetId: 'ja',
+    );
+    List<String> names(List<LlmTool> tools) => [
+      for (final tool in tools) tool.name,
+    ];
+
+    expect(names(lookups.toolsFor(scope)), isNot(contains('view_pdf_page')));
+    expect(
+      names(lookups.toolsFor(scope, images: true)),
+      containsAllInOrder(['read_pdf_page', 'view_card_image', 'view_pdf_page']),
+    );
+    expect(lookups.toolsFor(const AssistantScope(), images: true), isEmpty);
+  });
+
   group('reaches', () {
     test('a studyset with everything inside it, but neither archived nor '
         'private ones (D8)', () async {
@@ -213,6 +302,8 @@ void main() {
       ],
       cardsOf: (id) async => const [],
       notesOf: (id) async => const [],
+      pdfsOf: (id) async => const [],
+      files: FakeAssistantFiles(),
     );
     Future<List<String>> sent(AssistantScope scope) async => [
       for (final studyset in await withKorean.privateSent(scope)) studyset.id,
@@ -383,6 +474,8 @@ void main() {
           for (var i = 0; i < 40; i++) _card('c-$i', 'word $i', 'meaning'),
         ],
         notesOf: (_) async => const [],
+        pdfsOf: (_) async => const [],
+        files: FakeAssistantFiles(),
       );
       const scope = AssistantScope(reach: AssistantReach.library);
 
@@ -420,6 +513,233 @@ void main() {
     });
   });
 
+  group("reads a PDF's pages as text (D12)", () {
+    final files = FakeAssistantFiles()
+      ..pages['p-genki'] = {3: '  Verbs ending in -ru.  ', 4: 'x' * 7000}
+      ..pages['p-kanji'] = {1: 'secret'};
+    final reader = _lookups(files: files);
+    const scope = AssistantScope(
+      reach: AssistantReach.studyset,
+      studysetId: 'ja',
+    );
+
+    test('finding those within reach by their titles, or all', () async {
+      expect(await _answer(reader, 'find_pdfs', {}, scope), [
+        {
+          'id': 'p-genki',
+          'studyset': 'Verbs',
+          'title': 'Genki I',
+          'pages': 12,
+        },
+        {'id': 'p-drill', 'studyset': 'Verbs', 'title': 'Verb drills'},
+      ]);
+      expect(
+        _ids(await _answer(reader, 'find_pdfs', {'query': 'GENKI'}, scope)),
+        ['p-genki'],
+      );
+      expect(
+        _ids(
+          await _answer(
+            reader,
+            'find_pdfs',
+            {},
+            const AssistantScope(
+              reach: AssistantReach.studyset,
+              studysetId: 'ja',
+              includePrivate: true,
+            ),
+          ),
+        ),
+        ['p-genki', 'p-drill', 'p-kanji'],
+      );
+    });
+
+    test('a page, its text cut as a note is', () async {
+      expect(
+        await _answer(reader, 'read_pdf_page', {
+          'id': 'p-genki',
+          'page': 3,
+        }, scope),
+        {
+          'id': 'p-genki',
+          'studyset': 'Verbs',
+          'title': 'Genki I',
+          'page': 3,
+          'pages': 12,
+          'text': 'Verbs ending in -ru.',
+        },
+      );
+      final long =
+          (await _answer(reader, 'read_pdf_page', {
+                'id': 'p-genki',
+                'page': 4,
+              }, scope))!
+              as Map<String, Object?>;
+      expect(
+        (long['text']! as String).length,
+        AssistantLookups.maxPageLength,
+      );
+    });
+
+    test('never a PDF outside the scope, nor a page it cannot read', () async {
+      Future<LlmToolResult> read(String id, Object? page) => reader.answer(
+        LlmToolCall(
+          id: 'c',
+          name: 'read_pdf_page',
+          input: {
+            'id': id,
+            'page': page,
+          },
+        ),
+        scope,
+      );
+
+      for (final result in [
+        await read('p-es', 1),
+        // Private, and not switched on.
+        await read('p-kanji', 1),
+      ]) {
+        expect(result.isError, isTrue);
+        expect(result.content, contains('No PDF'));
+      }
+      expect((await read('p-genki', 9)).isError, isTrue);
+      expect((await read('p-genki', '3')).isError, isTrue);
+      expect(files.drawn, isEmpty);
+    });
+  });
+
+  group('images (D12)', () {
+    final files = FakeAssistantFiles()
+      ..images['img-eat'] = LlmImage(
+        mediaType: 'image/png',
+        bytes: Uint8List.fromList([1, 2, 3]),
+      )
+      ..pages['p-genki'] = {2: 'A table of verbs.'};
+    final pictured = AssistantLookups(
+      studysets: () async => _studysets,
+      cardsOf: (id) async => [...?_cards[id], if (id == 'verbs') _pictured],
+      notesOf: (id) async => _notes[id] ?? const [],
+      pdfsOf: (id) async => _pdfs[id] ?? const [],
+      files: files,
+    );
+    const scope = AssistantScope(
+      reach: AssistantReach.studyset,
+      studysetId: 'ja',
+    );
+    LlmToolCall call(String tool, Map<String, Object?> input) =>
+        LlmToolCall(id: 'c', name: tool, input: input);
+
+    test('a card found says how many it has, only to an assistant reading '
+        'them', () async {
+      Future<Object?> found({required bool images}) async => jsonDecode(
+        (await pictured.answer(
+          call('find_cards', {'query': '食べる'}),
+          scope,
+          images: images,
+        )).content,
+      );
+
+      expect(
+        ((await found(images: true))! as List).single,
+        containsPair(
+          'images',
+          2,
+        ),
+      );
+      expect(
+        ((await found(images: false))! as List).single,
+        isNot(contains('images')),
+      );
+    });
+
+    test('are asked for within reach, for the person to approve', () async {
+      expect(
+        await pictured.askedImage(
+          call('view_card_image', {'id': 'c-pictured', 'number': 2}),
+          scope,
+        ),
+        const AskedImage(
+          kind: LookupKind.cardImage,
+          title: '食べる',
+          number: 2,
+          assetId: 'img-bowl',
+        ),
+      );
+      expect(
+        await pictured.askedImage(
+          call('view_pdf_page', {'id': 'p-genki', 'page': 2}),
+          scope,
+        ),
+        const AskedImage(
+          kind: LookupKind.pageImage,
+          title: 'Genki I',
+          number: 2,
+          pdfId: 'p-genki',
+        ),
+      );
+    });
+
+    test('never outside reach, nor one there is not', () async {
+      for (final asked in [
+        call('view_card_image', {'id': 'c-pictured', 'number': 3}),
+        call('view_card_image', {'id': 'c-pictured', 'number': 0}),
+        call('view_card_image', {'id': 'c-es', 'number': 1}),
+        call('view_pdf_page', {'id': 'p-genki', 'page': 13}),
+        call('view_pdf_page', {'id': 'p-kanji', 'page': 1}),
+        call('view_pdf_page', {'id': 'p-es', 'page': 1}),
+        call('view_pdf_page', {'id': 'p-genki'}),
+        call('find_cards', {'id': 'c-pictured'}),
+      ]) {
+        expect(
+          await pictured.askedImage(asked, scope),
+          isNull,
+          reason: '$asked',
+        );
+      }
+    });
+
+    test(
+      'are never answered as a lookup is, before the person approves',
+      () async {
+        final result = await pictured.answer(
+          call('view_pdf_page', {'id': 'p-genki', 'page': 2}),
+          scope,
+          images: true,
+        );
+        expect(result.isError, isTrue);
+        expect(result.content, contains('approves'));
+        expect(files.drawn, isEmpty);
+      },
+    );
+
+    test('approved, are had from the files, a page drawn whole', () async {
+      final card = (await pictured.askedImage(
+        call('view_card_image', {'id': 'c-pictured', 'number': 1}),
+        scope,
+      ))!;
+      expect((await pictured.imageOf(card, scope))?.bytes, [1, 2, 3]);
+
+      final page = (await pictured.askedImage(
+        call('view_pdf_page', {'id': 'p-genki', 'page': 2}),
+        scope,
+      ))!;
+      expect((await pictured.imageOf(page, scope))?.mediaType, 'image/png');
+      expect(files.drawn, ['p-genki:2']);
+
+      // Out of reach by now: the scope no longer takes Verbs in.
+      expect(
+        await pictured.imageOf(
+          page,
+          const AssistantScope(
+            reach: AssistantReach.studyset,
+            studysetId: 'es',
+          ),
+        ),
+        isNull,
+      );
+    });
+  });
+
   group('refuses, as an error the model reads,', () {
     const scope = AssistantScope(
       reach: AssistantReach.studyset,

apps/learn/halkyon_learn_flutter/test/feature/assistant/data/conversation_payload_test.dart

@@ -4,6 +4,7 @@ import 'package:flutter_test/flutter_test.dart';
 import 'package:halkyon_learn_flutter/src/core/content/payload.dart';
 import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_payload.dart';
 import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
+import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_prompt.dart';
 import 'package:halkyon_llm/halkyon_llm.dart';
 
 // A conversation's and its turns' payloads (ADR 0021, D51): what a follow-up