halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

feat(learn): an assistant reads PDFs' pages, and sees images the person chooses or approves

Browse source 
ADR 0021, D12: text first. Lookups find a studyset's PDFs and read a page's
text as PDFium extracts it. An assistant that reads images is offered a
card's images as chips, and may ask to see a card's image or a PDF's page
drawn whole; the conversation asks the person each time (Show it, Don't
show) and sends nothing meanwhile. Images go only as png, jpeg, gif or webp
of at most 5 MB, are never kept in a turn, and a follow-up goes without them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-25 21:54:56 +01:00
commit 2b7d04fd70
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
2 changed files with 2030 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

37
apps/learn/halkyon_learn_flutter/test/support/fake_assistant_files.dart Normal file
View file

@ -0,0 +1,37 @@
import 'dart:typed_data';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_files.dart';
import 'package:halkyon_learn_flutter/src/feature/pdf_source/model/pdf_source.dart';
import 'package:halkyon_llm/halkyon_llm.dart';
/// [AssistantFiles] in memory: [pages]' text by PDF id and page, [images]
/// by asset id, and every page drawn as a PNG of its PDF's id and page.
class FakeAssistantFiles() implements AssistantFiles {
/// The text of each PDF's pages, by its id, then by page.
final pages = <String, Map<int, String>>{};
/// The images, by their asset rows' ids.
final images = <String, LlmImage>{};
/// The pages drawn, as `<pdf id>:<page>`.
final drawn = <String>[];
@override
Future<String?> pageText(PdfSource pdf, int page) async =>
pages[pdf.id]?[page];
@override
Future<LlmImage?> pageImage(PdfSource pdf, int page) async {
if (pages[pdf.id]?[page] == null) {
return null;
}
drawn.add('${pdf.id}:$page');
return LlmImage(
mediaType: 'image/png',
bytes: Uint8List.fromList('${pdf.id}:$page'.codeUnits),
);
}
@override
Future<LlmImage?> image(String assetId) async => images[assetId];
}

6
apps/learn/halkyon_learn_flutter/test/support/test_dependencies.dart
View file

@ -20,6 +20,7 @@ import 'package:halkyon_learn_flutter/src/feature/anki/data/anki_package.dart';
import 'package:halkyon_learn_flutter/src/feature/anki/data/anki_package_reader.dart';
import 'package:halkyon_learn_flutter/src/feature/archive/data/archive_reader.dart';
import 'package:halkyon_learn_flutter/src/feature/archive/model/import_archive.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_files.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_lookups.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
import 'package:halkyon_learn_flutter/src/feature/assistant/data/web_search.dart';
@ -194,6 +195,7 @@ Dependencies testDependencies({
final addresses = signInAddresses ?? MemorySignInAddresses();
final crypto = FakeAccountCrypto();
final keys = accountKeys ?? MemoryAccountKeyStore();
final engine = pdfEngine ?? FakePdfEngine();
return Dependencies(
serverName: 'learn.example.org',
auth: AuthRepository(auth, addresses),
@ -237,7 +239,7 @@ Dependencies testDependencies({
files: files ?? FakeFilePicker(),
fileSaver: fileSaver ?? FakeFileSaver(),
videos: videos ?? FakeYoutubeVideos(),
pdfEngine: pdfEngine ?? FakePdfEngine(),
pdfEngine: engine,
ankiPackages: InlineAnkiPackageReader(),
archives: InlineArchiveReader(),
exportArchiver: InlineExportArchiver(),
@ -350,6 +352,8 @@ Dependencies testDependencies({
studysets: studysets.withDeleted,
cardsOf: cards.cardsOf,
notesOf: notes.notesOf,
pdfsOf: pdfSources.sourcesOf,
files: DeviceAssistantFiles(assets, engine),
),
assets: assets,
deletions: deletions,