feat(learn): what an assistant may look up, and what it is told
The lookups a model may make in a conversation (ADR 0021, D7): the studysets within reach, cards and notes found by what they say, a note read whole; answered on the device, only within the scope the person chose (D49, D50), private studysets only where switched on (D8), never what is locked, never progress (D11). The question goes with each part ticked, labelled, and the system prompt says how far the model may look and that what it looks up is content, never instructions (D10). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
b965e05fdb
commit
df89ff9998
GPG key ID:
07F6FCF405488B64
1 changed files with 863 additions and 0 deletions
|
|
@ -0,0 +1,332 @@
|
|||
import 'dart:convert';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:halkyon_learn_flutter/src/core/content/rich_content.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_lookups.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/card/model/card.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/card/model/card_content.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/note/model/note.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/note/model/note_content.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset_content.dart';
|
||||
import 'package:halkyon_llm/halkyon_llm.dart';
|
||||
|
||||
// What a model may look up, and only within the scope the person chose
|
||||
// (ADR 0021, D7, D8, D11): read-only, on the device, never progress.
|
||||
|
||||
Studyset _studyset(
|
||||
String id,
|
||||
String title, {
|
||||
String? parent,
|
||||
String position = 'a0',
|
||||
bool private = false,
|
||||
bool locked = false,
|
||||
DateTime? archived,
|
||||
}) => Studyset(
|
||||
id: id,
|
||||
content: StudysetContent(title: title, description: RichContent.empty()),
|
||||
parentId: parent,
|
||||
position: position,
|
||||
isPrivate: private,
|
||||
isLocked: locked,
|
||||
archivedAt: archived,
|
||||
);
|
||||
|
||||
Card _card(
|
||||
String id,
|
||||
String front,
|
||||
String back, {
|
||||
List<String> hints = const [],
|
||||
}) => Card(
|
||||
id: id,
|
||||
position: 'a0',
|
||||
content: CardContent(
|
||||
front: RichContent.plain(front),
|
||||
back: RichContent.plain(back),
|
||||
hints: [for (final hint in hints) RichContent.plain(hint)],
|
||||
),
|
||||
heads: const {},
|
||||
);
|
||||
|
||||
Note _note(String id, String title, String body) => Note(
|
||||
id: id,
|
||||
position: 'a0',
|
||||
content: NoteContent(title: title, body: RichContent.plain(body)),
|
||||
heads: const {},
|
||||
);
|
||||
|
||||
// Japanese, with Verbs inside it; Kanji, private, inside it too; Spanish
|
||||
// beside them; Old, archived, inside Japanese; Locked, whose key isn't here.
|
||||
final _studysets = [
|
||||
_studyset('ja', 'Japanese'),
|
||||
_studyset('verbs', 'Verbs', parent: 'ja'),
|
||||
_studyset('kanji', 'Kanji', parent: 'ja', position: 'a1', private: true),
|
||||
_studyset(
|
||||
'old',
|
||||
'Old',
|
||||
parent: 'ja',
|
||||
position: 'a2',
|
||||
archived: DateTime.utc(2026),
|
||||
),
|
||||
_studyset('es', 'Spanish', position: 'a1'),
|
||||
_studyset('locked', 'Locked', position: 'a2', private: true, locked: true),
|
||||
];
|
||||
|
||||
final _cards = {
|
||||
'ja': [_card('c-ja', 'こんにちは', 'Hello')],
|
||||
'verbs': [
|
||||
_card('c-suru', 'する', 'to do', hints: ['irregular']),
|
||||
_card('c-kuru', '来る', 'to come', hints: ['irregular']),
|
||||
],
|
||||
'kanji': [_card('c-kanji', '日', 'sun, day')],
|
||||
'old': [_card('c-old', '古い', 'old (irregular?)')],
|
||||
'es': [_card('c-es', 'hacer', 'to do')],
|
||||
'locked': [_card('c-locked', 'secret', 'irregular')],
|
||||
};
|
||||
|
||||
final _notes = {
|
||||
'verbs': [
|
||||
_note(
|
||||
'n-groups',
|
||||
'Verb groups',
|
||||
'Ichidan, godan, and the irregular する and 来る.',
|
||||
),
|
||||
],
|
||||
'es': [_note('n-es', 'Irregular verbs', 'ser, ir, hacer')],
|
||||
};
|
||||
|
||||
AssistantLookups _lookups() => AssistantLookups(
|
||||
studysets: () async => _studysets,
|
||||
cardsOf: (id) async => _cards[id] ?? const [],
|
||||
notesOf: (id) async => _notes[id] ?? const [],
|
||||
);
|
||||
|
||||
Future<Object?> _answer(
|
||||
AssistantLookups lookups,
|
||||
String tool,
|
||||
Map<String, Object?> input,
|
||||
AssistantScope scope,
|
||||
) async {
|
||||
final result = await lookups.answer(
|
||||
LlmToolCall(id: 'call-1', name: tool, input: input),
|
||||
scope,
|
||||
);
|
||||
expect(result.callId, 'call-1');
|
||||
return jsonDecode(result.content);
|
||||
}
|
||||
|
||||
List<String> _ids(Object? json) => [
|
||||
for (final item in json! as List<Object?>) (item! as Map)['id']! as String,
|
||||
];
|
||||
|
||||
void main() {
|
||||
final lookups = _lookups();
|
||||
|
||||
test('offers no lookup where only what is ticked goes (D50)', () {
|
||||
expect(lookups.toolsFor(const AssistantScope()), isEmpty);
|
||||
});
|
||||
|
||||
test('offers its lookups for every wider scope', () {
|
||||
for (final reach in [
|
||||
AssistantReach.studyset,
|
||||
AssistantReach.tree,
|
||||
AssistantReach.library,
|
||||
]) {
|
||||
expect(
|
||||
[
|
||||
for (final tool in lookups.toolsFor(
|
||||
AssistantScope(reach: reach, studysetId: 'ja'),
|
||||
))
|
||||
tool.name,
|
||||
],
|
||||
['list_studysets', 'find_cards', 'find_notes', 'read_note'],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
group('reaches', () {
|
||||
test('a studyset with everything inside it, but neither archived nor '
|
||||
'private ones (D8)', () async {
|
||||
final json = await _answer(
|
||||
lookups,
|
||||
'list_studysets',
|
||||
{},
|
||||
const AssistantScope(
|
||||
reach: AssistantReach.studyset,
|
||||
studysetId: 'ja',
|
||||
),
|
||||
);
|
||||
expect(_ids(json), ['ja', 'verbs']);
|
||||
});
|
||||
|
||||
test('private ones once switched on', () async {
|
||||
final json = await _answer(
|
||||
lookups,
|
||||
'list_studysets',
|
||||
{},
|
||||
const AssistantScope(
|
||||
reach: AssistantReach.studyset,
|
||||
studysetId: 'ja',
|
||||
includePrivate: true,
|
||||
),
|
||||
);
|
||||
expect(_ids(json), ['ja', 'verbs', 'kanji']);
|
||||
});
|
||||
|
||||
test("a studyset's whole tree, from its top", () async {
|
||||
final json = await _answer(
|
||||
lookups,
|
||||
'list_studysets',
|
||||
{},
|
||||
const AssistantScope(
|
||||
reach: AssistantReach.tree,
|
||||
studysetId: 'verbs',
|
||||
),
|
||||
);
|
||||
expect(_ids(json), ['ja', 'verbs']);
|
||||
});
|
||||
|
||||
test('the library, but never what is locked here', () async {
|
||||
final json = await _answer(
|
||||
lookups,
|
||||
'list_studysets',
|
||||
{},
|
||||
const AssistantScope(
|
||||
reach: AssistantReach.library,
|
||||
includePrivate: true,
|
||||
),
|
||||
);
|
||||
expect(_ids(json), ['ja', 'verbs', 'kanji', 'es']);
|
||||
});
|
||||
});
|
||||
|
||||
test('finds cards by any of their parts, within the scope only', () async {
|
||||
final json = await _answer(
|
||||
lookups,
|
||||
'find_cards',
|
||||
{'query': 'IRREGULAR'},
|
||||
const AssistantScope(reach: AssistantReach.studyset, studysetId: 'verbs'),
|
||||
);
|
||||
|
||||
expect(_ids(json), ['c-suru', 'c-kuru']);
|
||||
expect((json! as List).first, {
|
||||
'id': 'c-suru',
|
||||
'studyset': 'Verbs',
|
||||
'front': 'する',
|
||||
'back': 'to do',
|
||||
'hints': ['irregular'],
|
||||
});
|
||||
});
|
||||
|
||||
test('sends no progress with a card (D11)', () async {
|
||||
final json = await _answer(
|
||||
lookups,
|
||||
'find_cards',
|
||||
{'query': 'する'},
|
||||
const AssistantScope(reach: AssistantReach.library),
|
||||
);
|
||||
|
||||
expect(
|
||||
((json! as List).single as Map).keys,
|
||||
unorderedEquals(['id', 'studyset', 'front', 'back', 'hints']),
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
'finds as many cards as asked for, ten unless said, 25 at most',
|
||||
() async {
|
||||
final many = AssistantLookups(
|
||||
studysets: () async => [_studyset('s', 'Many')],
|
||||
cardsOf: (_) async => [
|
||||
for (var i = 0; i < 40; i++) _card('c-$i', 'word $i', 'meaning'),
|
||||
],
|
||||
notesOf: (_) async => const [],
|
||||
);
|
||||
const scope = AssistantScope(reach: AssistantReach.library);
|
||||
|
||||
expect(
|
||||
await _answer(many, 'find_cards', {'query': 'word'}, scope),
|
||||
hasLength(10),
|
||||
);
|
||||
expect(
|
||||
await _answer(many, 'find_cards', {'query': 'word', 'limit': 3}, scope),
|
||||
hasLength(3),
|
||||
);
|
||||
expect(
|
||||
await _answer(many, 'find_cards', {
|
||||
'query': 'word',
|
||||
'limit': 100,
|
||||
}, scope),
|
||||
hasLength(25),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test('finds notes and reads one, within the scope only', () async {
|
||||
const scope = AssistantScope(reach: AssistantReach.tree, studysetId: 'ja');
|
||||
final found = await _answer(lookups, 'find_notes', {
|
||||
'query': 'irregular',
|
||||
}, scope);
|
||||
expect(_ids(found), ['n-groups']);
|
||||
|
||||
final note = await _answer(lookups, 'read_note', {'id': 'n-groups'}, scope);
|
||||
expect(note, {
|
||||
'id': 'n-groups',
|
||||
'studyset': 'Verbs',
|
||||
'title': 'Verb groups',
|
||||
'text': 'Ichidan, godan, and the irregular する and 来る.',
|
||||
});
|
||||
});
|
||||
|
||||
group('refuses, as an error the model reads,', () {
|
||||
const scope = AssistantScope(
|
||||
reach: AssistantReach.studyset,
|
||||
studysetId: 'ja',
|
||||
);
|
||||
|
||||
Future<LlmToolResult> call(String tool, Map<String, Object?> input) =>
|
||||
lookups.answer(LlmToolCall(id: 'c', name: tool, input: input), scope);
|
||||
|
||||
test('a note outside the scope, as if there were none', () async {
|
||||
final result = await call('read_note', {'id': 'n-es'});
|
||||
expect(result.isError, isTrue);
|
||||
expect(result.content, contains('No note'));
|
||||
});
|
||||
|
||||
test('a tool it never offered', () async {
|
||||
final result = await call('delete_cards', {'query': 'x'});
|
||||
expect(result.isError, isTrue);
|
||||
});
|
||||
|
||||
test('input the tool does not take', () async {
|
||||
expect((await call('find_cards', {})).isError, isTrue);
|
||||
expect((await call('find_cards', {'query': 7})).isError, isTrue);
|
||||
});
|
||||
|
||||
test('any lookup where only what is ticked goes (D50)', () async {
|
||||
final result = await lookups.answer(
|
||||
const LlmToolCall(id: 'c', name: 'find_cards', input: {'query': 'x'}),
|
||||
const AssistantScope(),
|
||||
);
|
||||
expect(result.isError, isTrue);
|
||||
});
|
||||
});
|
||||
|
||||
test('says what a lookup did, for the conversation to show', () {
|
||||
expect(
|
||||
describeLookup(
|
||||
const LlmToolCall(id: 'c', name: 'find_cards', input: {'query': 'する'}),
|
||||
const LlmToolResult(callId: 'c', content: '[{"id":"a"},{"id":"b"}]'),
|
||||
),
|
||||
const AssistantLookup(kind: LookupKind.cards, query: 'する', found: 2),
|
||||
);
|
||||
expect(
|
||||
describeLookup(
|
||||
const LlmToolCall(id: 'c', name: 'read_note', input: {'id': 'n'}),
|
||||
const LlmToolResult(callId: 'c', content: 'No note', isError: true),
|
||||
),
|
||||
const AssistantLookup(kind: LookupKind.note, found: 0),
|
||||
);
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue