feat(learn): asking an assistant in a conversation
ConversationCubit asks the assistant chosen each question with the conversation so far (ADR 0021, D47), answers the model's lookups on the device within the question's scope, eight rounds at most, the ninth told to answer with what it found and the tenth ending the turn, every call given its result so the conversation can still be sent; and keeps each turn, sealed, once it is answered (D51): nothing of a question stopped or failed, which can be asked again, and nothing at all on a device that can't seal, which still answers (D52). The question and the parts sent along are texts of their own, so a kept turn shows as it was asked, with its lookups and its answer (TurnView). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
df89ff9998
commit
a125215942
GPG key ID:
07F6FCF405488B64
1 changed files with 1250 additions and 124 deletions
|
|
@ -0,0 +1,456 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:halkyon_app/halkyon_app.dart';
|
||||
import 'package:halkyon_app/testing.dart';
|
||||
import 'package:halkyon_learn_flutter/src/core/content/rich_content.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/bloc/conversation_cubit.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/assistant_lookups.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/data/conversation_repository.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_conversation.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_lookup.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/assistant_topic.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/assistant/model/turn_view.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/card/model/card.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/card/model/card_content.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset_content.dart';
|
||||
import 'package:halkyon_llm/halkyon_llm.dart';
|
||||
|
||||
import '../../../support/fake_conversation_gateway.dart';
|
||||
|
||||
// Asking an assistant in a conversation (ADR 0021, D7, D47, D51, D52): the
|
||||
// conversation so far goes with every question, the model's lookups are
|
||||
// answered on the device within the scope, at most eight rounds of them,
|
||||
// and a turn is kept, sealed, only once answered; nowhere where the device
|
||||
// can't seal it.
|
||||
|
||||
const _claude = OfferedAssistant(
|
||||
id: 'claude',
|
||||
name: 'Claude',
|
||||
protocol: 'anthropic',
|
||||
host: 'api.anthropic.com',
|
||||
model: 'claude-sonnet',
|
||||
);
|
||||
|
||||
const _usage = LlmUsage(inputTokens: 100, outputTokens: 20);
|
||||
|
||||
/// The model answering [text], streamed in two pieces.
|
||||
List<LlmEvent> _says(String text) => [
|
||||
LlmTextDelta(text.substring(0, text.length ~/ 2)),
|
||||
LlmTextDelta(text.substring(text.length ~/ 2)),
|
||||
LlmFinished(
|
||||
message: LlmMessage.assistant([LlmText(text)]),
|
||||
stop: LlmStop.end,
|
||||
usage: _usage,
|
||||
),
|
||||
];
|
||||
|
||||
/// The model calling [calls].
|
||||
List<LlmEvent> _looksUp(List<LlmToolCall> calls) => [
|
||||
LlmFinished(
|
||||
message: LlmMessage.assistant(calls),
|
||||
stop: LlmStop.toolCalls,
|
||||
usage: _usage,
|
||||
),
|
||||
];
|
||||
|
||||
const _findIrregular = LlmToolCall(
|
||||
id: 'call-1',
|
||||
name: 'find_cards',
|
||||
input: {'query': 'irregular'},
|
||||
);
|
||||
|
||||
Card _card(String id, String front, String hint) => Card(
|
||||
id: id,
|
||||
position: 'a0',
|
||||
content: CardContent(
|
||||
front: RichContent.plain(front),
|
||||
back: RichContent.plain('…'),
|
||||
hints: [RichContent.plain(hint)],
|
||||
),
|
||||
heads: const {},
|
||||
);
|
||||
|
||||
final _verbs = Studyset(
|
||||
id: 'verbs',
|
||||
content: StudysetContent(title: 'Verbs', description: RichContent.empty()),
|
||||
);
|
||||
|
||||
/// A gateway whose answer comes only as the test sends it.
|
||||
class _SlowGateway() extends FakeAssistantGateway {
|
||||
final answer = StreamController<LlmEvent>();
|
||||
|
||||
@override
|
||||
Stream<LlmEvent> ask(String assistant, LlmRequest request) {
|
||||
asked.add((assistant, request));
|
||||
return answer.stream;
|
||||
}
|
||||
}
|
||||
|
||||
void main() {
|
||||
late FakeAssistantGateway gateway;
|
||||
late FakeConversationGateway kept;
|
||||
late ConversationRepository conversations;
|
||||
var lookups = AssistantLookups(
|
||||
studysets: () async => [_verbs],
|
||||
cardsOf: (_) async => [
|
||||
_card('c-suru', 'する', 'irregular'),
|
||||
_card('c-kuru', '来る', 'irregular'),
|
||||
],
|
||||
notesOf: (_) async => const [],
|
||||
);
|
||||
|
||||
setUp(() {
|
||||
gateway = FakeAssistantGateway();
|
||||
kept = FakeConversationGateway();
|
||||
conversations = ConversationRepository(kept);
|
||||
});
|
||||
tearDown(() => conversations.dispose());
|
||||
|
||||
ConversationCubit cubit({
|
||||
AssistantScope scope = const AssistantScope(),
|
||||
String? conversationId,
|
||||
AssistantChoice? choice = const GrantedAssistant(_claude),
|
||||
AssistantGateway? asking,
|
||||
}) => ConversationCubit(
|
||||
assistants: AssistantRepository(asking ?? gateway),
|
||||
lookups: lookups,
|
||||
conversations: conversations,
|
||||
choice: choice,
|
||||
assistant: const AssistantRef(id: 'claude'),
|
||||
scope: scope,
|
||||
conversationId: conversationId,
|
||||
);
|
||||
|
||||
/// The states [cubit] goes through while [act] runs.
|
||||
Future<List<ConversationState>> statesWhile(
|
||||
ConversationCubit cubit,
|
||||
Future<void> Function() act,
|
||||
) async {
|
||||
final states = <ConversationState>[];
|
||||
final listening = cubit.stream.listen(states.add);
|
||||
await act();
|
||||
await Future<void>.delayed(Duration.zero);
|
||||
await listening.cancel();
|
||||
return states;
|
||||
}
|
||||
|
||||
test('answers a question as it comes, then keeps the turn in a '
|
||||
'conversation of its own (D51)', () async {
|
||||
final conversation = cubit();
|
||||
await conversation.open();
|
||||
gateway.answers.add(_says('Because it joins.'));
|
||||
|
||||
final states = await statesWhile(
|
||||
conversation,
|
||||
() => conversation.ask('Why て-form?', const []),
|
||||
);
|
||||
|
||||
expect(
|
||||
[
|
||||
for (final state in states)
|
||||
if (state.asking case final asking?) asking.text,
|
||||
],
|
||||
['', 'Because ', 'Because it joins.'],
|
||||
);
|
||||
final last = states.last;
|
||||
expect(last.asking, isNull);
|
||||
expect(last.id, 'conversation-001');
|
||||
expect(TurnView.of(last.turns.single.messages).answer, 'Because it joins.');
|
||||
|
||||
final (assistant, request) = gateway.asked.single;
|
||||
expect(assistant, 'claude');
|
||||
expect(request.maxOutputTokens, ConversationCubit.maxOutputTokens);
|
||||
expect(request.tools, isEmpty, reason: 'only what is ticked goes (D50)');
|
||||
expect(request.system, contains('nothing to look up'));
|
||||
|
||||
final saved = (await conversations.all()).single;
|
||||
expect(saved.title, 'Why て-form?');
|
||||
expect(saved.assistant, const AssistantRef(id: 'claude'));
|
||||
final turn = (await conversations.turnsOf(saved.id)).single;
|
||||
expect(turn.usage, _usage);
|
||||
expect(turn.messages.last.text, 'Because it joins.');
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('sends the conversation so far with a follow-up, and keeps it in the '
|
||||
'same conversation', () async {
|
||||
final conversation = cubit();
|
||||
await conversation.open();
|
||||
gateway.answers
|
||||
..add(_says('One.'))
|
||||
..add(_says('Two.'));
|
||||
|
||||
await conversation.ask('First?', const []);
|
||||
await conversation.ask('And then?', const []);
|
||||
|
||||
final follow = gateway.asked.last.$2.messages;
|
||||
expect(
|
||||
[for (final message in follow) message.role],
|
||||
[
|
||||
LlmRole.user,
|
||||
LlmRole.assistant,
|
||||
LlmRole.user,
|
||||
],
|
||||
);
|
||||
expect(follow[1].text, 'One.');
|
||||
expect(conversation.state.turns, hasLength(2));
|
||||
expect(await conversations.all(), hasLength(1));
|
||||
expect(await conversations.turnsOf('conversation-001'), hasLength(2));
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('sends the parts ticked with the question', () async {
|
||||
final conversation = cubit();
|
||||
await conversation.open();
|
||||
gateway.answers.add(_says('It is する.'));
|
||||
|
||||
await conversation.ask('What is this?', const [
|
||||
AssistantPart(kind: AssistantPartKind.front, text: 'する'),
|
||||
]);
|
||||
|
||||
final view = TurnView.of(conversation.state.turns.single.messages);
|
||||
expect(view.question, 'What is this?');
|
||||
expect(view.sent, const [
|
||||
AssistantPart(kind: AssistantPartKind.front, text: 'する'),
|
||||
]);
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('answers the lookups within the scope, the results sent back, each '
|
||||
'shown in the turn (D7)', () async {
|
||||
final conversation = cubit(
|
||||
scope: const AssistantScope(
|
||||
reach: AssistantReach.studyset,
|
||||
studysetId: 'verbs',
|
||||
),
|
||||
);
|
||||
await conversation.open();
|
||||
gateway.answers
|
||||
..add(_looksUp([_findIrregular]))
|
||||
..add(_says('する and 来る.'));
|
||||
|
||||
await conversation.ask('Which verbs are irregular?', const []);
|
||||
|
||||
final (_, first) = gateway.asked.first;
|
||||
expect(
|
||||
[for (final tool in first.tools) tool.name],
|
||||
['list_studysets', 'find_cards', 'find_notes', 'read_note'],
|
||||
);
|
||||
expect(first.system, contains('the studyset “Verbs”'));
|
||||
final (_, second) = gateway.asked.last;
|
||||
final result = second.messages.last.parts.single as LlmToolResult;
|
||||
expect(result.callId, 'call-1');
|
||||
expect(result.content, contains('c-suru'));
|
||||
|
||||
final turn = conversation.state.turns.single;
|
||||
expect(TurnView.of(turn.messages).lookups, const [
|
||||
AssistantLookup(kind: LookupKind.cards, query: 'irregular', found: 2),
|
||||
]);
|
||||
expect(
|
||||
turn.usage,
|
||||
const LlmUsage(inputTokens: 200, outputTokens: 40),
|
||||
reason: 'both requests counted',
|
||||
);
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('a lookup that fails is an error the model reads', () async {
|
||||
final working = lookups;
|
||||
lookups = AssistantLookups(
|
||||
studysets: () async => [_verbs],
|
||||
cardsOf: (_) async => throw const FormatException('unreadable'),
|
||||
notesOf: (_) async => const [],
|
||||
);
|
||||
addTearDown(() => lookups = working);
|
||||
final conversation = cubit(
|
||||
scope: const AssistantScope(reach: AssistantReach.library),
|
||||
);
|
||||
await conversation.open();
|
||||
gateway.answers
|
||||
..add(_looksUp([_findIrregular]))
|
||||
..add(_says('I could not look.'));
|
||||
|
||||
await conversation.ask('Which?', const []);
|
||||
|
||||
final result =
|
||||
gateway.asked.last.$2.messages.last.parts.single as LlmToolResult;
|
||||
expect(result.isError, isTrue);
|
||||
expect(conversation.state.turns, hasLength(1));
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('stops looking up after eight rounds: the ninth is told to answer, '
|
||||
'the tenth ends the turn', () async {
|
||||
final conversation = cubit(
|
||||
scope: const AssistantScope(reach: AssistantReach.library),
|
||||
);
|
||||
await conversation.open();
|
||||
for (var round = 0; round < 12; round++) {
|
||||
gateway.answers.add(
|
||||
_looksUp([
|
||||
LlmToolCall(
|
||||
id: 'call-$round',
|
||||
name: 'find_cards',
|
||||
input: const {'query': 'irregular'},
|
||||
),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
await conversation.ask('Everything?', const []);
|
||||
|
||||
expect(gateway.asked, hasLength(10));
|
||||
LlmToolResult resultOf(int request) =>
|
||||
gateway.asked[request].$2.messages.last.parts.single as LlmToolResult;
|
||||
expect(resultOf(8).isError, isFalse, reason: 'the eighth round answered');
|
||||
expect(resultOf(9).isError, isTrue);
|
||||
expect(resultOf(9).content, contains('answer with what you found'));
|
||||
|
||||
final turn = conversation.state.turns.single;
|
||||
expect(TurnView.of(turn.messages).stoppedEarly, isTrue);
|
||||
expect(
|
||||
turn.messages.last.parts.single,
|
||||
isA<LlmToolResult>().having((result) => result.isError, 'isError', true),
|
||||
reason: 'every call answered, so a follow-up is still well formed',
|
||||
);
|
||||
expect(await conversations.turnsOf('conversation-001'), hasLength(1));
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test(
|
||||
"answers on a device that can't seal, but keeps nothing (D52)",
|
||||
() async {
|
||||
kept.sealing = false;
|
||||
final conversation = cubit();
|
||||
await conversation.open();
|
||||
expect(conversation.state.keeps, isFalse);
|
||||
gateway.answers
|
||||
..add(_says('One.'))
|
||||
..add(_says('Two.'));
|
||||
|
||||
await conversation.ask('First?', const []);
|
||||
await conversation.ask('And then?', const []);
|
||||
|
||||
expect(conversation.state.turns, hasLength(2));
|
||||
expect(conversation.state.id, isNull);
|
||||
expect(gateway.asked.last.$2.messages, hasLength(3));
|
||||
expect(kept.written, isEmpty);
|
||||
await conversation.close();
|
||||
},
|
||||
);
|
||||
|
||||
test('a question that fails keeps nothing and can be asked again', () async {
|
||||
final conversation = cubit();
|
||||
await conversation.open();
|
||||
gateway.failing = const AssistantFailed(AssistantProblem.limitReached);
|
||||
|
||||
await conversation.ask('Why?', const []);
|
||||
|
||||
final unanswered = conversation.state.unanswered!;
|
||||
expect(unanswered.failure?.problem, AssistantProblem.limitReached);
|
||||
expect(conversation.state.turns, isEmpty);
|
||||
expect(kept.written, isEmpty);
|
||||
|
||||
gateway
|
||||
..failing = null
|
||||
..answers.add(_says('Because.'));
|
||||
await conversation.retry();
|
||||
|
||||
expect(conversation.state.unanswered, isNull);
|
||||
expect(
|
||||
TurnView.of(conversation.state.turns.single.messages).question,
|
||||
'Why?',
|
||||
);
|
||||
expect(await conversations.all(), hasLength(1));
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('a question stopped keeps nothing and can be asked again', () async {
|
||||
final slow = _SlowGateway();
|
||||
final conversation = cubit(asking: slow);
|
||||
await conversation.open();
|
||||
|
||||
final asking = conversation.ask('Why?', const []);
|
||||
await Future<void>.delayed(Duration.zero);
|
||||
slow.answer.add(const LlmTextDelta('Becau'));
|
||||
await Future<void>.delayed(Duration.zero);
|
||||
expect(conversation.state.asking?.text, 'Becau');
|
||||
|
||||
await conversation.stop();
|
||||
await asking;
|
||||
|
||||
expect(conversation.state.asking, isNull);
|
||||
expect(conversation.state.unanswered?.failure, isNull);
|
||||
expect(conversation.state.turns, isEmpty);
|
||||
expect(kept.written, isEmpty);
|
||||
expect(slow.answer.hasListener, isFalse);
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('closing stops the answer on its way', () async {
|
||||
final slow = _SlowGateway();
|
||||
final conversation = cubit(asking: slow);
|
||||
await conversation.open();
|
||||
|
||||
unawaited(conversation.ask('Why?', const []));
|
||||
await Future<void>.delayed(Duration.zero);
|
||||
expect(slow.answer.hasListener, isTrue);
|
||||
|
||||
await conversation.close();
|
||||
|
||||
expect(slow.answer.hasListener, isFalse);
|
||||
expect(kept.written, isEmpty);
|
||||
});
|
||||
|
||||
test('continues a kept conversation, its turns sent along', () async {
|
||||
final id = await conversations.start(
|
||||
question: 'First?',
|
||||
assistant: const AssistantRef(id: 'claude'),
|
||||
scope: const AssistantScope(),
|
||||
);
|
||||
await conversations.keep(id, [
|
||||
LlmMessage.user(const [LlmText('First?')]),
|
||||
LlmMessage.assistant(const [LlmText('One.')]),
|
||||
], _usage);
|
||||
final conversation = cubit(conversationId: id);
|
||||
expect(conversation.state.reading, isTrue);
|
||||
|
||||
await conversation.open();
|
||||
|
||||
expect(conversation.state.reading, isFalse);
|
||||
expect(conversation.state.turns, hasLength(1));
|
||||
gateway.answers.add(_says('Two.'));
|
||||
await conversation.ask('And then?', const []);
|
||||
expect(gateway.asked.single.$2.messages, hasLength(3));
|
||||
expect(await conversations.all(), hasLength(1));
|
||||
expect(await conversations.turnsOf(id), hasLength(2));
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('asks nothing without an assistant to ask, as for a conversation '
|
||||
'whose assistant has gone', () async {
|
||||
final conversation = cubit(choice: null);
|
||||
await conversation.open();
|
||||
|
||||
await conversation.ask('Still there?', const []);
|
||||
|
||||
expect(gateway.asked, isEmpty);
|
||||
expect(conversation.state.asking, isNull);
|
||||
await conversation.close();
|
||||
});
|
||||
|
||||
test('a question of parts alone is titled by the first', () async {
|
||||
final conversation = cubit();
|
||||
await conversation.open();
|
||||
gateway.answers.add(_says('A verb.'));
|
||||
|
||||
await conversation.ask('', const [
|
||||
AssistantPart(kind: AssistantPartKind.front, text: 'する'),
|
||||
]);
|
||||
|
||||
expect((await conversations.all()).single.title, 'する');
|
||||
await conversation.close();
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue