Tizian 「ティツィアーン」
038aca5439
A pure Dart package, no SDKs, one interface (LlmClient) over Anthropic's Messages API, OpenAI's chat completions, in each known host's dialect and the local servers' (D45), and OpenAI's Responses API, stateless (D46). A model's turn keeps what its provider wants back with the tools' results: Anthropic's thinking blocks, the Responses API's reasoning items and message phase, Gemini's thought signatures. Failures come by kind, the provider's words cut and without keys. Tested on replies written from the providers' documentation, replayed in pieces of 1, 7 and 4,096 bytes; tool/record_llm.dart records real ones with a key of one's own, which the tests then run on too (D36). FakeLlmClient for the code that uses it. BSD-3-Clause, as halkyon_oidc. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
94 lines
3.2 KiB
Dart
94 lines
3.2 KiB
Dart
import 'dart:io';
|
|
|
|
import 'package:halkyon_llm/halkyon_llm.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
import 'support/fixture.dart';
|
|
|
|
/// Real replies, as `tool/record_llm.dart` recorded them (ADR 0021, D36):
|
|
/// their words are the model's, so only what every such reply must hold is
|
|
/// checked, in each folder of `test/fixtures/recorded/`, named
|
|
/// `<protocol>` or `<protocol>@<host>`.
|
|
void main() {
|
|
final root = Directory('test/fixtures/recorded');
|
|
final folders = root.existsSync()
|
|
? (root.listSync().whereType<Directory>().toList()
|
|
..sort((a, b) => a.path.compareTo(b.path)))
|
|
: <Directory>[];
|
|
if (folders.isEmpty) {
|
|
test(
|
|
'recorded replies',
|
|
() {},
|
|
skip: 'none recorded yet: dart run tool/record_llm.dart',
|
|
);
|
|
}
|
|
for (final folder in folders) {
|
|
final name = folder.uri.pathSegments.lastWhere((s) => s.isNotEmpty);
|
|
final protocol = LlmProtocol.named(name.split('@').first);
|
|
if (protocol == null) {
|
|
test(name, () => fail('$name names no protocol'));
|
|
continue;
|
|
}
|
|
final endpoint = LlmEndpoint(protocol: protocol, model: 'm', apiKey: 'k');
|
|
final request = LlmRequest(
|
|
messages: [
|
|
LlmMessage.user([const LlmText('?')]),
|
|
],
|
|
maxOutputTokens: 100,
|
|
);
|
|
|
|
Future<(List<LlmEvent>, LlmFailure?)> replay(Fixture fixture) async {
|
|
final events = <LlmEvent>[];
|
|
try {
|
|
await LlmClient(
|
|
httpClient: replying(fixture),
|
|
).send(endpoint, request).forEach(events.add);
|
|
} on LlmFailure catch (failure) {
|
|
return (events, failure);
|
|
}
|
|
return (events, null);
|
|
}
|
|
|
|
group('recorded from $name', () {
|
|
for (final MapEntry(key: scenario, value: fixture) in fixturesOf(
|
|
'recorded',
|
|
name,
|
|
).entries) {
|
|
test(scenario, () async {
|
|
final (events, failure) = await replay(fixture);
|
|
switch (scenario) {
|
|
case 'text' || 'tool_result':
|
|
expect(failure, isNull);
|
|
final finished = events.last as LlmFinished;
|
|
expect(
|
|
finished.stop,
|
|
scenario == 'text' ? LlmStop.end : isNot(LlmStop.refused),
|
|
);
|
|
expect(finished.message.text, isNotEmpty);
|
|
expect(
|
|
events.whereType<LlmTextDelta>().map((d) => d.text).join(),
|
|
finished.message.text,
|
|
);
|
|
expect(finished.usage.inputTokens, greaterThan(0));
|
|
expect(finished.usage.outputTokens, greaterThan(0));
|
|
case 'tool':
|
|
expect(failure, isNull);
|
|
final finished = events.last as LlmFinished;
|
|
expect(finished.stop, LlmStop.toolCalls);
|
|
expect(finished.message.toolCalls, isNotEmpty);
|
|
for (final call in finished.message.toolCalls) {
|
|
expect(call.id, isNotEmpty);
|
|
expect(call.name, 'find_cards');
|
|
expect(call.input['query'], isA<String>());
|
|
}
|
|
case 'bad_key':
|
|
expect(events, isEmpty);
|
|
expect(failure?.kind, LlmFailureKind.key);
|
|
default:
|
|
fail('No checks for $scenario');
|
|
}
|
|
});
|
|
}
|
|
});
|
|
}
|
|
}
|