import 'dart:io'; import 'package:halkyon_llm/halkyon_llm.dart'; import 'package:test/test.dart'; import 'support/fixture.dart'; /// Real replies, as `tool/record_llm.dart` recorded them (ADR 0021, D36): /// their words are the model's, so only what every such reply must hold is /// checked, in each folder of `test/fixtures/recorded/`, named /// `` or `@`. void main() { final root = Directory('test/fixtures/recorded'); final folders = root.existsSync() ? (root.listSync().whereType().toList() ..sort((a, b) => a.path.compareTo(b.path))) : []; if (folders.isEmpty) { test( 'recorded replies', () {}, skip: 'none recorded yet: dart run tool/record_llm.dart', ); } for (final folder in folders) { final name = folder.uri.pathSegments.lastWhere((s) => s.isNotEmpty); final protocol = LlmProtocol.named(name.split('@').first); if (protocol == null) { test(name, () => fail('$name names no protocol')); continue; } final endpoint = LlmEndpoint(protocol: protocol, model: 'm', apiKey: 'k'); final request = LlmRequest( messages: [ LlmMessage.user([const LlmText('?')]), ], maxOutputTokens: 100, ); Future<(List, LlmFailure?)> replay(Fixture fixture) async { final events = []; try { await LlmClient( httpClient: replying(fixture), ).send(endpoint, request).forEach(events.add); } on LlmFailure catch (failure) { return (events, failure); } return (events, null); } group('recorded from $name', () { for (final MapEntry(key: scenario, value: fixture) in fixturesOf( 'recorded', name, ).entries) { test(scenario, () async { final (events, failure) = await replay(fixture); switch (scenario) { case 'text' || 'tool_result': expect(failure, isNull); final finished = events.last as LlmFinished; expect( finished.stop, scenario == 'text' ? LlmStop.end : isNot(LlmStop.refused), ); expect(finished.message.text, isNotEmpty); expect( events.whereType().map((d) => d.text).join(), finished.message.text, ); expect(finished.usage.inputTokens, greaterThan(0)); expect(finished.usage.outputTokens, greaterThan(0)); case 'tool': expect(failure, isNull); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.toolCalls); expect(finished.message.toolCalls, isNotEmpty); for (final call in finished.message.toolCalls) { expect(call.id, isNotEmpty); expect(call.name, 'find_cards'); expect(call.input['query'], isA()); } case 'bad_key': expect(events, isEmpty); expect(failure?.kind, LlmFailureKind.key); default: fail('No checks for $scenario'); } }); } }); } }