// Records real replies of a provider beside the documented ones, which the // tests then run on too (ADR 0021, D36): // // cd packages/halkyon_llm // dart run tool/record_llm.dart --protocol anthropic // dart run tool/record_llm.dart --protocol openai --model // dart run tool/record_llm.dart --protocol openai-responses --model // // The key comes from LLM_API_KEY, or is asked for without echo; it is never // written anywhere. Four requests, each a few hundred tokens, cost a few // cents at most: a short answer; a tool call; the tool's result sent back // with the model's whole turn (its thinking blocks too), as the app's loop // does; and a wrong key. Each reply goes to // test/fixtures/recorded/[@]/.http, its status, // content type and body as they came, no request header. Read them before // committing them. import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'package:args/args.dart'; import 'package:halkyon_llm/halkyon_llm.dart'; import 'package:http/http.dart' as http; import '../test/support/fixture.dart'; /// The tool the recordings ask the model to call, as the tests expect it. const _findCards = LlmTool( name: 'find_cards', description: "Finds cards in the person's studyset whose front or back contains " 'the query.', inputSchema: { 'type': 'object', 'properties': { 'query': {'type': 'string', 'description': 'What to look for.'}, 'limit': {'type': 'integer', 'description': 'How many cards at most.'}, }, 'required': ['query'], }, ); const _system = 'You help someone learn Japanese with flashcards. Answer in one short ' 'sentence.'; Future main(List arguments) async { final options = ArgParser() ..addOption( 'protocol', allowed: [for (final protocol in LlmProtocol.values) protocol.key], mandatory: true, ) ..addOption('model', help: 'Anthropic: claude-haiku-5-5 unless given.') ..addOption('base-url', help: "The protocol's own provider unless given.") ..addFlag('help', abbr: 'h', negatable: false); final ArgResults given; try { given = options.parse(arguments); } on FormatException catch (error) { stderr ..writeln(error.message) ..writeln(options.usage); exitCode = 64; return; } if (given.flag('help')) { stdout.writeln(options.usage); return; } final protocol = LlmProtocol.named(given.option('protocol')!)!; final model = given.option('model') ?? (protocol == LlmProtocol.anthropic ? 'claude-haiku-5-5' : null); if (model == null) { stderr.writeln('Name the model with --model.'); exitCode = 64; return; } final key = Platform.environment['LLM_API_KEY'] ?? _askKey(); if (key.isEmpty) { stderr.writeln('No key.'); exitCode = 64; return; } final endpoint = LlmEndpoint( protocol: protocol, model: model, apiKey: key, baseUrl: given.option('base-url'), ); final folder = Directory( 'test/fixtures/recorded/${protocol.key}' '${given.option('base-url') == null ? '' : '@${endpoint.host}'}', )..createSync(recursive: true); Future record( String scenario, LlmRequest request, { LlmEndpoint? at, }) async { final recorder = _Recorder(File('${folder.path}/$scenario.http')); final client = LlmClient(httpClient: recorder); try { final finished = await client .send(at ?? endpoint, request) .where((event) => event is LlmFinished) .cast() .single; stdout.writeln('$scenario: ${finished.stop.name}, ${finished.usage}'); return finished; } on LlmFailure catch (failure) { stdout.writeln('$scenario: $failure'); return null; } finally { if (recorder.answered) { await recorder.written; } client.close(); recorder.close(); } } await record( 'text', LlmRequest( system: _system, messages: [ LlmMessage.user([const LlmText('What is "dog" in Japanese?')]), ], maxOutputTokens: 2048, personId: 'halkyon-record-tool', ), ); final question = LlmMessage.user([ const LlmText('Find up to three cards about "dog" in my studyset.'), ]); final tools = LlmRequest( system: '$_system Look things up in the studyset with find_cards.', messages: [question], tools: const [_findCards], maxOutputTokens: 2048, personId: 'halkyon-record-tool', ); final called = await record('tool', tools); final calls = called?.message.toolCalls ?? const []; if (calls.isEmpty) { stdout.writeln('tool_result: skipped, as the model called no tool.'); } else { await record( 'tool_result', LlmRequest( system: tools.system, messages: [ question, called!.message, LlmMessage.user([ for (final call in calls) LlmToolResult( callId: call.id, content: jsonEncode([ {'front': '犬', 'back': 'dog (いぬ, inu)'}, ]), ), ]), ], tools: tools.tools, maxOutputTokens: 2048, personId: 'halkyon-record-tool', ), ); } await record( 'bad_key', LlmRequest( messages: [ LlmMessage.user([const LlmText('Hello')]), ], maxOutputTokens: 16, ), at: LlmEndpoint( protocol: protocol, model: model, apiKey: 'sk-halkyon-wrong-key', baseUrl: given.option('base-url'), ), ); stdout.writeln('Written to ${folder.path}. Now: dart test'); } String _askKey() { stdout.write('API key (not shown): '); final echo = stdin.hasTerminal && stdin.echoMode; if (stdin.hasTerminal) { stdin.echoMode = false; } try { return stdin.readLineSync()?.trim() ?? ''; } finally { if (stdin.hasTerminal) { stdin.echoMode = echo; stdout.writeln(); } } } /// An HTTP client writing each reply to [file] as it streams through. final class _Recorder(final File file) extends http.BaseClient { final _inner = http.Client(); final _written = Completer(); /// Done once the reply is written. Future get written => _written.future; /// Whether a reply began, which is then written. bool answered = false; @override Future send(http.BaseRequest request) async { final response = await _inner.send(request); answered = true; final body = []; late final StreamSubscription> reading; final out = StreamController>(); out ..onListen = () { reading = response.stream.listen( (chunk) { body.addAll(chunk); out.add(chunk); }, onError: out.addError, onDone: () { _write(response, body); unawaited(out.close()); }, ); } // The adapter stops reading once the answer is over. ..onCancel = () async { await reading.cancel(); _write(response, body); }; return http.StreamedResponse( out.stream, response.statusCode, headers: response.headers, request: request, ); } void _write(http.StreamedResponse response, List body) { if (_written.isCompleted) { return; } file.writeAsBytesSync( Fixture.write(response.statusCode, { for (final name in ['content-type', 'retry-after']) name: ?response.headers[name], }, body), ); _written.complete(); } @override void close() => _inner.close(); }