import 'dart:convert'; import 'dart:typed_data'; import 'package:halkyon_llm/halkyon_llm.dart'; import 'package:test/test.dart'; import 'support/fixture.dart'; LlmEndpoint _at(String? baseUrl, {String apiKey = 'sk-test'}) => LlmEndpoint( protocol: LlmProtocol.openaiResponses, model: 'the-model', apiKey: apiKey, baseUrl: baseUrl, ); final _openAi = _at(null); final _ollama = _at('http://localhost:11434/v1', apiKey: ''); final _question = LlmRequest( system: 'Du hilfst beim Japanischlernen.', messages: [ LlmMessage.user([const LlmText('Was heißt „Hund“ auf Japanisch?')]), ], maxOutputTokens: 4096, personId: 'p_8Zq3', ); final _documented = fixturesOf('documented', 'openai-responses'); Future<(List, LlmFailure?)> _reply( Fixture fixture, { LlmEndpoint? endpoint, LlmRequest? request, Sent? sent, int chunk = 7, }) async { final client = LlmClient( httpClient: replying(fixture, sent: sent, chunk: chunk), ); final events = []; try { await client .send(endpoint ?? _openAi, request ?? _question) .forEach(events.add); } on LlmFailure catch (failure) { return (events, failure); } return (events, null); } Future> _bodyFor( LlmEndpoint endpoint, [ LlmRequest? request, ]) async { final sent = Sent(); await _reply( _documented['text']!, endpoint: endpoint, request: request, sent: sent, ); return sent.body; } String _text(List events) => events.whereType().map((delta) => delta.text).join(); Fixture _stream(String events) => Fixture.parse( 'inline', Uint8List.fromList( utf8.encode('HTTP 200\ncontent-type: text/event-stream\n\n$events'), ), ); String _event(Map data) => 'data: ${jsonEncode(data)}\n\n'; void main() { group('the request', () { test('goes to the responses with the key as a bearer token', () async { final sent = Sent(); await _reply(_documented['text']!, sent: sent); expect(sent.request.method, 'POST'); expect( sent.request.url, Uri.parse('https://api.openai.com/v1/responses'), ); expect( sent.request.headers, containsPair('authorization', 'Bearer sk-test'), ); }); test('to a local model without a key, carries none', () async { final sent = Sent(); await _reply(_documented['text']!, endpoint: _ollama, sent: sent); expect( sent.request.url, Uri.parse('http://localhost:11434/v1/responses'), ); expect(sent.request.headers.keys, isNot(contains('authorization'))); }); test('to OpenAI: the system as instructions, nothing stored, the ' 'reasoning asked for encrypted, the opaque ID as safety_identifier ' '(D30, D46)', () async { expect(await _bodyFor(_openAi), { 'model': 'the-model', 'instructions': 'Du hilfst beim Japanischlernen.', 'input': [ {'role': 'user', 'content': 'Was heißt „Hund“ auf Japanisch?'}, ], 'max_output_tokens': 4096, 'stream': true, 'store': false, 'include': ['reasoning.encrypted_content'], 'safety_identifier': 'p_8Zq3', }); }); test('to any other host: no opaque ID, which only OpenAI takes', () async { final body = await _bodyFor(_ollama); expect(body.keys, isNot(contains('safety_identifier'))); expect(body['store'], isFalse); }); test('asks for at least 16 tokens, the least the API takes', () async { final body = await _bodyFor( _openAi, LlmRequest(messages: _question.messages, maxOutputTokens: 1), ); expect(body['max_output_tokens'], 16); }); test('carries the effort as reasoning.effort, every level (D63)', () async { final body = await _bodyFor( _openAi, LlmRequest( messages: _question.messages, maxOutputTokens: 4096, effort: LlmEffort.max, ), ); expect(body['reasoning'], {'effort': 'max'}); expect(_openAi.efforts, LlmEffort.values.toSet()); }); final conversation = LlmRequest( messages: [ LlmMessage.user([ const LlmText('Was steht da? '), LlmImage( mediaType: 'image/png', bytes: Uint8List.fromList([1, 2, 3]), ), ]), LlmMessage.assistant([ const LlmProviderBlock( protocol: LlmProtocol.openaiResponses, block: { 'id': 'rs_1', 'type': 'reasoning', 'encrypted_content': 'gAAAA', 'summary': [], }, ), const LlmProviderBlock( protocol: LlmProtocol.anthropic, block: {'type': 'thinking'}, ), const LlmText('Ich sehe nach.', extra: {'phase': 'commentary'}), const LlmToolCall( id: 'call_1', name: 'find_cards', input: {'query': '犬'}, extra: { 'extra_content': { 'google': {'thought_signature': 'CiQB'}, }, }, ), const LlmToolCall(id: 'call_2', name: 'find_cards', input: {}), ]), LlmMessage.user([ const LlmToolResult(callId: 'call_1', content: '[]'), const LlmToolResult( callId: 'call_2', content: 'Kein Studienset', isError: true, ), const LlmText('Und?'), ]), LlmMessage.assistant([const LlmText('')]), ], tools: const [ LlmTool( name: 'find_cards', description: 'Finds cards.', inputSchema: {'type': 'object'}, ), ], maxOutputTokens: 100, ); test('holds images as data URLs, the reasoning back in its place as it ' 'came, the phase with the text, calls without their item ID, ' 'results right after, and leaves out empty turns and other ' "protocols' blocks and extras", () async { final body = await _bodyFor(_openAi, conversation); expect(body['input'], [ { 'role': 'user', 'content': [ {'type': 'input_text', 'text': 'Was steht da? '}, { 'type': 'input_image', 'image_url': 'data:image/png;base64,AQID', 'detail': 'auto', }, ], }, { 'id': 'rs_1', 'type': 'reasoning', 'encrypted_content': 'gAAAA', 'summary': [], }, { 'role': 'assistant', 'content': 'Ich sehe nach.', 'phase': 'commentary', }, { 'type': 'function_call', 'call_id': 'call_1', 'name': 'find_cards', 'arguments': '{"query":"犬"}', }, { 'type': 'function_call', 'call_id': 'call_2', 'name': 'find_cards', 'arguments': '{}', }, {'type': 'function_call_output', 'call_id': 'call_1', 'output': '[]'}, { 'type': 'function_call_output', 'call_id': 'call_2', 'output': 'Error: Kein Studienset', }, {'role': 'user', 'content': 'Und?'}, ]); expect(body['tools'], [ { 'type': 'function', 'name': 'find_cards', 'description': 'Finds cards.', 'parameters': {'type': 'object'}, 'strict': false, }, ]); }); }); group('replies', () { for (final chunk in [1, 7, 4096]) { test('the text streams as written, in pieces of $chunk bytes, the ' 'reasoning kept before it, usage from the response', () async { final (events, failure) = await _reply( _documented['text']!, chunk: chunk, ); expect(failure, isNull); expect(_text(events), '„Hund“ heißt auf Japanisch 犬 (いぬ, inu).'); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.end); expect( finished.usage, const LlmUsage(inputTokens: 412, outputTokens: 62), ); final [reasoning, text] = finished.message.parts; expect( reasoning, isA() .having( (b) => b.protocol, 'protocol', LlmProtocol.openaiResponses, ) .having((b) => b.block['id'], 'id', 'rs_0f3a91c2aa01') .having( (b) => b.block['encrypted_content'], 'the done item, whole', startsWith('gAAAAABo_Rk1nQ2'), ), ); expect( text, isA().having((t) => t.extra, 'extra', { 'phase': 'final_answer', }), ); expect(finished.message.text, _text(events)); }); } test('tool calls come after the reasoning, their arguments put together, ' 'and the stop says so', () async { final (events, failure) = await _reply(_documented['tool']!); expect(failure, isNull); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.toolCalls); expect( finished.usage, const LlmUsage(inputTokens: 472, outputTokens: 89), ); expect(finished.message.parts.first, isA()); final [first, second] = finished.message.toolCalls; expect(first.id, 'call_Hq2ZtX0aP4kLm9sB'); expect(first.name, 'find_cards'); expect(first.input, {'query': 'Hund', 'limit': 5}); expect(first.extra, isEmpty); expect(second.id, 'call_7DdmO9pD3xa9XTPN'); expect(second.input, {'query': '犬'}); expect(finished.message.parts.whereType(), isEmpty); }); test('an answer cut at the ceiling is incomplete: maxTokens, its text ' 'and reasoning kept', () async { final (events, failure) = await _reply(_documented['incomplete']!); expect(failure, isNull); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.maxTokens); expect(finished.message.text, '„Hund“ heißt'); expect( finished.usage, const LlmUsage(inputTokens: 412, outputTokens: 16), ); }); test('a quota used up mid-stream, its fields under error as OpenAI sends ' 'it, is the key', () async { final (events, failure) = await _reply(_documented['quota_error']!); expect(events, isEmpty); expect(failure!.kind, LlmFailureKind.key); expect(failure.detail, startsWith('You exceeded your current quota')); }); test('the documented flat error event fails the answer with the ' "provider's words, after the text so far", () async { final (events, failure) = await _reply( _documented['server_error_midstream']!, ); expect(_text(events), '„Hund“'); expect(failure!.kind, LlmFailureKind.provider); expect( failure.detail, 'The server had an error while processing your request.', ); }); final statuses = { 'status_401': (LlmFailureKind.key, null), 'status_429': (LlmFailureKind.rateLimited, const Duration(seconds: 20)), 'status_429_quota': (LlmFailureKind.key, null), }; for (final MapEntry(key: name, value: (kind, retryAfter)) in statuses.entries) { test('$name fails as ${kind.name}, as chat completions do', () async { final (events, failure) = await _reply(_documented[name]!); expect(events, isEmpty); expect(failure!.kind, kind); expect(failure.retryAfter, retryAfter); }); } }); group('replies off the documentation', () { Map message(String text, {String type = 'output_text'}) => { 'type': 'message', 'role': 'assistant', 'content': [ if (type == 'refusal') {'type': 'refusal', 'refusal': text} else {'type': type, 'text': text}, ], }; test( 'a server that streams no items gives them with the response', () async { final (events, failure) = await _reply( _stream( _event({ 'type': 'response.completed', 'response': { 'status': 'completed', 'output': [message('Hi')], 'usage': {'input_tokens': 3, 'output_tokens': 1}, }, }), ), ); expect(failure, isNull); expect((events.last as LlmFinished).message.text, 'Hi'); }, ); test('a refusal is text, and the stop says so', () async { final (events, _) = await _reply( _stream( '${_event({'type': 'response.refusal.delta', 'delta': 'Nein.'})}' '${_event({ 'type': 'response.output_item.done', 'output_index': 0, 'item': message('Nein.', type: 'refusal'), })}' '${_event({ 'type': 'response.completed', 'response': {'status': 'completed'}, })}', ), ); expect(_text(events), 'Nein.'); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.refused); expect(finished.message.text, 'Nein.'); }); test('a [DONE] ends it, as the SDKs allow', () async { final (events, failure) = await _reply( _stream( '${_event({ 'type': 'response.output_item.done', 'output_index': 0, 'item': message('Hi'), })}data: [DONE]\n\n', ), ); expect(failure, isNull); expect((events.last as LlmFinished).stop, LlmStop.other); }); test( 'a response failed without an error event fails by its code', () async { final (_, failure) = await _reply( _stream( _event({ 'type': 'response.failed', 'response': { 'status': 'failed', 'error': { 'code': 'rate_limit_exceeded', 'message': 'Slow down', }, 'usage': {'input_tokens': 9, 'output_tokens': 0}, }, }), ), ); expect(failure!.kind, LlmFailureKind.rateLimited); expect(failure.usage, const LlmUsage(inputTokens: 9, outputTokens: 0)); }, ); test( 'a stream that ends before the response does fails as malformed', () async { final (events, failure) = await _reply( _stream( _event({'type': 'response.output_text.delta', 'delta': 'Hi'}), ), ); expect(_text(events), 'Hi'); expect(failure!.kind, LlmFailureKind.malformed); }, ); test("a call's arguments that aren't JSON fail as malformed", () async { final (_, failure) = await _reply( _stream( '${_event({ 'type': 'response.output_item.done', 'output_index': 0, 'item': { 'type': 'function_call', 'call_id': 'c', 'name': 'n', 'arguments': '{"x":', }, })}' '${_event({ 'type': 'response.completed', 'response': {'status': 'completed'}, })}', ), ); expect(failure!.kind, LlmFailureKind.malformed); }); test('items of other kinds (a built-in tool) are passed over', () async { final (events, failure) = await _reply( _stream( '${_event({ 'type': 'response.output_item.done', 'output_index': 0, 'item': {'type': 'web_search_call', 'id': 'ws_1'}, })}' '${_event({ 'type': 'response.output_item.done', 'output_index': 1, 'item': message('Hi'), })}' '${_event({ 'type': 'response.completed', 'response': {'status': 'completed'}, })}', ), ); expect(failure, isNull); expect((events.last as LlmFinished).message.parts, [isA()]); }); }); }