import 'dart:convert'; import 'dart:typed_data'; import 'package:halkyon_llm/halkyon_llm.dart'; import 'package:test/test.dart'; import 'support/fixture.dart'; LlmEndpoint _at(String? baseUrl, {String apiKey = 'sk-test'}) => LlmEndpoint( protocol: LlmProtocol.openai, model: 'the-model', apiKey: apiKey, baseUrl: baseUrl, ); final _openAi = _at(null); final _mistral = _at('https://api.mistral.ai/v1'); final _gemini = _at('https://generativelanguage.googleapis.com/v1beta/openai/'); final _openRouter = _at('https://openrouter.ai/api/v1'); final _ollama = _at('http://localhost:11434/v1', apiKey: ''); final _question = LlmRequest( system: 'Du hilfst beim Japanischlernen.', messages: [ LlmMessage.user([const LlmText('Was heißt „Hund“ auf Japanisch?')]), ], maxOutputTokens: 4096, personId: 'p_8Zq3', ); final _documented = fixturesOf('documented', 'openai'); Future<(List, LlmFailure?)> _reply( Fixture fixture, { LlmEndpoint? endpoint, LlmRequest? request, Sent? sent, int chunk = 7, }) async { final client = LlmClient( httpClient: replying(fixture, sent: sent, chunk: chunk), ); final events = []; try { await client .send(endpoint ?? _openAi, request ?? _question) .forEach(events.add); } on LlmFailure catch (failure) { return (events, failure); } return (events, null); } Future> _bodyFor( LlmEndpoint endpoint, [ LlmRequest? request, ]) async { final sent = Sent(); await _reply( _documented['text']!, endpoint: endpoint, request: request, sent: sent, ); return sent.body; } String _text(List events) => events.whereType().map((delta) => delta.text).join(); Fixture _stream(String chunks) => Fixture.parse( 'inline', Uint8List.fromList( utf8.encode('HTTP 200\ncontent-type: text/event-stream\n\n$chunks'), ), ); void main() { group('the request', () { test( 'goes to the chat completions with the key as a bearer token', () async { final sent = Sent(); await _reply(_documented['text']!, sent: sent); expect(sent.request.method, 'POST'); expect( sent.request.url, Uri.parse('https://api.openai.com/v1/chat/completions'), ); expect( sent.request.headers, containsPair('authorization', 'Bearer sk-test'), ); }, ); test('to a local model without a key, carries none', () async { final sent = Sent(); await _reply(_documented['text']!, endpoint: _ollama, sent: sent); expect( sent.request.url, Uri.parse('http://localhost:11434/v1/chat/completions'), ); expect(sent.request.headers.keys, isNot(contains('authorization'))); }); test('to OpenAI: the ceiling its reasoning models take, usage asked for, ' 'nothing stored, the opaque ID as safety_identifier', () async { expect(await _bodyFor(_openAi), { 'model': 'the-model', 'messages': [ {'role': 'system', 'content': 'Du hilfst beim Japanischlernen.'}, {'role': 'user', 'content': 'Was heißt „Hund“ auf Japanisch?'}, ], 'max_completion_tokens': 4096, 'stream': true, 'stream_options': {'include_usage': true}, 'store': false, 'safety_identifier': 'p_8Zq3', }); }); test('to Mistral: only fields its strict schema has', () async { final body = await _bodyFor(_mistral); expect( body.keys, unorderedEquals(['model', 'messages', 'max_tokens', 'stream']), ); }); test('to Gemini: no store and no ID, which it refuses', () async { final body = await _bodyFor(_gemini); expect( body.keys, unorderedEquals([ 'model', 'messages', 'max_completion_tokens', 'stream', 'stream_options', ]), ); }); test('to OpenRouter: the ID as user, which it hashes on', () async { final body = await _bodyFor(_openRouter); expect( body.keys, unorderedEquals([ 'model', 'messages', 'max_completion_tokens', 'stream', 'user', ]), ); expect(body['user'], 'p_8Zq3'); }); group('the effort (D63)', () { LlmRequest at(LlmEffort effort) => LlmRequest( messages: _question.messages, maxOutputTokens: 4096, effort: effort, ); test( 'goes as reasoning_effort to OpenAI, Gemini and any other host', () async { for (final endpoint in [_openAi, _gemini, _ollama]) { final body = await _bodyFor(endpoint, at(LlmEffort.medium)); expect(body['reasoning_effort'], 'medium', reason: endpoint.host); expect(body.keys, isNot(contains('reasoning'))); } }, ); test('goes as reasoning.effort to OpenRouter', () async { final body = await _bodyFor(_openRouter, at(LlmEffort.max)); expect(body['reasoning'], {'effort': 'max'}); expect(body.keys, isNot(contains('reasoning_effort'))); }); test('is left out without one', () async { for (final endpoint in [_openAi, _gemini, _openRouter, _ollama]) { final body = await _bodyFor(endpoint); expect( body.keys, isNot(anyOf(contains('reasoning_effort'), contains('reasoning'))), reason: endpoint.host, ); } }); test("is each host's levels: Gemini's low to high, Mistral's none, " 'every level elsewhere', () { expect(_gemini.efforts, { LlmEffort.low, LlmEffort.medium, LlmEffort.high, }); expect(_mistral.efforts, isEmpty); for (final endpoint in [_openAi, _openRouter, _ollama]) { expect( endpoint.efforts, LlmEffort.values.toSet(), reason: endpoint.host, ); } }); test('a level the host takes not is refused before anything goes', () { final client = LlmClient(httpClient: replying(_documented['text']!)); expect( () => client.send(_mistral, at(LlmEffort.low)), throwsArgumentError, ); expect( () => client.send(_gemini, at(LlmEffort.xhigh)), throwsArgumentError, ); }); }); test( 'to any other host: what Ollama, vLLM and llama.cpp all take', () async { final body = await _bodyFor(_ollama); expect( body.keys, unorderedEquals([ 'model', 'messages', 'max_tokens', 'stream', 'stream_options', ]), ); }, ); final conversation = LlmRequest( messages: [ LlmMessage.user([ const LlmText('Was steht da? '), LlmImage( mediaType: 'image/png', bytes: Uint8List.fromList([1, 2, 3]), ), ]), LlmMessage.assistant([ const LlmProviderBlock( protocol: LlmProtocol.anthropic, block: {'type': 'thinking'}, ), const LlmToolCall( id: 'call_1', name: 'find_cards', input: {'query': '犬'}, extra: { 'extra_content': { 'google': {'thought_signature': 'CiQB'}, }, }, ), const LlmToolCall(id: 'call_2', name: 'find_cards', input: {}), ]), LlmMessage.user([ const LlmToolResult(callId: 'call_1', content: '[]'), const LlmToolResult( callId: 'call_2', content: 'Kein Studienset', isError: true, ), const LlmText('Und?'), ]), LlmMessage.assistant([const LlmText('')]), ], tools: const [ LlmTool( name: 'find_cards', description: 'Finds cards.', inputSchema: {'type': 'object'}, ), ], maxOutputTokens: 100, ); test('holds images as data URLs, calls without content, results as tool ' 'messages before the text, and leaves out empty turns and other ' "protocols' blocks", () async { final body = await _bodyFor(_openAi, conversation); expect(body['messages'], [ { 'role': 'user', 'content': [ {'type': 'text', 'text': 'Was steht da? '}, { 'type': 'image_url', 'image_url': {'url': 'data:image/png;base64,AQID'}, }, ], }, { 'role': 'assistant', 'tool_calls': [ { 'id': 'call_1', 'type': 'function', 'function': {'name': 'find_cards', 'arguments': '{"query":"犬"}'}, }, { 'id': 'call_2', 'type': 'function', 'function': {'name': 'find_cards', 'arguments': '{}'}, }, ], }, {'role': 'tool', 'tool_call_id': 'call_1', 'content': '[]'}, { 'role': 'tool', 'tool_call_id': 'call_2', 'content': 'Error: Kein Studienset', }, {'role': 'user', 'content': 'Und?'}, ]); expect(body['tools'], [ { 'type': 'function', 'function': { 'name': 'find_cards', 'description': 'Finds cards.', 'parameters': {'type': 'object'}, }, }, ]); }); test('gives Gemini its thought signature back with the call', () async { final body = await _bodyFor(_gemini, conversation); final calls = ((body['messages']! as List)[1] as Map)['tool_calls'] as List; expect((calls.first as Map)['extra_content'], { 'google': {'thought_signature': 'CiQB'}, }); expect((calls.last as Map).keys, isNot(contains('extra_content'))); }); }); group('replies', () { for (final chunk in [1, 7, 4096]) { test("OpenAI's text streams as written, in pieces of $chunk bytes, its " 'usage from the last chunk', () async { final (events, failure) = await _reply( _documented['text']!, chunk: chunk, ); expect(failure, isNull); expect(_text(events), '„Hund“ heißt auf Japanisch 犬 (いぬ, inu).'); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.end); expect( finished.usage, const LlmUsage(inputTokens: 412, outputTokens: 38), ); expect(finished.message.parts, [isA()]); }); } test("OpenAI's tool calls come in pieces, put together by index", () async { final (events, failure) = await _reply(_documented['tool']!); expect(failure, isNull); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.toolCalls); expect( finished.usage, const LlmUsage(inputTokens: 472, outputTokens: 89), ); expect(finished.message.parts, everyElement(isA())); final [first, second] = finished.message.toolCalls; expect(first.id, 'call_DdmO9pD3xa9XTPNJ32zg2hcA'); expect(first.name, 'find_cards'); expect(first.input, {'query': 'Hund', 'limit': 5}); expect(second.id, 'call_7Hq2ZtX0aP4kLm9sB1cD3eF5'); expect(second.input, {'query': '犬'}); }); test("Mistral's calls come whole, with usage on the last choice", () async { final (events, failure) = await _reply( _documented['mistral_tool']!, endpoint: _mistral, ); expect(failure, isNull); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.toolCalls); expect( finished.usage, const LlmUsage(inputTokens: 472, outputTokens: 89), ); final [first, second] = finished.message.toolCalls; expect(first.id, 'D681PevKs'); expect(first.input, {'query': 'Hund', 'limit': 5}); expect(second.id, 'Qk2Ty7Wb1'); expect(second.input, {'query': '犬'}); }); test("Gemini's call comes whole without an index, ends with stop, keeps " 'its thought signature, and counts its thinking as output', () async { final (events, failure) = await _reply( _documented['gemini_tool']!, endpoint: _gemini, ); expect(failure, isNull); expect(_text(events), 'Ich sehe nach.'); final finished = events.last as LlmFinished; expect(finished.stop, LlmStop.toolCalls); expect( finished.usage, const LlmUsage(inputTokens: 472, outputTokens: 89), ); final call = finished.message.toolCalls.single; expect(call.id, 'function-call-1'); expect(call.extra, { 'extra_content': { 'google': {'thought_signature': 'CiQBcsjafQ=='}, }, }); }); test("Ollama's lost error (a blank chunk, then the end) fails as " 'malformed', () async { final (events, failure) = await _reply( _documented['ollama_lost_error']!, endpoint: _ollama, ); expect(_text(events), '„Hund“'); expect(failure!.kind, LlmFailureKind.malformed); }); test("llama.cpp's error chunk fails the answer with its words", () async { final (_, failure) = await _reply( _documented['llamacpp_error']!, endpoint: _ollama, ); expect(failure!.kind, LlmFailureKind.provider); expect(failure.detail, 'the request exceeds the available context size'); }); test("OpenRouter's comments are passed over and its error chunk fails the " 'answer', () async { final (events, failure) = await _reply( _documented['openrouter_error']!, endpoint: _openRouter, ); expect(_text(events), '„Hund“'); expect(failure!.kind, LlmFailureKind.provider); expect(failure.detail, 'Provider disconnected'); }); final statuses = { 'status_400': (LlmFailureKind.request, null), 'status_401': (LlmFailureKind.key, null), 'status_429': (LlmFailureKind.rateLimited, const Duration(seconds: 20)), 'status_429_quota': (LlmFailureKind.key, null), 'mistral_status_422': (LlmFailureKind.request, null), 'gemini_status_400': (LlmFailureKind.request, null), }; for (final MapEntry(key: name, value: (kind, retryAfter)) in statuses.entries) { test('$name fails as ${kind.name}', () async { final (events, failure) = await _reply(_documented[name]!); expect(events, isEmpty); expect(failure!.kind, kind); expect(failure.retryAfter, retryAfter); }); } test("keeps OpenAI's words but the key", () async { final (_, failure) = await _reply(_documented['status_401']!); expect(failure!.detail, startsWith('Incorrect API key provided: sk-…')); expect(failure.detail, isNot(contains('wxyz'))); }); test("keeps of Mistral's refusal the field and why, never the value it " 'quotes', () async { final (_, failure) = await _reply(_documented['mistral_status_422']!); expect( failure!.detail, 'body.messages.0.content: Extra inputs are not permitted', ); }); test("reads Gemini's error in a list", () async { final (_, failure) = await _reply(_documented['gemini_status_400']!); expect(failure!.detail, startsWith('Invalid JSON payload received.')); }); }); group('replies off the documentation', () { String chunk(Map delta, {String? finish, Object? extra}) => 'data: ${jsonEncode({ 'choices': [ {'index': 0, 'delta': delta, 'finish_reason': finish}, ], ...?extra as Map?, })}\n\n'; test( 'an answer that ends with its finish reason but no [DONE] stands', () async { final (events, failure) = await _reply( _stream('${chunk({'content': 'Hi'})}${chunk({}, finish: 'stop')}'), ); expect(failure, isNull); expect((events.last as LlmFinished).message.text, 'Hi'); }, ); test("a call whose ID is Mistral's default 'null' gets one, and arguments " 'as an object are taken', () async { final (events, _) = await _reply( _stream( '${chunk({ 'tool_calls': [ { 'id': 'null', 'function': { 'name': 'find_cards', 'arguments': {'query': 'Hund'}, }, }, ], }, finish: 'tool_calls')}data: [DONE]\n\n', ), ); final call = (events.last as LlmFinished).message.toolCalls.single; expect(call.id, 'call_0'); expect(call.input, {'query': 'Hund'}); }); test('an index of -1 falls back to the ID', () async { final (events, _) = await _reply( _stream( '${chunk({ 'tool_calls': [ { 'index': -1, 'id': 'a', 'function': {'name': 'n', 'arguments': '{"x":'}, }, ], })}${chunk({ 'tool_calls': [ { 'index': -1, 'id': 'a', 'function': {'arguments': '1}'}, }, ], }, finish: 'tool_calls')}data: [DONE]\n\n', ), ); expect((events.last as LlmFinished).message.toolCalls.single.input, { 'x': 1, }); }); test('a refusal is text, and the stop says so', () async { final (events, _) = await _reply( _stream( '${chunk({'refusal': 'Dabei helfe ich nicht.'})}' '${chunk({}, finish: 'content_filter')}data: [DONE]\n\n', ), ); expect(_text(events), 'Dabei helfe ich nicht.'); expect((events.last as LlmFinished).stop, LlmStop.refused); }); test("Mistral's thinking parts are passed over, its text kept", () async { final (events, _) = await _reply( _stream( '${chunk({ 'content': [ {'type': 'thinking', 'thinking': []}, {'type': 'text', 'text': 'Hi'}, ], }, finish: 'stop')}data: [DONE]\n\n', ), ); expect(_text(events), 'Hi'); }); test('a length stop is maxTokens', () async { final (events, _) = await _reply( _stream( '${chunk({'content': 'Hi'}, finish: 'length')}data: [DONE]\n\n', ), ); expect((events.last as LlmFinished).stop, LlmStop.maxTokens); }); test('an error as a string in a chunk fails the answer', () async { final (_, failure) = await _reply( _stream('data: {"error":"model not loaded"}\n\n'), ); expect(failure!.detail, 'model not loaded'); }); test("a call's arguments that aren't JSON fail as malformed", () async { final (_, failure) = await _reply( _stream( '${chunk({ 'tool_calls': [ { 'index': 0, 'id': 'a', 'function': {'name': 'n', 'arguments': '{"x":'}, }, ], }, finish: 'tool_calls')}data: [DONE]\n\n', ), ); expect(failure!.kind, LlmFailureKind.malformed); }); }); }