halkyon/packages/halkyon_llm/test/openai_test.dart

import 'dart:convert';
import 'dart:typed_data';

import 'package:halkyon_llm/halkyon_llm.dart';
import 'package:test/test.dart';

import 'support/fixture.dart';

LlmEndpoint _at(String? baseUrl, {String apiKey = 'sk-test'}) => LlmEndpoint(
  protocol: LlmProtocol.openai,
  model: 'the-model',
  apiKey: apiKey,
  baseUrl: baseUrl,
);

final _openAi = _at(null);
final _mistral = _at('https://api.mistral.ai/v1');
final _gemini = _at('https://generativelanguage.googleapis.com/v1beta/openai/');
final _openRouter = _at('https://openrouter.ai/api/v1');
final _ollama = _at('http://localhost:11434/v1', apiKey: '');

final _question = LlmRequest(
  system: 'Du hilfst beim Japanischlernen.',
  messages: [
    LlmMessage.user([const LlmText('Was heißt „Hund“ auf Japanisch?')]),
  ],
  maxOutputTokens: 4096,
  personId: 'p_8Zq3',
);

final _documented = fixturesOf('documented', 'openai');

Future<(List<LlmEvent>, LlmFailure?)> _reply(
  Fixture fixture, {
  LlmEndpoint? endpoint,
  LlmRequest? request,
  Sent? sent,
  int chunk = 7,
}) async {
  final client = LlmClient(
    httpClient: replying(fixture, sent: sent, chunk: chunk),
  );
  final events = <LlmEvent>[];
  try {
    await client
        .send(endpoint ?? _openAi, request ?? _question)
        .forEach(events.add);
  } on LlmFailure catch (failure) {
    return (events, failure);
  }
  return (events, null);
}

Future<Map<String, Object?>> _bodyFor(
  LlmEndpoint endpoint, [
  LlmRequest? request,
]) async {
  final sent = Sent();
  await _reply(
    _documented['text']!,
    endpoint: endpoint,
    request: request,
    sent: sent,
  );
  return sent.body;
}

String _text(List<LlmEvent> events) =>
    events.whereType<LlmTextDelta>().map((delta) => delta.text).join();

Fixture _stream(String chunks) => Fixture.parse(
  'inline',
  Uint8List.fromList(
    utf8.encode('HTTP 200\ncontent-type: text/event-stream\n\n$chunks'),
  ),
);

void main() {
  group('the request', () {
    test(
      'goes to the chat completions with the key as a bearer token',
      () async {
        final sent = Sent();
        await _reply(_documented['text']!, sent: sent);
        expect(sent.request.method, 'POST');
        expect(
          sent.request.url,
          Uri.parse('https://api.openai.com/v1/chat/completions'),
        );
        expect(
          sent.request.headers,
          containsPair('authorization', 'Bearer sk-test'),
        );
      },
    );

    test('to a local model without a key, carries none', () async {
      final sent = Sent();
      await _reply(_documented['text']!, endpoint: _ollama, sent: sent);
      expect(
        sent.request.url,
        Uri.parse('http://localhost:11434/v1/chat/completions'),
      );
      expect(sent.request.headers.keys, isNot(contains('authorization')));
    });

    test('to OpenAI: the ceiling its reasoning models take, usage asked for, '
        'nothing stored, the opaque ID as safety_identifier', () async {
      expect(await _bodyFor(_openAi), {
        'model': 'the-model',
        'messages': [
          {'role': 'system', 'content': 'Du hilfst beim Japanischlernen.'},
          {'role': 'user', 'content': 'Was heißt „Hund“ auf Japanisch?'},
        ],
        'max_completion_tokens': 4096,
        'stream': true,
        'stream_options': {'include_usage': true},
        'store': false,
        'safety_identifier': 'p_8Zq3',
      });
    });

    test('to Mistral: only fields its strict schema has', () async {
      final body = await _bodyFor(_mistral);
      expect(
        body.keys,
        unorderedEquals(['model', 'messages', 'max_tokens', 'stream']),
      );
    });

    test('to Gemini: no store and no ID, which it refuses', () async {
      final body = await _bodyFor(_gemini);
      expect(
        body.keys,
        unorderedEquals([
          'model',
          'messages',
          'max_completion_tokens',
          'stream',
          'stream_options',
        ]),
      );
    });

    test('to OpenRouter: the ID as user, which it hashes on', () async {
      final body = await _bodyFor(_openRouter);
      expect(
        body.keys,
        unorderedEquals([
          'model',
          'messages',
          'max_completion_tokens',
          'stream',
          'user',
        ]),
      );
      expect(body['user'], 'p_8Zq3');
    });

    group('the effort (D63)', () {
      LlmRequest at(LlmEffort effort) => LlmRequest(
        messages: _question.messages,
        maxOutputTokens: 4096,
        effort: effort,
      );

      test(
        'goes as reasoning_effort to OpenAI, Gemini and any other host',
        () async {
          for (final endpoint in [_openAi, _gemini, _ollama]) {
            final body = await _bodyFor(endpoint, at(LlmEffort.medium));
            expect(body['reasoning_effort'], 'medium', reason: endpoint.host);
            expect(body.keys, isNot(contains('reasoning')));
          }
        },
      );

      test('goes as reasoning.effort to OpenRouter', () async {
        final body = await _bodyFor(_openRouter, at(LlmEffort.max));
        expect(body['reasoning'], {'effort': 'max'});
        expect(body.keys, isNot(contains('reasoning_effort')));
      });

      test('is left out without one', () async {
        for (final endpoint in [_openAi, _gemini, _openRouter, _ollama]) {
          final body = await _bodyFor(endpoint);
          expect(
            body.keys,
            isNot(anyOf(contains('reasoning_effort'), contains('reasoning'))),
            reason: endpoint.host,
          );
        }
      });

      test("is each host's levels: Gemini's low to high, Mistral's none, "
          'every level elsewhere', () {
        expect(_gemini.efforts, {
          LlmEffort.low,
          LlmEffort.medium,
          LlmEffort.high,
        });
        expect(_mistral.efforts, isEmpty);
        for (final endpoint in [_openAi, _openRouter, _ollama]) {
          expect(
            endpoint.efforts,
            LlmEffort.values.toSet(),
            reason: endpoint.host,
          );
        }
      });

      test('a level the host takes not is refused before anything goes', () {
        final client = LlmClient(httpClient: replying(_documented['text']!));
        expect(
          () => client.send(_mistral, at(LlmEffort.low)),
          throwsArgumentError,
        );
        expect(
          () => client.send(_gemini, at(LlmEffort.xhigh)),
          throwsArgumentError,
        );
      });
    });

    test(
      'to any other host: what Ollama, vLLM and llama.cpp all take',
      () async {
        final body = await _bodyFor(_ollama);
        expect(
          body.keys,
          unorderedEquals([
            'model',
            'messages',
            'max_tokens',
            'stream',
            'stream_options',
          ]),
        );
      },
    );

    final conversation = LlmRequest(
      messages: [
        LlmMessage.user([
          const LlmText('Was steht da? '),
          LlmImage(
            mediaType: 'image/png',
            bytes: Uint8List.fromList([1, 2, 3]),
          ),
        ]),
        LlmMessage.assistant([
          const LlmProviderBlock(
            protocol: LlmProtocol.anthropic,
            block: {'type': 'thinking'},
          ),
          const LlmToolCall(
            id: 'call_1',
            name: 'find_cards',
            input: {'query': '犬'},
            extra: {
              'extra_content': {
                'google': {'thought_signature': 'CiQB'},
              },
            },
          ),
          const LlmToolCall(id: 'call_2', name: 'find_cards', input: {}),
        ]),
        LlmMessage.user([
          const LlmToolResult(callId: 'call_1', content: '[]'),
          const LlmToolResult(
            callId: 'call_2',
            content: 'Kein Studienset',
            isError: true,
          ),
          const LlmText('Und?'),
        ]),
        LlmMessage.assistant([const LlmText('')]),
      ],
      tools: const [
        LlmTool(
          name: 'find_cards',
          description: 'Finds cards.',
          inputSchema: {'type': 'object'},
        ),
      ],
      maxOutputTokens: 100,
    );

    test('holds images as data URLs, calls without content, results as tool '
        'messages before the text, and leaves out empty turns and other '
        "protocols' blocks", () async {
      final body = await _bodyFor(_openAi, conversation);
      expect(body['messages'], [
        {
          'role': 'user',
          'content': [
            {'type': 'text', 'text': 'Was steht da? '},
            {
              'type': 'image_url',
              'image_url': {'url': 'data:image/png;base64,AQID'},
            },
          ],
        },
        {
          'role': 'assistant',
          'tool_calls': [
            {
              'id': 'call_1',
              'type': 'function',
              'function': {'name': 'find_cards', 'arguments': '{"query":"犬"}'},
            },
            {
              'id': 'call_2',
              'type': 'function',
              'function': {'name': 'find_cards', 'arguments': '{}'},
            },
          ],
        },
        {'role': 'tool', 'tool_call_id': 'call_1', 'content': '[]'},
        {
          'role': 'tool',
          'tool_call_id': 'call_2',
          'content': 'Error: Kein Studienset',
        },
        {'role': 'user', 'content': 'Und?'},
      ]);
      expect(body['tools'], [
        {
          'type': 'function',
          'function': {
            'name': 'find_cards',
            'description': 'Finds cards.',
            'parameters': {'type': 'object'},
          },
        },
      ]);
    });

    test('gives Gemini its thought signature back with the call', () async {
      final body = await _bodyFor(_gemini, conversation);
      final calls =
          ((body['messages']! as List)[1] as Map)['tool_calls'] as List;
      expect((calls.first as Map)['extra_content'], {
        'google': {'thought_signature': 'CiQB'},
      });
      expect((calls.last as Map).keys, isNot(contains('extra_content')));
    });
  });

  group('replies', () {
    for (final chunk in [1, 7, 4096]) {
      test("OpenAI's text streams as written, in pieces of $chunk bytes, its "
          'usage from the last chunk', () async {
        final (events, failure) = await _reply(
          _documented['text']!,
          chunk: chunk,
        );
        expect(failure, isNull);
        expect(_text(events), '„Hund“ heißt auf Japanisch 犬 (いぬ, inu).');
        final finished = events.last as LlmFinished;
        expect(finished.stop, LlmStop.end);
        expect(
          finished.usage,
          const LlmUsage(inputTokens: 412, outputTokens: 38),
        );
        expect(finished.message.parts, [isA<LlmText>()]);
      });
    }

    test("OpenAI's tool calls come in pieces, put together by index", () async {
      final (events, failure) = await _reply(_documented['tool']!);
      expect(failure, isNull);
      final finished = events.last as LlmFinished;
      expect(finished.stop, LlmStop.toolCalls);
      expect(
        finished.usage,
        const LlmUsage(inputTokens: 472, outputTokens: 89),
      );
      expect(finished.message.parts, everyElement(isA<LlmToolCall>()));
      final [first, second] = finished.message.toolCalls;
      expect(first.id, 'call_DdmO9pD3xa9XTPNJ32zg2hcA');
      expect(first.name, 'find_cards');
      expect(first.input, {'query': 'Hund', 'limit': 5});
      expect(second.id, 'call_7Hq2ZtX0aP4kLm9sB1cD3eF5');
      expect(second.input, {'query': '犬'});
    });

    test("Mistral's calls come whole, with usage on the last choice", () async {
      final (events, failure) = await _reply(
        _documented['mistral_tool']!,
        endpoint: _mistral,
      );
      expect(failure, isNull);
      final finished = events.last as LlmFinished;
      expect(finished.stop, LlmStop.toolCalls);
      expect(
        finished.usage,
        const LlmUsage(inputTokens: 472, outputTokens: 89),
      );
      final [first, second] = finished.message.toolCalls;
      expect(first.id, 'D681PevKs');
      expect(first.input, {'query': 'Hund', 'limit': 5});
      expect(second.id, 'Qk2Ty7Wb1');
      expect(second.input, {'query': '犬'});
    });

    test("Gemini's call comes whole without an index, ends with stop, keeps "
        'its thought signature, and counts its thinking as output', () async {
      final (events, failure) = await _reply(
        _documented['gemini_tool']!,
        endpoint: _gemini,
      );
      expect(failure, isNull);
      expect(_text(events), 'Ich sehe nach.');
      final finished = events.last as LlmFinished;
      expect(finished.stop, LlmStop.toolCalls);
      expect(
        finished.usage,
        const LlmUsage(inputTokens: 472, outputTokens: 89),
      );
      final call = finished.message.toolCalls.single;
      expect(call.id, 'function-call-1');
      expect(call.extra, {
        'extra_content': {
          'google': {'thought_signature': 'CiQBcsjafQ=='},
        },
      });
    });

    test("Ollama's lost error (a blank chunk, then the end) fails as "
        'malformed', () async {
      final (events, failure) = await _reply(
        _documented['ollama_lost_error']!,
        endpoint: _ollama,
      );
      expect(_text(events), '„Hund“');
      expect(failure!.kind, LlmFailureKind.malformed);
    });

    test("llama.cpp's error chunk fails the answer with its words", () async {
      final (_, failure) = await _reply(
        _documented['llamacpp_error']!,
        endpoint: _ollama,
      );
      expect(failure!.kind, LlmFailureKind.provider);
      expect(failure.detail, 'the request exceeds the available context size');
    });

    test("OpenRouter's comments are passed over and its error chunk fails the "
        'answer', () async {
      final (events, failure) = await _reply(
        _documented['openrouter_error']!,
        endpoint: _openRouter,
      );
      expect(_text(events), '„Hund“');
      expect(failure!.kind, LlmFailureKind.provider);
      expect(failure.detail, 'Provider disconnected');
    });

    final statuses = {
      'status_400': (LlmFailureKind.request, null),
      'status_401': (LlmFailureKind.key, null),
      'status_429': (LlmFailureKind.rateLimited, const Duration(seconds: 20)),
      'status_429_quota': (LlmFailureKind.key, null),
      'mistral_status_422': (LlmFailureKind.request, null),
      'gemini_status_400': (LlmFailureKind.request, null),
    };
    for (final MapEntry(key: name, value: (kind, retryAfter))
        in statuses.entries) {
      test('$name fails as ${kind.name}', () async {
        final (events, failure) = await _reply(_documented[name]!);
        expect(events, isEmpty);
        expect(failure!.kind, kind);
        expect(failure.retryAfter, retryAfter);
      });
    }

    test("keeps OpenAI's words but the key", () async {
      final (_, failure) = await _reply(_documented['status_401']!);
      expect(failure!.detail, startsWith('Incorrect API key provided: sk-…'));
      expect(failure.detail, isNot(contains('wxyz')));
    });

    test("keeps of Mistral's refusal the field and why, never the value it "
        'quotes', () async {
      final (_, failure) = await _reply(_documented['mistral_status_422']!);
      expect(
        failure!.detail,
        'body.messages.0.content: Extra inputs are not permitted',
      );
    });

    test("reads Gemini's error in a list", () async {
      final (_, failure) = await _reply(_documented['gemini_status_400']!);
      expect(failure!.detail, startsWith('Invalid JSON payload received.'));
    });
  });

  group('replies off the documentation', () {
    String chunk(Map<String, Object?> delta, {String? finish, Object? extra}) =>
        'data: ${jsonEncode({
          'choices': [
            {'index': 0, 'delta': delta, 'finish_reason': finish},
          ],
          ...?extra as Map<String, Object?>?,
        })}\n\n';

    test(
      'an answer that ends with its finish reason but no [DONE] stands',
      () async {
        final (events, failure) = await _reply(
          _stream('${chunk({'content': 'Hi'})}${chunk({}, finish: 'stop')}'),
        );
        expect(failure, isNull);
        expect((events.last as LlmFinished).message.text, 'Hi');
      },
    );

    test("a call whose ID is Mistral's default 'null' gets one, and arguments "
        'as an object are taken', () async {
      final (events, _) = await _reply(
        _stream(
          '${chunk({
            'tool_calls': [
              {
                'id': 'null',
                'function': {
                  'name': 'find_cards',
                  'arguments': {'query': 'Hund'},
                },
              },
            ],
          }, finish: 'tool_calls')}data: [DONE]\n\n',
        ),
      );
      final call = (events.last as LlmFinished).message.toolCalls.single;
      expect(call.id, 'call_0');
      expect(call.input, {'query': 'Hund'});
    });

    test('an index of -1 falls back to the ID', () async {
      final (events, _) = await _reply(
        _stream(
          '${chunk({
            'tool_calls': [
              {
                'index': -1,
                'id': 'a',
                'function': {'name': 'n', 'arguments': '{"x":'},
              },
            ],
          })}${chunk({
            'tool_calls': [
              {
                'index': -1,
                'id': 'a',
                'function': {'arguments': '1}'},
              },
            ],
          }, finish: 'tool_calls')}data: [DONE]\n\n',
        ),
      );
      expect((events.last as LlmFinished).message.toolCalls.single.input, {
        'x': 1,
      });
    });

    test('a refusal is text, and the stop says so', () async {
      final (events, _) = await _reply(
        _stream(
          '${chunk({'refusal': 'Dabei helfe ich nicht.'})}'
          '${chunk({}, finish: 'content_filter')}data: [DONE]\n\n',
        ),
      );
      expect(_text(events), 'Dabei helfe ich nicht.');
      expect((events.last as LlmFinished).stop, LlmStop.refused);
    });

    test("Mistral's thinking parts are passed over, its text kept", () async {
      final (events, _) = await _reply(
        _stream(
          '${chunk({
            'content': [
              {'type': 'thinking', 'thinking': <Object?>[]},
              {'type': 'text', 'text': 'Hi'},
            ],
          }, finish: 'stop')}data: [DONE]\n\n',
        ),
      );
      expect(_text(events), 'Hi');
    });

    test('a length stop is maxTokens', () async {
      final (events, _) = await _reply(
        _stream(
          '${chunk({'content': 'Hi'}, finish: 'length')}data: [DONE]\n\n',
        ),
      );
      expect((events.last as LlmFinished).stop, LlmStop.maxTokens);
    });

    test('an error as a string in a chunk fails the answer', () async {
      final (_, failure) = await _reply(
        _stream('data: {"error":"model not loaded"}\n\n'),
      );
      expect(failure!.detail, 'model not loaded');
    });

    test("a call's arguments that aren't JSON fail as malformed", () async {
      final (_, failure) = await _reply(
        _stream(
          '${chunk({
            'tool_calls': [
              {
                'index': 0,
                'id': 'a',
                'function': {'name': 'n', 'arguments': '{"x":'},
              },
            ],
          }, finish: 'tool_calls')}data: [DONE]\n\n',
        ),
      );
      expect(failure!.kind, LlmFailureKind.malformed);
    });
  });
}