halkyon/packages/halkyon_llm/test/openai_responses_test.dart

import 'dart:convert';
import 'dart:typed_data';

import 'package:halkyon_llm/halkyon_llm.dart';
import 'package:test/test.dart';

import 'support/fixture.dart';

LlmEndpoint _at(String? baseUrl, {String apiKey = 'sk-test'}) => LlmEndpoint(
  protocol: LlmProtocol.openaiResponses,
  model: 'the-model',
  apiKey: apiKey,
  baseUrl: baseUrl,
);

final _openAi = _at(null);
final _ollama = _at('http://localhost:11434/v1', apiKey: '');

final _question = LlmRequest(
  system: 'Du hilfst beim Japanischlernen.',
  messages: [
    LlmMessage.user([const LlmText('Was heißt „Hund“ auf Japanisch?')]),
  ],
  maxOutputTokens: 4096,
  personId: 'p_8Zq3',
);

final _documented = fixturesOf('documented', 'openai-responses');

Future<(List<LlmEvent>, LlmFailure?)> _reply(
  Fixture fixture, {
  LlmEndpoint? endpoint,
  LlmRequest? request,
  Sent? sent,
  int chunk = 7,
}) async {
  final client = LlmClient(
    httpClient: replying(fixture, sent: sent, chunk: chunk),
  );
  final events = <LlmEvent>[];
  try {
    await client
        .send(endpoint ?? _openAi, request ?? _question)
        .forEach(events.add);
  } on LlmFailure catch (failure) {
    return (events, failure);
  }
  return (events, null);
}

Future<Map<String, Object?>> _bodyFor(
  LlmEndpoint endpoint, [
  LlmRequest? request,
]) async {
  final sent = Sent();
  await _reply(
    _documented['text']!,
    endpoint: endpoint,
    request: request,
    sent: sent,
  );
  return sent.body;
}

String _text(List<LlmEvent> events) =>
    events.whereType<LlmTextDelta>().map((delta) => delta.text).join();

Fixture _stream(String events) => Fixture.parse(
  'inline',
  Uint8List.fromList(
    utf8.encode('HTTP 200\ncontent-type: text/event-stream\n\n$events'),
  ),
);

String _event(Map<String, Object?> data) => 'data: ${jsonEncode(data)}\n\n';

void main() {
  group('the request', () {
    test('goes to the responses with the key as a bearer token', () async {
      final sent = Sent();
      await _reply(_documented['text']!, sent: sent);
      expect(sent.request.method, 'POST');
      expect(
        sent.request.url,
        Uri.parse('https://api.openai.com/v1/responses'),
      );
      expect(
        sent.request.headers,
        containsPair('authorization', 'Bearer sk-test'),
      );
    });

    test('to a local model without a key, carries none', () async {
      final sent = Sent();
      await _reply(_documented['text']!, endpoint: _ollama, sent: sent);
      expect(
        sent.request.url,
        Uri.parse('http://localhost:11434/v1/responses'),
      );
      expect(sent.request.headers.keys, isNot(contains('authorization')));
    });

    test('to OpenAI: the system as instructions, nothing stored, the '
        'reasoning asked for encrypted, the opaque ID as safety_identifier '
        '(D30, D46)', () async {
      expect(await _bodyFor(_openAi), {
        'model': 'the-model',
        'instructions': 'Du hilfst beim Japanischlernen.',
        'input': [
          {'role': 'user', 'content': 'Was heißt „Hund“ auf Japanisch?'},
        ],
        'max_output_tokens': 4096,
        'stream': true,
        'store': false,
        'include': ['reasoning.encrypted_content'],
        'safety_identifier': 'p_8Zq3',
      });
    });

    test('to any other host: no opaque ID, which only OpenAI takes', () async {
      final body = await _bodyFor(_ollama);
      expect(body.keys, isNot(contains('safety_identifier')));
      expect(body['store'], isFalse);
    });

    test('asks for at least 16 tokens, the least the API takes', () async {
      final body = await _bodyFor(
        _openAi,
        LlmRequest(messages: _question.messages, maxOutputTokens: 1),
      );
      expect(body['max_output_tokens'], 16);
    });

    test('carries the effort as reasoning.effort, every level (D63)', () async {
      final body = await _bodyFor(
        _openAi,
        LlmRequest(
          messages: _question.messages,
          maxOutputTokens: 4096,
          effort: LlmEffort.max,
        ),
      );
      expect(body['reasoning'], {'effort': 'max'});
      expect(_openAi.efforts, LlmEffort.values.toSet());
    });

    final conversation = LlmRequest(
      messages: [
        LlmMessage.user([
          const LlmText('Was steht da? '),
          LlmImage(
            mediaType: 'image/png',
            bytes: Uint8List.fromList([1, 2, 3]),
          ),
        ]),
        LlmMessage.assistant([
          const LlmProviderBlock(
            protocol: LlmProtocol.openaiResponses,
            block: {
              'id': 'rs_1',
              'type': 'reasoning',
              'encrypted_content': 'gAAAA',
              'summary': <Object?>[],
            },
          ),
          const LlmProviderBlock(
            protocol: LlmProtocol.anthropic,
            block: {'type': 'thinking'},
          ),
          const LlmText('Ich sehe nach.', extra: {'phase': 'commentary'}),
          const LlmToolCall(
            id: 'call_1',
            name: 'find_cards',
            input: {'query': '犬'},
            extra: {
              'extra_content': {
                'google': {'thought_signature': 'CiQB'},
              },
            },
          ),
          const LlmToolCall(id: 'call_2', name: 'find_cards', input: {}),
        ]),
        LlmMessage.user([
          const LlmToolResult(callId: 'call_1', content: '[]'),
          const LlmToolResult(
            callId: 'call_2',
            content: 'Kein Studienset',
            isError: true,
          ),
          const LlmText('Und?'),
        ]),
        LlmMessage.assistant([const LlmText('')]),
      ],
      tools: const [
        LlmTool(
          name: 'find_cards',
          description: 'Finds cards.',
          inputSchema: {'type': 'object'},
        ),
      ],
      maxOutputTokens: 100,
    );

    test('holds images as data URLs, the reasoning back in its place as it '
        'came, the phase with the text, calls without their item ID, '
        'results right after, and leaves out empty turns and other '
        "protocols' blocks and extras", () async {
      final body = await _bodyFor(_openAi, conversation);
      expect(body['input'], [
        {
          'role': 'user',
          'content': [
            {'type': 'input_text', 'text': 'Was steht da? '},
            {
              'type': 'input_image',
              'image_url': 'data:image/png;base64,AQID',
              'detail': 'auto',
            },
          ],
        },
        {
          'id': 'rs_1',
          'type': 'reasoning',
          'encrypted_content': 'gAAAA',
          'summary': <Object?>[],
        },
        {
          'role': 'assistant',
          'content': 'Ich sehe nach.',
          'phase': 'commentary',
        },
        {
          'type': 'function_call',
          'call_id': 'call_1',
          'name': 'find_cards',
          'arguments': '{"query":"犬"}',
        },
        {
          'type': 'function_call',
          'call_id': 'call_2',
          'name': 'find_cards',
          'arguments': '{}',
        },
        {'type': 'function_call_output', 'call_id': 'call_1', 'output': '[]'},
        {
          'type': 'function_call_output',
          'call_id': 'call_2',
          'output': 'Error: Kein Studienset',
        },
        {'role': 'user', 'content': 'Und?'},
      ]);
      expect(body['tools'], [
        {
          'type': 'function',
          'name': 'find_cards',
          'description': 'Finds cards.',
          'parameters': {'type': 'object'},
          'strict': false,
        },
      ]);
    });
  });

  group('replies', () {
    for (final chunk in [1, 7, 4096]) {
      test('the text streams as written, in pieces of $chunk bytes, the '
          'reasoning kept before it, usage from the response', () async {
        final (events, failure) = await _reply(
          _documented['text']!,
          chunk: chunk,
        );
        expect(failure, isNull);
        expect(_text(events), '„Hund“ heißt auf Japanisch 犬 (いぬ, inu).');
        final finished = events.last as LlmFinished;
        expect(finished.stop, LlmStop.end);
        expect(
          finished.usage,
          const LlmUsage(inputTokens: 412, outputTokens: 62),
        );
        final [reasoning, text] = finished.message.parts;
        expect(
          reasoning,
          isA<LlmProviderBlock>()
              .having(
                (b) => b.protocol,
                'protocol',
                LlmProtocol.openaiResponses,
              )
              .having((b) => b.block['id'], 'id', 'rs_0f3a91c2aa01')
              .having(
                (b) => b.block['encrypted_content'],
                'the done item, whole',
                startsWith('gAAAAABo_Rk1nQ2'),
              ),
        );
        expect(
          text,
          isA<LlmText>().having((t) => t.extra, 'extra', {
            'phase': 'final_answer',
          }),
        );
        expect(finished.message.text, _text(events));
      });
    }

    test('tool calls come after the reasoning, their arguments put together, '
        'and the stop says so', () async {
      final (events, failure) = await _reply(_documented['tool']!);
      expect(failure, isNull);
      final finished = events.last as LlmFinished;
      expect(finished.stop, LlmStop.toolCalls);
      expect(
        finished.usage,
        const LlmUsage(inputTokens: 472, outputTokens: 89),
      );
      expect(finished.message.parts.first, isA<LlmProviderBlock>());
      final [first, second] = finished.message.toolCalls;
      expect(first.id, 'call_Hq2ZtX0aP4kLm9sB');
      expect(first.name, 'find_cards');
      expect(first.input, {'query': 'Hund', 'limit': 5});
      expect(first.extra, isEmpty);
      expect(second.id, 'call_7DdmO9pD3xa9XTPN');
      expect(second.input, {'query': '犬'});
      expect(finished.message.parts.whereType<LlmText>(), isEmpty);
    });

    test('an answer cut at the ceiling is incomplete: maxTokens, its text '
        'and reasoning kept', () async {
      final (events, failure) = await _reply(_documented['incomplete']!);
      expect(failure, isNull);
      final finished = events.last as LlmFinished;
      expect(finished.stop, LlmStop.maxTokens);
      expect(finished.message.text, '„Hund“ heißt');
      expect(
        finished.usage,
        const LlmUsage(inputTokens: 412, outputTokens: 16),
      );
    });

    test('a quota used up mid-stream, its fields under error as OpenAI sends '
        'it, is the key', () async {
      final (events, failure) = await _reply(_documented['quota_error']!);
      expect(events, isEmpty);
      expect(failure!.kind, LlmFailureKind.key);
      expect(failure.detail, startsWith('You exceeded your current quota'));
    });

    test('the documented flat error event fails the answer with the '
        "provider's words, after the text so far", () async {
      final (events, failure) = await _reply(
        _documented['server_error_midstream']!,
      );
      expect(_text(events), '„Hund“');
      expect(failure!.kind, LlmFailureKind.provider);
      expect(
        failure.detail,
        'The server had an error while processing your request.',
      );
    });

    final statuses = {
      'status_401': (LlmFailureKind.key, null),
      'status_429': (LlmFailureKind.rateLimited, const Duration(seconds: 20)),
      'status_429_quota': (LlmFailureKind.key, null),
    };
    for (final MapEntry(key: name, value: (kind, retryAfter))
        in statuses.entries) {
      test('$name fails as ${kind.name}, as chat completions do', () async {
        final (events, failure) = await _reply(_documented[name]!);
        expect(events, isEmpty);
        expect(failure!.kind, kind);
        expect(failure.retryAfter, retryAfter);
      });
    }
  });

  group('replies off the documentation', () {
    Map<String, Object?> message(String text, {String type = 'output_text'}) =>
        {
          'type': 'message',
          'role': 'assistant',
          'content': [
            if (type == 'refusal')
              {'type': 'refusal', 'refusal': text}
            else
              {'type': type, 'text': text},
          ],
        };

    test(
      'a server that streams no items gives them with the response',
      () async {
        final (events, failure) = await _reply(
          _stream(
            _event({
              'type': 'response.completed',
              'response': {
                'status': 'completed',
                'output': [message('Hi')],
                'usage': {'input_tokens': 3, 'output_tokens': 1},
              },
            }),
          ),
        );
        expect(failure, isNull);
        expect((events.last as LlmFinished).message.text, 'Hi');
      },
    );

    test('a refusal is text, and the stop says so', () async {
      final (events, _) = await _reply(
        _stream(
          '${_event({'type': 'response.refusal.delta', 'delta': 'Nein.'})}'
          '${_event({
            'type': 'response.output_item.done',
            'output_index': 0,
            'item': message('Nein.', type: 'refusal'),
          })}'
          '${_event({
            'type': 'response.completed',
            'response': {'status': 'completed'},
          })}',
        ),
      );
      expect(_text(events), 'Nein.');
      final finished = events.last as LlmFinished;
      expect(finished.stop, LlmStop.refused);
      expect(finished.message.text, 'Nein.');
    });

    test('a [DONE] ends it, as the SDKs allow', () async {
      final (events, failure) = await _reply(
        _stream(
          '${_event({
            'type': 'response.output_item.done',
            'output_index': 0,
            'item': message('Hi'),
          })}data: [DONE]\n\n',
        ),
      );
      expect(failure, isNull);
      expect((events.last as LlmFinished).stop, LlmStop.other);
    });

    test(
      'a response failed without an error event fails by its code',
      () async {
        final (_, failure) = await _reply(
          _stream(
            _event({
              'type': 'response.failed',
              'response': {
                'status': 'failed',
                'error': {
                  'code': 'rate_limit_exceeded',
                  'message': 'Slow down',
                },
                'usage': {'input_tokens': 9, 'output_tokens': 0},
              },
            }),
          ),
        );
        expect(failure!.kind, LlmFailureKind.rateLimited);
        expect(failure.usage, const LlmUsage(inputTokens: 9, outputTokens: 0));
      },
    );

    test(
      'a stream that ends before the response does fails as malformed',
      () async {
        final (events, failure) = await _reply(
          _stream(
            _event({'type': 'response.output_text.delta', 'delta': 'Hi'}),
          ),
        );
        expect(_text(events), 'Hi');
        expect(failure!.kind, LlmFailureKind.malformed);
      },
    );

    test("a call's arguments that aren't JSON fail as malformed", () async {
      final (_, failure) = await _reply(
        _stream(
          '${_event({
            'type': 'response.output_item.done',
            'output_index': 0,
            'item': {
              'type': 'function_call',
              'call_id': 'c',
              'name': 'n',
              'arguments': '{"x":',
            },
          })}'
          '${_event({
            'type': 'response.completed',
            'response': {'status': 'completed'},
          })}',
        ),
      );
      expect(failure!.kind, LlmFailureKind.malformed);
    });

    test('items of other kinds (a built-in tool) are passed over', () async {
      final (events, failure) = await _reply(
        _stream(
          '${_event({
            'type': 'response.output_item.done',
            'output_index': 0,
            'item': {'type': 'web_search_call', 'id': 'ws_1'},
          })}'
          '${_event({
            'type': 'response.output_item.done',
            'output_index': 1,
            'item': message('Hi'),
          })}'
          '${_event({
            'type': 'response.completed',
            'response': {'status': 'completed'},
          })}',
        ),
      );
      expect(failure, isNull);
      expect((events.last as LlmFinished).message.parts, [isA<LlmText>()]);
    });
  });
}