feat(llm): halkyon_llm, language models' APIs streamed over plain HTTP (ADR 0021, step 2)
A pure Dart package, no SDKs, one interface (LlmClient) over Anthropic's Messages API, OpenAI's chat completions, in each known host's dialect and the local servers' (D45), and OpenAI's Responses API, stateless (D46). A model's turn keeps what its provider wants back with the tools' results: Anthropic's thinking blocks, the Responses API's reasoning items and message phase, Gemini's thought signatures. Failures come by kind, the provider's words cut and without keys. Tested on replies written from the providers' documentation, replayed in pieces of 1, 7 and 4,096 bytes; tool/record_llm.dart records real ones with a key of one's own, which the tests then run on too (D36). FakeLlmClient for the code that uses it. BSD-3-Clause, as halkyon_oidc. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
e4c7259df3
commit
038aca5439
GPG key ID:
07F6FCF405488B64
1 changed files with 4841 additions and 0 deletions
513
packages/halkyon_llm/test/openai_responses_test.dart
Normal file
513
packages/halkyon_llm/test/openai_responses_test.dart
Normal file
|
|
@ -0,0 +1,513 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:halkyon_llm/halkyon_llm.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'support/fixture.dart';
|
||||
|
||||
LlmEndpoint _at(String? baseUrl, {String apiKey = 'sk-test'}) => LlmEndpoint(
|
||||
protocol: LlmProtocol.openaiResponses,
|
||||
model: 'the-model',
|
||||
apiKey: apiKey,
|
||||
baseUrl: baseUrl,
|
||||
);
|
||||
|
||||
final _openAi = _at(null);
|
||||
final _ollama = _at('http://localhost:11434/v1', apiKey: '');
|
||||
|
||||
final _question = LlmRequest(
|
||||
system: 'Du hilfst beim Japanischlernen.',
|
||||
messages: [
|
||||
LlmMessage.user([const LlmText('Was heißt „Hund“ auf Japanisch?')]),
|
||||
],
|
||||
maxOutputTokens: 4096,
|
||||
personId: 'p_8Zq3',
|
||||
);
|
||||
|
||||
final _documented = fixturesOf('documented', 'openai-responses');
|
||||
|
||||
Future<(List<LlmEvent>, LlmFailure?)> _reply(
|
||||
Fixture fixture, {
|
||||
LlmEndpoint? endpoint,
|
||||
LlmRequest? request,
|
||||
Sent? sent,
|
||||
int chunk = 7,
|
||||
}) async {
|
||||
final client = LlmClient(
|
||||
httpClient: replying(fixture, sent: sent, chunk: chunk),
|
||||
);
|
||||
final events = <LlmEvent>[];
|
||||
try {
|
||||
await client
|
||||
.send(endpoint ?? _openAi, request ?? _question)
|
||||
.forEach(events.add);
|
||||
} on LlmFailure catch (failure) {
|
||||
return (events, failure);
|
||||
}
|
||||
return (events, null);
|
||||
}
|
||||
|
||||
Future<Map<String, Object?>> _bodyFor(
|
||||
LlmEndpoint endpoint, [
|
||||
LlmRequest? request,
|
||||
]) async {
|
||||
final sent = Sent();
|
||||
await _reply(
|
||||
_documented['text']!,
|
||||
endpoint: endpoint,
|
||||
request: request,
|
||||
sent: sent,
|
||||
);
|
||||
return sent.body;
|
||||
}
|
||||
|
||||
String _text(List<LlmEvent> events) =>
|
||||
events.whereType<LlmTextDelta>().map((delta) => delta.text).join();
|
||||
|
||||
Fixture _stream(String events) => Fixture.parse(
|
||||
'inline',
|
||||
Uint8List.fromList(
|
||||
utf8.encode('HTTP 200\ncontent-type: text/event-stream\n\n$events'),
|
||||
),
|
||||
);
|
||||
|
||||
String _event(Map<String, Object?> data) => 'data: ${jsonEncode(data)}\n\n';
|
||||
|
||||
void main() {
|
||||
group('the request', () {
|
||||
test('goes to the responses with the key as a bearer token', () async {
|
||||
final sent = Sent();
|
||||
await _reply(_documented['text']!, sent: sent);
|
||||
expect(sent.request.method, 'POST');
|
||||
expect(
|
||||
sent.request.url,
|
||||
Uri.parse('https://api.openai.com/v1/responses'),
|
||||
);
|
||||
expect(
|
||||
sent.request.headers,
|
||||
containsPair('authorization', 'Bearer sk-test'),
|
||||
);
|
||||
});
|
||||
|
||||
test('to a local model without a key, carries none', () async {
|
||||
final sent = Sent();
|
||||
await _reply(_documented['text']!, endpoint: _ollama, sent: sent);
|
||||
expect(
|
||||
sent.request.url,
|
||||
Uri.parse('http://localhost:11434/v1/responses'),
|
||||
);
|
||||
expect(sent.request.headers.keys, isNot(contains('authorization')));
|
||||
});
|
||||
|
||||
test('to OpenAI: the system as instructions, nothing stored, the '
|
||||
'reasoning asked for encrypted, the opaque ID as safety_identifier '
|
||||
'(D30, D46)', () async {
|
||||
expect(await _bodyFor(_openAi), {
|
||||
'model': 'the-model',
|
||||
'instructions': 'Du hilfst beim Japanischlernen.',
|
||||
'input': [
|
||||
{'role': 'user', 'content': 'Was heißt „Hund“ auf Japanisch?'},
|
||||
],
|
||||
'max_output_tokens': 4096,
|
||||
'stream': true,
|
||||
'store': false,
|
||||
'include': ['reasoning.encrypted_content'],
|
||||
'safety_identifier': 'p_8Zq3',
|
||||
});
|
||||
});
|
||||
|
||||
test('to any other host: no opaque ID, which only OpenAI takes', () async {
|
||||
final body = await _bodyFor(_ollama);
|
||||
expect(body.keys, isNot(contains('safety_identifier')));
|
||||
expect(body['store'], isFalse);
|
||||
});
|
||||
|
||||
test('asks for at least 16 tokens, the least the API takes', () async {
|
||||
final body = await _bodyFor(
|
||||
_openAi,
|
||||
LlmRequest(messages: _question.messages, maxOutputTokens: 1),
|
||||
);
|
||||
expect(body['max_output_tokens'], 16);
|
||||
});
|
||||
|
||||
final conversation = LlmRequest(
|
||||
messages: [
|
||||
LlmMessage.user([
|
||||
const LlmText('Was steht da? '),
|
||||
LlmImage(
|
||||
mediaType: 'image/png',
|
||||
bytes: Uint8List.fromList([1, 2, 3]),
|
||||
),
|
||||
]),
|
||||
LlmMessage.assistant([
|
||||
const LlmProviderBlock(
|
||||
protocol: LlmProtocol.openaiResponses,
|
||||
block: {
|
||||
'id': 'rs_1',
|
||||
'type': 'reasoning',
|
||||
'encrypted_content': 'gAAAA',
|
||||
'summary': <Object?>[],
|
||||
},
|
||||
),
|
||||
const LlmProviderBlock(
|
||||
protocol: LlmProtocol.anthropic,
|
||||
block: {'type': 'thinking'},
|
||||
),
|
||||
const LlmText('Ich sehe nach.', extra: {'phase': 'commentary'}),
|
||||
const LlmToolCall(
|
||||
id: 'call_1',
|
||||
name: 'find_cards',
|
||||
input: {'query': '犬'},
|
||||
extra: {
|
||||
'extra_content': {
|
||||
'google': {'thought_signature': 'CiQB'},
|
||||
},
|
||||
},
|
||||
),
|
||||
const LlmToolCall(id: 'call_2', name: 'find_cards', input: {}),
|
||||
]),
|
||||
LlmMessage.user([
|
||||
const LlmToolResult(callId: 'call_1', content: '[]'),
|
||||
const LlmToolResult(
|
||||
callId: 'call_2',
|
||||
content: 'Kein Studienset',
|
||||
isError: true,
|
||||
),
|
||||
const LlmText('Und?'),
|
||||
]),
|
||||
LlmMessage.assistant([const LlmText('')]),
|
||||
],
|
||||
tools: const [
|
||||
LlmTool(
|
||||
name: 'find_cards',
|
||||
description: 'Finds cards.',
|
||||
inputSchema: {'type': 'object'},
|
||||
),
|
||||
],
|
||||
maxOutputTokens: 100,
|
||||
);
|
||||
|
||||
test('holds images as data URLs, the reasoning back in its place as it '
|
||||
'came, the phase with the text, calls without their item ID, '
|
||||
'results right after, and leaves out empty turns and other '
|
||||
"protocols' blocks and extras", () async {
|
||||
final body = await _bodyFor(_openAi, conversation);
|
||||
expect(body['input'], [
|
||||
{
|
||||
'role': 'user',
|
||||
'content': [
|
||||
{'type': 'input_text', 'text': 'Was steht da? '},
|
||||
{
|
||||
'type': 'input_image',
|
||||
'image_url': 'data:image/png;base64,AQID',
|
||||
'detail': 'auto',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
'id': 'rs_1',
|
||||
'type': 'reasoning',
|
||||
'encrypted_content': 'gAAAA',
|
||||
'summary': <Object?>[],
|
||||
},
|
||||
{
|
||||
'role': 'assistant',
|
||||
'content': 'Ich sehe nach.',
|
||||
'phase': 'commentary',
|
||||
},
|
||||
{
|
||||
'type': 'function_call',
|
||||
'call_id': 'call_1',
|
||||
'name': 'find_cards',
|
||||
'arguments': '{"query":"犬"}',
|
||||
},
|
||||
{
|
||||
'type': 'function_call',
|
||||
'call_id': 'call_2',
|
||||
'name': 'find_cards',
|
||||
'arguments': '{}',
|
||||
},
|
||||
{'type': 'function_call_output', 'call_id': 'call_1', 'output': '[]'},
|
||||
{
|
||||
'type': 'function_call_output',
|
||||
'call_id': 'call_2',
|
||||
'output': 'Error: Kein Studienset',
|
||||
},
|
||||
{'role': 'user', 'content': 'Und?'},
|
||||
]);
|
||||
expect(body['tools'], [
|
||||
{
|
||||
'type': 'function',
|
||||
'name': 'find_cards',
|
||||
'description': 'Finds cards.',
|
||||
'parameters': {'type': 'object'},
|
||||
'strict': false,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
group('replies', () {
|
||||
for (final chunk in [1, 7, 4096]) {
|
||||
test('the text streams as written, in pieces of $chunk bytes, the '
|
||||
'reasoning kept before it, usage from the response', () async {
|
||||
final (events, failure) = await _reply(
|
||||
_documented['text']!,
|
||||
chunk: chunk,
|
||||
);
|
||||
expect(failure, isNull);
|
||||
expect(_text(events), '„Hund“ heißt auf Japanisch 犬 (いぬ, inu).');
|
||||
final finished = events.last as LlmFinished;
|
||||
expect(finished.stop, LlmStop.end);
|
||||
expect(
|
||||
finished.usage,
|
||||
const LlmUsage(inputTokens: 412, outputTokens: 62),
|
||||
);
|
||||
final [reasoning, text] = finished.message.parts;
|
||||
expect(
|
||||
reasoning,
|
||||
isA<LlmProviderBlock>()
|
||||
.having(
|
||||
(b) => b.protocol,
|
||||
'protocol',
|
||||
LlmProtocol.openaiResponses,
|
||||
)
|
||||
.having((b) => b.block['id'], 'id', 'rs_0f3a91c2aa01')
|
||||
.having(
|
||||
(b) => b.block['encrypted_content'],
|
||||
'the done item, whole',
|
||||
startsWith('gAAAAABo_Rk1nQ2'),
|
||||
),
|
||||
);
|
||||
expect(
|
||||
text,
|
||||
isA<LlmText>().having((t) => t.extra, 'extra', {
|
||||
'phase': 'final_answer',
|
||||
}),
|
||||
);
|
||||
expect(finished.message.text, _text(events));
|
||||
});
|
||||
}
|
||||
|
||||
test('tool calls come after the reasoning, their arguments put together, '
|
||||
'and the stop says so', () async {
|
||||
final (events, failure) = await _reply(_documented['tool']!);
|
||||
expect(failure, isNull);
|
||||
final finished = events.last as LlmFinished;
|
||||
expect(finished.stop, LlmStop.toolCalls);
|
||||
expect(
|
||||
finished.usage,
|
||||
const LlmUsage(inputTokens: 472, outputTokens: 89),
|
||||
);
|
||||
expect(finished.message.parts.first, isA<LlmProviderBlock>());
|
||||
final [first, second] = finished.message.toolCalls;
|
||||
expect(first.id, 'call_Hq2ZtX0aP4kLm9sB');
|
||||
expect(first.name, 'find_cards');
|
||||
expect(first.input, {'query': 'Hund', 'limit': 5});
|
||||
expect(first.extra, isEmpty);
|
||||
expect(second.id, 'call_7DdmO9pD3xa9XTPN');
|
||||
expect(second.input, {'query': '犬'});
|
||||
expect(finished.message.parts.whereType<LlmText>(), isEmpty);
|
||||
});
|
||||
|
||||
test('an answer cut at the ceiling is incomplete: maxTokens, its text '
|
||||
'and reasoning kept', () async {
|
||||
final (events, failure) = await _reply(_documented['incomplete']!);
|
||||
expect(failure, isNull);
|
||||
final finished = events.last as LlmFinished;
|
||||
expect(finished.stop, LlmStop.maxTokens);
|
||||
expect(finished.message.text, '„Hund“ heißt');
|
||||
expect(
|
||||
finished.usage,
|
||||
const LlmUsage(inputTokens: 412, outputTokens: 16),
|
||||
);
|
||||
});
|
||||
|
||||
test('a quota used up mid-stream, its fields under error as OpenAI sends '
|
||||
'it, is the key', () async {
|
||||
final (events, failure) = await _reply(_documented['quota_error']!);
|
||||
expect(events, isEmpty);
|
||||
expect(failure!.kind, LlmFailureKind.key);
|
||||
expect(failure.detail, startsWith('You exceeded your current quota'));
|
||||
});
|
||||
|
||||
test('the documented flat error event fails the answer with the '
|
||||
"provider's words, after the text so far", () async {
|
||||
final (events, failure) = await _reply(
|
||||
_documented['server_error_midstream']!,
|
||||
);
|
||||
expect(_text(events), '„Hund“');
|
||||
expect(failure!.kind, LlmFailureKind.provider);
|
||||
expect(
|
||||
failure.detail,
|
||||
'The server had an error while processing your request.',
|
||||
);
|
||||
});
|
||||
|
||||
final statuses = {
|
||||
'status_401': (LlmFailureKind.key, null),
|
||||
'status_429': (LlmFailureKind.rateLimited, const Duration(seconds: 20)),
|
||||
'status_429_quota': (LlmFailureKind.key, null),
|
||||
};
|
||||
for (final MapEntry(key: name, value: (kind, retryAfter))
|
||||
in statuses.entries) {
|
||||
test('$name fails as ${kind.name}, as chat completions do', () async {
|
||||
final (events, failure) = await _reply(_documented[name]!);
|
||||
expect(events, isEmpty);
|
||||
expect(failure!.kind, kind);
|
||||
expect(failure.retryAfter, retryAfter);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
group('replies off the documentation', () {
|
||||
Map<String, Object?> message(String text, {String type = 'output_text'}) =>
|
||||
{
|
||||
'type': 'message',
|
||||
'role': 'assistant',
|
||||
'content': [
|
||||
if (type == 'refusal')
|
||||
{'type': 'refusal', 'refusal': text}
|
||||
else
|
||||
{'type': type, 'text': text},
|
||||
],
|
||||
};
|
||||
|
||||
test(
|
||||
'a server that streams no items gives them with the response',
|
||||
() async {
|
||||
final (events, failure) = await _reply(
|
||||
_stream(
|
||||
_event({
|
||||
'type': 'response.completed',
|
||||
'response': {
|
||||
'status': 'completed',
|
||||
'output': [message('Hi')],
|
||||
'usage': {'input_tokens': 3, 'output_tokens': 1},
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(failure, isNull);
|
||||
expect((events.last as LlmFinished).message.text, 'Hi');
|
||||
},
|
||||
);
|
||||
|
||||
test('a refusal is text, and the stop says so', () async {
|
||||
final (events, _) = await _reply(
|
||||
_stream(
|
||||
'${_event({'type': 'response.refusal.delta', 'delta': 'Nein.'})}'
|
||||
'${_event({
|
||||
'type': 'response.output_item.done',
|
||||
'output_index': 0,
|
||||
'item': message('Nein.', type: 'refusal'),
|
||||
})}'
|
||||
'${_event({
|
||||
'type': 'response.completed',
|
||||
'response': {'status': 'completed'},
|
||||
})}',
|
||||
),
|
||||
);
|
||||
expect(_text(events), 'Nein.');
|
||||
final finished = events.last as LlmFinished;
|
||||
expect(finished.stop, LlmStop.refused);
|
||||
expect(finished.message.text, 'Nein.');
|
||||
});
|
||||
|
||||
test('a [DONE] ends it, as the SDKs allow', () async {
|
||||
final (events, failure) = await _reply(
|
||||
_stream(
|
||||
'${_event({
|
||||
'type': 'response.output_item.done',
|
||||
'output_index': 0,
|
||||
'item': message('Hi'),
|
||||
})}data: [DONE]\n\n',
|
||||
),
|
||||
);
|
||||
expect(failure, isNull);
|
||||
expect((events.last as LlmFinished).stop, LlmStop.other);
|
||||
});
|
||||
|
||||
test(
|
||||
'a response failed without an error event fails by its code',
|
||||
() async {
|
||||
final (_, failure) = await _reply(
|
||||
_stream(
|
||||
_event({
|
||||
'type': 'response.failed',
|
||||
'response': {
|
||||
'status': 'failed',
|
||||
'error': {
|
||||
'code': 'rate_limit_exceeded',
|
||||
'message': 'Slow down',
|
||||
},
|
||||
'usage': {'input_tokens': 9, 'output_tokens': 0},
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(failure!.kind, LlmFailureKind.rateLimited);
|
||||
expect(failure.usage, const LlmUsage(inputTokens: 9, outputTokens: 0));
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'a stream that ends before the response does fails as malformed',
|
||||
() async {
|
||||
final (events, failure) = await _reply(
|
||||
_stream(
|
||||
_event({'type': 'response.output_text.delta', 'delta': 'Hi'}),
|
||||
),
|
||||
);
|
||||
expect(_text(events), 'Hi');
|
||||
expect(failure!.kind, LlmFailureKind.malformed);
|
||||
},
|
||||
);
|
||||
|
||||
test("a call's arguments that aren't JSON fail as malformed", () async {
|
||||
final (_, failure) = await _reply(
|
||||
_stream(
|
||||
'${_event({
|
||||
'type': 'response.output_item.done',
|
||||
'output_index': 0,
|
||||
'item': {
|
||||
'type': 'function_call',
|
||||
'call_id': 'c',
|
||||
'name': 'n',
|
||||
'arguments': '{"x":',
|
||||
},
|
||||
})}'
|
||||
'${_event({
|
||||
'type': 'response.completed',
|
||||
'response': {'status': 'completed'},
|
||||
})}',
|
||||
),
|
||||
);
|
||||
expect(failure!.kind, LlmFailureKind.malformed);
|
||||
});
|
||||
|
||||
test('items of other kinds (a built-in tool) are passed over', () async {
|
||||
final (events, failure) = await _reply(
|
||||
_stream(
|
||||
'${_event({
|
||||
'type': 'response.output_item.done',
|
||||
'output_index': 0,
|
||||
'item': {'type': 'web_search_call', 'id': 'ws_1'},
|
||||
})}'
|
||||
'${_event({
|
||||
'type': 'response.output_item.done',
|
||||
'output_index': 1,
|
||||
'item': message('Hi'),
|
||||
})}'
|
||||
'${_event({
|
||||
'type': 'response.completed',
|
||||
'response': {'status': 'completed'},
|
||||
})}',
|
||||
),
|
||||
);
|
||||
expect(failure, isNull);
|
||||
expect((events.last as LlmFinished).message.parts, [isA<LlmText>()]);
|
||||
});
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue