halkyon/packages/halkyon_llm/test/recorded_test.dart

import 'dart:io';

import 'package:halkyon_llm/halkyon_llm.dart';
import 'package:test/test.dart';

import 'support/fixture.dart';

/// Real replies, as `tool/record_llm.dart` recorded them (ADR 0021, D36):
/// their words are the model's, so only what every such reply must hold is
/// checked, in each folder of `test/fixtures/recorded/`, named
/// `<protocol>` or `<protocol>@<host>`.
void main() {
  final root = Directory('test/fixtures/recorded');
  final folders = root.existsSync()
      ? (root.listSync().whereType<Directory>().toList()
          ..sort((a, b) => a.path.compareTo(b.path)))
      : <Directory>[];
  if (folders.isEmpty) {
    test(
      'recorded replies',
      () {},
      skip: 'none recorded yet: dart run tool/record_llm.dart',
    );
  }
  for (final folder in folders) {
    final name = folder.uri.pathSegments.lastWhere((s) => s.isNotEmpty);
    final protocol = LlmProtocol.named(name.split('@').first);
    if (protocol == null) {
      test(name, () => fail('$name names no protocol'));
      continue;
    }
    final endpoint = LlmEndpoint(protocol: protocol, model: 'm', apiKey: 'k');
    final request = LlmRequest(
      messages: [
        LlmMessage.user([const LlmText('?')]),
      ],
      maxOutputTokens: 100,
    );

    Future<(List<LlmEvent>, LlmFailure?)> replay(Fixture fixture) async {
      final events = <LlmEvent>[];
      try {
        await LlmClient(
          httpClient: replying(fixture),
        ).send(endpoint, request).forEach(events.add);
      } on LlmFailure catch (failure) {
        return (events, failure);
      }
      return (events, null);
    }

    group('recorded from $name', () {
      for (final MapEntry(key: scenario, value: fixture) in fixturesOf(
        'recorded',
        name,
      ).entries) {
        test(scenario, () async {
          final (events, failure) = await replay(fixture);
          switch (scenario) {
            case 'text' || 'tool_result':
              expect(failure, isNull);
              final finished = events.last as LlmFinished;
              expect(
                finished.stop,
                scenario == 'text' ? LlmStop.end : isNot(LlmStop.refused),
              );
              expect(finished.message.text, isNotEmpty);
              expect(
                events.whereType<LlmTextDelta>().map((d) => d.text).join(),
                finished.message.text,
              );
              expect(finished.usage.inputTokens, greaterThan(0));
              expect(finished.usage.outputTokens, greaterThan(0));
            case 'tool':
              expect(failure, isNull);
              final finished = events.last as LlmFinished;
              expect(finished.stop, LlmStop.toolCalls);
              expect(finished.message.toolCalls, isNotEmpty);
              for (final call in finished.message.toolCalls) {
                expect(call.id, isNotEmpty);
                expect(call.name, 'find_cards');
                expect(call.input['query'], isA<String>());
              }
            case 'bad_key':
              expect(events, isEmpty);
              expect(failure?.kind, LlmFailureKind.key);
            default:
              fail('No checks for $scenario');
          }
        });
      }
    });
  }
}