import 'dart:async'; import 'dart:convert'; import 'dart:typed_data'; import 'package:clock/clock.dart'; import 'package:halkyon_base_server/base.dart'; import 'package:halkyon_llm/halkyon_llm.dart'; import 'package:halkyon_llm/testing.dart'; import 'package:serverpod/serverpod.dart'; import 'package:test/test.dart'; import '../support/fixtures.dart'; import '../support/password_support.dart'; import '../test_tools/serverpod_test_tools.dart'; // The assistants an operator grants (Halkyon Learn's ADR 0021, step 2): what // a person is offered and agrees to (D40), a question forwarded to the // provider and its answer streamed back (D35) at the effort chosen (D63), // refused without a grant, an agreement, room within the person's budget // (D58, D59) or a question the provider can take; what each request counts // (D38, D42), kept 3 months (D41) and deleted with the person; and the // operator's grants, budgets and figures, refused to anyone else. // The provider is a FakeLlmClient: halkyon_llm's own tests hold the // protocols. /// 22:30 UTC on 25 Oct 2026: the day's budget resets at midnight, the /// month's on 1 Nov. final _now = DateTime.utc(2026, 10, 25, 22, 30); final _settings = { 'ASSISTANTS': 'claude local', 'ASSISTANT_ID_KEY': 'k' * assistantIdKeyMinLength, 'ASSISTANT_CURRENCY': 'EUR', 'ASSISTANT_CLAUDE_NAME': 'Claude Sonnet', 'ASSISTANT_CLAUDE_PROTOCOL': 'anthropic', 'ASSISTANT_CLAUDE_MODEL': 'claude-sonnet-5-5', 'ASSISTANT_CLAUDE_KEY': 'sk-ant-test', 'ASSISTANT_CLAUDE_IMAGES': 'on', 'ASSISTANT_CLAUDE_PRICE_INPUT': '3', 'ASSISTANT_CLAUDE_PRICE_OUTPUT': '15', 'ASSISTANT_LOCAL_NAME': 'Qwen at home', 'ASSISTANT_LOCAL_PROTOCOL': 'openai', 'ASSISTANT_LOCAL_BASE_URL': 'http://ollama:11434/v1', 'ASSISTANT_LOCAL_MODEL': 'qwen3:8b', }; Assistants _assistants([Map changes = const {}]) => assistantsFrom({..._settings, ...changes}.get); AssistantMessage _user(List parts) => AssistantMessage(role: AssistantRole.user, parts: parts); AssistantMessage _model(List parts) => AssistantMessage(role: AssistantRole.assistant, parts: parts); AssistantPart _text(String text) => AssistantPart(kind: AssistantPartKind.text, text: text); AssistantQuestion _question({ String assistant = 'claude', List? messages, List tools = const [], int maxOutputTokens = 2048, AssistantEffort? effort, }) => AssistantQuestion( assistant: assistant, system: 'Du hilfst beim Japanischlernen.', messages: messages ?? [ _user([_text('Was heißt „Hund“?')]), ], tools: tools, maxOutputTokens: maxOutputTokens, effort: effort, ); /// A budget of [dailyCost] and [monthlyCost] in millionths, and /// [dailyTokens] and [monthlyTokens]; none of them: no limits. AssistantLimits _budget({ int? dailyCost, int? monthlyCost, int? dailyTokens, int? monthlyTokens, }) => AssistantLimits( dailyCostMicros: dailyCost, monthlyCostMicros: monthlyCost, dailyTokens: dailyTokens, monthlyTokens: monthlyTokens, ); /// The limits of [budget], to compare. (int?, int?, int?, int?) _limitsOf(AssistantLimits budget) => ( budget.dailyCostMicros, budget.monthlyCostMicros, budget.dailyTokens, budget.monthlyTokens, ); /// What [use] says, to compare. (int, int) _useOf(AssistantUse use) => (use.costMicros, use.tokens); Matcher _refused(AssistantRefusal refusal) => throwsA( isA().having( (e) => e.refusal, 'refusal', refusal, ), ); Matcher _refusedForOperator(OperatorRefusal reason) => throwsA( isA().having((e) => e.reason, 'reason', reason), ); void main() { withServerpod( 'Assistants', // The figures are counted on a session of their own, outside any test's // transaction: these tests delete what they make instead. rollbackDatabase: RollbackDatabase.disabled, (sessionBuilder, endpoints) { late Session session; late Serverpod pod; late FakeLlmClient provider; /// The instance's operator. late UuidValue olga; late UuidValue olgasLaptop; /// Identified; no operator. late UuidValue ben; late UuidValue bensPhone; setUp(() async { useBase(sessionBuilder, app: 'learn'); session = sessionBuilder.build(); pod = session.serverpod; await _deleteAll(session); provider = FakeLlmClient(); useAssistants(pod, _assistants()); useLlmClient(pod, provider); olga = await newPersonWithAddress( session, 'olga@example.org', handle: 'Olga', ); olgasLaptop = await signInNewDevice( session, olga, scopes: {identified, instanceOperator}, ); ben = await newPersonWithAddress( session, 'ben@example.org', handle: 'ben', ); bensPhone = await signInNewDevice(session, ben, scopes: {identified}); }); tearDown(() async { await assistantUsageCounted(); useAssistants(pod, null); useLlmClient(pod, null); await _deleteAll(sessionBuilder.build()); }); TestSessionBuilder notSignedIn() => sessionBuilder.copyWith( authentication: AuthenticationOverride.unauthenticated(), ); TestSessionBuilder asOlga() => sessionBuilder.copyWith( authentication: AuthenticationOverride.authenticationInfo( olga.uuid, {identified, instanceOperator}, authId: olgasLaptop.uuid, ), ); TestSessionBuilder asBen() => sessionBuilder.copyWith( authentication: AuthenticationOverride.authenticationInfo( ben.uuid, {identified}, authId: bensPhone.uuid, ), ); Future atNow(Future Function() body) => withClock(Clock.fixed(_now), body); /// Grants Ben [assistant] with [budget] (none: no limits), as Olga. Future grantBen({ String assistant = 'claude', AssistantLimits? budget, }) => endpoints.assistantOperator.grant( asOlga(), accounts: [ben], assistants: [assistant], budget: budget ?? _budget(), ); /// Grants Ben [assistant], and he agrees to it. Future grantAndAgree({ String assistant = 'claude', AssistantLimits? budget, }) async { await grantBen(assistant: assistant, budget: budget); await endpoints.assistant.agree(asBen(), assistant: assistant); } /// A grant of [assistant] to Ben from before, which the instance may /// set no more (D44). Future grantedBefore(String assistant) => AssistantGrant.db.insertRow( session, AssistantGrant( authUserId: ben, assistant: assistant, grantedAt: _now, ), ); /// What Ben's question is answered with, whole. Future> askAsBen(AssistantQuestion question) => endpoints.assistant.ask(asBen(), question: question).toList(); /// Ben's figures of [assistant], day by day. Future> figuresOfBen([ String assistant = 'claude', ]) => AssistantFigures.db.find( session, where: (t) => t.authUserId.equals(ben) & t.assistant.equals(assistant), orderBy: (t) => t.day, ); /// Ben's use of [assistant] on [day], as counted before. Future usedBefore( DateTime day, { String assistant = 'claude', int inputTokens = 0, int outputTokens = 0, int costMicros = 0, }) => AssistantFigures.db.insertRow( session, AssistantFigures( authUserId: ben, assistant: assistant, day: day, requests: 1, inputTokens: inputTokens, outputTokens: outputTokens, costMicros: costMicros, ), ); group('asking (D35, point 7)', () { test('streams the answer as the model writes it, then the whole turn ' "with what was counted, forwarded with the operator's endpoint and " "the person's opaque ID (D30)", () async { await grantAndAgree(); provider.replyText( '„Hund“ heißt 犬.', usage: const LlmUsage(inputTokens: 1000, outputTokens: 200), ); final answers = await askAsBen(_question()); expect( [for (final answer in answers) ?answer.delta].join(), '„Hund“ heißt 犬.', ); final last = answers.last; expect(last.delta, isNull); expect(last.stop, AssistantStop.end); expect(last.message!.role, AssistantRole.assistant); expect(last.message!.parts.single.text, '„Hund“ heißt 犬.'); expect((last.inputTokens, last.outputTokens), (1000, 200)); final (endpoint, request) = provider.sent.single; expect(endpoint.protocol, LlmProtocol.anthropic); expect(endpoint.model, 'claude-sonnet-5-5'); expect(endpoint.apiKey, 'sk-ant-test'); expect(request.system, 'Du hilfst beim Japanischlernen.'); expect(request.messages.single.text, 'Was heißt „Hund“?'); expect(request.maxOutputTokens, 2048); expect(request.personId, _assistants().personIdOf(ben)); expect(request.personId, isNot(contains(ben.uuid))); }); test('counts each request into the UTC day, at the prices of the ' 'time (D38, D42)', () async { await grantAndAgree(); provider ..replyText( 'a', usage: const LlmUsage(inputTokens: 1000, outputTokens: 200), ) ..replyText( 'b', usage: const LlmUsage(inputTokens: 500, outputTokens: 100), ); await atNow(() => askAsBen(_question())); await atNow(() => askAsBen(_question())); final day = (await figuresOfBen()).single; expect(day.day, DateTime.utc(2026, 10, 25)); expect(day.requests, 2); expect((day.inputTokens, day.outputTokens), (1500, 300)); // 1,500 input tokens at 3 and 300 output at 15 per million. expect(day.costMicros, 1500 * 3 + 300 * 15); }); test("sends the conversation as the person's app holds it: images to " 'an assistant that reads them, tool calls with what the provider ' 'wants back, their results, blocks only the provider reads; and ' 'hands the tool calls back alike', () async { await grantAndAgree(); provider.reply([ LlmFinished( message: LlmMessage.assistant(const [ LlmProviderBlock( protocol: LlmProtocol.anthropic, block: {'type': 'thinking', 'signature': 'EqQB'}, ), LlmToolCall( id: 'toolu_2', name: 'find_cards', input: {'query': 'Katze'}, extra: {'x': 1}, ), ]), stop: LlmStop.toolCalls, usage: const LlmUsage(inputTokens: 10, outputTokens: 5), ), ]); final picture = Uint8List.fromList([1, 2, 3]); final answers = await askAsBen( _question( messages: [ _user([ _text('Was steht da?'), AssistantPart( kind: AssistantPartKind.image, mediaType: 'image/png', bytes: ByteData.sublistView(picture), ), ]), _model([ AssistantPart( kind: AssistantPartKind.providerBlock, protocol: 'anthropic', json: '{"type":"thinking","signature":"EqQA"}', ), AssistantPart( kind: AssistantPartKind.text, text: 'Ich sehe nach.', extra: '{"phase":"commentary"}', ), AssistantPart( kind: AssistantPartKind.toolCall, callId: 'toolu_1', name: 'find_cards', json: '{"query":"Hund"}', ), ]), _user([ AssistantPart( kind: AssistantPartKind.toolResult, callId: 'toolu_1', text: 'Keine Karte', isError: true, ), ]), ], tools: [ AssistantTool( name: 'find_cards', description: 'Finds cards.', inputSchema: '{"type":"object"}', ), ], ), ); final request = provider.sent.single.$2; final [first, second, third] = request.messages; expect(first.role, LlmRole.user); final [text, image] = first.parts; expect((text as LlmText).text, 'Was steht da?'); expect((image as LlmImage).mediaType, 'image/png'); expect(image.bytes, picture); expect(second.role, LlmRole.assistant); final [block, said, call] = second.parts; expect((block as LlmProviderBlock).protocol, LlmProtocol.anthropic); expect(block.block, {'type': 'thinking', 'signature': 'EqQA'}); expect((said as LlmText).extra, {'phase': 'commentary'}); expect((call as LlmToolCall).id, 'toolu_1'); expect(call.input, {'query': 'Hund'}); final result = third.parts.single as LlmToolResult; expect((result.callId, result.content), ('toolu_1', 'Keine Karte')); expect(result.isError, isTrue); expect(request.tools.single.name, 'find_cards'); expect(request.tools.single.inputSchema, {'type': 'object'}); final last = answers.single; expect(last.stop, AssistantStop.toolCalls); final [kept, called] = last.message!.parts; expect(kept.kind, AssistantPartKind.providerBlock); expect(kept.protocol, 'anthropic'); expect(jsonDecode(kept.json!), { 'type': 'thinking', 'signature': 'EqQB', }); expect(called.kind, AssistantPartKind.toolCall); expect((called.callId, called.name), ('toolu_2', 'find_cards')); expect(jsonDecode(called.json!), {'query': 'Katze'}); expect(jsonDecode(called.extra!), {'x': 1}); }); test("holds the ceiling on output to the effort's, the model's own " "default's without one (D63)", () async { useAssistants( pod, _assistants({ 'ASSISTANT_CLAUDE_EFFORT': 'medium', 'ASSISTANT_CLAUDE_MAX_EFFORT': 'xhigh', }), ); await grantAndAgree(); await grantAndAgree(assistant: 'local'); provider ..replyText('a') ..replyText('b') ..replyText('c') ..replyText('d'); await askAsBen(_question(maxOutputTokens: 100000)); await askAsBen( _question(maxOutputTokens: 100000, effort: AssistantEffort.xhigh), ); await askAsBen(_question(maxOutputTokens: 0)); await askAsBen( _question(assistant: 'local', maxOutputTokens: 100000), ); expect( [for (final (_, request) in provider.sent) request.maxOutputTokens], [ LlmEffort.medium.outputCeiling, LlmEffort.xhigh.outputCeiling, 1, LlmEffort.defaultOutputCeiling, ], ); }); test("asks at the effort the person picks within the operator's " "range, else at the operator's default, else at the model's own " '(D63)', () async { useAssistants( pod, _assistants({ 'ASSISTANT_CLAUDE_EFFORT': 'medium', 'ASSISTANT_CLAUDE_MAX_EFFORT': 'high', }), ); await grantAndAgree(); await grantAndAgree(assistant: 'local'); provider ..replyText('a') ..replyText('b') ..replyText('c') ..replyText('d'); await askAsBen(_question()); await askAsBen(_question(effort: AssistantEffort.low)); await askAsBen(_question(effort: AssistantEffort.high)); await askAsBen(_question(assistant: 'local')); expect( [for (final (_, request) in provider.sent) request.effort], [LlmEffort.medium, LlmEffort.low, LlmEffort.high, null], ); }); test('money stops only the assistants with prices; tokens stop every ' 'one (D59)', () async { // Each grant sets the person's budget, so the second sets it again. await grantAndAgree(budget: _budget(dailyCost: 100)); await grantAndAgree( assistant: 'local', budget: _budget(dailyCost: 100), ); await usedBefore(DateTime.utc(2026, 10, 25), costMicros: 100); provider.replyText('犬'); await expectLater( atNow(() => askAsBen(_question())), _refused(AssistantRefusal.limitReached), ); await atNow(() => askAsBen(_question(assistant: 'local'))); expect(provider.sent.single.$1.model, 'qwen3:8b'); }); test('an answer cancelled is counted to its end all the same, so ' "cancelling can't slip past a limit", () async { await grantAndAgree(); final held = _HeldClient(); useLlmClient(pod, held); final answers = endpoints.assistant.ask( asBen(), question: _question(), ); final first = Completer(); final subscription = answers.listen((answer) { if (!first.isCompleted) { first.complete(answer); } }); held.add(const LlmTextDelta('„Hund“')); await first.future; await subscription.cancel(); held ..add(const LlmTextDelta(' heißt 犬.')) ..finish(const LlmUsage(inputTokens: 800, outputTokens: 400)); await assistantUsageCounted(); expect(held.cancelled, isFalse); final day = (await figuresOfBen()).single; expect( (day.requests, day.inputTokens, day.outputTokens), (1, 800, 400), ); }); group( "a provider's failure is told by kind, what it counted counted", () { final failures = { LlmFailureKind.key: AssistantRefusal.providerKey, LlmFailureKind.rateLimited: AssistantRefusal.providerBusy, LlmFailureKind.unavailable: AssistantRefusal.providerBusy, LlmFailureKind.request: AssistantRefusal.providerRefused, LlmFailureKind.provider: AssistantRefusal.providerFailed, LlmFailureKind.network: AssistantRefusal.providerFailed, LlmFailureKind.malformed: AssistantRefusal.providerFailed, }; for (final MapEntry(key: kind, value: refusal) in failures.entries) { test('${kind.name}: ${refusal.name}', () async { await grantAndAgree(); provider.fail( kind, usage: const LlmUsage(inputTokens: 300, outputTokens: 0), ); await expectLater(askAsBen(_question()), _refused(refusal)); final day = (await figuresOfBen()).single; expect((day.requests, day.inputTokens), (1, 300)); }); } test("with the provider's wait, where it says", () async { await grantAndAgree(); provider.reply( const [], failure: const LlmFailure( kind: LlmFailureKind.rateLimited, detail: 'slow down', retryAfter: Duration(seconds: 30), ), ); await expectLater( askAsBen(_question()), throwsA( isA().having( (e) => e.retryAfterSeconds, 'retryAfterSeconds', 30, ), ), ); }); }, ); }); group('asking is refused, and nothing goes to the provider', () { tearDown(() => expect(provider.sent, isEmpty)); test('to someone not signed in', () async { await expectLater( endpoints.assistant .ask(notSignedIn(), question: _question()) .toList(), throwsA(isA()), ); }); test('without a grant, for an assistant not set, and for one set no ' 'more (D44)', () async { await expectLater( askAsBen(_question()), _refused(AssistantRefusal.notGranted), ); await expectLater( askAsBen(_question(assistant: 'gpt')), _refused(AssistantRefusal.notGranted), ); await grantAndAgree(); useAssistants(pod, _assistants({'ASSISTANTS': 'local'})); await expectLater( askAsBen(_question()), _refused(AssistantRefusal.notGranted), ); }); test('until the person agreed, and again once the assistant moved to ' 'another provider (D40)', () async { await grantBen(); await expectLater( askAsBen(_question()), _refused(AssistantRefusal.notAgreed), ); await endpoints.assistant.agree(asBen(), assistant: 'claude'); useAssistants( pod, _assistants({ 'ASSISTANT_CLAUDE_BASE_URL': 'https://claude.proxy.example.org', }), ); await expectLater( askAsBen(_question()), _refused(AssistantRefusal.notAgreed), ); }); test( "past a day's budget in money, all of the person's assistants " "counted together, saying it resets at UTC's midnight (D42, D58)", () async { await grantAndAgree(budget: _budget(dailyCost: 10000)); await usedBefore(DateTime.utc(2026, 10, 25), costMicros: 6000); await usedBefore( DateTime.utc(2026, 10, 25), assistant: 'haiku', costMicros: 4000, ); await expectLater( atNow(() => askAsBen(_question())), throwsA( isA() .having( (e) => e.refusal, 'refusal', AssistantRefusal.limitReached, ) .having( (e) => e.resetsAt, 'resetsAt', DateTime.utc(2026, 10, 26), ), ), ); }, ); test("past a month's budget, saying it resets on the next month's " "first, which a day's budget doesn't change", () async { await grantAndAgree( budget: _budget(dailyCost: 10000, monthlyCost: 25000), ); await usedBefore(DateTime.utc(2026, 10, 3), costMicros: 15000); await usedBefore(DateTime.utc(2026, 10, 25), costMicros: 10000); await expectLater( atNow(() => askAsBen(_question())), throwsA( isA() .having( (e) => e.refusal, 'refusal', AssistantRefusal.limitReached, ) .having( (e) => e.resetsAt, 'resetsAt', DateTime.utc(2026, 11), ), ), ); }); test('past a budget in tokens, every assistant counted, priced or ' 'not (D59)', () async { await grantAndAgree(budget: _budget(dailyTokens: 1000)); await grantAndAgree( assistant: 'local', budget: _budget(dailyTokens: 1000), ); await usedBefore( DateTime.utc(2026, 10, 25), assistant: 'local', inputTokens: 500, outputTokens: 100, ); await usedBefore( DateTime.utc(2026, 10, 25), inputTokens: 300, outputTokens: 100, ); await expectLater( atNow(() => askAsBen(_question(assistant: 'local'))), _refused(AssistantRefusal.limitReached), ); await expectLater( atNow(() => askAsBen(_question())), _refused(AssistantRefusal.limitReached), ); }); test("with an effort above the operator's highest, or to an " 'assistant without a range (D63)', () async { useAssistants( pod, _assistants({ 'ASSISTANT_CLAUDE_EFFORT': 'medium', 'ASSISTANT_CLAUDE_MAX_EFFORT': 'high', }), ); await grantAndAgree(); await grantAndAgree(assistant: 'local'); await expectLater( askAsBen(_question(effort: AssistantEffort.xhigh)), _refused(AssistantRefusal.invalidEffort), ); await expectLater( askAsBen( _question(assistant: 'local', effort: AssistantEffort.low), ), _refused(AssistantRefusal.invalidEffort), ); }); test('with an image, to an assistant that reads none (D12)', () async { await grantAndAgree(assistant: 'local'); await expectLater( askAsBen( _question( assistant: 'local', messages: [ _user([ AssistantPart( kind: AssistantPartKind.image, mediaType: 'image/png', bytes: ByteData(3), ), ]), ], ), ), _refused(AssistantRefusal.imagesNotRead), ); }); group('for a question that cannot be sent', () { final malformed = { 'no turn': _question(messages: []), "the last turn the model's": _question( messages: [ _user([_text('?')]), _model([_text('!')]), ], ), 'a turn without parts': _question(messages: [_user([])]), "a tool call in the person's turn": _question( messages: [ _user([ AssistantPart( kind: AssistantPartKind.toolCall, callId: 'c', name: 'n', json: '{}', ), ]), ], ), "an image in the model's turn": _question( messages: [ _user([_text('?')]), _model([ AssistantPart( kind: AssistantPartKind.image, mediaType: 'image/png', bytes: ByteData(3), ), ]), _user([_text('?')]), ], ), 'a text without its text': _question( messages: [ _user([AssistantPart(kind: AssistantPartKind.text)]), ], ), "a tool call's input that is no JSON object": _question( messages: [ _user([_text('?')]), _model([ AssistantPart( kind: AssistantPartKind.toolCall, callId: 'c', name: 'n', json: '[1]', ), ]), _user([ AssistantPart( kind: AssistantPartKind.toolResult, callId: 'c', text: '', ), ]), ], ), 'a block of a protocol unknown': _question( messages: [ _user([_text('?')]), _model([ AssistantPart( kind: AssistantPartKind.providerBlock, protocol: 'lumo', json: '{}', ), ]), _user([_text('?')]), ], ), 'an image of a type the providers refuse': _question( messages: [ _user([ AssistantPart( kind: AssistantPartKind.image, mediaType: 'image/svg+xml', bytes: ByteData(3), ), ]), ], ), 'a tool named as no provider takes': _question( tools: [ AssistantTool( name: 'find cards', description: '', inputSchema: '{"type":"object"}', ), ], ), "a tool's schema that is no JSON object": _question( tools: [ AssistantTool( name: 'find_cards', description: '', inputSchema: 'object', ), ], ), }; for (final MapEntry(key: name, value: question) in malformed.entries) { test(name, () async { await grantAndAgree(); await expectLater( askAsBen(question), _refused(AssistantRefusal.malformed), ); }); } }); test('for a question too large: its text, or an image', () async { await grantAndAgree(); await expectLater( askAsBen( _question( messages: [ _user([_text('犬' * (assistantQuestionMaxCharacters + 1))]), ], ), ), _refused(AssistantRefusal.tooLarge), ); await expectLater( askAsBen( _question( messages: [ _user([ AssistantPart( kind: AssistantPartKind.image, mediaType: 'image/jpeg', bytes: ByteData(assistantImageMaxBytes + 1), ), ]), ], ), ), _refused(AssistantRefusal.tooLarge), ); }); test('on a member, which offers no assistant yet (D43)', () async { await grantAndAgree(); useBase( sessionBuilder, app: 'todo', accounts: MemberOfRegister(Uri.parse('http://learn.test:8082')), ); await expectLater( askAsBen(_question()), _refused(AssistantRefusal.notGranted), ); expect( (await endpoints.assistant.offered(asBen())).assistants, isEmpty, ); }); }); test('a person has at most $maxQuestionsAtOnce questions on their way ' 'at once, so parallel ones never pass a limit together', () async { await grantAndAgree(); final held = [ for (var i = 0; i < maxQuestionsAtOnce; i++) _HeldClient(), ]; final client = _Clients(held); useLlmClient(pod, client); final asked = [ for (var i = 0; i < maxQuestionsAtOnce; i++) askAsBen(_question()), ]; await client.allSent; await expectLater( askAsBen(_question()), _refused(AssistantRefusal.tooManyAtOnce), ); for (final one in held) { one.finish(const LlmUsage(inputTokens: 1, outputTokens: 1)); } await Future.wait(asked); expect(client.sent, maxQuestionsAtOnce); }); group('what a person is offered (point 6, D27)', () { test("the assistants granted that are set, in the operator's order, " 'with what goes where and their effort, and the budget with what ' 'is used of it, every assistant together', () async { useAssistants( pod, _assistants({ 'ASSISTANT_CLAUDE_EFFORT': 'medium', 'ASSISTANT_CLAUDE_MAX_EFFORT': 'high', }), ); await grantBen(assistant: 'local'); await grantAndAgree( budget: _budget( dailyCost: 10000, monthlyCost: 50000, monthlyTokens: 100000, ), ); await grantedBefore('gone'); await usedBefore( DateTime.utc(2026, 10, 3), costMicros: 7000, inputTokens: 600, outputTokens: 100, ); await usedBefore( DateTime.utc(2026, 10, 25), costMicros: 2500, inputTokens: 100, outputTokens: 20, ); await usedBefore( DateTime.utc(2026, 10, 25), assistant: 'local', inputTokens: 250, outputTokens: 50, ); await usedBefore(DateTime.utc(2026, 9, 30), costMicros: 1); final offered = await atNow( () => endpoints.assistant.offered(asBen()), ); expect(offered.currency, 'EUR'); expect(offered.ownKeys, isFalse); final [claude, local] = offered.assistants; expect(claude.id, 'claude'); expect(claude.name, 'Claude Sonnet'); expect(claude.protocol, 'anthropic'); expect(claude.host, 'api.anthropic.com'); expect(claude.model, 'claude-sonnet-5-5'); expect(claude.images, isTrue); expect(claude.agreedAt, isNotNull); expect(claude.effort, AssistantEffort.medium); expect(claude.efforts, [ AssistantEffort.low, AssistantEffort.medium, AssistantEffort.high, ]); expect(local.id, 'local'); expect(local.host, 'ollama'); expect(local.agreedAt, isNull); expect(local.effort, isNull); expect(local.efforts, isEmpty); expect(_limitsOf(offered.budget), (10000, 50000, null, 100000)); expect(_useOf(offered.usedToday), (2500, 420)); expect(_useOf(offered.usedThisMonth), (9500, 1120)); }); test( 'no budget: no limits, what is used counted all the same', () async { await grantAndAgree(); await usedBefore(DateTime.utc(2026, 10, 25), costMicros: 30); final offered = await atNow( () => endpoints.assistant.offered(asBen()), ); expect(_limitsOf(offered.budget), (null, null, null, null)); expect(_useOf(offered.usedToday), (30, 0)); }, ); test('an agreement to another provider counts as none', () async { await grantAndAgree(); useAssistants( pod, _assistants({ 'ASSISTANT_CLAUDE_BASE_URL': 'https://claude.proxy.example.org', }), ); final claude = (await endpoints.assistant.offered(asBen())) .assistants .single; expect(claude.host, 'claude.proxy.example.org'); expect(claude.agreedAt, isNull); }); group('with two assistants of one provider (D66)', () { setUp( () => useAssistants( pod, _assistants({ 'ASSISTANTS': 'claude opus local', 'ASSISTANT_OPUS_NAME': 'Claude Opus', 'ASSISTANT_OPUS_PROTOCOL': 'anthropic', 'ASSISTANT_OPUS_MODEL': 'claude-opus-5-5', 'ASSISTANT_OPUS_KEY': 'sk-ant-test', }), ), ); Future> agreed() async => { for (final assistant in (await endpoints.assistant.offered( asBen(), )).assistants) assistant.id: assistant.agreedAt != null, }; test('one agreement holds for every assistant of its provider, one ' 'granted later too; another provider asks its own', () async { await endpoints.assistantOperator.grant( asOlga(), accounts: [ben], assistants: ['claude', 'local'], budget: _budget(), ); await endpoints.assistant.agree(asBen(), assistant: 'claude'); await grantBen(assistant: 'opus'); expect(await agreed(), { 'claude': true, 'opus': true, 'local': false, }); provider.replyText('犬'); await askAsBen(_question(assistant: 'opus')); await expectLater( askAsBen(_question(assistant: 'local')), _refused(AssistantRefusal.notAgreed), ); }); test('taking back the assistant agreed to keeps the agreement for ' 'the others of its provider', () async { await endpoints.assistantOperator.grant( asOlga(), accounts: [ben], assistants: ['claude', 'opus'], budget: _budget(), ); await endpoints.assistant.agree(asBen(), assistant: 'claude'); await endpoints.assistantOperator.change( asOlga(), account: ben, assistants: ['opus'], budget: _budget(), ); expect(await agreed(), {'opus': true}); }); test( "the operator's list shows it agreed to for each of them", () async { await endpoints.assistantOperator.grant( asOlga(), accounts: [ben], assistants: ['claude', 'opus'], budget: _budget(), ); await endpoints.assistant.agree(asBen(), assistant: 'opus'); final grantee = (await endpoints.assistantOperator.granteesOf( asOlga(), accounts: [ben], )).single; expect( [for (final each in grantee.assistants) each.agreedAt != null], [true, true], ); }, ); }); test('a new model of the same provider keeps the agreement', () async { await grantAndAgree(); useAssistants( pod, _assistants({'ASSISTANT_CLAUDE_MODEL': 'claude-opus-5-5'}), ); provider.replyText('犬'); final offered = await endpoints.assistant.offered(asBen()); expect(offered.assistants.single.agreedAt, isNotNull); await askAsBen(_question()); }); test('whether personal keys are allowed (D5, D6)', () async { useAssistants(pod, _assistants({'ASSISTANT_OWN_KEYS': 'on'})); expect((await endpoints.assistant.offered(asBen())).ownKeys, isTrue); expect( (await endpoints.instance.basics(notSignedIn())).assistantOwnKeys, isTrue, ); }); test('agreeing is refused without a grant, or for an assistant not ' 'set', () async { await expectLater( endpoints.assistant.agree(asBen(), assistant: 'claude'), _refused(AssistantRefusal.notGranted), ); await grantedBefore('gone'); await expectLater( endpoints.assistant.agree(asBen(), assistant: 'gone'), _refused(AssistantRefusal.notGranted), ); }); test( "a person's own figures, by month, without anyone else's", () async { await grantAndAgree(); await usedBefore( DateTime.utc(2026, 9, 12), inputTokens: 10, outputTokens: 2, costMicros: 60, ); await usedBefore( DateTime.utc(2026, 10, 3), inputTokens: 100, outputTokens: 20, costMicros: 600, ); await usedBefore( DateTime.utc(2026, 10, 25), inputTokens: 50, outputTokens: 10, costMicros: 300, ); await AssistantFigures.db.insertRow( session, AssistantFigures( authUserId: olga, assistant: 'claude', day: DateTime.utc(2026, 10, 25), requests: 9, inputTokens: 9, outputTokens: 9, costMicros: 9, ), ); final figures = await endpoints.assistant.figures(asBen()); expect( [ for (final month in figures) ( month.month, month.requests, month.inputTokens, month.outputTokens, month.costMicros, ), ], [ (DateTime.utc(2026, 10), 2, 150, 30, 900), (DateTime.utc(2026, 9), 1, 10, 2, 60), ], ); expect(figures.map((month) => month.account), everyElement(ben)); expect(figures.map((month) => month.handle), everyElement(isNull)); }, ); test('are refused to someone not signed in', () async { await expectLater( endpoints.assistant.offered(notSignedIn()), throwsA(isA()), ); await expectLater( endpoints.assistant.agree(notSignedIn(), assistant: 'claude'), throwsA(isA()), ); await expectLater( endpoints.assistant.figures(notSignedIn()), throwsA(isA()), ); }); }); group("the operator's calls (points 6 and 8, D58)", () { test( 'are refused to anyone but an operator, signed in or not', () async { await grantBen(); final calls = Function(TestSessionBuilder)>{ 'setUp': (as) => endpoints.assistantOperator.setUp(as), 'grantees': (as) => endpoints.assistantOperator.grantees( as, startingWith: '', offset: 0, ), 'granteesOf': (as) => endpoints.assistantOperator.granteesOf( as, accounts: [ben], ), 'grant': (as) => endpoints.assistantOperator.grant( as, accounts: [ben], assistants: ['local'], budget: _budget(), ), 'change': (as) => endpoints.assistantOperator.change( as, account: ben, assistants: ['local'], budget: _budget(), ), 'revoke': (as) => endpoints.assistantOperator.revoke(as, account: ben), 'figures': (as) => endpoints.assistantOperator.figures( as, month: DateTime.utc(2026, 10), ), }; for (final MapEntry(key: name, value: call) in calls.entries) { await expectLater( call(asBen()), throwsA(isA()), reason: name, ); await expectLater( call(notSignedIn()), throwsA(isA()), reason: name, ); } final grants = await AssistantGrant.db.find(session); expect([for (final grant in grants) grant.assistant], ['claude']); }, ); test('see the assistants set, without their keys, with their effort ' 'and how many people have any', () async { useAssistants( pod, _assistants({ 'ASSISTANT_CLAUDE_EFFORT': 'medium', 'ASSISTANT_CLAUDE_MAX_EFFORT': 'high', }), ); await grantBen(); await grantBen(assistant: 'local'); final setUp = await endpoints.assistantOperator.setUp(asOlga()); expect(setUp.currency, 'EUR'); expect(setUp.ownKeys, isFalse); expect(setUp.people, 1); final [claude, local] = setUp.assistants; expect( ( claude.id, claude.name, claude.protocol, claude.host, claude.model, ), ( 'claude', 'Claude Sonnet', 'anthropic', 'api.anthropic.com', 'claude-sonnet-5-5', ), ); expect((claude.priceInput, claude.priceOutput), (3.0, 15.0)); expect(claude.images, isTrue); expect( (claude.effort, claude.maxEffort), (AssistantEffort.medium, AssistantEffort.high), ); expect((local.protocol, local.host), ('openai', 'ollama')); expect(local.priceInput, isNull); expect((local.effort, local.maxEffort), (null, null)); expect(jsonEncode(setUp.toJson()), isNot(contains('sk-ant-test'))); }); test('grant several people several assistants at once, each with the ' 'budget set; granted before, the assistants add up, the ' 'agreements stay and the budget is the new one', () async { await endpoints.assistantOperator.grant( asOlga(), accounts: [ben, olga], assistants: ['claude'], budget: _budget(dailyCost: 10000), ); await endpoints.assistant.agree(asBen(), assistant: 'claude'); await endpoints.assistantOperator.grant( asOlga(), accounts: [ben], assistants: ['local', 'claude'], budget: _budget(monthlyCost: 90000, monthlyTokens: 200000), ); final grantees = await endpoints.assistantOperator.grantees( asOlga(), startingWith: '', offset: 0, ); final [bens, olgas] = grantees; expect((bens.account, bens.handle), (ben, 'ben')); expect( [for (final each in bens.assistants) each.assistant], [ 'claude', 'local', ], ); final [claude, local] = bens.assistants; expect(claude.assistantSet, isTrue); expect(claude.agreedAt, isNotNull); expect(local.agreedAt, isNull); expect(_limitsOf(bens.budget), (null, 90000, null, 200000)); expect((olgas.account, olgas.handle), (olga, 'Olga')); expect( [for (final each in olgas.assistants) each.assistant], [ 'claude', ], ); expect(_limitsOf(olgas.budget), (10000, null, null, null)); }); test( "change a person's assistants and budget to exactly those given: " 'the assistants left out go, the agreements of those kept stay', () async { await grantAndAgree(budget: _budget(dailyCost: 10000)); await grantBen(assistant: 'local'); await endpoints.assistantOperator.change( asOlga(), account: ben, assistants: ['claude'], budget: _budget(dailyTokens: 5000), ); final [bens] = await endpoints.assistantOperator.granteesOf( asOlga(), accounts: [ben, olga], ); final claude = bens.assistants.single; expect(claude.assistant, 'claude'); expect(claude.agreedAt, isNotNull); expect(_limitsOf(bens.budget), (null, null, 5000, null)); }, ); test('grantees are found by the start of their handle, whatever its ' 'case, by handle, a page at a time, with what they used, those ' 'without a grant left out', () async { final cleo = await newPersonWithAddress( session, 'cleo@example.org', handle: 'Cleo', ); await newPersonWithAddress( session, 'bert@example.org', handle: 'bert', ); await endpoints.assistantOperator.grant( asOlga(), accounts: [ben, cleo, olga], assistants: ['claude'], budget: _budget(), ); await usedBefore( DateTime.utc(2026, 10, 25), costMicros: 300, inputTokens: 30, outputTokens: 5, ); Future> handles(String start, int offset) async => [ for (final each in await atNow( () => endpoints.assistantOperator.grantees( asOlga(), startingWith: start, offset: offset, ), )) each.handle, ]; expect(await handles('', 0), ['ben', 'Cleo', 'Olga']); expect(await handles('', 2), ['Olga']); expect(await handles('B', 0), ['ben']); expect(await handles('%', 0), isEmpty); final bens = (await atNow( () => endpoints.assistantOperator.grantees( asOlga(), startingWith: 'ben', offset: 0, ), )).single; expect(_useOf(bens.usedToday), (300, 35)); expect(_useOf(bens.usedThisMonth), (300, 35)); }); test('a grant of an assistant set no more is shown so (D44)', () async { await grantBen(); await grantBen(assistant: 'local'); useAssistants(pod, _assistants({'ASSISTANTS': 'claude'})); final [bens] = await endpoints.assistantOperator.granteesOf( asOlga(), accounts: [ben], ); final [claude, local] = bens.assistants; expect(claude.assistantSet, isTrue); expect((local.assistant, local.assistantSet), ('local', false)); }); test( 'granting and changing are refused for an assistant not set, a ' 'limit below 0, and an account unknown, changing nothing', () async { final nobody = UuidValue.fromString( '0193a3d4-5c6e-7f80-9a1b-2c3d4e5f6a7b', ); await expectLater( endpoints.assistantOperator.grant( asOlga(), accounts: [ben], assistants: ['claude', 'gpt'], budget: _budget(), ), _refusedForOperator(OperatorRefusal.assistantNotSet), ); for (final budget in [ _budget(dailyCost: -1), _budget(monthlyCost: -1), _budget(dailyTokens: -1), _budget(monthlyTokens: -1), ]) { await expectLater( grantBen(budget: budget), _refusedForOperator(OperatorRefusal.invalidLimit), ); } await expectLater( endpoints.assistantOperator.grant( asOlga(), accounts: [ben, nobody], assistants: ['claude'], budget: _budget(), ), _refusedForOperator(OperatorRefusal.notFound), ); await expectLater( endpoints.assistantOperator.change( asOlga(), account: nobody, assistants: ['claude'], budget: _budget(), ), _refusedForOperator(OperatorRefusal.notFound), ); await expectLater( endpoints.assistantOperator.change( asOlga(), account: ben, assistants: ['gpt'], budget: _budget(), ), _refusedForOperator(OperatorRefusal.assistantNotSet), ); expect(await AssistantGrant.db.count(session), 0); expect(await AssistantBudget.db.count(session), 0); }, ); test("revoke a person's assistants and budget, which then refuses; " 'a person without any is refused', () async { await grantAndAgree(budget: _budget(dailyCost: 1000)); await grantBen(assistant: 'local'); await endpoints.assistantOperator.revoke(asOlga(), account: ben); await expectLater( askAsBen(_question()), _refused(AssistantRefusal.notGranted), ); expect(await AssistantGrant.db.count(session), 0); expect(await AssistantBudget.db.count(session), 0); await expectLater( endpoints.assistantOperator.revoke(asOlga(), account: ben), _refusedForOperator(OperatorRefusal.notFound), ); }); test("see everyone's figures of a month, per person and assistant, " 'never what was asked', () async { await usedBefore(DateTime.utc(2026, 10, 3), costMicros: 600); await usedBefore(DateTime.utc(2026, 10, 25), costMicros: 300); await usedBefore(DateTime.utc(2026, 9, 30), costMicros: 1); await usedBefore( DateTime.utc(2026, 10, 25), assistant: 'local', inputTokens: 70, ); await AssistantFigures.db.insertRow( session, AssistantFigures( authUserId: olga, assistant: 'claude', day: DateTime.utc(2026, 10), requests: 4, inputTokens: 40, outputTokens: 4, costMicros: 180, ), ); final figures = await endpoints.assistantOperator.figures( asOlga(), month: DateTime.utc(2026, 10, 17, 9), ); expect( { for (final row in figures) (row.handle, row.assistant): ( row.month, row.requests, row.inputTokens, row.costMicros, ), }, { ('ben', 'claude'): (DateTime.utc(2026, 10), 2, 0, 900), ('ben', 'local'): (DateTime.utc(2026, 10), 1, 70, 0), ('Olga', 'claude'): (DateTime.utc(2026, 10), 4, 40, 180), }, ); }); }); group('the figures (D41)', () { test('are kept for the UTC month and the two before it, the older ' 'days forgotten daily', () async { await usedBefore(DateTime.utc(2026, 7, 31)); await usedBefore(DateTime.utc(2026, 8)); await usedBefore(DateTime.utc(2026, 10, 25)); final forgotten = await forgetOldAssistantFigures(session, now: _now); expect(forgotten, 1); expect( [for (final day in await figuresOfBen()) day.day], [DateTime.utc(2026, 8), DateTime.utc(2026, 10, 25)], ); }); test('go with the person, as do their grants and budget', () async { await grantAndAgree(budget: _budget(dailyCost: 1000)); await usedBefore(DateTime.utc(2026, 10, 25)); await deletePerson(session, ben); expect(await AssistantGrant.db.count(session), 0); expect(await AssistantBudget.db.count(session), 0); expect(await AssistantFigures.db.count(session), 0); }); }); }, ); } /// A provider whose one answer the test writes as it goes. final class _HeldClient implements LlmClient { final _events = StreamController(); final _text = StringBuffer(); final sentOnce = Completer(); bool _finished = false; /// Whether the answer was cancelled before it ended. bool cancelled = false; void add(LlmTextDelta delta) { _text.write(delta.text); _events.add(delta); } void finish(LlmUsage usage) { _events.add( LlmFinished( message: LlmMessage.assistant([LlmText(_text.toString())]), stop: LlmStop.end, usage: usage, ), ); _finished = true; unawaited(_events.close()); } @override Stream send(LlmEndpoint endpoint, LlmRequest request) { // Called as the stream ends, too. _events.onCancel = () => cancelled = !_finished; sentOnce.complete(); return _events.stream; } @override void close() {} } /// Providers answering one request each, in turn. final class _Clients implements LlmClient { _Clients(this._clients); final List<_HeldClient> _clients; int sent = 0; Future get allSent => Future.wait([for (final client in _clients) client.sentOnce.future]); @override Stream send(LlmEndpoint endpoint, LlmRequest request) => _clients[sent++].send(endpoint, request); @override void close() {} } Future _deleteAll(Session session) async { await AssistantGrant.db.deleteWhere(session, where: (t) => t.id > 0); await AssistantBudget.db.deleteWhere(session, where: (t) => t.id > 0); await AssistantFigures.db.deleteWhere(session, where: (t) => t.id > 0); await deleteAll(session); } extension on Map { String? get(String name) => this[name]; }