halkyon/test/deploy/runner_test.dart

import 'dart:io';

import 'package:test/test.dart';
import 'package:yaml/yaml.dart';

/// The CI runner on halkyon.li's host at home (ADR 0039, D6, D16): jobs in
/// the nightly window alone, while the heavy sidecars rest, and a sandbox
/// that keeps them away from the instance, the home network and the tunnel.
void main() {
  String read(String path) => File(path).readAsStringSync();

  /// [unit]'s value of [key], the last where it is set more than once.
  String? setting(String unit, String key) =>
      RegExp('^$key=(.*)\$', multiLine: true).allMatches(unit).lastOrNull?[1];

  group("the runner's sandbox (D16)", () {
    final unit = read('deploy/runner/forgejo-runner.service');

    test('reaches no IPv6 address at all, so no home address, however its '
        'prefix changes', () {
      expect(setting(unit, 'IPAddressDeny')!.split(' '), contains('::/0'));
    });

    test('reaches no private IPv4 network: the home network, Docker, the '
        'tunnel, link-local, CGNAT', () {
      expect(
        setting(unit, 'IPAddressDeny')!.split(' '),
        containsAll([
          '10.0.0.0/8',
          '172.16.0.0/12',
          '192.168.0.0/16',
          '100.64.0.0/10',
          '169.254.0.0/16',
        ]),
      );
    });

    test('sees nothing of /home, nor of /data but its own folder', () {
      expect(setting(unit, 'ProtectHome'), 'yes');
      expect(setting(unit, 'TemporaryFileSystem'), '/data:ro');
      expect(
        setting(unit, 'BindPaths'),
        '/data/forgejo-runner:/var/lib/forgejo-runner',
      );
      expect(unit, contains('RequiresMountsFor=/data'));
    });

    test("can't reach Docker or containerd", () {
      final hidden = setting(unit, 'InaccessiblePaths')!.split(' ');
      expect(
        hidden,
        containsAll([
          '-/run/docker.sock',
          '-/run/containerd',
          '-/var/lib/docker',
          '-/var/lib/containerd',
        ]),
      );
    });

    test('leaves the instance its share of memory, swapping rather than '
        'starving it', () {
      expect(setting(unit, 'MemoryMax'), '5G');
      expect(setting(unit, 'MemoryHigh'), '4.5G');
      expect(setting(unit, 'MemorySwapMax'), isNull);
      expect(setting(unit, 'OOMScoreAdjust'), '1000');
    });
  });

  group('the nightly window (D6)', () {
    final unit = read('deploy/runner/forgejo-runner.service');
    final timer = read('deploy/runner/forgejo-runner.timer');

    test('starts at 02:00 alone, never at boot or to catch up', () {
      expect(setting(timer, 'OnCalendar'), '*-*-* 02:00:00');
      expect(setting(timer, 'Persistent'), 'false');
      expect(unit, isNot(contains('WantedBy=multi-user.target')));
    });

    test('ends three hours later, whatever runs then', () {
      expect(setting(unit, 'RuntimeMaxSec'), '3h');
    });

    group('rests the heavy sidecars around it, as deploy', () {
      final night = read('deploy/night.sh');
      final rest = RegExp(
        r'^heavy="(.+)"$',
        multiLine: true,
      ).firstMatch(night)![1]!.split(' ');

      test('the ones that take memory or build', () {
        expect(
          rest,
          unorderedEquals([
            'jellyfin',
            'languagetool',
            'dictionaries',
            'brouter',
            'brouter-segments',
            'map-tiles',
            'map-tiles-refresh',
            'photon',
            'photon-data',
          ]),
        );
      });

      test('never Caddy, PostgreSQL or an app server', () {
        expect(
          rest,
          isNot(
            anyOf(
              contains('caddy'),
              contains('postgres'),
              contains('server'),
              contains('train-server'),
              contains('todo-server'),
              contains('meditate-server'),
            ),
          ),
        );
      });

      test('before the runner starts and after it ends', () {
        expect(
          setting(read('deploy/runner/halkyon-night.timer'), 'OnCalendar'),
          '*-*-* 01:55:00',
        );
        expect(
          setting(read('deploy/runner/halkyon-morning.timer'), 'OnCalendar'),
          '*-*-* 05:00:00',
        );
        final evening = read('deploy/runner/halkyon-night.service');
        final morning = read('deploy/runner/halkyon-morning.service');
        expect(setting(evening, 'User'), 'deploy');
        expect(setting(morning, 'User'), 'deploy');
        expect(setting(evening, 'ExecStart'), contains('night.sh stop'));
        expect(setting(morning, 'ExecStart'), contains('night.sh start'));
      });

      test('starts the heavy sidecars after every boot too', () {
        // A reboot or a power cut between 01:55 and 05:00 would otherwise leave them
        // stopped (unless-stopped keeps a stopped container so) until the next morning;
        // the runner's own timer never catches up, so nothing clashes with it.
        expect(
          setting(read('deploy/runner/halkyon-morning.timer'), 'OnBootSec'),
          isNotNull,
        );
        expect(
          setting(read('deploy/runner/forgejo-runner.timer'), 'Persistent'),
          'false',
        );
      });
    });
  });

  group('a window by hand (halkyon-ci-now)', () {
    final script = read('deploy/runner/ci-now.sh');

    test('rests the heavy sidecars around the runner, unless asked to keep '
        'them', () {
      expect(script, contains('systemctl start halkyon-night'));
      expect(script, contains('systemctl start halkyon-morning'));
      expect(script, contains('--keep-sidecars'));
    });

    test('stops the runner and starts the sidecars again however it ends: '
        'done, interrupted, stopped or its SSH session gone', () {
      final trap = RegExp(
        r'^trap (\S+) (.*)$',
        multiLine: true,
      ).firstMatch(script);
      expect(trap, isNotNull);
      expect(trap![2]!.split(' '), containsAll(['EXIT', 'INT', 'TERM', 'HUP']));
      final restore = RegExp(
        '^${trap[1]}\\(\\) \\{([^}]*)\\}',
        multiLine: true,
      ).firstMatch(script)![1]!;
      expect(restore, contains('systemctl stop forgejo-runner'));
      expect(restore, contains('systemctl start halkyon-morning'));
    });

    test('leaves the nightly window alone: refuses while the runner runs', () {
      expect(script, contains('systemctl is-active --quiet forgejo-runner'));
    });

    test(
      "is installed by host step 7, root's own, never deploy's checkout",
      () {
        final readme = read('deploy/host/README.md');
        expect(
          readme,
          contains('deploy/runner/ci-now.sh /usr/local/sbin/halkyon-ci-now'),
        );
      },
    );
  });

  group('the workflow', () {
    final workflow = loadYaml(read('.forgejo/workflows/ci.yaml')) as YamlMap;
    final jobs = workflow['jobs'] as YamlMap;

    test('checks on the runner at home', () {
      expect((jobs['check'] as YamlMap)['runs-on'], 'balthasar');
      expect(read('deploy/runner/config.yml'), contains('"balthasar:host"'));
    });

    test('deploys nothing: halkyon.li deploys by hand (D13)', () {
      expect(jobs.keys, ['check']);
    });

    test('first proves it reaches neither the tunnel, the home network nor '
        'any IPv6 address', () {
      final sandbox =
          ((jobs['check'] as YamlMap)['steps'] as YamlList)
                  .cast<YamlMap>()
                  .singleWhere((step) => step['name'] == 'Sandbox')['run']
              as String;
      expect(sandbox, contains('/dev/tcp/10.61.97.2/443'));
      expect(sandbox, contains('/dev/tcp/192.168.178.1/80'));
      expect(sandbox, contains('/dev/tcp/2606:4700:4700::1111/443'));
      expect(sandbox, contains('/data/home'));
      expect(sandbox, isNot(contains('10.61.96.2')));
    });
  });
}