build(check): the images' resolution replayed as the deploy builds them, before any push
Only the deploy builds the images, on the server, after every check has passed, so an image that pub refuses cost a push each time (twice on 4 Nov 2026). A step of tool/check.dart, right after the packages, replays each Dockerfile's build stage up to its `dart pub get` from the working tree's files in an empty folder, with Dart alone at the image's version (downloaded once into ~/.cache/halkyon-check/, its SHA-256 checked), offline, in an environment holding nothing of the machine's Dart or Flutter: `dart run` from Flutter sets DART_ROOT, through which pub found Flutter and accepted what the image refuses. Checked on both failed deploys' states, which fail it as they did, and on the fix, which passes. --images-only runs it alone, in seconds. The list of images, dartImages, moves into tool/src/images.dart, and a test keeps every Dockerfile building with Dart on it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
2ec58bfaee
commit
1ba532d6ef
2 changed files with 478 additions and 16 deletions
194
tool/check.dart
194
tool/check.dart
|
|
@ -1,12 +1,15 @@
|
|||
// The checks every push has to pass: the packages the lock names, the
|
||||
// generated code, formatting, static analysis (infos are fatal too), and every
|
||||
// package's tests. The pre-push hook (tool/pre-push) runs it, and so does CI on
|
||||
// The checks every push has to pass: the packages the lock names, the images'
|
||||
// resolution, the generated code, formatting, static analysis (infos are fatal
|
||||
// too), and every package's tests. The pre-push hook (tool/pre-push) runs it, and so does CI on
|
||||
// every push (.forgejo/workflows/ci.yaml).
|
||||
//
|
||||
// dart run tool/check.dart everything
|
||||
// dart run tool/check.dart --changed-since=origin/main
|
||||
// the tests of the packages changed since then (in
|
||||
// the working tree too), and of what depends on them
|
||||
// dart run tool/check.dart --images-only
|
||||
// the images' resolution alone, in seconds: after a
|
||||
// change to a Dockerfile, the root pubspec or the lock
|
||||
//
|
||||
// The tests run through Melos' scripts (`test:dart`, `test:flutter`, the root
|
||||
// pubspec), Melos itself a dependency of the root.
|
||||
|
|
@ -14,10 +17,14 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:archive/archive_io.dart';
|
||||
import 'package:args/args.dart';
|
||||
import 'package:crypto/crypto.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:path/path.dart' as p;
|
||||
|
||||
import 'src/check_files.dart';
|
||||
import 'src/images.dart';
|
||||
|
||||
Future<void> main(List<String> arguments) async {
|
||||
final options =
|
||||
|
|
@ -28,11 +35,17 @@ Future<void> main(List<String> arguments) async {
|
|||
'Test only the packages changed since this commit, and those '
|
||||
'depending on them.',
|
||||
)
|
||||
..addFlag(
|
||||
'images-only',
|
||||
negatable: false,
|
||||
help: "Check only the images' resolution.",
|
||||
)
|
||||
..addFlag('help', abbr: 'h', negatable: false))
|
||||
.parse(arguments);
|
||||
if (options.flag('help')) {
|
||||
stdout.writeln(
|
||||
'Usage: dart run tool/check.dart [--changed-since=<commit>]',
|
||||
'Usage: dart run tool/check.dart [--changed-since=<commit>] '
|
||||
'[--images-only]',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
|
@ -57,6 +70,12 @@ Future<void> main(List<String> arguments) async {
|
|||
('flutter', ['pub', 'get', '--enforce-lockfile']),
|
||||
], quiet: true);
|
||||
|
||||
await _images();
|
||||
if (options.flag('images-only')) {
|
||||
stdout.writeln("✓ The images' resolution passed.");
|
||||
return;
|
||||
}
|
||||
|
||||
await _generatedCode();
|
||||
|
||||
final own = (await _output('git', [
|
||||
|
|
@ -150,6 +169,173 @@ Future<void> _generatedCode() async {
|
|||
}
|
||||
}
|
||||
|
||||
/// Fails the check unless every image of [dartImages] resolves its packages
|
||||
/// as its build stage does on the server: its steps up to `dart pub get`
|
||||
/// replayed in an empty folder from the working tree's files, with a Dart
|
||||
/// SDK of the image's version alone, without Flutter (pub refused the
|
||||
/// overrides of Flutter plugins there and accepted them here, 4 Nov 2026),
|
||||
/// offline from the pub cache the step before filled. Only the deploy builds
|
||||
/// the images, after every check; a failure there cost a push each time.
|
||||
Future<void> _images() async {
|
||||
stdout.writeln('▸ Images');
|
||||
final repository = Directory.current.path;
|
||||
final tracked = (await _output('git', [
|
||||
'ls-files',
|
||||
'-z',
|
||||
'--cached',
|
||||
'--others',
|
||||
'--exclude-standard',
|
||||
])).split('\x00').where((path) => path.isNotEmpty).toList();
|
||||
final work = await Directory.systemTemp.createTemp('halkyon-images-');
|
||||
try {
|
||||
for (final dockerfile in dartImages) {
|
||||
final ImageResolution resolution;
|
||||
try {
|
||||
resolution = imageResolution(File(dockerfile).readAsStringSync());
|
||||
} on FormatException catch (error) {
|
||||
stderr.writeln('✗ $dockerfile: ${error.message}');
|
||||
exit(1);
|
||||
}
|
||||
final sdk = await _dartSdk(resolution.dartVersion);
|
||||
final app = Directory(p.join(work.path, p.dirname(dockerfile)));
|
||||
var workdir = '/app';
|
||||
final environment = _imageEnvironment(sdk);
|
||||
for (final step in resolution.steps) {
|
||||
final ok = switch (step) {
|
||||
WorkdirStep(:final path) => () {
|
||||
workdir = p.posix.normalize(p.posix.join(workdir, path));
|
||||
return true;
|
||||
}(),
|
||||
CopyStep() => () {
|
||||
for (final MapEntry(key: from, value: to) in copiedFiles(
|
||||
step,
|
||||
tracked,
|
||||
workdir: workdir,
|
||||
).entries) {
|
||||
final source = File(p.join(repository, from));
|
||||
if (source.existsSync()) {
|
||||
final target = File(
|
||||
p.join(app.path, p.relative(to, from: '/app')),
|
||||
)..parent.createSync(recursive: true);
|
||||
source.copySync(target.path);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}(),
|
||||
RunStep(:final command) => await _runIn(
|
||||
p.join(app.path, p.relative(workdir, from: '/app')),
|
||||
command == 'dart pub get' ? 'dart pub get --offline' : command,
|
||||
environment,
|
||||
),
|
||||
};
|
||||
if (!ok) {
|
||||
stderr.writeln(
|
||||
"✗ Images failed: $dockerfile's build stage, as the deploy builds "
|
||||
'it, fails here.',
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
stdout.writeln(' $dockerfile');
|
||||
}
|
||||
} finally {
|
||||
work.deleteSync(recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// What a build stage's commands see: little but the SDK on the PATH, as in
|
||||
/// Dart's image. Nothing of the machine's Dart or Flutter passes: `dart run`
|
||||
/// from Flutter sets `DART_ROOT` to Flutter's Dart, through which pub finds
|
||||
/// Flutter and accepts what the image refuses (seen 4 Nov 2026), and
|
||||
/// `FLUTTER_ROOT` would do the same. Only the home and the pub cache, the
|
||||
/// temporary folder, the language and the proxies are kept.
|
||||
Map<String, String> _imageEnvironment(String sdk) => {
|
||||
for (final name in [
|
||||
'HOME',
|
||||
'PUB_CACHE',
|
||||
'TMPDIR',
|
||||
'LANG',
|
||||
'http_proxy',
|
||||
'https_proxy',
|
||||
'no_proxy',
|
||||
'HTTP_PROXY',
|
||||
'HTTPS_PROXY',
|
||||
'NO_PROXY',
|
||||
])
|
||||
name: ?Platform.environment[name],
|
||||
'PATH': '${p.join(sdk, 'bin')}:/usr/local/bin:/usr/bin:/bin',
|
||||
};
|
||||
|
||||
/// Runs [command] through `sh -c` in [directory] with [environment] alone,
|
||||
/// its output shown only should it fail; whether it succeeded.
|
||||
Future<bool> _runIn(
|
||||
String directory,
|
||||
String command,
|
||||
Map<String, String> environment,
|
||||
) async {
|
||||
Directory(directory).createSync(recursive: true);
|
||||
final result = await Process.run(
|
||||
'sh',
|
||||
['-c', command],
|
||||
workingDirectory: directory,
|
||||
environment: environment,
|
||||
includeParentEnvironment: false,
|
||||
);
|
||||
if (result.exitCode != 0) {
|
||||
stderr
|
||||
..writeln(' \$ $command')
|
||||
..write(result.stdout)
|
||||
..write(result.stderr);
|
||||
}
|
||||
return result.exitCode == 0;
|
||||
}
|
||||
|
||||
/// The Dart SDK at [version] alone, as Dart's images hold it: kept in the
|
||||
/// user's cache (`~/.cache/halkyon-check/`), downloaded from Google's archive
|
||||
/// the first time and checked against the archive's SHA-256. Its folder.
|
||||
Future<String> _dartSdk(String version) async {
|
||||
final home =
|
||||
Platform.environment['XDG_CACHE_HOME'] ??
|
||||
p.join(Platform.environment['HOME']!, '.cache');
|
||||
final folder = Directory(p.join(home, 'halkyon-check', 'dart-$version'));
|
||||
final sdk = p.join(folder.path, 'dart-sdk');
|
||||
if (File(p.join(sdk, 'bin', 'dart')).existsSync()) {
|
||||
return sdk;
|
||||
}
|
||||
final name = dartSdkArchive(Platform.operatingSystem, Platform.version);
|
||||
final url =
|
||||
'https://storage.googleapis.com/dart-archive/channels/stable/release/'
|
||||
'$version/sdk/$name';
|
||||
stdout.writeln(' Dart $version alone, downloaded once: $url');
|
||||
final archive = await http.get(Uri.parse(url));
|
||||
final sum = await http.get(Uri.parse('$url.sha256sum'));
|
||||
if (archive.statusCode != 200 || sum.statusCode != 200) {
|
||||
stderr.writeln('✗ Images failed: Dart $version not downloaded ($url).');
|
||||
exit(1);
|
||||
}
|
||||
if ('${sha256.convert(archive.bodyBytes)}' != sum.body.split(' ').first) {
|
||||
stderr.writeln("✗ Images failed: Dart $version doesn't match its SHA-256.");
|
||||
exit(1);
|
||||
}
|
||||
final partial = Directory('${folder.path}.partial');
|
||||
if (partial.existsSync()) {
|
||||
partial.deleteSync(recursive: true);
|
||||
}
|
||||
await extractArchiveToDisk(
|
||||
ZipDecoder().decodeBytes(archive.bodyBytes),
|
||||
partial.path,
|
||||
);
|
||||
// The archive's modes aren't kept by every extraction; the SDK's
|
||||
// executables are its bin folder's.
|
||||
await _output('chmod', [
|
||||
'-R',
|
||||
'u+x',
|
||||
p.join(partial.path, 'dart-sdk', 'bin'),
|
||||
]);
|
||||
partial.renameSync(folder.path);
|
||||
return sdk;
|
||||
}
|
||||
|
||||
/// The working tree's files that differ from the committed ones, untracked
|
||||
/// files not ignored among them, each with a digest of its content, or
|
||||
/// [absentFile] where it was deleted.
|
||||
|
|
|
|||
124
tool/src/images.dart
Normal file
124
tool/src/images.dart
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
// The images built from the workspace, as tool/check.dart replays their
|
||||
// resolution: pub refuses in an image what it accepts on a laptop with Flutter
|
||||
// (the deploys of 4 Nov 2026: overrides of Flutter plugins, exit 69), and only
|
||||
// the deploy builds the images, on the server, after every check has passed.
|
||||
import 'package:path/path.dart' as p;
|
||||
|
||||
/// The Dockerfiles whose build stage resolves the workspace with Dart's own
|
||||
/// image (`FROM dart:<version> AS build`), built by the deploy.
|
||||
const dartImages = [
|
||||
'apps/learn/halkyon_learn_server/Dockerfile',
|
||||
'apps/learn/halkyon_learn_curator/Dockerfile',
|
||||
'apps/train/halkyon_train_server/Dockerfile',
|
||||
'apps/todo/halkyon_todo_server/Dockerfile',
|
||||
];
|
||||
|
||||
/// One instruction of a Dockerfile's build stage that the replay follows.
|
||||
sealed class const BuildStep();
|
||||
|
||||
/// `COPY <sources…> <destination>`, from the build's context.
|
||||
final class const CopyStep(final List<String> sources, final String destination)
|
||||
extends BuildStep;
|
||||
|
||||
/// `RUN <command>`, through `sh -c`.
|
||||
final class const RunStep(final String command) extends BuildStep;
|
||||
|
||||
/// `WORKDIR <path>`.
|
||||
final class const WorkdirStep(final String path) extends BuildStep;
|
||||
|
||||
/// The resolution of an image: the Dart its build stage runs on, and the
|
||||
/// stage's steps up to and including `RUN dart pub get`.
|
||||
typedef ImageResolution = ({String dartVersion, List<BuildStep> steps});
|
||||
|
||||
final _from = RegExp(r'^FROM dart:(\S+) AS build$');
|
||||
|
||||
/// The build stage of [dockerfile] (its text) up to and including its
|
||||
/// `RUN dart pub get`. Throws a [FormatException] where it has no such stage,
|
||||
/// or an instruction the replay doesn't know comes before `pub get`, so that
|
||||
/// a changed Dockerfile fails the check rather than being replayed wrongly.
|
||||
ImageResolution imageResolution(String dockerfile) {
|
||||
String? version;
|
||||
final steps = <BuildStep>[];
|
||||
for (final line in _instructions(dockerfile)) {
|
||||
if (version == null) {
|
||||
version = _from.firstMatch(line)?[1];
|
||||
continue;
|
||||
}
|
||||
final [instruction, ...words] = line.split(RegExp(r'\s+'));
|
||||
final argument = line.substring(instruction.length).trim();
|
||||
switch (instruction) {
|
||||
case 'COPY' when words.length >= 2 && !argument.startsWith('--'):
|
||||
steps.add(CopyStep(words.sublist(0, words.length - 1), words.last));
|
||||
case 'RUN':
|
||||
steps.add(RunStep(argument));
|
||||
if (argument == 'dart pub get') {
|
||||
return (dartVersion: version, steps: steps);
|
||||
}
|
||||
case 'WORKDIR':
|
||||
steps.add(WorkdirStep(argument));
|
||||
default:
|
||||
throw FormatException(
|
||||
"The replay of an image's resolution doesn't know: $line",
|
||||
);
|
||||
}
|
||||
}
|
||||
throw FormatException(
|
||||
version == null
|
||||
? 'No build stage FROM dart:<version> AS build'
|
||||
: 'No RUN dart pub get in the build stage',
|
||||
);
|
||||
}
|
||||
|
||||
/// The Dockerfile's instructions, comments and blank lines left out, lines
|
||||
/// continued with a backslash joined.
|
||||
Iterable<String> _instructions(String dockerfile) sync* {
|
||||
var pending = '';
|
||||
for (final line in dockerfile.split('\n')) {
|
||||
final trimmed = line.trim();
|
||||
if (trimmed.isEmpty || trimmed.startsWith('#')) {
|
||||
continue;
|
||||
}
|
||||
if (trimmed.endsWith(r'\')) {
|
||||
pending += '${trimmed.substring(0, trimmed.length - 1).trim()} ';
|
||||
continue;
|
||||
}
|
||||
yield pending + trimmed;
|
||||
pending = '';
|
||||
}
|
||||
}
|
||||
|
||||
/// Where [step] puts each of [files] (the repository's files, paths relative
|
||||
/// to the build's context), relative to [workdir], as Docker's `COPY` does:
|
||||
/// a folder's content into the destination, a file into it where it ends in
|
||||
/// a slash or several sources are named, else as the destination itself.
|
||||
Map<String, String> copiedFiles(
|
||||
CopyStep step,
|
||||
Iterable<String> files, {
|
||||
String workdir = '/app',
|
||||
}) {
|
||||
final destination = p.posix.normalize(
|
||||
p.posix.join(workdir, step.destination),
|
||||
);
|
||||
final intoFolder =
|
||||
step.destination.endsWith('/') ||
|
||||
step.destination == '.' ||
|
||||
step.sources.length > 1;
|
||||
return {
|
||||
for (final source in step.sources.map(p.posix.normalize))
|
||||
for (final file in files)
|
||||
if (file == source)
|
||||
file: intoFolder
|
||||
? p.posix.join(destination, p.posix.basename(file))
|
||||
: destination
|
||||
else if (p.posix.isWithin(source, file))
|
||||
file: p.posix.join(destination, p.posix.relative(file, from: source)),
|
||||
};
|
||||
}
|
||||
|
||||
/// The name of the Dart SDK's archive for this machine, as Google's archive
|
||||
/// names it (`dartsdk-linux-x64-release.zip`); [version] is
|
||||
/// `Platform.version`, and [os] `Platform.operatingSystem`.
|
||||
String dartSdkArchive(String os, String version) {
|
||||
final arch = RegExp(r'"\w+_(\w+)"').firstMatch(version)?[1] ?? 'x64';
|
||||
return 'dartsdk-${os == 'macos' ? 'macos' : 'linux'}-$arch-release.zip';
|
||||
}
|
||||
Loading…
Reference in a new issue