chore: Halkyon Learn moves into apps/learn
Its server, client, app and curator, and the tools that write its assets (maps, Anki fixtures, icons), as they are; paths are fixed in the commits that follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
11abd9afce
commit
649499be23
1 changed files with 0 additions and 0 deletions
|
|
@ -1,133 +0,0 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:halkyon_learn_curator/src/sources/cedict.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/tatoeba.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
void main() {
|
||||
late Directory into;
|
||||
|
||||
setUp(() => into = Directory.systemTemp.createTempSync('tatoeba'));
|
||||
tearDown(() => into.deleteSync(recursive: true));
|
||||
|
||||
Stream<String> lines(String fixture) =>
|
||||
File('test/fixtures/$fixture')
|
||||
.openRead()
|
||||
.transform(utf8.decoder)
|
||||
.transform(const LineSplitter());
|
||||
|
||||
/// [file]'s lines with the fixtures' sentences of [code] as examples.
|
||||
Future<List<Map<String, Object?>>> enriched(File file, String code) async {
|
||||
final links = await readTatoebaLinks(lines('tatoeba-$code-eng_links.tsv'));
|
||||
final sentences = await readTatoebaSentences(
|
||||
lines('tatoeba-${code}_sentences.tsv'),
|
||||
wanted: links.containsKey,
|
||||
);
|
||||
final linked = {for (final ids in links.values) ...ids};
|
||||
final translations = await readTatoebaSentences(
|
||||
lines('tatoeba-eng_sentences.tsv'),
|
||||
wanted: linked.contains,
|
||||
);
|
||||
return [
|
||||
for (final line in await withTatoebaExamples(
|
||||
file,
|
||||
sentences: sentences,
|
||||
translations: translations,
|
||||
links: links,
|
||||
).join().then((text) => text.split('\n')))
|
||||
if (line.isNotEmpty) jsonDecode(line) as Map<String, Object?>,
|
||||
];
|
||||
}
|
||||
|
||||
List<Map<String, Object?>> examplesOf(Map<String, Object?> line) => [
|
||||
for (final example
|
||||
in ((((line['entry']! as Map)['senses']! as List).first
|
||||
as Map)['examples']
|
||||
as List? ??
|
||||
const []))
|
||||
example as Map<String, Object?>,
|
||||
];
|
||||
|
||||
test("a German word's sentences come first in its first sense, those "
|
||||
'without a translation left out, the header crediting Tatoeba', () async {
|
||||
final [german, ...] = await wiktionaryDictionaries(
|
||||
lines('wiktionary-sample.jsonl'),
|
||||
into: into,
|
||||
sourceVersion: '2026-10-03',
|
||||
languages: {'de'},
|
||||
);
|
||||
|
||||
final result = await enriched(german, 'deu');
|
||||
|
||||
expect([
|
||||
for (final credit in result.first['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], contains('Tatoeba'));
|
||||
final haus = result
|
||||
.skip(1)
|
||||
.singleWhere((line) => line['id'] == 'Haus/noun');
|
||||
final examples = examplesOf(haus);
|
||||
// Three of Tatoeba's, before Wiktionary's own.
|
||||
// Those nearest seven words long first, then the shortest.
|
||||
expect(examples.take(3).map((example) => example['tatoeba']), [
|
||||
9984782,
|
||||
1550095,
|
||||
605418,
|
||||
]);
|
||||
expect(examples.first['text'], 'Haus');
|
||||
expect(examples.first['de'], 'Er ist im Haus.');
|
||||
expect(examples.first['en'], isA<String>());
|
||||
expect(examples[3]['de'], 'In dem Haus haben wir mal gewohnt.');
|
||||
// A form finds its word: aß is essen's.
|
||||
final essen = result
|
||||
.skip(1)
|
||||
.singleWhere((line) => line['id'] == 'essen/verb');
|
||||
expect(examplesOf(essen).first['text'], 'aß');
|
||||
});
|
||||
|
||||
test(
|
||||
'Chinese sentences are cut by the longest words the dictionary has',
|
||||
() async {
|
||||
final file = File('${into.path}/cc-cedict-hans-en.jsonl');
|
||||
await file.writeAsString(
|
||||
cedictDictionary(
|
||||
await File('test/fixtures/cedict-sample.txt').readAsString(),
|
||||
traditional: false,
|
||||
).join(),
|
||||
);
|
||||
|
||||
final result = await enriched(file, 'cmn');
|
||||
|
||||
final xuexi = result
|
||||
.skip(1)
|
||||
.singleWhere((line) => line['id'] == '學習 学习 [xue2 xi2]');
|
||||
expect(examplesOf(xuexi), isNotEmpty);
|
||||
expect(examplesOf(xuexi).first['text'], '学习');
|
||||
expect(examplesOf(xuexi).first['zh-Hans'], contains('学习'));
|
||||
},
|
||||
);
|
||||
|
||||
test('Korean words are found with their particles taken off', () async {
|
||||
final [korean, ...] = await wiktionaryDictionaries(
|
||||
lines('wiktionary-sample.jsonl'),
|
||||
into: into,
|
||||
sourceVersion: '2026-10-03',
|
||||
languages: {'ko'},
|
||||
);
|
||||
|
||||
final result = await enriched(korean, 'kor');
|
||||
|
||||
final home = result.skip(1).singleWhere((line) => line['id'] == '집/noun/1');
|
||||
expect(examplesOf(home), isNotEmpty);
|
||||
expect(examplesOf(home).first['text'], '집');
|
||||
expect(examplesOf(home).first['ko'], contains('집에서'));
|
||||
});
|
||||
|
||||
test("Tatoeba's codes of the languages", () {
|
||||
expect(tatoebaCode('de'), 'deu');
|
||||
expect(tatoebaCode('zh-Hant'), 'cmn');
|
||||
expect(tatoebaCode('ja'), isNull);
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue