fix(dictionary): a long kana word stands, and a word offers the longer ones

Cutting a line into words, kana that is no common word gave way to a
shorter common one whatever its length, so もうろく (耄碌, senility) was
taken as もう and ろく. The rule now holds only for kana of up to three
characters (が and まだ still win over がま, a cattail, and だ); a longer
kana word stands even when uncommon, as ichi.moe's segmenter weighs long
kana words by their length. And as no cut is always right, each word now
knows the longer words running over it into its neighbours, worked out
from the same lookup, which the Watch screen's panel offers beneath its
entries ("Longer words here"), each to add with the line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-13 19:04:38 +02:00
commit 8fb5e105c9
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
11 changed files with 232 additions and 38 deletions

File diff suppressed because one or more lines are too long

View file

@ -1797,6 +1797,7 @@
"watchProgressDue": "Fällig", "watchProgressDue": "Fällig",
"watchProgressLearned": "Gelernt · {days, plural, =0{heute wieder} =1{morgen wieder} other{in {days} Tagen wieder}}", "watchProgressLearned": "Gelernt · {days, plural, =0{heute wieder} =1{morgen wieder} other{in {days} Tagen wieder}}",
"watchJisho": "Auf Jisho nachschlagen", "watchJisho": "Auf Jisho nachschlagen",
"watchWider": "Längere Wörter hier",
"watchFilter": "Wörter filtern", "watchFilter": "Wörter filtern",
"watchFilterFresh": "Neue Wörter", "watchFilterFresh": "Neue Wörter",
"watchFilterHeld": "Wörter im Lernset", "watchFilterHeld": "Wörter im Lernset",

View file

@ -2111,6 +2111,7 @@
"watchProgressLearned": "Learned · {days, plural, =0{again today} =1{again tomorrow} other{again in {days} days}}", "watchProgressLearned": "Learned · {days, plural, =0{again today} =1{again tomorrow} other{again in {days} days}}",
"@watchProgressLearned": {"placeholders": {"days": {"type": "int"}}}, "@watchProgressLearned": {"placeholders": {"days": {"type": "int"}}},
"watchJisho": "Look up on Jisho", "watchJisho": "Look up on Jisho",
"watchWider": "Longer words here",
"watchFilter": "Filter the words", "watchFilter": "Filter the words",
"watchFilterFresh": "New words", "watchFilterFresh": "New words",
"watchFilterHeld": "Words in the studyset", "watchFilterHeld": "Words in the studyset",

View file

@ -10623,6 +10623,12 @@ abstract class AppLocalizations {
/// **'Look up on Jisho'** /// **'Look up on Jisho'**
String get watchJisho; String get watchJisho;
/// No description provided for @watchWider.
///
/// In en, this message translates to:
/// **'Longer words here'**
String get watchWider;
/// No description provided for @watchFilter. /// No description provided for @watchFilter.
/// ///
/// In en, this message translates to: /// In en, this message translates to:

View file

@ -6682,6 +6682,9 @@ class AppLocalizationsDe extends AppLocalizations {
@override @override
String get watchJisho => 'Auf Jisho nachschlagen'; String get watchJisho => 'Auf Jisho nachschlagen';
@override
String get watchWider => 'Längere Wörter hier';
@override @override
String get watchFilter => 'Wörter filtern'; String get watchFilter => 'Wörter filtern';

View file

@ -6624,6 +6624,9 @@ class AppLocalizationsEn extends AppLocalizations {
@override @override
String get watchJisho => 'Look up on Jisho'; String get watchJisho => 'Look up on Jisho';
@override
String get watchWider => 'Longer words here';
@override @override
String get watchFilter => 'Filter the words'; String get watchFilter => 'Filter the words';

View file

@ -6,20 +6,27 @@ import 'package:halkyon_learn_flutter/src/feature/dictionary/model/japanese_dein
/// A part of a sentence: a word a dictionary has, as it stands there /// A part of a sentence: a word a dictionary has, as it stands there
/// ([text], from [start]), with the entries it may be, the likeliest first, /// ([text], from [start]), with the entries it may be, the likeliest first,
/// and how it is conjugated ([steps], when its [entries] list it in another /// and how it is conjugated ([steps], when its [entries] list it in another
/// form); or text between words, without entries. /// form); or text between words, without entries. [wider] are the longer
/// words running over it into its neighbours, the longest first: what the
/// cut may have missed (decided 2026-10-13: もう tapped in a line cut as
/// もう and ろく also offers 耄碌, もうろく).
final class const SentencePart({ final class const SentencePart({
required final String text, required final String text,
required final int start, required final int start,
final List<DictionaryWord> entries = const [], final List<DictionaryWord> entries = const [],
final List<String> steps = const [], final List<String> steps = const [],
final List<SentencePart> wider = const [],
}) with Equatable { }) with Equatable {
/// Whether a dictionary has it. /// Whether a dictionary has it.
bool get isWord => entries.isNotEmpty; bool get isWord => entries.isNotEmpty;
@override @override
List<Object?> get props => [text, start, entries, steps]; List<Object?> get props => [text, start, entries, steps, wider];
} }
/// How many longer words a word offers at most.
const widerWords = 4;
/// The longest a word of a sentence is taken to be, in characters. /// The longest a word of a sentence is taken to be, in characters.
const longestWord = 12; const longestWord = 12;
@ -78,15 +85,20 @@ List<SentencePart> sentenceParts(
} }
} }
// Every word starting at each place, worked out once.
final at = <int, List<_Word>>{};
List<_Word> wordsAt(int from) => at[from] ??= _inWord(characters[from])
? _wordsAt(characters, from, found)
: const [];
final wordStarts = <int>[];
while (start < characters.length) { while (start < characters.length) {
final word = _inWord(characters[start]) final word = _pick(wordsAt(start));
? _longestWordAt(characters, start, found)
: null;
if (word == null) { if (word == null) {
start++; start++;
continue; continue;
} }
plainUntil(start); plainUntil(start);
wordStarts.add(start);
parts.add( parts.add(
SentencePart( SentencePart(
text: String.fromCharCodes( text: String.fromCharCodes(
@ -101,16 +113,84 @@ List<SentencePart> sentenceParts(
plainFrom = start; plainFrom = start;
} }
plainUntil(characters.length); plainUntil(characters.length);
return parts; // Each word's longer words: those starting at or before it and ending at
// or after it, larger than it.
var next = 0;
return [
for (final part in parts)
if (!part.isWord)
part
else
() {
final from = wordStarts[next++];
final to = from + part.text.runes.length;
final wider = <({int from, _Word word})>[
for (
var other = from;
other >= 0 && from - other < longestWord;
other--
)
for (final word in wordsAt(other))
if (other + word.length >= to &&
(other < from || other + word.length > to))
(from: other, word: word),
]..sort((a, b) => b.word.length.compareTo(a.word.length));
final seen = <String>{};
return SentencePart(
text: part.text,
start: part.start,
entries: part.entries,
steps: part.steps,
wider: [
for (final (:from, :word) in wider)
if (seen.add(
String.fromCharCodes(
characters.sublist(from, from + word.length),
),
))
SentencePart(
text: String.fromCharCodes(
characters.sublist(from, from + word.length),
),
start: offsets[from],
entries: word.entries,
steps: word.steps,
),
].take(widerWords).toList(),
);
}(),
];
} }
typedef _Word = ({ typedef _Word = ({
int length, int length,
List<DictionaryWord> entries, List<DictionaryWord> entries,
List<String> steps, List<String> steps,
bool kana,
}); });
_Word? _longestWordAt( /// Of the words starting at a place, the longest, except that kana of up
/// to three characters that is no common word gives way to a shorter one
/// that is: が and まだ rather than がま (a cattail) and だ. A longer kana
/// word stands even when uncommon (decided 2026-10-13): もうろく (耄碌), not
/// もう and ろく; ichi.moe's segmenter likewise counts long kana words by
/// their length before their commonness.
_Word? _pick(List<_Word> words) {
_Word? uncommon;
for (final word in words) {
if (word.kana &&
word.length <= 3 &&
!word.entries.any((entry) => entry.labels.contains('common'))) {
uncommon ??= word;
continue;
}
return word;
}
return uncommon;
}
/// Every word starting at [start], the longest first.
List<_Word> _wordsAt(
List<int> characters, List<int> characters,
int start, int start,
Map<String, List<DictionaryItem>> found, Map<String, List<DictionaryItem>> found,
@ -121,7 +201,7 @@ _Word? _longestWordAt(
_inWord(characters[start + longest])) { _inWord(characters[start + longest])) {
longest++; longest++;
} }
_Word? uncommon; final words = <_Word>[];
for (var length = longest; length > 0; length--) { for (var length = longest; length > 0; length--) {
final text = String.fromCharCodes( final text = String.fromCharCodes(
characters.sublist(start, start + length), characters.sublist(start, start + length),
@ -158,20 +238,14 @@ _Word? _longestWordAt(
if (entries.isEmpty) { if (entries.isEmpty) {
continue; continue;
} }
final word = ( words.add((
length: length, length: length,
entries: entries, entries: entries,
steps: candidates.first.steps, steps: candidates.first.steps,
); kana: kana,
// Kana that is no common word gives way to a shorter one that is: が ));
// and まだ rather than がま (a cattail) and だ.
if (kana && !entries.any((entry) => entry.labels.contains('common'))) {
uncommon ??= word;
continue;
}
return word;
} }
return uncommon; return words;
} }
/// Whether [entry], found by [key] for text written in kana alone, is a /// Whether [entry], found by [key] for text written in kana alone, is a

View file

@ -422,8 +422,10 @@ class WatchCubit(
void unselect() => emit(state.copyWith(selected: null)); void unselect() => emit(state.copyWith(selected: null));
/// Adds [entry] to the studyset as a card, with the chosen word's line /// Adds [entry] to the studyset as a card, with the chosen word's line
/// (and its translation) as its first example, tagged with the show. /// (and its translation) as its first example, tagged with the show; the
Future<void> add(DictionaryWord entry) async { /// word marked in the line is [form], a longer word running over the one
/// chosen, or the one chosen.
Future<void> add(DictionaryWord entry, {String? form}) async {
final selected = state.selected; final selected = state.selected;
final part = state.selectedPart; final part = state.selectedPart;
if (selected == null || part == null) { if (selected == null || part == null) {
@ -435,7 +437,7 @@ class WatchCubit(
studysetId, studysetId,
[ [
entry.withSentence(( entry.withSentence((
form: part.text, form: form ?? part.text,
japanese: oneLine(line), japanese: oneLine(line),
english: switch (state.translationOf(selected.line)) { english: switch (state.translationOf(selected.line)) {
final text? => oneLine(text), final text? => oneLine(text),

View file

@ -607,12 +607,13 @@ class const _ChosenWord({required final Dictionary dictionary})
return const SizedBox(); return const SizedBox();
} }
final translation = state.translationOf(selected.line); final translation = state.translationOf(selected.line);
Widget? trailing(DictionaryItem entry) => state.known.contains(entry.ref) Widget? trailing(DictionaryWord entry, String form) =>
state.known.contains(entry.ref)
? null ? null
: FilledButton.tonal( : FilledButton.tonal(
onPressed: () async { onPressed: () async {
final messenger = ScaffoldMessenger.of(context); final messenger = ScaffoldMessenger.of(context);
await cubit().add(entry as DictionaryWord); await cubit().add(entry, form: form);
messenger.showSnackBar( messenger.showSnackBar(
SnackBar(content: Text(l10n.watchAdded(entry.headword))), SnackBar(content: Text(l10n.watchAdded(entry.headword))),
); );
@ -627,6 +628,20 @@ class const _ChosenWord({required final Dictionary dictionary})
_JishoLink(word: word), _JishoLink(word: word),
], ],
); );
Widget row(DictionaryWord entry, String form) => _Unfolding(
word: entry,
folded: DictionaryWordTile(
word: entry,
progress: beside(entry),
trailing: trailing(entry, form),
),
whole: DictionaryWordTile(
word: entry,
whole: true,
progress: beside(entry),
trailing: trailing(entry, form),
),
);
return ListView( return ListView(
padding: const EdgeInsets.fromLTRB(16, 8, 8, 16), padding: const EdgeInsets.fromLTRB(16, 8, 8, 16),
children: [ children: [
@ -661,20 +676,17 @@ class const _ChosenWord({required final Dictionary dictionary})
const SizedBox(height: 8), const SizedBox(height: 8),
for (final (index, entry) in part.entries.indexed) ...[ for (final (index, entry) in part.entries.indexed) ...[
if (index > 0) const Divider(height: 1), if (index > 0) const Divider(height: 1),
_Unfolding( row(entry, part.text),
word: entry, ],
folded: DictionaryWordTile( // What the cut may have missed (decided 2026-10-13).
word: entry, if (part.wider.isNotEmpty) ...[
progress: beside(entry), const SizedBox(height: 16),
trailing: trailing(entry), Text(l10n.watchWider, style: theme.textTheme.titleSmall),
), for (final wider in part.wider)
whole: DictionaryWordTile( for (final entry in wider.entries) ...[
word: entry, const Divider(height: 1),
whole: true, row(entry, wider.text),
progress: beside(entry), ],
trailing: trailing(entry),
),
),
], ],
], ],
); );

View file

@ -112,6 +112,72 @@ void main() {
expect(price.single.entries.single.headword, '値段'); expect(price.single.entries.single.headword, '値段');
}); });
test('a kana word of four or more is one word even if uncommon: '
'もうろく, not もう and ろく; が and まだ stay apart', () async {
final repository = DictionaryRepository(
FakeDictionaryGateway(
words: [
word('30', 'お前', reading: 'おまえ', pos: ['pn'], labels: ['common']),
word('31', 'が', pos: ['prt'], labels: ['common']),
word('32', 'もう', pos: ['adv'], labels: ['common']),
word('33', 'ろく', pos: ['adj-na']),
word('34', '耄碌', reading: 'もうろく', pos: ['n', 'vs']),
word('35', 'だ', pos: ['cop'], labels: ['common']),
word('36', 'よ', pos: ['prt'], labels: ['common']),
word('37', 'がま', pos: ['n']),
const DictionaryWord(
ref: (source: 'jmdict', id: '38'),
headword: '未だ',
reading: 'まだ',
labels: ['common'],
senses: [
VocabularySense(glosses: ['still'], pos: ['adv'], misc: ['uk']),
],
),
],
),
);
final [senile, still] = await repository.wordsOf(jmdict, [
'お前がもうろくだよ',
'がまだ',
]);
expect(
[for (final part in senile) part.text],
['お前', 'が', 'もうろく', 'だ', 'よ'],
);
expect(senile[2].entries.first.headword, '耄碌');
expect([for (final part in still) part.text], ['が', 'まだ']);
});
test('a word knows the longer words running over it into its '
'neighbours, the longest first: が in がまだ is also がま', () async {
final repository = DictionaryRepository(
FakeDictionaryGateway(
words: [
word('31', 'が', pos: ['prt'], labels: ['common']),
word('37', 'がま', pos: ['n']),
const DictionaryWord(
ref: (source: 'jmdict', id: '38'),
headword: '未だ',
reading: 'まだ',
labels: ['common'],
senses: [
VocabularySense(glosses: ['still'], pos: ['adv'], misc: ['uk']),
],
),
],
),
);
final [still] = await repository.wordsOf(jmdict, ['がまだ']);
expect([for (final wider in still.first.wider) wider.text], ['がま']);
expect(still.first.wider.single.entries.single.headword, 'がま');
expect(still.last.wider, isEmpty);
});
test( test(
'asks once for a sentence, and for several sentences at once', 'asks once for a sentence, and for several sentences at once',
() async { () async {

View file

@ -558,6 +558,32 @@ void main() {
expect(find.byType(VocabularySensesView), findsNothing); expect(find.byType(VocabularySensesView), findsNothing);
}); });
testWidgets('a word tapped also offers the longer words running over it, '
'to add with the line', (tester) async {
await watch(
tester,
vtt: {
('episode-1', 2): 'WEBVTT\n\n00:00:01.000 --> 00:00:05.000\nがまだ\n',
('episode-1', 3): 'WEBVTT\n\n00:00:01.000 --> 00:00:05.000\nStill.\n',
},
words: [
word('31', 'が', pos: ['prt'], labels: ['common']),
word('37', 'がま', pos: ['n'], glosses: ['bulrush']),
word('38', 'まだ', pos: ['adv'], labels: ['common']),
],
);
await playTo(tester, const Duration(seconds: 2));
await tester.tap(find.text('が'));
await tester.pumpAndSettle();
expect(find.text('Longer words here'), findsOneWidget);
expect(find.textContaining('bulrush', findRichText: true), findsOneWidget);
await tester.tap(find.text('Add').last);
await tester.pumpAndSettle();
final card = (await cards.cardsOf('japanese')).single;
expect(card.content.entry, (source: 'jmdict', id: '37'));
});
testWidgets('on a phone the words are below the video', (tester) async { testWidgets('on a phone the words are below the video', (tester) async {
await watch(tester, size: const Size(412, 915)); await watch(tester, size: const Size(412, 915));
await playTo(tester, const Duration(seconds: 2)); await playTo(tester, const Duration(seconds: 2));