From efd2f04596fa535cbe935ddedad2e6dc93907871 Mon Sep 17 00:00:00 2001 From: sokcuri Date: Mon, 9 Nov 2026 08:03:14 +0900 Subject: [PATCH 1/2] fix(names): transliterate kana speaker labels in captions A hiragana run of two to six kana in a speaker label at the line start, in round or square brackets, before a colon or before a closing quote, is read as a name by its sound instead of as the word it spells; stage directions, roles, whole common words and verb continuatives keep their reading (spec LEX-5800). An artist's kana name before feat. or joined by / to a Latin name in a credit reads as a name too. --- spec/words.md | 30 +++ src/core/names/cues.cpp | 176 +++++++++++++++++- src/core/names/document.cpp | 1 + src/core/names/labels.h | 2 + src/core/names/name_cues.cpp | 4 + src/core/names/name_cues.h | 5 + src/core/names/registry.h | 7 + .../cases/names-kana-speaker-labels-5800.tsv | 30 +++ .../names-kana-speaker-labels-5800.tsv | 4 + tests/rules/wrapped/cut-word.tsv | 2 +- 10 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 tests/rules/cases/names-kana-speaker-labels-5800.tsv create mode 100644 tests/rules/documents/names-kana-speaker-labels-5800.tsv diff --git a/spec/words.md b/spec/words.md index b6e7a2ebe..5e55f98cc 100644 --- a/spec/words.md +++ b/spec/words.md @@ -11653,3 +11653,33 @@ name.common-predicate에 둔다(高い, 強い, 眩しい). 쌍점이나 【】 | やめろって言っただろ | 그만두라고 했지 | 유지 | 사례: `tests/rules/cases/dam-temi-te-iu.tsv` + +## LEX-5800 줄 머리 화자 표시의 히라가나 이름은 소리대로 쓴다 + +자막과 대본의 줄 머리에서 말을 붙이는 화자 표시에 든 히라가나 두 글자에서 여섯 글자 덩어리는 이름이다. 둥근 괄호와 각괄호(전각, +반각: `(のぼる)`, `（のぼる）`, `[のぼる]`, `［のぼる］`), 콜론 앞(`のぼる：`), 줄을 닫는 「 앞(`のぼる「…」`)이 모두 그렇다. 덩어리가 +동사 꼴(まもる 守る, ひかる 光る)이거나 사전에 없는 말(たまお)이어도 낱말로 옮기지 않고, 이름 행이 있으면 그 읽기로, 없으면 문서의 +히라가나 이름처럼 장음 한 쌍을 줄여(ゆう `유`) 소리대로 쓴다. 덩어리 뒤 경칭은 이름 뒤에서 그대로 읽는다(かけるくん `카케루 군`). +문서 모드와 줄마다 모드가 같다. + +- 사전의 온전한 일반 낱말(ざわめき, くしゃみ, ひそひそ), 사람 명사와 대명사(おばあさん, あなた), 경칭까지 한 낱말인 말(おかあさん, + みなさん)과 お나 ご에 경칭이 붙은 말(おきゃくさん), 동사의 연용형(わらい, こごえ: 몸짓이나 소리를 적은 지문), 같은 반을 되풀이한 + 소리, っ로 끝나는 덩어리, 역할 표시(noun.label-role)는 이름이 아니다. +- 괄호 뒤에 말이 없거나(（笑い）) 조사가 괄호를 받으면(（まもる）は) 화자 표시가 아니고, 줄 가운데 괄호(昨日は(たぶん))도 아니다. +- 콜론 뒤에 다른 콜론이 또 오거나 가나도 문장 부호도 없는 값만 오는 줄은 항목이다(けもみみ：兎, けもみみ：兎　美徳：純潔). +- 각괄호 화자 표시는 줄을 연다. 다른 괄호 묶음 뒤의 각괄호는 목록의 낱말이다([ガンガンいこうぜ] [いのちだいじ]). 일곱 글자 이상의 + 덩어리와 낱말에 조사가 붙은 덩어리는 말이다(でもいつのまにか「). +- 크레딧에서 줄 머리 이름 바로 뒤에 feat. 이나 ft. 가 오면(ぽんた(Feat.IA)), 줄 머리 이름 뒤 괄호에서 /로 라틴 문자 이름과 묶인 첫 + 이름이면(山田太郎(たろー/T-BOX)) 사전 낱말이어도 이름이다. + +| 원문 | 출력 | 구분 | +|---|---|---| +| (ひかる)待って！ | (히카루) 기다려! | 고침 | +| ［のぼる］行くぞ！ | [노보루]가겠어! | 고침 | +| かける「早く来て」 | 카케루 「빨리 와」 | 고침 | +| ぽんた(Feat.IA) | 폰타(Feat.IA) | 고침 | +| (ざわめき)皆さん 静かに。 | (웅성거림) 여러분 조용하게. | 유지 | +| (おかあさん)ごはんよ。 | (어머니) 밥이야. | 유지 | +| 昨日は(たぶん)晴れだった。 | 어제는(아마) 맑음이었다. | 유지 | + +사례: `tests/rules/cases/names-kana-speaker-labels-5800.tsv`, `tests/rules/documents/names-kana-speaker-labels-5800.tsv` diff --git a/src/core/names/cues.cpp b/src/core/names/cues.cpp index c6aab5da0..e53ec6bd2 100644 --- a/src/core/names/cues.cpp +++ b/src/core/names/cues.cpp @@ -418,29 +418,37 @@ std::size_t CaptionLabelEnd(std::string_view line) { return SpeechAfterLabel(line.substr(end)) ? end : 0; } -bool BracketLabelAt(std::string_view before, std::string_view after) { +namespace { + +// BracketLabelAt for the brackets of opens (( and （, or [ and ［), each closed by its pair. +bool LabelInBracketsAt(std::string_view before, std::string_view after, bool square) { static constexpr std::string_view kLabelSeparators = "／/・･＆&，,：: 　"; - static constexpr std::string_view kStops = "()（）\n「」『』、。！？!?…‥"; + const std::string_view kStops = square ? "[]［］()（）\n「」『』、。！？!?…‥" : "()（）\n「」『』、。！？!?…‥"; + const char32_t half = square ? U'[' : U'(', full = square ? U'［' : U'（'; std::size_t open = before.size(); for (int n = 0; open > 0; n++) { const text::Rune r = RuneBefore(before, open); - if (r.cp == U'(' || r.cp == U'（') break; + if (r.cp == half || r.cp == full) break; if (InRunes(r.cp, kStops) || n >= kMaxCaptionGroup) return false; open -= r.size; } if (open == 0) return false; const char32_t bracket = RuneBefore(before, open).cp; + const char32_t closing = bracket == U'(' ? U')' : bracket == U'（' ? U'）' : bracket == U'[' ? U']' : U'］'; const std::string_view parts = before.substr(open); if (!parts.empty() && !InRunes(RuneBefore(parts, parts.size()).cp, kLabelSeparators)) return false; - if (!CaptionLead(before.substr(0, open - RuneBefore(before, open).size))) return false; + const std::string_view lead = before.substr(0, open - RuneBefore(before, open).size); + if (!CaptionLead(lead)) return false; + // A label in square brackets opens the line: one after another group is a word of a list ([ガンガン] [いのち]). + if (square && AnyRuneOf(lead, [](char32_t c) { return InRunes(c, ")）]］"); })) return false; std::string_view rest = after; rest.remove_prefix(dict::StartsWithMember(HDTRANS_CLASS("honorific.name"), rest)); - if (!rest.empty() && !InRunes(FirstRune(rest).cp, kLabelSeparators) && FirstRune(rest).cp != (bracket == U'(' ? U')' : U'）')) { + if (!rest.empty() && !InRunes(FirstRune(rest).cp, kLabelSeparators) && FirstRune(rest).cp != closing) { return false; } for (std::size_t i = 0, n = 0; i < rest.size() && n <= static_cast(kMaxCaptionGroup); n++) { const text::Rune r = text::DecodeRune(rest.substr(i)); - if (r.cp == (bracket == U'(' ? U')' : U'）')) { + if (r.cp == closing) { // Speech follows a label; a word in brackets alone is a heading or an aside ((春)), and // one that a particle takes is a word of the sentence (（アヤメ）と呼ばれた). return SpeechAfterLabel(rest.substr(i + r.size)); @@ -451,6 +459,12 @@ bool BracketLabelAt(std::string_view before, std::string_view after) { return false; } +} // namespace + +bool BracketLabelAt(std::string_view before, std::string_view after) { return LabelInBracketsAt(before, after, false); } + +bool SquareLabelAt(std::string_view before, std::string_view after) { return LabelInBracketsAt(before, after, true); } + bool SpeechAfterLabel(std::string_view rest) { while (rest.starts_with(" ") || rest.starts_with("　")) rest.remove_prefix(rest.starts_with(" ") ? 1 : std::string_view("　").size()); if (rest.empty() || rest.starts_with("\n") || rest.starts_with("\r") || rest.starts_with("と") || rest.starts_with("って")) return false; @@ -952,6 +966,156 @@ bool Registry::InflectedRun(std::string_view t) { return false; } +namespace { + +// Whether after starts with s, ASCII letters compared without case. +bool StartsWithFolded(std::string_view after, std::string_view s) { + if (after.size() < s.size()) return false; + for (std::size_t i = 0; i < s.size(); i++) { + const char a = after[i], b = s[i]; + if (a != b && !(a >= 'A' && a <= 'Z' && a - 'A' + 'a' == b)) return false; + } + return true; +} + +// Whether the word between before and after is the name of an artist in a credit: alone at the line start +// before feat. or ft. (ぽんた(Feat.IA), ゆう feat. ミク), or the first of the names of a group in brackets that +// joins it with / to a name in Latin letters (山田太郎(たろー/T-BOX)). +bool CreditNameAt(std::string_view before, std::string_view after) { + if (CaptionLead(before)) { + std::string_view a = after; + if (a.starts_with("(")) { + a.remove_prefix(1); + } else if (a.starts_with("（")) { + a.remove_prefix(std::string_view("（").size()); + } else if (a.starts_with(" ") || a.starts_with("　")) { + a = TrimSpace(a); + } else { + return false; + } + return StartsWithFolded(a, "feat.") || StartsWithFolded(a, "feat ") || StartsWithFolded(a, "ft."); + } + if (!(before.ends_with("(") || before.ends_with("（"))) return false; + // The group follows a name in kanji or katakana that opens the line. + std::string_view head = before.substr(0, before.size() - RuneBefore(before, before.size()).size); + const std::size_t named = head.size(); + while (!head.empty() && (IsKanji(RuneBefore(head, head.size()).cp) || IsKatakana(RuneBefore(head, head.size()).cp) || + InRunes(RuneBefore(head, head.size()).cp, " 　・"))) { + head.remove_suffix(RuneBefore(head, head.size()).size); + } + if (head.size() == named || !CaptionLead(head)) return false; + std::string_view a = after; + if (a.starts_with("/")) { + a.remove_prefix(1); + } else if (a.starts_with("／")) { + a.remove_prefix(std::string_view("／").size()); + } else { + return false; + } + return !a.empty() && ((a[0] >= 'A' && a[0] <= 'Z') || (a[0] >= 'a' && a[0] <= 'z')); +} + +} // namespace + +bool Registry::KanaLabelName(std::string_view t) { + const std::size_t n = RuneCount(t); + // A name in hiragana runs to six kana at most; a longer run is words (でもいつのまにか). + if (n < 2 || n > 6 || !IsHiragana(FirstRune(t).cp) || RoleLike(t) || LabelRole(t) || WordThenParticle(t)) return false; + // A small kana or a long mark cannot open the name, and a run that ends in っ or repeats its half is a sound + // (えっ, ざわざわ, ひそひそ). + if (InRunes(FirstRune(t).cp, "ぁぃぅぇぉっゃゅょゎ") || t.ends_with("っ")) return false; + // Kana are three bytes each in UTF-8. + if (n >= 4 && n % 2 == 0 && t.size() == 3 * n && t.substr(0, t.size() / 2) == t.substr(t.size() / 2)) return false; + std::string_view core = t; + while (core.ends_with("ー") || core.ends_with("～") || core.ends_with("〜")) core.remove_suffix(RuneBefore(core, core.size()).size); + if (RuneCount(core) < 2 || !AllRunes(core, IsHiragana)) return false; + // A person of the dictionaries or the name layer is the name (みのり 미노리); a whole common word, a + // person noun or a pronoun is the word (ざわめき, くしゃみ, せんせい, おばあさん, あなた). + const DictFlags f = Flags(core); + if (f.role || f.pronoun || f.human || f.thing) return false; + if (f.person || f.layerSurname || f.layerGiven) return true; + if (f.common && f.word) return false; + // The continuative of a verb names an act, as a stage direction does (わらい, こごえ, せきこみ). + dict::LookupContext ctx; + ctx.plain = &snap_->Plain(); + for (const dict::Candidate& c : snap_->GetWordInfo(core, dict::LookupMode::General, ctx).list) { + if (c.user != nullptr || c.codes == nullptr || c.codes->partText.size() != 4 || c.codes->partText[0] != 'M' || + c.source.size() > core.size()) { + continue; + } + const std::string_view tail = core.substr(c.source.size()); + const std::string_view stem = word::Conjugation(c.codes->partText[3] - '0', 10); + if (stem.find_first_of("(X") == std::string_view::npos && tail == stem) return false; + } + return true; +} + +std::string Registry::ReadKanaLabels(std::string_view line) { + if (snap_ == nullptr) return std::string(line); + // The label stands in the first characters of the line, after caption marks (⚟, ≫) and asides. + constexpr std::size_t kLabelReach = 4 * static_cast(kMaxCaptionGroup); + std::string out; + std::size_t at = 0; + for (std::size_t i = 0; i < line.size() && i < kLabelReach;) { + const text::Rune r = RuneAt(line, i); + if (r.cp == U'\n') break; + if (!IsHiragana(r.cp) || (i > 0 && (IsHiragana(RuneBefore(line, i).cp) || IsKatakana(RuneBefore(line, i).cp) || + IsKanji(RuneBefore(line, i).cp)))) { + i += r.size == 0 ? 1 : r.size; + continue; + } + std::size_t end = i; + while (end < line.size()) { + const text::Rune c = RuneAt(line, end); + if (!IsHiragana(c.cp) && !(end > i && InRunes(c.cp, "ー～〜"))) break; + end += c.size; + } + if (end < line.size() && (IsKatakana(RuneAt(line, end).cp) || IsKanji(RuneAt(line, end).cp))) { + i = end; + continue; + } + std::string_view run = line.substr(i, end - i); + // A label may end in an honorific (あげるくん), which the engine reads after the name; not a word with + // one (おかあさん, みなさん, おじょうさん), nor お or ご and an honorific, which make a person by their + // work (おきゃくさん, おまわりさん). + if (const std::size_t h = dict::EndsWithMember(HDTRANS_CLASS("honorific.name"), run); h > 0 && h < run.size()) { + const DictFlags whole = Flags(run); + if (whole.common || whole.role || whole.human || whole.pronoun || run.starts_with("お") || run.starts_with("ご")) { + i = end; + continue; + } + run.remove_suffix(h); + } + const std::string_view before = line.substr(0, i), after = line.substr(i + run.size()); + // An artist's name in a credit is a name whatever word it spells (ぽんた, すずらん). + const bool credit = run.size() == end - i && RuneCount(run) <= 8 && !RoleLike(run) && !LabelRole(run) && + !InRunes(FirstRune(run).cp, "ぁぃぅぇぉっゃゅょゎ") && CreditNameAt(before, after); + // A colon that another field follows on the line opens a value of a list of fields, not speech + // (けもみみ：兎　美徳：純潔). + const std::size_t colon = after.starts_with(":") ? 1 : after.starts_with("：") ? std::string_view("：").size() : 0; + const std::string_view value = after.substr(colon); + // Nor one that holds no kana nor mark of a sentence: a value (けもみみ：兎). + const bool field = colon > 0 && (value.find(':') != std::string_view::npos || value.find("：") != std::string_view::npos || + !AnyRuneOf(value, [](char32_t c) { return IsHiragana(c) || IsKatakana(c) || InRunes(c, "。！？!?…‥～〜"); })); + if (credit || (!field && SpeakerLabelAt(*snap_, before, after) && KanaLabelName(run))) { + // A person row's reading (ゆうこ 유우코), else the kana as the names of the document are read, one long + // vowel pair dropped (ゆう 유, しどう 시도). + const DictFlags f = Flags(run); + std::string written = f.person || f.layerSurname || f.layerGiven ? RenderPart(run, true) : std::string(); + if (!IsHangulText(written)) written = Written(run); + if (!written.empty()) { + out.append(line.substr(at, i - at)); + out.append(written); + at = i + run.size(); + } + } + i = end; + } + if (at == 0) return std::string(line); + out.append(line.substr(at)); + return out; +} + // Whether the kanji k with い is an adjective of the dictionaries (太い), whose く (太く) is no verb's. bool Registry::AdjectiveKanji(std::string_view k) { if (snap_ == nullptr) return false; diff --git a/src/core/names/document.cpp b/src/core/names/document.cpp index eb3bb7231..536f4af25 100644 --- a/src/core/names/document.cpp +++ b/src/core/names/document.cpp @@ -633,6 +633,7 @@ LineFacts ReadLine(DocumentContext& doc, std::string_view line) { std::string cueText; const std::string kin = pipeline::internal::WriteKinTerms(line, doc.female, KanjiWordHolds, doc.snap); facts.text = r.DropReadings(names::StripRuby(kin), &cueText, &facts.cued); + facts.text = r.ReadKanaLabels(facts.text); const std::string_view cues = cueText.empty() ? std::string_view(facts.text) : std::string_view(cueText); dict::KeySet plain, guarded; for (const names::Occurrence& o : r.Occurrences(facts.text)) { diff --git a/src/core/names/labels.h b/src/core/names/labels.h index c57c71391..376761bf4 100644 --- a/src/core/names/labels.h +++ b/src/core/names/labels.h @@ -32,6 +32,8 @@ std::size_t CaptionLabelEnd(std::string_view line); // Whether the word between before and after is a part of a closed round-bracket speaker // label at the start of a caption, followed by speech. bool BracketLabelAt(std::string_view before, std::string_view after); +// The same in square brackets ([のぼる]はい, ［たまお］うん; spec LEX-5800). +bool SquareLabelAt(std::string_view before, std::string_view after); // Whether rest, the text after a speaker label in brackets, starts an utterance: there is text // before the line ends, and it neither quotes the brackets with と or って nor starts with a diff --git a/src/core/names/name_cues.cpp b/src/core/names/name_cues.cpp index 4ecfe88dc..da9c2a4a2 100644 --- a/src/core/names/name_cues.cpp +++ b/src/core/names/name_cues.cpp @@ -938,6 +938,10 @@ std::vector NameCues(const dict::Snapshot& snap, std::string_view text) return out; } +bool SpeakerLabelAt(const dict::Snapshot& snap, std::string_view before, std::string_view after) { + return LabelAt(snap, before, after) || internal::SquareLabelAt(before, after); +} + bool PersonSlotAt(const dict::Snapshot& snap, std::string_view key, std::string_view before, std::string_view after) { if (CommonSenseAt(after)) return false; // An honorific: the document has named the word already, so a sense of an animal does not stand in the diff --git a/src/core/names/name_cues.h b/src/core/names/name_cues.h index 283c6c203..f8db8777c 100644 --- a/src/core/names/name_cues.h +++ b/src/core/names/name_cues.h @@ -43,6 +43,11 @@ std::vector NameCues(const dict::Snapshot& snap, std::string_view text) // or as the subject or topic of what a person does or says (spec LEX-5175). bool PersonSlotAt(const dict::Snapshot& snap, std::string_view key, std::string_view before, std::string_view after); +// Whether the word between before and after is a speaker label at the start of a line: a part of a label in +// round or square brackets followed by speech ((のぼる)うん, ［たまお］はい), or the word alone before a colon, +// ≫, a quote that ends the line or a role in round brackets, after caption marks only (spec LEX-5800). +bool SpeakerLabelAt(const dict::Snapshot& snap, std::string_view before, std::string_view after); + // Whether after opens the value of a field of a status window: a colon and a value with no kana or kanji // (力：35, 敏捷：Ｂ+; spec LEX-5175). bool StatusFieldAt(std::string_view after); diff --git a/src/core/names/registry.h b/src/core/names/registry.h index 8ca2a7e15..d3ea3fe92 100644 --- a/src/core/names/registry.h +++ b/src/core/names/registry.h @@ -131,6 +131,11 @@ class Registry { std::string DropReadings(std::string_view line, std::string* cueText = nullptr, std::vector* captionNames = nullptr); + // line with each hiragana name of a speaker label at its start written in Hangul by its sound + // ((のぼる)うん 노보루, ［たまお］はい, ゆう：うん, ひかる「待って」): a run of two to six kana that the + // dictionaries have as no whole common word, role or sound, read as the katakana names are (spec LEX-5800). + std::string ReadKanaLabels(std::string_view line); + text::Preset preset = text::Preset::FanShort; // The rendering of ちゃん written to Dict; "" leaves ちゃん to the engine. std::string chan = "짱"; @@ -176,6 +181,8 @@ class Registry { std::vector LooseKanaUses(std::string_view line); // Whether the hiragana t is a form of a verb or an adjective of the dictionaries (いい, 来た). bool InflectedRun(std::string_view t); + // Whether the hiragana run t of a speaker label is a name (ReadKanaLabels). + bool KanaLabelName(std::string_view t); // Whether the hiragana t is a word of the dictionary and a particle after it (どこまで: どこ まで, // それから), which is no name however often the document writes it (spec LEX-4828). bool WordThenParticle(std::string_view t); diff --git a/tests/rules/cases/names-kana-speaker-labels-5800.tsv b/tests/rules/cases/names-kana-speaker-labels-5800.tsv new file mode 100644 index 000000000..bcf93f8f8 --- /dev/null +++ b/tests/rules/cases/names-kana-speaker-labels-5800.tsv @@ -0,0 +1,30 @@ +# 규칙: LEX-5800 +rule input expected kind layer +names:kana-label (ひかる)待って！ (히카루) 기다려! fix names +names:kana-label （まもる）大丈夫だよ。 (마모루) 괜찮아. fix names +names:kana-label ［のぼる］行くぞ！ has:[노보루] fix names +names:kana-label [あゆむ]うん。 has:[아유무] fix names +names:kana-label かける「早く来て」 has:카케루 「 fix names +names:kana-label (ひかる・まもる)せーの！ has:(히카루·마모루) fix names +names:kana-label (会場 笑い)(のぼる)ほんと？ has:(회장 웃음)(노보루) fix names +names:kana-label ぽんた(Feat.IA) has:폰타( fix names +names:kana-label いさむ：それは違う。 has:이사무： keep names +names:kana-label ⚟（すすむ）もしもし？ has:⚟(스스무) keep names +names:kana-label (たまお)はい！ has:(타마오) keep names +names:kana-label (かけるくん)おはよう。 has:(카케루 군) keep names +names:kana-label 山田太郎(たろー/T-BOX) has:(타로/T-BOX) keep names +names:kana-label （笑い）そうだね。 has:(웃음) keep names +names:kana-label (ざわめき)皆さん 静かに。 has:(웅성거림) keep names +names:kana-label (くしゃみ)ああ… has:(재채기) keep names +names:kana-label (おかあさん)ごはんよ。 has:(어머니) keep names +names:kana-label (おきゃくさん)すみません。 has:(손님) keep names +names:kana-label (わらい)そうだね。 lacks:와라이 keep names +names:kana-label (ひそひそ)聞いた？ has:(소곤소곤) keep names +names:kana-label 昨日は(たぶん)晴れだった。 has:(아마) keep names +names:kana-label (はい)そうです。 has:(네) keep names +names:kana-label ねえ「待って」 has:저기 keep names +names:kana-label （ナレーション）その日、 has:(내레이션) keep names +names:kana-label ほら「見て」 has:자 「 keep names +names:kana-label それは(まもる)ためだ。 lacks:마모루 keep names +names:kana-label けもみみ：兎　美徳：純潔 lacks:케모미미 keep names +names:kana-label これは(とても/very)いい lacks:토테모 keep names diff --git a/tests/rules/documents/names-kana-speaker-labels-5800.tsv b/tests/rules/documents/names-kana-speaker-labels-5800.tsv new file mode 100644 index 000000000..096171b01 --- /dev/null +++ b/tests/rules/documents/names-kana-speaker-labels-5800.tsv @@ -0,0 +1,4 @@ +# 규칙: LEX-5800 +rule input expected kind layer +names:kana-label (のぼる)行くぞ！\n(あゆむ)うん。\n（笑い）\n(のぼる)待って！ has:(노보루) 가겠어!\n(아유무) 응.\n(웃음)\n(노보루) 기다려! fix names +names:kana-label かける「早く来て」\nかける「ほら」 has:카케루 「빨리 와」\n카케루 「 fix names diff --git a/tests/rules/wrapped/cut-word.tsv b/tests/rules/wrapped/cut-word.tsv index ac5e4e883..c7929d6d7 100644 --- a/tests/rules/wrapped/cut-word.tsv +++ b/tests/rules/wrapped/cut-word.tsv @@ -65,7 +65,7 @@ wrapped:cut-word (患者)はあ\nおいくらでしょうか？ (환자) 하아\ wrapped:cut-word これで間違いないと思います\nよう　さっきは電話でどうも 이것으로 틀림없다고 생각합니다\n요우 조금 전은 전화로 아무래도 keep driver wrapped:cut-word 含まれております\nよ～う　考えた 포함되어 있습니다\n요우~ 생각했다 keep driver wrapped:cut-word 処刑されます\nそ… そんな！ 처형당합니다\n그… 그런! keep driver -wrapped:cut-word キスしたくせに\n（るな）しおちゃん差し入れ　持ってきたよ～。 키스했으면서\n(루인) 시오짱 간식 가져왔어~. keep driver +wrapped:cut-word キスしたくせに\n（るな）しおちゃん差し入れ　持ってきたよ～。 키스했으면서\n(루나) 시오짱 간식 가져왔어~. fix driver wrapped:cut-word 彼はゲームを始め\n\nるとか、そういう話。 그는 게임을 시작해\n\n루라든지, 그런 이야기. keep driver wrapped:cut-word もう逃\n「げるしかない」 이제 도\n「밖에 없어」 keep driver wrapped:cut-word 試合が始ま\nりです 시합이 시작\n입니다 keep driver From 209ad4f8947350d4e0ad9e7772e41b4717e6b8db Mon Sep 17 00:00:00 2001 From: sokcuri Date: Mon, 9 Nov 2026 08:20:08 +0900 Subject: [PATCH 2/2] fix(names): keep kana words, verb forms and sounds out of kana speaker labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A label run is a word, not a name, when the dictionaries have it whole: a kana spelling of a kanji word, a common noun, a sound or interjection, a する stem, a verb form other than the dictionary form, an adjective, or a word with a sentence-final particle; two kana need a person row or no row at all. Before 「, a colon or ≫ a name-layer person still wins; in brackets the word does. A copula after the brackets keeps them a word of the sentence. --- spec/words.md | 11 +- src/core/names/cues.cpp | 115 ++++++++++++++---- src/core/names/registry.h | 7 +- .../cases/names-kana-speaker-labels-5800.tsv | 13 ++ 4 files changed, 117 insertions(+), 29 deletions(-) diff --git a/spec/words.md b/spec/words.md index 5e55f98cc..aa7b47682 100644 --- a/spec/words.md +++ b/spec/words.md @@ -11662,9 +11662,14 @@ name.common-predicate에 둔다(高い, 強い, 眩しい). 쌍점이나 【】 히라가나 이름처럼 장음 한 쌍을 줄여(ゆう `유`) 소리대로 쓴다. 덩어리 뒤 경칭은 이름 뒤에서 그대로 읽는다(かけるくん `카케루 군`). 문서 모드와 줄마다 모드가 같다. -- 사전의 온전한 일반 낱말(ざわめき, くしゃみ, ひそひそ), 사람 명사와 대명사(おばあさん, あなた), 경칭까지 한 낱말인 말(おかあさん, - みなさん)과 お나 ご에 경칭이 붙은 말(おきゃくさん), 동사의 연용형(わらい, こごえ: 몸짓이나 소리를 적은 지문), 같은 반을 되풀이한 - 소리, っ로 끝나는 덩어리, 역할 표시(noun.label-role)는 이름이 아니다. +- 사전이 덩어리 전체를 낱말로 가지면 이름 층에 같은 이름이 있어도 낱말이다. 사전의 온전한 일반 낱말과 한자 낱말의 가나 표기(ざわめき, + うた, おとこ, ははおや, てんいん), 사람 명사와 대명사(おばあさん, あなた), 소리 사전의 감탄사와 소리(てか), する 동사의 어간(てき), + 기본형이 아닌 동사 꼴(まって, やめて, わらい, こごえ: て형, 연용형)과 형용사 꼴(いい), 낱말과 문장 끝 조사(いいよ, あるぞ)가 그렇다. + 동사 기본형은 이름에 흔해서(ひかる, まもる) 이름으로 읽고, 사전의 사람 행은 언제나 이름이다(みのり). 「, 콜론, ≫ 앞에는 화자만 + 서므로 이름 층의 사람 행이 낱말보다 먼저다(さくら「…」 `사쿠라`). 괄호는 소리나 목소리도 적으므로 괄호 안에서는 낱말이 먼저다((うた) `노래`). +- 두 글자 덩어리는 사람 행이 있거나 어느 사전에도 없을 때만 이름이다(おと는 낱말, もず는 이름). +- 경칭까지 한 낱말인 말(おかあさん, みなさん)과 お나 ご에 경칭이 붙은 말(おきゃくさん), 같은 반을 되풀이한 소리, っ로 끝나는 덩어리, + 역할 표시(noun.label-role)는 이름이 아니다. 괄호 뒤에 だ, です, じゃ가 오면 괄호는 문장의 낱말이다(（ないしょ）だよ). - 괄호 뒤에 말이 없거나(（笑い）) 조사가 괄호를 받으면(（まもる）は) 화자 표시가 아니고, 줄 가운데 괄호(昨日は(たぶん))도 아니다. - 콜론 뒤에 다른 콜론이 또 오거나 가나도 문장 부호도 없는 값만 오는 줄은 항목이다(けもみみ：兎, けもみみ：兎　美徳：純潔). - 각괄호 화자 표시는 줄을 연다. 다른 괄호 묶음 뒤의 각괄호는 목록의 낱말이다([ガンガンいこうぜ] [いのちだいじ]). 일곱 글자 이상의 diff --git a/src/core/names/cues.cpp b/src/core/names/cues.cpp index e53ec6bd2..6fd5ac4e9 100644 --- a/src/core/names/cues.cpp +++ b/src/core/names/cues.cpp @@ -14,6 +14,7 @@ #include "core/names/labels.h" #include "core/names/registry.h" #include "core/names/name_cues.h" +#include "core/pipeline/sound.h" #include "core/text/unicode.h" #include "core/text/utf8.h" #include "core/word/conjugation.h" @@ -1017,36 +1018,91 @@ bool CreditNameAt(std::string_view before, std::string_view after) { } // namespace -bool Registry::KanaLabelName(std::string_view t) { +bool Registry::KanaWordRun(std::string_view t) { + if (snap_ == nullptr || t.empty()) return false; + const DictFlags f = Flags(t); + if (f.role || f.pronoun || f.human || f.thing) return true; + // A common word of its own: no ending, particle, verb or adjective row (るな 구나 ends a verb) and no reading of + // a kanji standing alone. + for (const dict::Candidate& c : snap_->RetrieveRecords(t, dict::LookupMode::General)) { + if (c.codes == nullptr || c.source != t || c.codes->partText.empty() || c.codes->gram.Contains("On")) continue; + if (c.codes->sem.HasPrefix("QPN") || c.codes->sem.HasPrefix("QCT")) continue; + if (!InRunes(static_cast(c.codes->partText[0]), "STUVPMH")) return true; + } + // An interjection or a sound of the sound lexicon (てか, うん, ざわざわ). + std::u32string katakana; + for (std::size_t i = 0; i < t.size();) { + const text::Rune r = RuneAt(t, i); + katakana.push_back(IsHiragana(r.cp) ? r.cp + 0x60 : r.cp); + i += r.size == 0 ? 1 : r.size; + } + if (pipeline::internal::IsSoundKey(katakana)) return true; + // A kana spelling of a word of the base dictionary (おとこ 男, ははおや 母親, てんいん 店員). + for (const dict::UserEntry* e : snap_->TextPlain().Search(t, dict::LookupMode::General)) { + if (e->key == t && e->kana && !e->nameLayer) return true; + } + dict::LookupContext ctx; + ctx.plain = &snap_->Plain(); + for (const dict::Candidate& c : snap_->GetWordInfo(t, dict::LookupMode::General, ctx).list) { + if (c.codes == nullptr || c.source.size() > t.size()) continue; + const std::string_view part = c.codes->partText; + if (part.empty()) continue; + // A word the engine reads whole that is no verb: a noun, an adverb or an interjection (てか, ないしょ). + if (c.source == t && part[0] != 'M' && !c.codes->sem.HasPrefix("QPN")) return true; + if (part.size() != 4) continue; + const std::string_view tail = t.substr(c.source.size()); + // An adjective (いい, すごく). + if (part[0] == 'H' && (tail == "い" || tail == "く" || tail == "かった" || tail == "くて")) return true; + if (part[0] != 'M') continue; + // The stem of a する verb is a noun (てき 適). + if (c.source == t && part[3] == '9') return true; + // A verb in any form but its dictionary form, which given names take (ひかる, まもる): the て form, the + // continuative or the past (まって, やめて, わらい, こごえ). + const int kind = part[3] - '0'; + if (kind < 0 || kind > word::kKindIku) continue; + for (int branch = 1; branch < word::kConjugationBranches; branch++) { + const std::string_view conj = word::Conjugation(kind, branch); + if (conj.find_first_of("(X") == std::string_view::npos && tail == conj && (!conj.empty() || branch == 10)) return true; + } + } + return false; +} + +bool Registry::KanaLabelName(std::string_view t, bool bracketed) { const std::size_t n = RuneCount(t); // A name in hiragana runs to six kana at most; a longer run is words (でもいつのまにか). if (n < 2 || n > 6 || !IsHiragana(FirstRune(t).cp) || RoleLike(t) || LabelRole(t) || WordThenParticle(t)) return false; - // A small kana or a long mark cannot open the name, and a run that ends in っ or repeats its half is a sound - // (えっ, ざわざわ, ひそひそ). + // A small kana cannot open the name, and a run that ends in っ is a sound (えっ). if (InRunes(FirstRune(t).cp, "ぁぃぅぇぉっゃゅょゎ") || t.ends_with("っ")) return false; - // Kana are three bytes each in UTF-8. - if (n >= 4 && n % 2 == 0 && t.size() == 3 * n && t.substr(0, t.size() / 2) == t.substr(t.size() / 2)) return false; std::string_view core = t; while (core.ends_with("ー") || core.ends_with("～") || core.ends_with("〜")) core.remove_suffix(RuneBefore(core, core.size()).size); - if (RuneCount(core) < 2 || !AllRunes(core, IsHiragana)) return false; - // A person of the dictionaries or the name layer is the name (みのり 미노리); a whole common word, a - // person noun or a pronoun is the word (ざわめき, くしゃみ, せんせい, おばあさん, あなた). + const std::size_t m = RuneCount(core); + if (m < 2 || !AllRunes(core, IsHiragana)) return false; + // A run that repeats its half is a sound (ざわざわ, ひそひそ); kana are three bytes each in UTF-8. + if (m >= 4 && m % 2 == 0 && core.size() == 3 * m && core.substr(0, core.size() / 2) == core.substr(core.size() / 2)) return false; + // A person row of the dictionaries is the name (みのり 미노리). const DictFlags f = Flags(core); - if (f.role || f.pronoun || f.human || f.thing) return false; - if (f.person || f.layerSurname || f.layerGiven) return true; - if (f.common && f.word) return false; - // The continuative of a verb names an act, as a stage direction does (わらい, こごえ, せきこみ). - dict::LookupContext ctx; - ctx.plain = &snap_->Plain(); - for (const dict::Candidate& c : snap_->GetWordInfo(core, dict::LookupMode::General, ctx).list) { - if (c.user != nullptr || c.codes == nullptr || c.codes->partText.size() != 4 || c.codes->partText[0] != 'M' || - c.source.size() > core.size()) { - continue; - } - const std::string_view tail = core.substr(c.source.size()); - const std::string_view stem = word::Conjugation(c.codes->partText[3] - '0', 10); - if (stem.find_first_of("(X") == std::string_view::npos && tail == stem) return false; - } + if (f.person) return true; + // Before 「, a colon or ≫ only a speaker stands, and a person of the name layer is the name there (さくら「). + // Brackets also tag a sound or a voice, so a word there stays the word though the name layer has a person of + // that name ((うた), (おと)). + if (!bracketed && (f.layerSurname || f.layerGiven)) return true; + // A word the dictionaries have whole is the word: a common word or its kana spelling, a person noun or a + // pronoun, an adverb, an interjection or a sound, a verb or an adjective in a form a name never takes + // (ざわめき, おとこ, ははおや, うた, おばあさん, てか, まって, わらい, いい). + if (KanaWordRun(core)) return false; + if (f.layerSurname || f.layerGiven) return true; + // Two kana are most often a word: they name a person only on a person row or where no dictionary has them + // (おと, てき, but ゆう, もず). + if (m == 2 && (f.records > 0 || (snap_ != nullptr && !snap_->NameRecords(core).empty()))) return false; + // A word and a particle that ends a sentence are a phrase (いいよ, そうね, あるぞ). + const text::Rune last = RuneBefore(core, core.size()); + if (InRunes(last.cp, "よねぞぜわさなのか") && m >= 3) { + const std::string_view head = core.substr(0, core.size() - last.size); + if (KanaWordRun(head) || InflectedRun(head)) return false; + } + // A run that ends in the て of a verb's て form is one (まって, よんで) though the dictionaries spell it otherwise. + if ((core.ends_with("って") || core.ends_with("んで") || core.ends_with("いて") || core.ends_with("いで")) && m >= 3) return false; return true; } @@ -1097,7 +1153,18 @@ std::string Registry::ReadKanaLabels(std::string_view line) { // Nor one that holds no kana nor mark of a sentence: a value (けもみみ：兎). const bool field = colon > 0 && (value.find(':') != std::string_view::npos || value.find("：") != std::string_view::npos || !AnyRuneOf(value, [](char32_t c) { return IsHiragana(c) || IsKatakana(c) || InRunes(c, "。！？!?…‥～〜"); })); - if (credit || (!field && SpeakerLabelAt(*snap_, before, after) && KanaLabelName(run))) { + // A copula after the brackets makes them a word of the sentence ((ないしょ)だよ). + bool copula = false; + { + std::string_view rest = after.substr(dict::StartsWithMember(HDTRANS_CLASS("honorific.name"), after)); + const char32_t close = rest.empty() ? 0 : FirstRune(rest).cp; + if (InRunes(close, ")）]］")) { + rest = TrimSpace(rest.substr(FirstRune(rest).size)); + copula = rest.starts_with("だ") || rest.starts_with("です") || rest.starts_with("でし") || rest.starts_with("じゃ"); + } + } + const bool bracketed = BracketLabelAt(before, after) || SquareLabelAt(before, after); + if (credit || (!field && !copula && SpeakerLabelAt(*snap_, before, after) && KanaLabelName(run, bracketed))) { // A person row's reading (ゆうこ 유우코), else the kana as the names of the document are read, one long // vowel pair dropped (ゆう 유, しどう 시도). const DictFlags f = Flags(run); diff --git a/src/core/names/registry.h b/src/core/names/registry.h index d3ea3fe92..d6ee04dd0 100644 --- a/src/core/names/registry.h +++ b/src/core/names/registry.h @@ -181,8 +181,11 @@ class Registry { std::vector LooseKanaUses(std::string_view line); // Whether the hiragana t is a form of a verb or an adjective of the dictionaries (いい, 来た). bool InflectedRun(std::string_view t); - // Whether the hiragana run t of a speaker label is a name (ReadKanaLabels). - bool KanaLabelName(std::string_view t); + // Whether the hiragana run t of a speaker label is a name (ReadKanaLabels); bracketed: the label is in brackets, + // which also tag sounds and voices. + bool KanaLabelName(std::string_view t, bool bracketed); + // Whether the dictionaries have the hiragana run t as a word whole (KanaLabelName). + bool KanaWordRun(std::string_view t); // Whether the hiragana t is a word of the dictionary and a particle after it (どこまで: どこ まで, // それから), which is no name however often the document writes it (spec LEX-4828). bool WordThenParticle(std::string_view t); diff --git a/tests/rules/cases/names-kana-speaker-labels-5800.tsv b/tests/rules/cases/names-kana-speaker-labels-5800.tsv index bcf93f8f8..27b1ceaa7 100644 --- a/tests/rules/cases/names-kana-speaker-labels-5800.tsv +++ b/tests/rules/cases/names-kana-speaker-labels-5800.tsv @@ -28,3 +28,16 @@ names:kana-label ほら「見て」 has:자 「 keep names names:kana-label それは(まもる)ためだ。 lacks:마모루 keep names names:kana-label けもみみ：兎　美徳：純潔 lacks:케모미미 keep names names:kana-label これは(とても/very)いい lacks:토테모 keep names +names:kana-label （おとこ）待て！ has:(남자) keep names +names:kana-label （ははおや）ごはんよ。 has:(어머니) keep names +names:kana-label （てんいん）いらっしゃいませ。 has:(점원) keep names +names:kana-label （うた）ラララ has:(노래) keep names +names:kana-label （てき）来るぞ！ lacks:테키 keep names +names:kana-label （ないしょ）だよ lacks:나이쇼 keep names +names:kana-label まって「行かないで」 has:기다려 「 keep names +names:kana-label やめて「お願い」 lacks:야메테 keep names +names:kana-label てか≫マジ？ has:그보다 keep names +names:kana-label てか「マジ？」 has:그보다 「 keep names +names:kana-label いいよ「どうぞ」 has:좋아 「 keep names +names:kana-label あるぞ「おう」 lacks:아루조 keep names +names:kana-label さくら「早く来て」 has:사쿠라 「 fix names