#!/usr/bin/env python3 """Generate the xjyutping data files from the upstream sources. python3 tools/build-data.py [--sources DIR] [--py-data DIR] Run it from anywhere; paths are resolved from this file. The sources are third-party data kept outside the repository, by default in the directory that contains this repository (tools/fetch-sources.sh fetches them): jyutping-table-master/list.tsv LSHK Jyutping table, CC BY 4.0 rime-cantonese/jyut6ping3.chars.dict.yaml rime-cantonese, CC BY 4.0 rime-cantonese/jyut6ping3.words.dict.yaml rime-cantonese, CC BY 4.0 opencc/HKVariants.txt, opencc/TWVariants.txt OpenCC, Apache-2.0 jyut-dict/src/dictionaries/cedict/data/ CC-CANTO.txt CC-Canto (Pleco Inc.), CC BY-SA 3.0 READINGS.txt Cantonese readings of CC-CEDICT words (Pleco), CC BY-SA 3.0 cantonese-books-data//*資料.json 粵音資料集叢 book data (jyut.net, by 石見田), no licence stated rime-cantonese is authoritative. CC-Canto and the CC-CEDICT readings only add words that are not in its list and that the list would read otherwise; the books only add characters that LSHK and rime lack (see BOOKS). Outputs: xjyutping-chars.def (this repository) default reading of every character, polyphone flags, variant -> canonical character map, run-final readings xjyutping-words.def (this repository) word readings, max word length per final character --py-data DIR the same tables as UTF-8 TSV for the Python package xjyutping-py (default: ../xjyutping-py/src/xjyutping/ data next to this repository, if it exists): chars.tsv char, default, other readings (space separated), 1 if polyphone else 0 finals.tsv char, reading at the end of a run variants.tsv variant, canonical character words.tsv word (and canonical-spelling alias), readings """ import collections import json import math import pathlib import re import sys import unicodedata REPO = pathlib.Path(__file__).resolve().parent.parent # xjyutping-tex ROOT = REPO.parent # the sources PY_DATA = ROOT / 'xjyutping-py' / 'src' / 'xjyutping' / 'data' SYL = re.compile(r'^[a-z]+[1-6]$') VERSION = '2026/09/28 v1.2.0' # Standalone default readings. The first block settles characters that # rime-cantonese leaves undecided (every reading has the same weight); the # second overrides literary or rare defaults whose standalone use in Hong Kong # text is clearly something else. Words and phrases come from the word # list; these only apply to a character standing alone. CURATED_DEFAULTS = { '會': 'wui5', # 我會去 (will/can); 開會, 社會 ... come from the word list '生': 'saang1', # 生仔, 生嘅; 生活/學生 come from the word list '行': 'haang4', # 行去 (walk); 銀行/行為 come from the word list '畫': 'waa2', # 一幅畫 '料': 'liu2', # 有料, 材料 via words '咪': 'mai6', # 咪係囉 '咯': 'lok3', '率': 'leot2', '刊': 'hon2', '撈': 'lou1', '彙': 'wui6', '嗎': 'maa3', '嘎': 'gaa4', '咧': 'le4', '量': 'loeng6', # 廢物量, 交易量; the verb 量 comes from words '返': 'faan1', # 得返, 返中環; literary 往返/返回 are words '呀': 'aa3', # the ordinary particle (LSHK has no aa4) '驚': 'geng1', # 好驚, 唔使驚; 驚喜/震驚 are words '吓': 'haa5', # aspect marker 睇吓, 試吓 '爭': 'zaang1', # 爭我一百蚊; 爭取/競爭 are words '划': 'waa4', # 划算, 划船 (劃 carries waak6) '幢': 'zong6', # classifier; LSHK has no dung6 '呢': 'ni1', # demonstrative 呢張床; see CURATED_FINALS } # A different reading when the character ends a run of Chinese characters # (before punctuation, Latin text or the end): the particle 呢 in 你呢？ CURATED_FINALS = { '呢': 'ne1', } # Variant shapes missing from the OpenCC tables. CURATED_VARIANTS = { '恒': '恆', } # Word readings to add or correct on top of rime-cantonese. Several are # guards: longer words that keep a new entry from capturing a phrase it # should not (集中咗 for 中咗, 絕種咗 for 種咗 ...). CURATED_WORDS = { # rime errors '畀咗': 'bei2 zo2', '畀咗你': 'bei2 zo2 nei5', '畀咗佢': 'bei2 zo2 keoi5', '畀咗我': 'bei2 zo2 ngo5', '畀咗未': 'bei2 zo2 mei6', '畀咗錢': 'bei2 zo2 cin2', '畀咗錢未': 'bei2 zo2 cin2 mei6', '有朋自遠方來': 'jau5 pang4 zi6 jyun5 fong1 loi4', '相處': 'soeng1 cyu5', '共處': 'gung6 cyu5', '前人種樹': 'cin4 jan4 zung3 syu6', '少壯': 'siu3 zong3', '少壯不努力': 'siu3 zong3 bat1 nou5 lik6', '世上無難事': 'sai3 soeng6 mou4 naan4 si6', '屢見不鮮': 'leoi5 gin3 bat1 sin1', '外銷量': 'ngoi6 siu1 loeng6', '碳排放量': 'taan3 paai4 fong3 loeng6', '輸送量': 'syu1 sung3 loeng6', '會唔會': 'wui5 m4 wui5', '幾度': 'gei2 dou6', '揣度': 'cyun2 dok6', '間房': 'gaan1 fong2', '成日': 'seng4 jat6', '係呀': 'hai6 aa3', # literary words that must keep their reading after a default change '返回': 'faan2 wui4', '返鄉': 'faan2 hoeng1', '返航': 'faan2 hong4', '返程': 'faan2 cing4', '返還': 'faan2 waan4', '返國': 'faan2 gwok3', '積重難返': 'zik1 zung6 naan4 faan2', '驚聞': 'ging1 man4', '驚見': 'ging1 gin3', '驚爆': 'ging1 baau3', '驚現': 'ging1 jin6', '驚叫': 'ging1 giu3', '驚傳': 'ging1 cyun4', '驚變': 'ging1 bin3', '驚艷': 'ging1 jim6', '驚悉': 'ging1 sik1', '大吃一驚': 'daai6 hek3 jat1 ging1', '量體溫': 'loeng4 tai2 wan1', '量血壓': 'loeng4 hyut3 aat3', # missing words '好正': 'hou2 zeng3', '好精': 'hou2 zeng1', '把聲': 'baa2 seng1', '好平': 'hou2 peng4', '咁平': 'gam3 peng4', '最平': 'zeoi3 peng4', '平嘢': 'peng4 je5', '瞓唔到覺': 'fan3 m4 dou2 gaau3', '成個鐘': 'seng4 go3 zung1', '成身': 'seng4 san1', '成間': 'seng4 gaan1', '彈琴': 'taan4 kam4', '彈鋼琴': 'taan4 gong3 kam4', '彈結他': 'taan4 git3 taa1', '彈吉他': 'taan4 gat1 taa1', '彈古箏': 'taan4 gu2 zang1', '張相': 'zoeng1 soeng2', '相集': 'soeng2 zaap6', '着咗件': 'zoek3 zo2 gin6', '着衫': 'zoek3 saam1', '着多件褸': 'zoek3 do1 gin6 lau1', '着多件衫': 'zoek3 do1 gin6 saam1', '著住': 'zoek3 zyu6', '訂位': 'deng6 wai2', '訂枱': 'deng6 toi2', '冇位': 'mou5 wai2', '問下': 'man6 haa5', '問下個月': 'man6 haa6 go3 jyut6', '問下個禮拜': 'man6 haa6 go3 lai5 baai3', '今晚會': 'gam1 maan5 wui5', '聽晚會': 'ting1 maan5 wui5', '琴晚會': 'kam4 maan5 wui5', '行車線': 'hang4 ce1 sin3', '而行': 'ji4 hang4', '恒生': 'hang4 sang1', '不亦說乎': 'bat1 jik6 jyut6 fu4', '中咗': 'zung3 zo2', '集中咗': 'zaap6 zung1 zo2', '人話': 'jan4 waa6', '講人話': 'gong2 jan4 waa2', '唔講人話': 'm4 gong2 jan4 waa2', '發佈會': 'faat3 bou3 wui2', '發布會': 'faat3 bou3 wui2', '調高': 'tiu4 gou1', '強調高': 'koeng4 diu6 gou1', '數吓': 'sou2 haa5', '數完': 'sou2 jyun4', '應聘': 'jing3 ping3', '個名': 'go3 meng2', '頂帽': 'deng2 mou2', '上咗': 'soeng5 zo2', '乾隆': 'kin4 lung4', '朝住': 'ciu4 zyu6', '種咗': 'zung3 zo2', '絕種咗': 'zyut6 zung2 zo2', '畫咗': 'waak6 zo2', '屏住': 'bing2 zyu6', '屏息': 'bing2 sik1', '盛飯': 'sing4 faan6', '行長': 'hong4 zoeng2', '做邊行': 'zou6 bin1 hong4', '轉咗工': 'zyun3 zo2 gung1', '一唱一和': 'jat1 coeng3 jat1 wo6', '小心地滑': 'siu2 sam1 dei6 waat6', '成個人': 'seng4 go3 jan4', '成個人生': 'seng4 go3 jan4 sang1', '成件事': 'seng4 gin6 si6', '件事': 'gin6 si6', '成頭家': 'seng4 tau4 gaa1', '輕輕': 'hing1 hing1', '輕輕地': 'hing1 hing1 dei2', '輕輕哋': 'heng1 heng1 dei2', '為配合': 'wai6 pui3 hap6', '為確保': 'wai6 kok3 bou2', '為免': 'wai6 min5', # 平啲 'cheaper', with guards for X平 words followed by 啲 '平啲': 'peng4 di1', '公平啲': 'gung1 ping4 di1', '不公平啲': 'bat1 gung1 ping4 di1', '持平啲': 'ci4 ping4 di1', '擺平啲': 'baai2 ping4 di1', '扁平啲': 'bin2 ping4 di1', '躺平啲': 'tong2 ping4 di1', '燙平啲': 'tong3 ping4 di1', '鏟平啲': 'caan2 ping4 di1', '填平啲': 'tin4 ping4 di1', '水平啲': 'seoi2 ping4 di1', '太平啲': 'taai3 ping4 di1', '和平啲': 'wo4 ping4 di1', '開平啲': 'hoi1 ping4 di1', '攤平啲': 'taan1 ping4 di1', '整平啲': 'zing2 ping4 di1', '壓平啲': 'aat3 ping4 di1', # 1.1.0: 慢慢行 is wrong in rime; the others lose a tie to the longer # final word (屋|企住, 都|會話, 生|詞表) '慢慢行': 'maan6 maan2 haang4', '屋企住': 'uk1 kei2 zyu6', '都會話': 'dou1 wui5 waa6', '生詞表': 'saang1 ci4 biu2', # 1.2.0: rime reads 合 gap3 here (its own 化合 is faa3 hap6), and 會否 # wui2 (the modal is wui5); CC-Canto and the CC-CEDICT readings agree '化合物': 'faa3 hap6 mat6', '會否': 'wui5 fau2', } # Words of CC-Canto and the CC-CEDICT Cantonese readings that are not added # (every added word was checked by hand). Most give a character a reading # that Hong Kong usage and rime do not use (乙 jyut3, 這 ze5, 陷 haam6, # 寺 zi6, 購 gau3, 擾 jiu5, 噶 gaa1 ...), drop a colloquial changed tone # (慳錢 cin4, 練習簿 bou6, 黃大仙廟 miu6), read a Hong Kong name otherwise # (柯士甸 din6, 楊千嬅 waa4, 許冠傑 gun1) or are wrong (同學會 wui5, # 左行 haang1); others would take characters from their neighbours (方法|會 # would become 方|法會, 中學|到中|學到, 家長|會擔心家長會|擔心). EXCLUDED_WORDS = set(''' 一年生一打一更一毛錢七姊妹七姊妹星團七姊妹道三相三相點上甘嶺上甘嶺區不勝其擾不對勁不擴散核武器條約不暇不蒸饅頭爭口氣丕績丙烷乙丑乙二醇乙亥乙型乙基乙太乙巳乙方乙未乙炔乙烯乙種乙肝乙部乙酉乙酰乙醛了當二年生亞喀巴交易會人肉搜索伊媚兒併捲機併系群侵擾保有修會修道張偷雞不著蝕把米傣族傳幫帶優惠券元江哈尼族彞族傣族自治縣兄弟會光孝寺光明頂全會八達嶺公主道公立醫院兼並與收購兼併出勤率划船初生利什曼病副將加勁努庫阿洛法勇悍勘測勛績北京烤鴨區間十六烷值半生南普陀寺博士買驢卡洛馳吃拿卡要同和站同學會含蘊吳興周會咄咄咄咄稱奇和勝和和達清夫哄抬物價哲蚌寺唐招提寺唐老鴨商洛商洛市商洽單立文噶倫噶哈巫族噶嗒噶嘣噶噶噶布倫噶拉噶爾噶爾縣噶瑪蘭噶瑪蘭族噶舉派噶隆噶霏國會議員國會議長圓明園坎塔布連坎塔布連山脈坎塔布連海坎大哈坎大哈省坎帕拉坎特伯雷坎特伯雷大主教坎貝爾坎貝爾侏儒倉鼠坦噶尼喀坦噶尼喀湖報告會塌陷塔公寺塔爾寺塗乙塗漿檯塗鴨多年生夢到大慈恩寺大昭寺大雅鄉大麥町太僕寺太空探索奈洛比女修道張好奇會吃苦頭的好心倒做了驢肝肺好整以暇好玩兒媽閣廟嬌媚孔乙己孔雀女孟連傣族拉祜族佤族自治縣季會學到學舌宗喀巴室町家長會密會富蘊富蘊縣寬帶寶達邨對乙酰氨基酚小昭寺小雅少林寺尤坎尼雅尼雅河屈指可數崇洋媚外崗頂站嶺上開花左行巴伐洛堤市議會布坎南布氏桿菌病布洛陀席不暇暖帳簾干擾素平頂山平頂山市幹將幾遠店錢庫珀帶廣雅張惠妹强制性彈盡糧絕彩擴待乙妥德宏傣族景頗族自治州快刀斷亂麻情陷夜中環惠遠寺慈善機構慈照寺慳錢戎行戒斷戛然戛然而止手彈折中抹去抹平捷安特捷爾任斯克捷爾梅茲捷達航空貨運掏錢探測器接待處掩映搔擾撟舌撤併擴大化收購要約敬上敵將數學符號斯捷潘新平彞族傣族自治縣新界地產和富大埔新紀新藝綜合體斷路器日喀則日喀則地區日喀則市日無暇晷星光行星架坡春試昨早景谷傣族彞族自治縣智力測驗書會朗拿甸奴木柵木柵線李斯特氏桿菌東區走廊東協板鴨林和西站柯士甸柯士甸道校友會核不擴散核擴散桿菌梳刷棉襖楊千嬅橄欖球欲取姑予欺哄歐洲核研究組織歪瓜劣棗氯乙烯沉陷沒溜兒沖涼涼法會洛寧縣洛川縣洛扎縣洛江區洛浦縣洛溪站洛隆縣洛龍區洪廟村海幢寺海底擴張海底擴張說淫猥深不可測準噶爾準將溜之大吉滇藏川滬綜指激忿灌腸炭疽桿菌無暇煎魚照會熱水澡燈會爽捷牙縫刷狗玩兒的猥褻獷悍玉皇頂玩兒不轉玩兒命玩兒壞玩兒完玩兒得轉玩兒花招玩失蹤玩藝兒甄綜甲乙甲亢甲狀腺功能亢進申購登位百分目及直行瞞哄知會知識論秋試科舉考試第二處筆會約分紅寺堡紅寺堡區紅寺堡鎮納什維爾紛擾結核桿菌練習簿織田信長繚繞繞一圈繞遠兒纏擾羊腸小道老爺嶺耍錢耿馬傣族佤族自治縣聖路易斯聚乙烯聚氯乙烯肉毒桿菌能說會道臥位自愧不如自愧弗如興都庫什船到橋門自會直苯乙烯莫洛尼莰烷菏蘭華嚴經萌渚嶺萬難葉蘊儀蒙娜麗莎蒙特卡洛法蕉嶺蕉嶺縣蕾哈娜薩噶達娃節薩斯喀徹溫藍妹蠡測行衰運袷襖製表複試西門町西雙版納傣族自治州覆蓋面觥籌交錯許冠傑許冠英調嘴學舌諂媚豬舍貝聿銘買斷賞錢購物券購物大廈購物廣場購物袋購物車購買者贖錢赤坎區超高速乙太網路越演越烈越陷越深趙構跨地區跨學科跨海大橋跨線橋跨鶴揚州跨鶴西遊路易十四路易威登路易港身陷身陷囹圄軍團桿菌軍帽農會返券黃牛送客檯送返這刻這是這次達嚕噶齊那打素醫院酥酪醃漬量販店金平苗族瑤族傣族自治縣金平苗瑤傣自治縣金銀膶閉塞眼睛捉麻雀閑暇關帝廟關廟防水表阿喀琉斯陜西院試陷於癱瘓階地雙江拉祜族佤族布朗族傣族自治縣電位電玩震中霍亂桿菌靈谷寺靈雀寺青年會非微擾非誠勿擾面試會頤和園頻帶首重香港考試及評核局馬噶爾尼馬噶爾尼使團馬戛爾尼馬戛爾尼使團馬拉喀什馬斯喀特騎驢找驢騷擾客蚤騷驢驍悍驢友驢年馬月驢脣馬觜驢駒子驢騾高大上高蹺鬧哄哄鬧著玩兒魂牽夢繞鮑威爾鮑德里亞鮑羅丁鱖魚鵠立黃大仙廟黎曼面點將龍舌蘭酒 '''.split()) # Sentence-final particles: their tone follows the intonation, so a new word # may not change the default reading of one at its end (你好嗎 maa1). PARTICLES = '呀啊吖喇啦嘞囉咯喎㗎嘅咩嗎呢噃啫咋嘛喔哦吓啩' # The character dictionaries of cantonese-books-data used for characters that # LSHK and rime-cantonese lack, most authoritative first. The last five # files are four pre-1940 books, used through the modern Jyutping their # digitiser derived (粵拼讀音). Not used: the older books, which carry # reconstructed readings only (粵拼擬音), and 1962_廣州音字彙, withdrawn from # the collection as inaccurate. BOOKS = [ '2004_廣州話正音字典/B01_資料.json', '1992_常用字廣州話讀音表/C01_資料.json', '1992_香港中學生中文詞典/B01_資料.json', '1988_廣州話標準音字彙/B01_資料.json', '1974_1996_粵語同音字典/B01_資料.json', '1941_粵音韻彙/B01_資料.json', '1985_粵語查音識字字典/B01_資料.json', '1971_同音字彙/B01_資料.json', '1947_The_Students_Cantonese_English_Dictionary/B01_資料.json', '1967_部身字典/B01_資料.json', '1941_道字典/B01_資料.json', '1939_道漢字音/C01_卷一_道漢字典_資料.json', '1939_道漢字音/D01_卷二_粵語音典_資料.json', '1931_民眾識字粵語拼音字彙/B01_資料.json', '1916_廣話國語一貫未定稿/B01_資料.json', '1914_分部分音廣話九聲字宗/B01_資料.json', ] # Labels (讀音標記) of readings that never become the default: popular but # incorrect, old, archaic, original, wrong, rare, proper names only. NOT_DEFAULT = ('俗', '舊', '古', '本', '原', '誤', '罕', '專名') def is_han(c): o = ord(c) return (0x3400 <= o <= 0x4DBF or 0x4E00 <= o <= 0x9FFF or 0xF900 <= o <= 0xFAFF or 0x20000 <= o <= 0x323AF or o == 0x3007) def rime_rows(path): """Yield the tab-separated rows of a rime dict.yaml body.""" body = False for line in open(path, encoding='utf8'): line = line.rstrip('\n') if line == '...': body = True continue if body and line and not line.startswith('#'): yield line.split('\t') def load_lshk(): readings = collections.defaultdict(list) with open(ROOT / 'jyutping-table-master/list.tsv', encoding='utf8') as f: next(f) for line in f: ch, _, jp = line.split('\t')[:3] if is_han(ch) and SYL.match(jp) and jp not in readings[ch]: readings[ch].append(jp) return readings def load_rime_chars(): """char -> [(reading, weight)], weight None for the primary reading.""" chars = collections.defaultdict(list) for row in rime_rows(ROOT / 'rime-cantonese/jyut6ping3.chars.dict.yaml'): ch, jp = row[0], row[1] weight = float(row[2].rstrip('%')) if len(row) > 2 and row[2] else None if len(ch) == 1 and is_han(ch) and SYL.match(jp): chars[ch].append((jp, weight)) return chars def load_variants(freq): """Map HK/TW variant forms to the OpenCC standard form rime uses. freq counts how often each character occurs in rime's word list; a mapping is kept only when rime prefers the standard form (so 參 is not folded into the rare 蔘, nor 針 into 鍼).""" standard = set() targets = collections.defaultdict(set) for name in ('HKVariants.txt', 'TWVariants.txt'): for line in open(ROOT / 'opencc' / name, encoding='utf8'): if line.startswith('#') or not line.strip(): continue key, values = line.rstrip('\n').split('\t') standard.add(key) for v in values.split(): if v != key: targets[v].add(key) # A form that is itself standard (才, 煙, 核 ...) or ambiguous (么) stays. canon = {v: s for v, (s,) in ((v, tuple(ks)) for v, ks in targets.items() if len(ks) == 1) if v not in standard and len(v) == 1 and freq[s] >= freq[v]} canon.update(CURATED_VARIANTS) return canon def load_book(path): """char -> [(reading, label)] of a cantonese-books-data file, in book order; the label (讀音標記) is '' when there is none.""" def walk(x, out): if isinstance(x, dict): if isinstance(x.get('粵拼讀音'), str): label = x.get('讀音標記', x.get('標記')) or '' out.append((x['粵拼讀音'], ''.join(label))) # a list of labels too for v in x.values(): walk(v, out) elif isinstance(x, list): for v in x: walk(v, out) return out book = collections.defaultdict(list) for entry in json.load(open(path, encoding='utf8')): heads = entry.get('字頭') for ch in [heads] if isinstance(heads, str) else heads or []: if isinstance(ch, str) and len(ch) == 1: book[ch] += [r for r in walk(entry, []) if r not in book[ch]] return book def load_cedict(path): """word -> readings of a CC-Canto style file (the Jyutping in braces after the pinyin). Of alternatives 'a / b' the first is taken; a changed tone, marked cin4*2 or hang4*haang4, is read as the changed form.""" words = collections.defaultdict(list) for line in open(path, encoding='utf8'): m = re.match(r'(\S+) \S+ \[[^]]*\] \{([^}]*)\}', line) if not m: continue syls = [] for s in m.group(2).split('/')[0].lower().split(): s, _, new = s.partition('*') syls.append(s[:-1] + new if new.isdigit() else new or s) if syls not in words[m.group(1)]: words[m.group(1)].append(syls) return words def with_aliases(words, canon): """The word list plus aliases spelt with canonical characters, so 因為 finds rime's 因爲; also the number of aliases dropped as clashes.""" aliases, clashes = {}, 0 for w, syls in words.items(): cw = ''.join(canon.get(c, c) for c in w) if cw == w or cw in words: continue if cw in aliases and aliases[cw] != syls: clashes += 1 continue aliases[cw] = syls return {**words, **aliases}, len(aliases), clashes def reader(chars, canon, lex): """The readings of a run of text with the word list lex, segmented as xjyutping.sty and xjyutping-py do: the fewest words, then the fewest single characters; on a tie the longer final word wins.""" longest = collections.defaultdict(int) for w in lex: longest[w[-1]] = max(longest[w[-1]], len(w)) def read(run): cr = ''.join(canon.get(c, c) for c in run) cost, back = [0] * (len(run) + 1), [1] * (len(run) + 1) for i in range(1, len(run) + 1): best = cost[i - 1] + 100001 for size in range(2, min(i, max(longest[run[i - 1]], longest[cr[i - 1]])) + 1): if cost[i - size] + 100000 <= best and (run[i - size:i] in lex or cr[i - size:i] in lex): best, back[i] = cost[i - size] + 100000, size cost[i] = best out, j = [], len(run) while j: i = j - back[j] out[:0] = (lex.get(run[i:j]) or lex[cr[i:j]]) if j - i > 1 else [chars[run[i]][0]] j = i return out return read def pick_default(ch, rime, lshk): if ch in CURATED_DEFAULTS: return CURATED_DEFAULTS[ch] order = lshk.get(ch, []) rank = lambda r: order.index(r) if r in order else len(order) if rime: primary = [r for r, w in rime if w is None] if not primary: top = max(w for _, w in rime) primary = [r for r, w in rime if w == top] return min(primary, key=rank) # stable: ties keep rime order return order[0] def main(): global ROOT import argparse ap = argparse.ArgumentParser(description='Generate the xjyutping data files.') ap.add_argument('--sources', type=pathlib.Path, default=ROOT, help='directory holding the source folders (default: %(default)s)') ap.add_argument('--py-data', type=pathlib.Path, default=None, help='data directory of the Python package (default: %s, ' 'skipped if that package is not there)' % PY_DATA) args = ap.parse_args() ROOT = args.sources.resolve() py_data = args.py_data or (PY_DATA if PY_DATA.parent.is_dir() else None) lshk = load_lshk() rime = load_rime_chars() rows = [(r[0], r[1].split()) for r in rime_rows(ROOT / 'rime-cantonese/jyut6ping3.words.dict.yaml')] canon = load_variants(collections.Counter(c for w, _ in rows for c in w)) chars = {} # char -> (default, other readings, flag as polyphone?) known = {} # char -> every reading a source gives it for ch in sorted(set(lshk) | set(rime) | set(canon)): r = rime.get(ch) or rime.get(canon.get(ch, '')) l = lshk.get(ch) or lshk.get(canon.get(ch, ''), []) if not r and not l: continue default = pick_default(ch, r, {ch: l}) if r: # rime weights: none = primary, 5% common, 3% uncommon, 0% rare weight = {jp: 100 if w is None else w for jp, w in r} else: weight = {jp: 100 for jp in l} others = sorted((jp for jp, w in weight.items() if w >= 3 and jp != default), key=lambda jp: l.index(jp) if jp in l else len(l)) # A changed tone (人 jan4 -> jan2, 姨 ji4 -> ji1) belongs to words, # so it alone does not make a character ambiguous. changed = lambda jp: (jp[:-1] == default[:-1] and jp[-1] in '12' and default[-1] not in '12') flag = any(weight[jp] >= 5 and not (changed(jp) and weight[jp] < weight.get(default, 100)) for jp in others) chars[ch] = (default, others, flag) known[ch] = set(weight) | set(l) # Characters LSHK and rime lack, from the first book that has them: its # first reading not labelled as incorrect, old or rare. Compatibility # ideographs (the same characters as unified ones) are left out. from_books = collections.Counter() for name in BOOKS: for ch, rs in sorted(load_book(ROOT / 'cantonese-books-data' / name).items()): rs = [(jp, label) for jp, label in rs if SYL.match(jp)] if (ch in chars or not rs or not is_han(ch) or unicodedata.normalize('NFC', ch) != ch): continue good = [jp for jp, label in rs if not any(x in label for x in NOT_DEFAULT)] default = (good or [rs[0][0]])[0] others = [jp for jp in dict.fromkeys(good) if jp != default] chars[ch] = (default, others, bool(others)) known[ch] = {jp for jp, _ in rs} from_books[name.split('/')[0]] += 1 chars = dict(sorted(chars.items())) # --- words ----------------------------------------------------------- entries = collections.defaultdict(list) usage = collections.defaultdict(collections.Counter) for word, syls in rows: if (len(word) < 2 or len(syls) != len(word) or not all(c in chars for c in word) or not all(SYL.match(s) for s in syls)): continue if syls not in entries[word]: entries[word].append(syls) for c, s in zip(word, syls): usage[c][s] += 1 def score(word, syls): # how usual each syllable is for its char return sum(math.log(usage[c][s] + 1) for c, s in zip(word, syls)) words = {w: max(rs, key=lambda s: score(w, s)) for w, rs in entries.items()} for w, jp in CURATED_WORDS.items(): assert len(jp.split()) == len(w) and all(SYL.match(x) for x in jp.split()), w words[w] = jp.split() dupes = sum(1 for rs in entries.values() if len(rs) > 1) def adverbs(ws): # The adverb suffix 地 (慢慢地, 麻麻地) is often written 哋. for w, syls in list(ws.items()): if w.endswith('地') and syls[-1] == 'dei2': words.setdefault(w[:-1] + '哋', syls) adverbs(words) # Words of CC-Canto and the CC-CEDICT Cantonese readings that the list # (by raw or canonical spelling) does not have. A reading is taken if # - each syllable is a reading LSHK, rime or the book gives its # character, or a changed tone of one (tone 1 or 2); # - it keeps a reading rime gives to every word and every pair of # characters of the list inside it (rime stays authoritative); # - it does not change the reading of a final particle (你好嗎 maa1); # - for two characters AB: no word of the list ends in A with another # reading, as a tie goes to the later word (屋企|住 -> 屋|企住). # CC-Canto, the Cantonese dictionary, wins over the readings of Mandarin # words; then the most usual syllables. cedict = ROOT / 'jyut-dict/src/dictionaries/cedict/data' sources = {'CC-Canto': load_cedict(cedict / 'CC-CANTO.txt'), 'CC-CEDICT readings': load_cedict(cedict / 'READINGS.txt')} cw = lambda w: ''.join(canon.get(c, c) for c in w) listed = {w: entries[w] if w in entries and w not in CURATED_WORDS else [s] for w, s in words.items()} listed = {**{cw(w): rs for w, rs in listed.items()}, **listed} pairs = collections.defaultdict(set) # two characters -> readings in the list ends = collections.defaultdict(set) # char -> readings ending a word of the list for w, rs in listed.items(): for s in rs: ends[w[-1]].add(s[-1]) for i in range(len(w) - 1): pairs[w[i:i + 2]].add((s[i], s[i + 1])) def problem(w, syls): """Why the reading syls of the new word w is not taken, or None.""" if len(syls) != len(w) or not all( SYL.match(s) and (s in known[c] or s[-1] in '12' and any(k[:-1] == s[:-1] for k in known[c])) for c, s in zip(w, syls)): return 'unknown readings' if not all(syls[i:j] in listed.get(w[i:j], listed.get(cw(w[i:j]), [syls[i:j]])) for i in range(len(w)) for j in range(i + 2, len(w) + 1)): return 'a word inside read otherwise' if not all(tuple(syls[i:i + 2]) in pairs.get(cw(w[i:i + 2]), {tuple(syls[i:i + 2])}) for i in range(len(w) - 1)): return 'two characters read otherwise' if w[-1] in PARTICLES and syls[-1] != chars[w[-1]][0]: return 'final particle' if len(w) == 2 and ends[cw(w[0])] - {syls[0]}: return 'first character ends a word read otherwise' return None rejected = collections.Counter() new = {} # word -> (reading, source) for w in sorted(set().union(*sources.values())): if len(w) < 2 or w in listed or cw(w) in listed: continue if w in EXCLUDED_WORDS: rejected['excluded by hand'] += 1 continue if not all(c in chars for c in w): rejected['characters outside the table'] += 1 continue why = {(name, tuple(s)): problem(w, s) for name, src in sources.items() for s in src.get(w, [])} ok = [(name, list(s)) for (name, s), p in why.items() if not p] if not ok: rejected[next(iter(why.values()))] += 1 continue name = ok[0][0] new[w] = (max((s for n, s in ok if n == name), key=lambda s: score(w, s)), name) # Only the words that change a reading: shortest first (only shorter # words can change how a word is read), add those that the list, with # the shorter words added, reads otherwise. added = {} for size in sorted({len(w) for w in new}): read = reader(chars, canon, with_aliases({**words, **added}, canon)[0]) added.update({w: s for w, (s, _) in new.items() if len(w) == size and read(w) != s}) rejected['read so already'] = len(new) - len(added) words.update(added) adverbs(added) from_cedict = collections.Counter(new[w][1] for w in added) allwords, n_aliases, clashes = with_aliases(words, canon) longest = collections.defaultdict(int) # final char -> longest word for w in allwords: longest[w[-1]] = max(longest[w[-1]], len(w)) # --- output ---------------------------------------------------------- header = [ '%% Generated by tools/build-data.py -- do not edit.', '%% Distributed under CC BY-SA 4.0 (it adapts CC BY-SA material);', '%% see README.md for the sources and their authors.', '%% Sources: LSHK Jyutping table (CC BY 4.0),', '%% rime-cantonese jyut6ping3 dictionaries (CC BY 4.0),', '%% OpenCC HK/TW variant tables (Apache-2.0),', ] with open(REPO / 'xjyutping-chars.def', 'w', encoding='utf8') as f: f.write('\n'.join(header) + '\n%% cantonese-books-data readings of ' 'characters (no licence stated).\n') f.write('\\ProvidesFile{xjyutping-chars.def}[%s xjyutping character data]\n' % VERSION) for ch, (default, others, flag) in chars.items(): if flag: # polyphone: default, then the other readings f.write('\\xjp@M %s%s %s;\n' % (ch, default, ' '.join(others))) else: f.write('\\xjp@C %s%s;\n' % (ch, default)) for v, s in sorted(canon.items()): if v in chars: f.write('\\xjp@V %s%s;\n' % (v, s)) for ch, jp in CURATED_FINALS.items(): f.write('\\xjp@F %s%s;\n' % (ch, jp)) with open(REPO / 'xjyutping-words.def', 'w', encoding='utf8') as f: f.write('\n'.join(header) + '\n%% CC-Canto and CC-CEDICT Cantonese readings ' '(CC BY-SA 3.0).\n') f.write('\\ProvidesFile{xjyutping-words.def}[%s xjyutping word data]\n' % VERSION) for w in sorted(allwords): f.write('\\xjp@W %s=%s;\n' % (w, ' '.join(allwords[w]))) for c in sorted(longest): f.write('\\xjp@E %s%d;\n' % (c, longest[c])) # The same tables for the Python package (it derives the longest-word # bound from words.tsv itself). if py_data: py_data.mkdir(parents=True, exist_ok=True) def tsv(name, rows): with open(py_data / name, 'w', encoding='utf8', newline='\n') as f: f.writelines('\t'.join(row) + '\n' for row in rows) tsv('chars.tsv', ((ch, d, ' '.join(o), '1' if fl else '0') for ch, (d, o, fl) in chars.items())) tsv('finals.tsv', CURATED_FINALS.items()) tsv('variants.tsv', ((v, s) for v, s in sorted(canon.items()) if v in chars)) tsv('words.tsv', ((w, ' '.join(allwords[w])) for w in sorted(allwords))) else: print('Python data skipped: %s not found' % PY_DATA.parent, file=sys.stderr) print('chars %d (polyphonic %d), variants %d, words %d (+%d aliases, ' '%d alias clashes, %d words with several readings)' % ( len(chars), sum(f for _, _, f in chars.values()), len(canon), len(words), n_aliases, clashes, dupes), file=sys.stderr) print('chars added from cantonese-books-data: %d (%s)' % ( sum(from_books.values()), ', '.join('%s %d' % kv for kv in from_books.items())), file=sys.stderr) print('words added from jyut-dict: %d (%s); not added: %s' % ( len(added), ', '.join('%s %d' % kv for kv in sorted(from_cedict.items())), ', '.join('%d %s' % (n, why) for why, n in rejected.most_common())), file=sys.stderr) if __name__ == '__main__': main()