添加声母、韵母频次权重

This commit is contained in:
zhiyang7 2022-02-24 11:03:54 +08:00
parent 494a13827b
commit 8e2ed8923e
1 changed files with 73 additions and 11 deletions

84
main.py
View File

@ -5,6 +5,8 @@ import os
import re
import sys
import traceback
from numpy import NaN
import requests, time
import pandas as pd
from flask import Flask, jsonify, render_template, request
@ -20,14 +22,55 @@ TONE_TABLE = [
set(['á', 'é', 'í', 'ó', 'ú', 'ǘ']),
set(['ǎ', 'ě', 'ǐ', 'ǒ', 'ǔ', 'ǚ']),
set(['à', 'è', 'ì', 'ò', 'ù', 'ǜ']),
set(['a', 'e', 'i', 'o', 'u', 'ü'])
]
TONE_MAP = {
'ā': 'a', 'ē': 'e', 'ī': 'i', 'ō': 'o', 'ū': 'u', 'ǖ': 'v',
'á': 'a', 'é': 'e', 'í': 'i', 'ó': 'o', 'ú': 'u', 'ǘ': 'v',
'ǎ': 'a', 'ě': 'e', 'ǐ': 'i', 'ǒ': 'o', 'ǔ': 'u', 'ǚ': 'v',
'à': 'a', 'è': 'e', 'ì': 'i', 'ò': 'o', 'ù': 'u', 'ǜ': 'v',
'a': 'a', 'e': 'e', 'i': 'i', 'o': 'o', 'u': 'u', 'ü': 'v'
}
INITIAL_WEIGHT = {"y":12789,"j":9523,"sh":9043,"zh":7735,"x":7656,"b":7649,"l":7240,"d":6712,"h":5992,"q":5711,"g":5614,"w":5475,"m":5111,"f":5059,"ch":4965,"t":4680,"r":3906,"z":3186,"s":3149,"c":2236,"k":2210,"":2142,"n":2079,"p":1866,"":549,"":1}
FINAL_WEIGHT = {"i":21826,"u":18074,"an":7550,"ing":5519,"ian":5083,"ou":4657,"en":4652,"eng":4648,"ong":4567,"ang":4460,"ao":4450,"e":4446,"in":3827,"ai":3721,"uan":3337,"uo":3279,"ei":3205,"a":3141,"ui":2936,"un":2366,"iao":2299,"iu":1937,"iang":1845,"ie":1786,"ue":1367,"o":1356,"ia":1231,"uang":1061,"er":1008,"ua":864,"":549,"v":417,"iong":381,"uai":373,"ve":59,"":1}
def tone_only(pinyin, **kwargs):
for i in range(4):
for i in range(5):
if len(set(pinyin).intersection(TONE_TABLE[i])) > 0:
return str(i + 1)
return pinyin
def pinyin_to_tone(pinyin, **kwargs):
pinyin = ''.join(filter(str.isdigit, pinyin))
if(len(pinyin) <= 0):
return '5'
if(pinyin[0] == '1'):
return '1'
elif (pinyin[0] == '2'):
return '2'
elif (pinyin[0] == '3'):
return '3'
elif (pinyin[0] == '4'):
return '4'
return '5'
def pinyin_only(pinyin, **kwargs):
for i in range(5):
if len(set(pinyin).intersection(TONE_TABLE[i])) > 0:
for s in TONE_TABLE[i]:
pinyin = pinyin.replace(s, TONE_MAP[s])
return pinyin
def merge_pinyin_tone(pinyin, tone):
plist = pinyin.split()
for i, t in enumerate(tone):
if(tone != '5'):
plist[i] = plist[i] + t
return ' '.join(plist)
style.register('TONE_ONLY', tone_only)
def trim_space(s):
@ -44,19 +87,31 @@ def compute_pinyin(s, style=None):
all_idiom = None
def query_word(word):
# {"tipsList":[{"fpindex":"andmkbblbeoojhcp","fpinyin":"bā miàn zhāng luo","fname":"八面张罗"}],"found":"true"}
ret = requests.get("https://cy.hwxnet.com/searchtips.do?wd=%s&timestamp=%d&qt=1" % (word,int(time.time() * 1000)))
time.sleep(0.1)
data = ret.json()
if len(data["tipsList"]) > 0:
return data["tipsList"][0]['fpinyin']
return ''
def init():
global all_idiom
if os.path.exists("all_idiom.csv"):
all_idiom = pd.read_csv('all_idiom.csv')
all_idiom["pinyin_initials"].str.split(",").explode("splited").value_counts().to_json()
all_idiom["pinyin_finals"].str.split(",").explode("splited").value_counts().to_json()
else:
all_idiom = pd.read_json('idiom.json')
idiom_frequency = pd.read_csv('idiom_frequency.csv')
all_idiom = all_idiom.merge(idiom_frequency, how='outer', on='word')
all_idiom['frequency'] = all_idiom['frequency'].fillna(1).astype(int)
all_idiom['pinyin_r'] = all_idiom.apply(lambda x: compute_pinyin(x['word']) if x['pinyin_r'] else x['pinyin_r'], axis=1)
all_idiom['pinyin'] = all_idiom.apply(lambda x: compute_pinyin(x['word'], style=Style.TONE) if x['pinyin'] != NaN else x['pinyin'], axis=1)
all_idiom['pinyin_r'] = all_idiom.apply(lambda x: pinyin_only(x['pinyin']) if x['pinyin_r'] != NaN else x['pinyin_r'], axis=1)
all_idiom['pinyin_rt'] = all_idiom.apply(lambda x: int(''.join(map(lambda y: str(len(y)), re.split('[ ,]',x['pinyin_r'])))), axis=1)
all_idiom['pinyin'] = all_idiom.apply(lambda x: compute_pinyin(x['word'], style=Style.TONE) if x['pinyin'] else x['pinyin'], axis=1)
all_idiom['pinyin_tone'] = all_idiom.apply(lambda x: ''.join(lazy_pinyin(x['word'], style="TONE_ONLY")), axis=1)
all_idiom['pinyin_tone'] = all_idiom.apply(lambda x: ''.join([tone_only(y) for y in x['pinyin'].split()]), axis=1)
all_idiom['pinyin_initials' ] = all_idiom.apply(lambda x: ','.join(list(lazy_pinyin(x['word'], style=Style.INITIALS, strict=False))), axis=1)
all_idiom['pinyin_finals'] = all_idiom.apply(lambda x: ','.join(list(lazy_pinyin(x['word'], style=Style.FINALS, strict=False))), axis=1)
all_idiom.to_csv("all_idiom.csv")
@ -216,11 +271,18 @@ def print_max_group(all_idiom, group, num):
for item in get_max_group(all_idiom, group, num):
print(item)
def get_pinyin_weight(initials, finals):
initials = set(initials.split(','))
weight = sum([INITIAL_WEIGHT[w] for w in initials])
finals = set(finals.split(','))
weight += sum([FINAL_WEIGHT[w] for w in finals])
return weight
def get_max_group(all_idiom, group, num):
result = list()
if(len(group) == 0):
return result
group['pinyin_c'] = group.apply(lambda x: (math.log(x['frequency'], 2)/16 + 1) * len(set(trim_space(x['pinyin_r']))), axis=1)
group['pinyin_c'] = group.apply(lambda x: (math.log(x['frequency'], 2)/16 + 1) * get_pinyin_weight(x['pinyin_initials'], x['pinyin_finals']), axis=1)
ret_list = group.nlargest(num, ['pinyin_c', 'frequency']).index.tolist()
for i in ret_list:
result.append(json.loads(all_idiom.loc[i].to_json(orient = 'index',force_ascii=False)))
@ -250,10 +312,10 @@ def main(argv):
# parameter = '3323'
# mode = '1'
# parameter = 'bai vvv vv vvv,012 000 00 000;bai tou er xin,012 010 10 001'
# mode = '2'
# parameter = '2134'
mode = '3'
parameter = '行之有效,00 11 10 00 0100 1101;各执一词,00 11 12 22 0010 1022'
mode = '2'
parameter = '2134'
# mode = '3'
# parameter = '行之有效,00 11 10 00 0100 1101;各执一词,00 11 12 22 0010 1022'
num = 6
try:
opts, args = getopt.getopt(argv, "hm:p:n:", ["mode=", "parameter=", "num="])
@ -280,5 +342,5 @@ if __name__ == '__main__':
current_work_dir = os.path.dirname(__file__)
os.chdir(current_work_dir)
init()
# main(sys.argv[1:])
app.run(debug=True, host="0.0.0.0")
main(sys.argv[1:])
# app.run(debug=True, host="0.0.0.0")