這幾天在看輸入法的東西，資料比較少，特此記錄一下，這里使用到了前向概率Viterbi算法，公式是
P = 初始狀態(tài)概率*轉(zhuǎn)移概率*發(fā)射概率
這里會得到很多概率值，我們只獲取最大概率的中文漢字鏈。
定義一個漢字節(jié)點(diǎn)
type Node struct {
Word string //漢字
Py string //拼音
Emission float64 //漢字出現(xiàn)的概率
MaxScore float64 //最大分?jǐn)?shù)
PreNode *Node //下一個漢字節(jié)點(diǎn)
}
定義需要的數(shù)據(jù)
/**
讀取整理好的漢字拼音數(shù)據(jù)，獲取的map數(shù)據(jù)，數(shù)據(jù)格式類型為
{
"ni":{"你"：0.91，"尼"：0.789,...},
"wo":{"我"：0.91，"窩"：0.189,...},
.....
}
**/
var emissionMap = make(map[string]map[string]float64)
/**
讀取大量詞語數(shù)據(jù)獲取的詞語數(shù)組
**/
var wordsArray = make([]string, 1)
/**
分析漢字拼音數(shù)據(jù)，獲取的數(shù)組數(shù)據(jù)。格式如下：
[
{
"你"：Node ，
"好"：Node，
},
...
]
**/
var inputSequence = make([]map[string]*Node, 1)
/**
詞語出現(xiàn)的概率值，這里會將詞語位置顛倒
｛
"好你"：0.789，
"興高":0.567,
....
｝
**/
var freqMap = make(map[string]float64)
/**
viterbi算法的數(shù)據(jù)緩存
**/
var viterbi_cache = make(map[string]float64)
/**
輸入的已經(jīng)切分的拼音數(shù)組
**/
var pinyins []string
讀取拼音漢字?jǐn)?shù)據(jù)
func readPinyinData() {
f, err := os.Open("googlepinyin.txt")
if err != nil {
fmt.Println(err.Error())
return
}
buf := bufio.NewReader(f)
for {
line, err := buf.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
fmt.Println(err.Error())
break
}
line = strings.TrimSpace(line)
if len(line) <= 1 {
continue
}
PinyinArray := strings.Split(line, " ")
word := PinyinArray[0]
f, _ := strconv.ParseFloat(PinyinArray[1], 64)
//因為這里獲取是個大于1的值，所以求正切值
em := math.Atan(f) / (math.Pi / 2)
py := PinyinArray[len(PinyinArray)-1]
if len(strings.Split(word, "")) > 1 {
continue
}
if emissionMap[py] == nil {
emissionMap[py] = make(map[string]float64)
}
emissionMap[py][word] = em
pyArray := strings.Split(py, "")
py = pyArray[0]
if len(strings.Split(word, "")) > 1 {
break
}
if emissionMap[py] == nil {
emissionMap[py] = make(map[string]float64)
}
emissionMap[py][word] = em
}
讀取詞語數(shù)據(jù)
func readWords() {
f, err := os.Open("RenMinData.txt")
if err != nil {
fmt.Println(err.Error())
return
}
buf := bufio.NewReader(f)
for {
line, err := buf.ReadString('\n')
if err != nil {
if err == io.EOF {
break
}
fmt.Println(err.Error())
break
}
line = strings.TrimSpace(line)
if len(line) <= 1 {
continue
}
wordsArray = append(wordsArray, "<s>")
words := strings.Split(line, "")
for j := 0; j < len(words); j++ {
wordsArray = append(wordsArray, strings.TrimSpace(words[j]))
}
wordsArray = append(wordsArray, "</s>")
}
wordsArray = wordsArray[1:]
for k, _ := range wordsArray {
i := k
key := ""
for j := i; (i-j < 6) && (j >= 0); j-- {
key += wordsArray[j]
freqMap[key] = freqMap[key] + 1
}
}
}
獲取轉(zhuǎn)移概率
func get_trans_prop(args ...string) float64 {
key := ""
for _, v := range args {
key += v
}
C_2 := float64(freqMap[key] + 1.0)
C_1 := float64(len(wordsArray))
return C_2 / C_1
}
獲取初始概率
func get_init_prop(word string) float64 {
return get_trans_prop(word, "<s>")
}
獲取inputSequence 數(shù)據(jù)
func get_inputSequence(pys []string) {
pinyins = pys
for _, v := range pinyins {
mymap := make(map[string]*Node)
if emissionMap[v] == nil {
continue
}
for w, r := range emissionMap[v] {
mymap[w] = &Node{Word: w, Emission: r, Py: v}
}
inputSequence = append(inputSequence, mymap)
}
inputSequence = inputSequence[1:]
}
???
viterbi算法
func get_key(t int, k string) string {
return strings.Join([]string{strconv.Itoa(t), k}, "_")
}
func viterbi(t int, k string) float64 {
//如果有緩存數(shù)據(jù)，直接返回數(shù)據(jù)值
if viterbi_cache[get_key(t, k)] != 0 {
return viterbi_cache[get_key(t, k)]
}
node := inputSequence[t][k]
if t == 0 {
state_transfer := get_init_prop(k)
emission_prop := emissionMap[pinyins[t]][k]
node.MaxScore = 1.0 * state_transfer * emission_prop
viterbi_cache[get_key(t, k)] = node.MaxScore
return node.MaxScore
}
n := t - 1
for i, v := range inputSequence[n] {
state_transfer := get_trans_prop(k, i)
emission_prop := emissionMap[pinyins[n]][i]
if len(pinyins)-1 == t {
emission_prop *= emissionMap[pinyins[t]][k]
}
score := viterbi(n, i) * state_transfer * emission_prop
if score > node.MaxScore {
node.MaxScore = score
node.PreNode = v
}
}
viterbi_cache[get_key(t, k)] = node.MaxScore
return node.MaxScore
}
獲取最大概率的數(shù)據(jù)
func translate(pys []string) {
get_inputSequence(pys)
// 使用viterbi算法求解最大路徑
var max_node *Node
max_score := 0.0
for k, node := range inputSequence[len(inputSequence)-1] {
score := viterbi(len(pys)-1, k)
if score > max_score {
max_score = score
max_node = node
}
}
// 回溯輸出最大路徑
results := make([]string, 1)
for {
results = append(results, max_node.Word)
if max_node.PreNode != nil {
max_node = max_node.PreNode
} else {
break
}
}
results = results[1:]
for i := len(results) - 1; i >= 0; i-- {
fmt.Print(results[i])
}
}
使用
func main() {
readPinyinData()
readWords()
translate([]string{"ni", "zai", "na", "li"})
}
