示例#1
0
文件: tnt.go 项目: uohzoaix/gonlp
func (tnt *TnT) Tag(data []string) []pojo.Result {
	//log.Println("tnt", tnt)
	tags := []pojo.Tag{}
	tags = append(tags, pojo.InitTag(pojo.InitPre("BOS", "BOS"), 0.0, ""))
	stage := make(map[string]pojo.StageValue)

	for _, ch := range data {
		stage = make(map[string]pojo.StageValue)
		samples := tnt.Status
		if val, ok := tnt.Word[ch]; ok {
			samples = val
		}
		for _, s := range samples {
			wd := math.Log(tnt.Wd.Get(s+"-"+ch)) - math.Log(tnt.Uni.Get(s))
			for _, tag := range tags {
				p := tag.GetScore() + wd + tnt.Trans[tag.GetPre().ToString()+"-"+s]
				pre := pojo.Pre{tag.GetPre().GetTwo(), s}.ToString()
				if sv, ok := stage[pre]; !ok || p > sv.GetScore() {
					if tag.GetSuffix() == "" {
						stage[pre] = pojo.StageValue{p, s}
					} else {
						stage[pre] = pojo.StageValue{p, tag.GetSuffix() + "-" + s}
					}
				}
			}
		}
		tags = []pojo.Tag{}
		for key, val := range stage {
			tags = append(tags, pojo.Tag{pojo.InitPre(strings.Split(key, "-")[0], strings.Split(key, "-")[1]), val.GetScore(), val.GetValue()})
		}
		temp := make(map[float64]pojo.Tag)
		for _, val := range tags {
			temp[val.GetScore()] = val
		}
		mk := []float64{}
		i := 0
		for k, _ := range temp {
			mk = append(mk, k)
			i++
		}
		sort.Sort(sort.Reverse(sort.Float64Slice(mk)))
		tags = []pojo.Tag{}
		i = 0
		for _, v := range mk {
			if i < tnt.getNum() {
				tags = append(tags, temp[v])
			}
			i++
		}
	}
	tags = []pojo.Tag{}
	for key, val := range stage {
		score := val.GetScore() + tnt.getEos(strings.Split(key, "-")[1])
		tags = append(tags, pojo.Tag{pojo.InitPre(strings.Split(key, "-")[0], strings.Split(key, "-")[1]), score, val.GetValue()})
	}
	temp := make(map[float64]pojo.Tag)
	for _, val := range tags {
		temp[val.GetScore()] = val
	}
	mk := []float64{}
	for k, _ := range temp {
		mk = append(mk, k)
	}
	sort.Sort(sort.Reverse(sort.Float64Slice(mk)))
	tags = []pojo.Tag{}
	for _, v := range mk {
		tags = append(tags, temp[v])
	}
	results := []pojo.Result{}
	tagArr := strings.Split(tags[0].GetSuffix(), "-")
	if len(tagArr) != len(data) {
		panic("出错了!")
	}
	for i, val := range data {
		results = append(results, pojo.Result{val, tagArr[i]})
	}
	return results
}
示例#2
0
文件: test.go 项目: uohzoaix/gonlp
func main() {
	arr := make([]string, 0)
	arr = append(arr, "fdf")
	arr = append(arr, "gfgg")
	test := Test{"aa", arr}
	t := reflect.TypeOf(test)
	v := reflect.ValueOf(test)
	fmt.Println(reflect.TypeOf(test).Name())
	test1 := (*Test)(unsafe.Pointer(&test))
	//test.(Test)
	fmt.Println(test1.Name)
	for i := 0; i < t.NumField(); i++ {
		//根据索引获取属性 StructField
		f := t.Field(i)
		//返回属性的对应值 相当于取值操作
		val := v.Field(i).Interface()
		testval := [2]string{"fdf", "123"}
		fmt.Println([]reflect.Value{reflect.ValueOf(testval)})
		fmt.Println(v.Field(1).Kind() == reflect.Slice)
		//data[f.Name] = val
		fmt.Printf("%6s : %v = %v \n", f.Name, f.Type, val)
	}

	fname := "testfile.txt"
	fout, err := os.Create(fname)
	defer fout.Close()
	if err != nil {
		fmt.Println(fname, err)
		return
	}
	ret, _ := json.Marshal(test)
	fout.Write([]byte(ret))

	fin, err := os.Open(fname)
	defer fin.Close()
	if err != nil {
		fmt.Println(fname, err)
		return
	}
	var result, temp []byte
	var beginPos int
	buf := make([]byte, 2048*10000)
	for {
		num, _ := fin.Read(buf)
		if 0 == num {
			break
		}
		if len(result) > 0 {
			temp = result
			result = make([]byte, beginPos+num)
			copy(result[0:len(temp)], temp[:])
		} else {
			result = make([]byte, num)
		}
		copy(result[beginPos:beginPos+num], buf[:num])
		//os.Stdout.Write(buf[:num])
	}
	//fmt.Println(result)
	testfunc(ret, test)

	data := make(map[string]interface{})
	var testinter interface{}
	testinter = test
	json.Unmarshal([]byte(ret), &testinter)
	fmt.Println("testinter", testinter)
	t = reflect.TypeOf(test)
	v = reflect.ValueOf(&test).Elem()
	for key, val := range data {
		for i := 0; i < t.NumField(); i++ {
			f := t.Field(i)
			if f.Name == key {
				field := v.FieldByName(key)
				if field.IsValid() {
					if field.CanSet() {
						if field.Kind() == reflect.String {
							field.Set(reflect.ValueOf(val.(string)))
						} else if field.Kind() == reflect.Slice {
							val, ok := val.([]interface{})
							if ok {
							}
							fmt.Println(ok, val)
						}
					}
				}
				break
			}
		}
	}
	fmt.Println(test.Teles)

	addOne := util.InitAddOne()
	fmt.Println(addOne.None)
	fmt.Println(addOne.Exist("fdf"))

	dicts := list.New()
	wordTags := list.New()
	wordTags.PushBack(pojo.InitWT("fdf", "gfgg"))
	dicts.PushBack(wordTags)
	fmt.Println(dicts.Front().Value.(*list.List).Front().Value.(*pojo.WordTag).ToString())

	testmaplist := make(map[string]*list.List)
	testmaplist["fdf"] = list.New()

	tags := make([]pojo.Tag, 1)
	tags = append(tags, pojo.InitTag(pojo.InitPre("BOS", "BOS"), 0.0, ""))

	teststring := "你好啊,abcd"
	for _, val := range teststring {
		fmt.Println(string(val))
	}

	teststring = "  fdf fd gfhgh "
	fmt.Println(strings.Fields(teststring))
	fmt.Println(regexp.MustCompile("[^\\s]+").FindAllString(teststring, -1))

	fmt.Println(regexp.MustCompile("([\u4E00-\u9FA5]+)").FindAllString("你好 fdf 费大幅度发", -1))
	fmt.Println(strings.Trim("    fdf     ", " "))

	testsliceappend := []string{"dff", "dgf"}
	testappended := []string{"fdg", "ghhh"}
	testsliceappend = append(testsliceappend, testappended...)
	fmt.Println(testsliceappend)

	fmt.Println(strings.Trim(" fdfd   fdfd   fdf ", " "))
	fmt.Println(strings.SplitN(strings.Trim(" fdfd fdfd fdf ", " "), " ", 2)[1])

	sentences := []string{}
	delimiter := regexp.MustCompile("[,。?!;]")
	for _, line := range strings.SplitN("dfd,fddd\r\nfdgf。gg\r\n", "\r\n", -1) {
		fmt.Println(line)
		line = strings.Trim(line, " ")
		if line == "" {
			continue
		}
		line = delimiter.ReplaceAllString(line, " ")
		for _, sent := range regexp.MustCompile("[^\\s]+").FindAllString(line, -1) {
			sent = strings.Trim(sent, " ")
			fmt.Println(sent)
			if sent == "" {
				continue
			}
			sentences = append(sentences, sent)
		}
	}
	fmt.Println(sentences)

	ma := make(map[string]float64)
	ma["aa"] = 0.1

	tttt := &TTT{ma, "dd"}
	log.Println(tttt)
	log.Println(json.Marshal(tttt))

	tnt := &util.TnT{10, 0.0, 0.0, 0.0, []string{}, util.InitAddOne(), util.InitAddOne(), util.InitAddOne(), util.InitNormal(), util.InitNormal(), util.InitNormal(), make(map[string]([]string)), nil}
	log.Println(tnt)
	log.Println(json.Marshal(tnt))
	tnt.Save("1.marshal")
	ttt := &Test{"aa", arr}
	log.Println(json.Marshal(ttt))
	log.Println(ttt)

	bufs := new(bytes.Buffer)
	var datas = []byte{
		47,
		125,
		125,
	}
	for _, v := range datas {
		log.Println(v)
		err := binary.Write(bufs, binary.LittleEndian, v)
		if err != nil {
			fmt.Println("binary.Write failed:", err)
		}
	}
	log.Println(bufs.Bytes())

	fout, err = os.Create("2.marshal")
	defer fout.Close()
	if err != nil {
		return
	}
	fout.Write(bufs.Bytes())

	fin, err = os.Open("2.marshal")
	defer fin.Close()
	var results, temps []byte
	var begin int
	buf = make([]byte, 2048*10000)
	for {
		num, _ := fin.Read(buf)
		if 0 == num {
			break
		}
		if len(results) > 0 {
			temps = results
			results = make([]byte, begin+num)
			copy(results[0:len(temps)], temp[:])
		} else {
			results = make([]byte, num)
		}
		copy(results[begin:begin+num], buf[:num])
		//os.Stdout.Write(buf[:num])
	}
	log.Println(results)
}