示例#1
0
文件: lattice.go 项目: postfix/kagome
func Main(input string) {
	if input == "" {
		usage()
	}
	var out = os.Stdout
	if *fOutputFile != "" {
		var err error
		out, err = os.OpenFile(*fOutputFile, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		defer out.Close()
	}

	t := kagome.NewTokenizer()
	if *fUserDicFile != "" {
		if udic, err := kagome.NewUserDic(*fUserDicFile); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		} else {
			t.SetUserDic(udic)
		}
	}

	tokens := t.Dot(input, out)
	if *fVerbose {
		for i, size := 1, len(tokens); i < size; i++ {
			tok := tokens[i]
			f := tok.Features()
			if tok.Class == kagome.DUMMY {
				fmt.Fprintf(os.Stderr, "%s\n", tok.Surface)
			} else {

				fmt.Fprintf(os.Stderr, "%s\t%v\n", tok.Surface, strings.Join(f, ","))
			}
		}
	}
}
示例#2
0
文件: kagome.go 项目: postfix/kagome
func Main() {
	if *fHttp != "" && *fInputFile != "" {
		usage()
	}

	var udic *kagome.UserDic
	if *fUserDicFile != "" {
		var err error
		udic, err = kagome.NewUserDic(*fUserDicFile)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

	if *fHttp != "" {
		t := kagome.NewTokenizer()
		if udic != nil {
			t.SetUserDic(udic)
		}
		hTok := &KagomeHandler{tokenizer: t}
		hDem := &KagomeDemoHandler{tokenizer: t}
		mux := http.NewServeMux()
		mux.Handle("/", hTok)
		mux.Handle("/_demo", hDem)
		log.Fatal(http.ListenAndServe(*fHttp, mux))
		os.Exit(0)
	}

	var inputFile = os.Stdin
	if *fInputFile != "" {
		var err error
		inputFile, err = os.Open(*fInputFile)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		defer inputFile.Close()
	}

	t := kagome.NewTokenizer()
	if udic != nil {
		t.SetUserDic(udic)
	}

	var tokenize = t.Tokenize
	switch {
	case *fTokenizeMode == "normal":
		break
	case *fTokenizeMode == "search":
		tokenize = t.SearchModeTokenize
	case *fTokenizeMode == "extended":
		tokenize = t.ExtendedModeTokenize
	case *fTokenizeMode != "":
		fmt.Fprintf(os.Stderr, "invalid argument: -mode %v\n", *fTokenizeMode)
		usage()
	}

	scanner := bufio.NewScanner(inputFile)
	for scanner.Scan() {
		line := scanner.Text()
		tokens := tokenize(line)
		for i, size := 1, len(tokens); i < size; i++ {
			tok := tokens[i]
			c := tok.Features()
			if tok.Class == kagome.DUMMY {
				fmt.Printf("%s\n", tok.Surface)
			} else {
				fmt.Printf("%s\t%v\n", tok.Surface, strings.Join(c, ","))
			}
		}
	}
	if err := scanner.Err(); err != nil {
		log.Println(err)
	}
}