// command main
func command(opt *option) error {
var dic tokenizer.Dic
if opt.dic == "" {
dic = tokenizer.SysDic()
} else {
var err error
dic, err = tokenizer.NewDic(opt.dic)
if err != nil {
return err
}
}
var udic tokenizer.UserDic
if opt.udic != "" {
var err error
udic, err = tokenizer.NewUserDic(opt.udic)
if err != nil {
return err
}
}
var fp = os.Stdin
if opt.file != "" {
var err error
fp, err = os.Open(opt.file)
if err != nil {
return err
}
defer fp.Close()
}
t := tokenizer.NewWithDic(dic)
t.SetUserDic(udic)
mode := tokenizer.Normal
switch opt.mode {
case "normal":
mode = tokenizer.Normal
break
case "search":
mode = tokenizer.Search
case "extended":
mode = tokenizer.Extended
}
scanner := bufio.NewScanner(fp)
for scanner.Scan() {
line := scanner.Text()
tokens := t.Analyze(line, mode)
for i, size := 1, len(tokens); i < size; i++ {
tok := tokens[i]
c := tok.Features()
if tok.Class == tokenizer.DUMMY {
fmt.Printf("%s\n", tok.Surface)
} else {
fmt.Printf("%s\t%v\n", tok.Surface, strings.Join(c, ","))
}
}
}
return scanner.Err()
}
func init() {
_ = tokenizer.SysDic() // prepare system dictionary
}
func init() {
tokenizer.SysDic()
}
func init() {
rand.Seed(time.Now().UnixNano())
_ = tokenizer.SysDic()
}
