// command main
func command(opt *option) error {
var dic tokenizer.Dic
if opt.dic == "" {
dic = tokenizer.SysDic()
} else {
var err error
dic, err = tokenizer.NewDic(opt.dic)
if err != nil {
return err
}
}
var udic tokenizer.UserDic
if opt.udic != "" {
var err error
udic, err = tokenizer.NewUserDic(opt.udic)
if err != nil {
return err
}
}
var fp = os.Stdin
if opt.file != "" {
var err error
fp, err = os.Open(opt.file)
if err != nil {
return err
}
defer fp.Close()
}
t := tokenizer.NewWithDic(dic)
t.SetUserDic(udic)
mode := tokenizer.Normal
switch opt.mode {
case "normal":
mode = tokenizer.Normal
break
case "search":
mode = tokenizer.Search
case "extended":
mode = tokenizer.Extended
}
scanner := bufio.NewScanner(fp)
for scanner.Scan() {
line := scanner.Text()
tokens := t.Analyze(line, mode)
for i, size := 1, len(tokens); i < size; i++ {
tok := tokens[i]
c := tok.Features()
if tok.Class == tokenizer.DUMMY {
fmt.Printf("%s\n", tok.Surface)
} else {
fmt.Printf("%s\t%v\n", tok.Surface, strings.Join(c, ","))
}
}
}
return scanner.Err()
}
// command main
func command(opt *option) error {
t := tokenizer.New()
var out = os.Stdout
if opt.output != "" {
var err error
out, err = os.OpenFile(opt.output, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666)
if err != nil {
fmt.Fprintln(ErrorWriter, err)
os.Exit(1)
}
defer out.Close()
}
var udic tokenizer.UserDic
if opt.udic != "" {
var err error
udic, err = tokenizer.NewUserDic(opt.udic)
if err != nil {
return err
}
t.SetUserDic(udic)
}
if opt.udic != "" {
if udic, err := tokenizer.NewUserDic(opt.udic); err != nil {
fmt.Fprintln(ErrorWriter, err)
os.Exit(1)
} else {
t.SetUserDic(udic)
}
}
tokens := t.Dot(opt.input, out)
if opt.verbose {
for i, size := 1, len(tokens); i < size; i++ {
tok := tokens[i]
f := tok.Features()
if tok.Class == tokenizer.DUMMY {
fmt.Fprintf(ErrorWriter, "%s\n", tok.Surface)
} else {
fmt.Fprintf(ErrorWriter, "%s\t%v\n", tok.Surface, strings.Join(f, ","))
}
}
}
return nil
}
// command main
func command(opt *option) error {
var udic tokenizer.UserDic
if opt.udic != "" {
var err error
if udic, err = tokenizer.NewUserDic(opt.udic); err != nil {
return err
}
}
t := tokenizer.New()
t.SetUserDic(udic)
mux := http.NewServeMux()
mux.Handle("/", &TokenizeDemoHandler{tokenizer: t})
mux.Handle("/a", &TokenizeHandler{tokenizer: t})
log.Fatal(http.ListenAndServe(opt.http, mux))
return nil
}