func main() {
if len(os.Args) < 2 {
showUsage()
}
scanner := tokenizer.NewScanner(nil)
res, err := scanner.ScanBytes([]byte(os.Args[1]))
if err != nil {
fmt.Println(err)
return
}
b, err := json.Marshal(res)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(b))
}
func TokenizeHander(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeError(w, http.StatusMethodNotAllowed, "Method not allowed")
return
}
req := &TokenizeReq{}
err := json.NewDecoder(r.Body).Decode(req)
if err != nil {
writeError(w, http.StatusBadRequest, "Bad request")
return
}
scanner := tokenizer.NewScanner(nil)
resp, err := scanner.ScanBytes([]byte(req.Input))
if err != nil {
writeError(w, http.StatusInternalServerError, "Internal error")
return
}
json.NewEncoder(w).Encode(resp)
}