func main() {
var Stdin = os.NewFile(uintptr(syscall.Stdin), "/dev/stdin")
var s scanner.Scanner
s.Init(Stdin)
s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
tok := s.Scan()
for tok != scanner.EOF {
switch tok {
case scanner.Ident:
fmt.Println("Ident")
case scanner.Int:
fmt.Println("Int")
case scanner.Float:
fmt.Println("Float")
case scanner.Char:
fmt.Println("Char")
case scanner.String:
fmt.Println("String")
case scanner.RawString:
fmt.Println("RawString")
case scanner.Comment:
fmt.Println("Comment")
}
tok = s.Scan()
}
}
func Tokenize(text string, stopwords map[string]int, words map[string]int) {
var s scanner.Scanner
s.Init(strings.NewReader(text))
tok := s.Scan()
for tok != scanner.EOF {
if tok == scanner.String {
Tokenize(strings.Trim(s.TokenText(), "\"`"), stopwords, words)
} else if tok == scanner.Char {
Tokenize(strings.Trim(s.TokenText(), "'"), stopwords, words)
} else if tok == scanner.Ident {
word := s.TokenText()
if _, ok := stopwords[word]; !ok && len(word) > 2 {
stem, err := snowball.Stem(word, "english", true)
if err != nil {
fmt.Errorf("Couldnt stem word: %s", word)
stem = word
}
if _, ok := stopwords[stem]; !ok {
words[stem] += 1
}
}
}
tok = s.Scan()
}
}
// gofmtFlags looks for a comment of the form
//
// //gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
f, err := os.Open(filename)
if err != nil {
return "" // ignore errors - they will be found later
}
defer f.Close()
// initialize scanner
var s scanner.Scanner
s.Init(f)
s.Error = func(*scanner.Scanner, string) {} // ignore errors
s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments
// look for //gofmt comment
for s.Line <= maxLines {
switch s.Scan() {
case scanner.Comment:
const prefix = "//gofmt "
if t := s.TokenText(); strings.HasPrefix(t, prefix) {
return strings.TrimSpace(t[len(prefix):])
}
case scanner.EOF:
return ""
}
}
return ""
}
func Example() {
const src = `
// This is scanned code.
if a > 10 {
someParsable = text
}`
var s scanner.Scanner
s.Init(strings.NewReader(src))
var tok rune
for tok != scanner.EOF {
tok = s.Scan()
fmt.Println("At position", s.Pos(), ":", s.TokenText())
}
// Output:
// At position 3:4 : if
// At position 3:6 : a
// At position 3:8 : >
// At position 3:11 : 10
// At position 3:13 : {
// At position 4:15 : someParsable
// At position 4:17 : =
// At position 4:22 : text
// At position 5:3 : }
// At position 5:3 :
}
func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {
var s scanner.Scanner
s.Error = func(s *scanner.Scanner, msg string) {
log.Printf("command scan error: %s [%s]", msg, path)
}
s.Init(strings.NewReader(path))
parent := root
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
//log.Printf("cmdFind: token: [%s]", s.TokenText())
label := s.TokenText()
children := matchChildren(parent.Children, label)
size := len(children)
if size < 1 {
return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
}
if size > 1 {
return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
}
parent = children[0]
}
//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)
return parent, nil
}
func TestReduceScanner(t *testing.T) {
in := make(chan interface{})
go func() {
reader := strings.NewReader(mapInput)
s := scanner.Scanner{}
s.Init(reader)
tok := s.Scan()
for tok != scanner.EOF {
tok = s.Scan()
in <- s.TokenText()
}
close(in)
}()
mapOut := Map(in, func(input interface{}) interface{} {
key := input.(string)
if len(key) == 0 {
return ""
} else {
return string(key[0])
}
})
redOut := Reduce(mapOut, func(key interface{}, values []interface{}) interface{} {
return len(values)
})
for tok := range redOut {
if tok.Value.(int) == 0 {
t.Error("expect reduce should be positive")
}
}
}
func TestMapScanner(t *testing.T) {
in := make(chan interface{})
go func() {
reader := strings.NewReader(mapInput)
s := scanner.Scanner{}
s.Init(reader)
tok := s.Scan()
for tok != scanner.EOF {
tok = s.Scan()
in <- s.TokenText()
}
close(in)
}()
out := Map(in, func(input interface{}) interface{} {
key := input.(string)
if len(key) == 0 {
return ""
} else {
return string(key[0])
}
})
for tok := range out {
for _, val := range tok.Values {
if !strings.HasPrefix(val.(string), tok.Key.(string)) {
t.Error(val.(string) + ` should has "` + tok.Key.(string) + `" prefix`)
}
}
}
}
func split(expr string) (keys []string, err error) {
var msgs []string
var s scanner.Scanner
s.Init(strings.NewReader(expr))
s.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings
s.Error = func(s *scanner.Scanner, msg string) { msgs = append(msgs, fmt.Sprintf("%s %s", s.Pos(), msg)) }
key := ""
keys = []string{}
for err == nil {
t := s.Peek()
// fmt.Printf(">>> %s: %s %s\n", s.Pos(), scanner.TokenString(t), s.TokenText())
switch t {
case '[':
key, err = scanBracketedKey(&s)
case '.':
s.Scan()
continue
case scanner.EOF:
goto end
default:
key, err = scanKey(&s)
}
if len(msgs) > 0 {
err = errors.New(strings.Join(msgs, "\n"))
}
if err == nil {
keys = append(keys, key)
}
}
end:
return
}
// Parse updates an inverted index with the file's words
func (index *InvertedIndex) Parse(filename string) {
if index.alreadyParsed(filename) {
return
}
file, err := os.Open(filename)
if err != nil {
cwd, _ := os.Getwd()
fmt.Printf("The file named %s doesn't exist, the current directory is %s\n",
filename, cwd)
return
}
defer file.Close()
var s scanner.Scanner
s.Init(file)
var token rune
for token != scanner.EOF {
token = s.Scan()
tokenText := s.TokenText()
_, found := index.indexMap[tokenText]
pos := s.Pos()
result := Result{File: filename,
Line: pos.Line,
Index: pos.Column - len(tokenText)}
if !found {
index.indexMap[tokenText] = []Result{result}
} else {
index.indexMap[tokenText] = append(index.indexMap[tokenText],
result)
}
}
index.files = append(index.files, filename)
}
// Read contents of a Go file up to the package declaration. This can be used
// to find the the build tags.
func readGoContents(fp string) ([]byte, error) {
f, err := os.Open(fp)
defer f.Close()
if err != nil {
return []byte{}, err
}
var s scanner.Scanner
s.Init(f)
var tok rune
var pos scanner.Position
for tok != scanner.EOF {
tok = s.Scan()
// Getting the token text will skip comments by default.
tt := s.TokenText()
// build tags will not be after the package declaration.
if tt == "package" {
pos = s.Position
break
}
}
buf := bytes.NewBufferString("")
f.Seek(0, 0)
_, err = io.CopyN(buf, f, int64(pos.Offset))
if err != nil {
return []byte{}, err
}
return buf.Bytes(), nil
}
func Parser(s string) []string {
var items []string
var sc scanner.Scanner
sc.Init(strings.NewReader(s))
var tok rune
for tok != scanner.EOF {
tok = sc.Scan()
item := sc.TokenText()
if tok != scanner.EOF {
i := len(items)
if i > 0 {
if item == "=" && (items[i-1] == "!" || items[i-1] == ">" || items[i-1] == "<" || items[i-1] == "=") {
items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
continue
}
if item == "|" && (items[i-1]) == "|" {
items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
continue
}
if item == "&" && (items[i-1]) == "&" {
items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
continue
}
}
items = append(items, item)
}
}
return items
}
func (t Tokenizer) CreateTokens(request *messaging.ParserRequest) (response messaging.ParserResponse) {
response.Body = make(map[string]string)
b := bytes.NewBufferString(request.Query)
var s scanner.Scanner
s.Init(b)
index := 0
for {
tok := s.Scan()
if tok != scanner.EOF {
response.Body[strconv.Itoa(index)] = s.TokenText()
index++
} else {
break
}
}
if index == 0 {
response.IsSuccess = false
response.Message = "Error! Nil Query Allocated!"
} else {
response.IsSuccess = false
response.Message = "Success! Query Tokenized Successfully!"
}
return
}
func (this limitedStringSpan) scanString(s *scanner.Scanner) (string, error) {
tok := s.Scan()
if tok != scanner.EOF && tok == -2 {
return s.TokenText(), nil
}
return "", NewInvalidTypeError()
}
func TestUnwindScanner(t *testing.T) {
in := make(chan interface{})
go func() {
reader := strings.NewReader(unwindInput)
s := scanner.Scanner{}
s.Init(reader)
tok := s.Scan()
for tok != scanner.EOF {
tok = s.Scan()
in <- s.TokenText()
}
close(in)
}()
out := Unwind(in, func(input interface{}, output chan<- interface{}) {
output <- strings.ToLower(input.(string))
output <- strings.ToUpper(input.(string))
})
index := 0
for tok := range out {
if index%2 == 0 {
if strings.ToLower(tok.(string)) != tok.(string) {
t.Error(tok, " should be lower cased")
}
} else {
if strings.ToUpper(tok.(string)) != tok.(string) {
t.Error(tok, " should be upper cased")
}
}
index++
}
}
// ParseKeySet parses a string of comma separated keys
// made of dot separated sections into a KeySet.
func ParseKeySet(s string) (KeySet, error) {
sc := new(scanner.Scanner).Init(strings.NewReader(s))
sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
var k Key
var ks KeySet
for {
// scan section
switch sc.Scan() {
case scanner.String:
u, _ := strconv.Unquote(sc.TokenText())
if len(u) == 0 {
return nil, fmt.Errorf("part of key missing in %q", s)
}
k = append(k, u)
case scanner.Ident, scanner.Int:
k = append(k, sc.TokenText())
default:
return nil, fmt.Errorf("bad formatting in %q", s)
}
// scan separator
switch sc.Scan() {
case '.':
continue
case ',':
ks = append(ks, k)
k = nil
case scanner.EOF:
return append(ks, k), nil
default:
return nil, fmt.Errorf("invalid separator in %q", s)
}
}
}
/**
* This function parses a value string parameter and returns Number value
* embedded within the string. It returns nil if it doesn't find any
* Number value in the value string.
* Example: "some4.56more" would return 4.56
*/
func extractFloat32(value string) Number {
var sc scanner.Scanner
var tok rune
var valFloat64 float64
var valFloat32 Number
var err error
var isFound bool
if len(value) > 0 {
sc.Init(strings.NewReader(value))
sc.Mode = scanner.ScanFloats
for tok != scanner.EOF {
tok = sc.Scan()
// fmt.Println("At position", sc.Pos(), ":", sc.TokenText())
valFloat64, err = strconv.ParseFloat(sc.TokenText(), 64)
if err == nil {
isFound = true
break
}
}
}
if isFound {
valFloat32 = Number(valFloat64)
}
return valFloat32
}
func parseText(s *scanner.Scanner, depth int) ([]ast, error) {
var slice []ast
for {
switch s.Scan() {
case '+', '-', '/', '%', '*', '=', '<', '>', '!':
slice = append(slice, parseIdent(s.TokenText()))
case scanner.Ident:
ident := s.TokenText()
// Periods are allowed in package names.
for s.Peek() == '.' {
s.Next()
ident += "."
if s.Scan() != scanner.Ident {
return nil, stitchError{pos: s.Pos(),
err: fmt.Errorf("bad ident name: %s",
ident)}
}
ident += s.TokenText()
}
slice = append(slice, parseIdent(ident))
case scanner.Float:
x, _ := strconv.ParseFloat(s.TokenText(), 64)
slice = append(slice, astFloat(x))
case scanner.Int:
x, _ := strconv.Atoi(s.TokenText())
slice = append(slice, astInt(x))
case scanner.String:
str := strings.Trim(s.TokenText(), "\"")
slice = append(slice, astString(str))
case '(':
// We need to save our position before recursing because the
// scanner will have moved on by the time the recursive call
// returns.
pos := s.Pos()
sexp, err := parseText(s, depth+1)
if err != nil {
return nil, err
}
slice = append(slice, astSexp{sexp: sexp, pos: pos})
case ')':
if depth == 0 {
return nil, stitchError{s.Pos(), errUnbalancedParens}
}
return slice, nil
case scanner.EOF:
if depth != 0 {
return nil, stitchError{s.Pos(), errUnbalancedParens}
}
return slice, nil
default:
return nil, stitchError{s.Pos(), fmt.Errorf("bad element: %s",
s.TokenText())}
}
}
}
func main() {
s := new(scanner.Scanner)
s.Init(os.Stdin)
for {
spew.Printf("> ")
x := s.Scan()
spew.Println(x, s.TokenText())
}
}
func main() {
s := new(scanner.Scanner)
s.Init(os.Stdin)
s.Error = err
parse := start
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
parse = parse(s, tok)
}
}
// generate cuda wrapper for file.
func cuda2go(fname string) {
// open cuda file
f, err := os.Open(fname)
util.PanicErr(err)
defer f.Close()
// read tokens
var token []string
var s scanner.Scanner
s.Init(f)
tok := s.Scan()
for tok != scanner.EOF {
if !filter(s.TokenText()) {
token = append(token, s.TokenText())
}
tok = s.Scan()
}
// find function name and arguments
funcname := ""
argstart, argstop := -1, -1
for i := 0; i < len(token); i++ {
if token[i] == "__global__" {
funcname = token[i+2]
argstart = i + 4
}
if argstart > 0 && token[i] == ")" {
argstop = i + 1
break
}
}
argl := token[argstart:argstop]
// isolate individual arguments
var args [][]string
start := 0
for i, a := range argl {
if a == "," || a == ")" {
args = append(args, argl[start:i])
start = i + 1
}
}
// separate arg names/types and make pointers Go-style
argn := make([]string, len(args))
argt := make([]string, len(args))
for i := range args {
if args[i][1] == "*" {
args[i] = []string{args[i][0] + "*", args[i][2]}
}
argt[i] = typemap(args[i][0])
argn[i] = args[i][1]
}
wrapgen(fname, funcname, argt, argn)
}
func main() {
values := url.Values{}
//リクエストパラメータ用のkey-valueを追加
values.Add("city", "200010")
fmt.Println("パラメータ:\n", values.Encode())
/***
//時間制限ありGET
client := &http.Client{Timeout: time.Duration(10 * time.Second)}
get(client, values)
****/
//普通のGET
get(values)
//普通のPOST
post(values)
fmt.Println("GETリクエスト:\n", responseGet)
fmt.Println("POSTリクエスト:\n", responsePost)
//受け取るJSONファイルの解析
dec := json.NewDecoder(bytes.NewBufferString(responseGet))
var d data
dec.Decode(&d)
b, err := json.Marshal(d)
if err != nil {
fmt.Errorf("%s", err)
}
jsonData := string(b)
fmt.Println("【date: 日付, title: 天気, url: URL】")
fmt.Println(jsonData)
//字句解析
lexArray := make([]string, 1)
lexical := strings.NewReader(jsonData)
var scan scanner.Scanner
scan.Init(lexical)
for {
x := scan.Scan()
//fmt.Println(x, scan.TokenText())
if x == -1 {
break
} else if x == -6 {
lexArray = append(lexArray, scan.TokenText())
}
}
fmt.Println("明日は", lexArray[10], "です. 天気は", lexArray[13], "です. URLは", lexArray[15], "です.")
}
func main() {
const src = "hello 1"
var s scanner.Scanner
s.Init(bytes.NewBufferString(src))
tok := s.Scan()
for tok != scanner.EOF {
fmt.Println(scanner.TokenString(tok))
// do something with tok
tok = s.Scan()
}
}
func cc_Calc(r io.Reader) int {
cc := 1 // default
var s scanner.Scanner
s.Init(r)
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
if tok == scanner.Ident && isBranch(s.TokenText()) {
cc++
}
}
return cc
}
func (this delimiterSpan) scanString(s *scanner.Scanner) (string, error) {
tok := s.Scan()
if tok != scanner.EOF && tok != -2 && tok != 45 && tok != -3 {
return s.TokenText(), nil
} else {
if debugLayoutDef {
fmt.Printf("expected tok:=!(-2,-3,45), received:%d ('%s')\n", tok, s.TokenText())
}
}
return "", NewInvalidTypeError()
}
func scanBracketedKey(s *scanner.Scanner) (key string, err error) {
s.Scan() // scan the '['
key, err = scanKey(s)
if err == nil {
t := s.Scan()
if t != ']' {
err = fmt.Errorf("Unexpected token at %s. Expected ']', had %s", s.Pos(), scanner.TokenString(t))
}
}
return
}
func main() {
var s scanner.Scanner
s.Init(os.Stdin)
for {
switch s.Scan() {
case scanner.EOF:
return // all done
case scanner.Ident:
fmt.Println(s.TokenText())
}
}
}
func scanKey(s *scanner.Scanner) (key string, err error) {
t := s.Scan()
switch t {
case scanner.Ident, scanner.Int, scanner.Float:
key = s.TokenText()
case scanner.String:
key = strings.Trim(s.TokenText(), "\"")
default:
err = fmt.Errorf("Unexpected token at %s. Expected ident, number or string, had %s", s.Pos(), scanner.TokenString(t))
}
return
}
func Map(fileName string, intermediate chan Partial) {
file, err := os.Open(fileName)
if err == nil {
var s scanner.Scanner
s.Init(file)
tok := s.Scan()
for tok != scanner.EOF {
intermediate <- Partial{s.TokenText(), fileName}
tok = s.Scan()
}
}
intermediate <- Partial{"", ""}
}
func StringToMap(value string) (map[string]string, error) {
var tokenizer scanner.Scanner
tokenizer.Init(strings.NewReader(value))
tokenizer.Error = func(*scanner.Scanner, string) {}
result := make(map[string]string)
next := func() string {
tokenizer.Scan()
return tokenizer.TokenText()
}
var lvalue, rvalue, expression string
for {
lvalue = next()
if lvalue == "" {
return result, errors.New(fmt.Sprintf("Expected key at pos '%d' but found none; "+
"map values should be 'key=value' separated by commas", tokenizer.Pos().Offset))
}
if lvalue == "{" {
// Assume this is JSON format and attempt to un-marshal
return JSONToMap(value)
}
expression = next()
if expression != "=" {
return result, errors.New(fmt.Sprintf("Expected '=' after '%s' but found '%s'; "+
"map values should be 'key=value' separated by commas", lvalue, expression))
}
rvalue = next()
if rvalue == "" {
return result, errors.New(fmt.Sprintf("Expected value after '%s' but found none; "+
"map values should be 'key=value' separated by commas", expression))
}
// TODO: Handle quoted strings and escaped double quotes
result[lvalue] = rvalue
// Are there anymore tokens?
delimiter := next()
if delimiter == "" {
break
}
// Should be a comma next
if delimiter != "," {
return result, errors.New(fmt.Sprintf("Expected ',' after '%s' but found '%s'; "+
"map values should be 'key=value' separated by commas", rvalue, delimiter))
}
}
return result, nil
}
func main() {
//get a single integer
var a int
fmt.Scanf("%d", &a)
fmt.Println(a)
//get a string (no spaces)
var b string
fmt.Scanf("%s", &b)
fmt.Println(b)
//list of n ints
var myList []int
fmt.Print("Enter list: ")
var input string
fmt.Scanln(&input)
fmt.Print(input)
var s scanner.Scanner
s.Init(input)
tok := s.Scan()
for tok != scanner.EOF {
// do something with tok
tok = s.Scan()
asInt, _ := strconv.Atoi(tok)
myList = append(myList, asInt)
}
fmt.Println(myList)
}
