Esempio n. 1
3
func main() {
	var Stdin = os.NewFile(uintptr(syscall.Stdin), "/dev/stdin")
	var s scanner.Scanner
	s.Init(Stdin)
	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
	tok := s.Scan()
	for tok != scanner.EOF {
		switch tok {
		case scanner.Ident:
			fmt.Println("Ident")
		case scanner.Int:
			fmt.Println("Int")
		case scanner.Float:
			fmt.Println("Float")
		case scanner.Char:
			fmt.Println("Char")
		case scanner.String:
			fmt.Println("String")
		case scanner.RawString:
			fmt.Println("RawString")
		case scanner.Comment:
			fmt.Println("Comment")
		}
		tok = s.Scan()
	}
}
func Tokenize(text string, stopwords map[string]int, words map[string]int) {
	var s scanner.Scanner
	s.Init(strings.NewReader(text))
	tok := s.Scan()
	for tok != scanner.EOF {
		if tok == scanner.String {
			Tokenize(strings.Trim(s.TokenText(), "\"`"), stopwords, words)
		} else if tok == scanner.Char {
			Tokenize(strings.Trim(s.TokenText(), "'"), stopwords, words)
		} else if tok == scanner.Ident {
			word := s.TokenText()
			if _, ok := stopwords[word]; !ok && len(word) > 2 {
				stem, err := snowball.Stem(word, "english", true)
				if err != nil {
					fmt.Errorf("Couldnt stem word: %s", word)
					stem = word
				}
				if _, ok := stopwords[stem]; !ok {
					words[stem] += 1
				}
			}
		}
		tok = s.Scan()
	}
}
Esempio n. 3
0
// gofmtFlags looks for a comment of the form
//
//	//gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
	f, err := os.Open(filename)
	if err != nil {
		return "" // ignore errors - they will be found later
	}
	defer f.Close()

	// initialize scanner
	var s scanner.Scanner
	s.Init(f)
	s.Error = func(*scanner.Scanner, string) {}       // ignore errors
	s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments

	// look for //gofmt comment
	for s.Line <= maxLines {
		switch s.Scan() {
		case scanner.Comment:
			const prefix = "//gofmt "
			if t := s.TokenText(); strings.HasPrefix(t, prefix) {
				return strings.TrimSpace(t[len(prefix):])
			}
		case scanner.EOF:
			return ""
		}

	}

	return ""
}
Esempio n. 4
0
func Example() {
	const src = `
	// This is scanned code.
	if a > 10 {
		someParsable = text
	}`
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()
		fmt.Println("At position", s.Pos(), ":", s.TokenText())
	}

	// Output:
	// At position 3:4 : if
	// At position 3:6 : a
	// At position 3:8 : >
	// At position 3:11 : 10
	// At position 3:13 : {
	// At position 4:15 : someParsable
	// At position 4:17 : =
	// At position 4:22 : text
	// At position 5:3 : }
	// At position 5:3 :
}
Esempio n. 5
0
File: cmd.go Progetto: udhos/nexthop
func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {

	var s scanner.Scanner
	s.Error = func(s *scanner.Scanner, msg string) {
		log.Printf("command scan error: %s [%s]", msg, path)
	}
	s.Init(strings.NewReader(path))

	parent := root
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		//log.Printf("cmdFind: token: [%s]", s.TokenText())
		label := s.TokenText()
		children := matchChildren(parent.Children, label)
		size := len(children)
		if size < 1 {
			return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
		}
		if size > 1 {
			return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
		}
		parent = children[0]
	}

	//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)

	return parent, nil
}
Esempio n. 6
0
func TestReduceScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	mapOut := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	redOut := Reduce(mapOut, func(key interface{}, values []interface{}) interface{} {
		return len(values)
	})
	for tok := range redOut {
		if tok.Value.(int) == 0 {
			t.Error("expect reduce should be positive")
		}
	}
}
Esempio n. 7
0
func TestMapScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	for tok := range out {
		for _, val := range tok.Values {
			if !strings.HasPrefix(val.(string), tok.Key.(string)) {
				t.Error(val.(string) + ` should has "` + tok.Key.(string) + `" prefix`)
			}
		}
	}
}
Esempio n. 8
0
func split(expr string) (keys []string, err error) {

	var msgs []string
	var s scanner.Scanner
	s.Init(strings.NewReader(expr))
	s.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings
	s.Error = func(s *scanner.Scanner, msg string) { msgs = append(msgs, fmt.Sprintf("%s %s", s.Pos(), msg)) }

	key := ""
	keys = []string{}
	for err == nil {
		t := s.Peek()
		// fmt.Printf(">>> %s: %s %s\n", s.Pos(), scanner.TokenString(t), s.TokenText())
		switch t {
		case '[':
			key, err = scanBracketedKey(&s)
		case '.':
			s.Scan()
			continue
		case scanner.EOF:
			goto end
		default:
			key, err = scanKey(&s)
		}
		if len(msgs) > 0 {
			err = errors.New(strings.Join(msgs, "\n"))
		}
		if err == nil {
			keys = append(keys, key)
		}
	}
end:
	return
}
// Parse updates an inverted index with the file's words
func (index *InvertedIndex) Parse(filename string) {
	if index.alreadyParsed(filename) {
		return
	}
	file, err := os.Open(filename)
	if err != nil {
		cwd, _ := os.Getwd()
		fmt.Printf("The file named %s doesn't exist, the current directory is %s\n",
			filename, cwd)
		return
	}
	defer file.Close()
	var s scanner.Scanner
	s.Init(file)
	var token rune
	for token != scanner.EOF {
		token = s.Scan()
		tokenText := s.TokenText()
		_, found := index.indexMap[tokenText]
		pos := s.Pos()
		result := Result{File: filename,
			Line:  pos.Line,
			Index: pos.Column - len(tokenText)}
		if !found {
			index.indexMap[tokenText] = []Result{result}
		} else {
			index.indexMap[tokenText] = append(index.indexMap[tokenText],
				result)
		}
	}
	index.files = append(index.files, filename)
}
Esempio n. 10
0
// Read contents of a Go file up to the package declaration. This can be used
// to find the the build tags.
func readGoContents(fp string) ([]byte, error) {
	f, err := os.Open(fp)
	defer f.Close()
	if err != nil {
		return []byte{}, err
	}

	var s scanner.Scanner
	s.Init(f)
	var tok rune
	var pos scanner.Position
	for tok != scanner.EOF {
		tok = s.Scan()

		// Getting the token text will skip comments by default.
		tt := s.TokenText()
		// build tags will not be after the package declaration.
		if tt == "package" {
			pos = s.Position
			break
		}
	}

	buf := bytes.NewBufferString("")
	f.Seek(0, 0)
	_, err = io.CopyN(buf, f, int64(pos.Offset))
	if err != nil {
		return []byte{}, err
	}

	return buf.Bytes(), nil
}
Esempio n. 11
0
func Parser(s string) []string {
	var items []string
	var sc scanner.Scanner
	sc.Init(strings.NewReader(s))
	var tok rune
	for tok != scanner.EOF {
		tok = sc.Scan()
		item := sc.TokenText()
		if tok != scanner.EOF {
			i := len(items)
			if i > 0 {
				if item == "=" && (items[i-1] == "!" || items[i-1] == ">" || items[i-1] == "<" || items[i-1] == "=") {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "|" && (items[i-1]) == "|" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "&" && (items[i-1]) == "&" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
			}
			items = append(items, item)
		}
	}
	return items
}
Esempio n. 12
0
func (t Tokenizer) CreateTokens(request *messaging.ParserRequest) (response messaging.ParserResponse) {

	response.Body = make(map[string]string)
	b := bytes.NewBufferString(request.Query)
	var s scanner.Scanner
	s.Init(b)

	index := 0
	for {
		tok := s.Scan()
		if tok != scanner.EOF {
			response.Body[strconv.Itoa(index)] = s.TokenText()
			index++
		} else {
			break
		}

	}

	if index == 0 {
		response.IsSuccess = false
		response.Message = "Error! Nil Query Allocated!"
	} else {
		response.IsSuccess = false
		response.Message = "Success! Query Tokenized Successfully!"
	}

	return
}
Esempio n. 13
0
func (this limitedStringSpan) scanString(s *scanner.Scanner) (string, error) {
	tok := s.Scan()
	if tok != scanner.EOF && tok == -2 {
		return s.TokenText(), nil
	}
	return "", NewInvalidTypeError()
}
Esempio n. 14
0
func TestUnwindScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(unwindInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Unwind(in, func(input interface{}, output chan<- interface{}) {
		output <- strings.ToLower(input.(string))
		output <- strings.ToUpper(input.(string))
	})
	index := 0
	for tok := range out {
		if index%2 == 0 {
			if strings.ToLower(tok.(string)) != tok.(string) {
				t.Error(tok, " should be lower cased")
			}
		} else {
			if strings.ToUpper(tok.(string)) != tok.(string) {
				t.Error(tok, " should be upper cased")
			}
		}
		index++
	}
}
Esempio n. 15
0
File: parse.go Progetto: simleb/span
// ParseKeySet parses a string of comma separated keys
// made of dot separated sections into a KeySet.
func ParseKeySet(s string) (KeySet, error) {
	sc := new(scanner.Scanner).Init(strings.NewReader(s))
	sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
	var k Key
	var ks KeySet
	for {
		// scan section
		switch sc.Scan() {
		case scanner.String:
			u, _ := strconv.Unquote(sc.TokenText())
			if len(u) == 0 {
				return nil, fmt.Errorf("part of key missing in %q", s)
			}
			k = append(k, u)
		case scanner.Ident, scanner.Int:
			k = append(k, sc.TokenText())
		default:
			return nil, fmt.Errorf("bad formatting in %q", s)
		}
		// scan separator
		switch sc.Scan() {
		case '.':
			continue
		case ',':
			ks = append(ks, k)
			k = nil
		case scanner.EOF:
			return append(ks, k), nil
		default:
			return nil, fmt.Errorf("invalid separator in %q", s)
		}
	}
}
Esempio n. 16
0
/**
 * This function parses a value string parameter and returns Number value
 * embedded within the string. It returns nil if it doesn't find any
 * Number value in the value string.
 * Example: "some4.56more" would return 4.56
 */
func extractFloat32(value string) Number {
	var sc scanner.Scanner
	var tok rune
	var valFloat64 float64
	var valFloat32 Number
	var err error
	var isFound bool

	if len(value) > 0 {
		sc.Init(strings.NewReader(value))
		sc.Mode = scanner.ScanFloats

		for tok != scanner.EOF {
			tok = sc.Scan()
			// fmt.Println("At position", sc.Pos(), ":", sc.TokenText())
			valFloat64, err = strconv.ParseFloat(sc.TokenText(), 64)
			if err == nil {
				isFound = true
				break
			}
		}
	}

	if isFound {
		valFloat32 = Number(valFloat64)
	}

	return valFloat32
}
Esempio n. 17
0
func parseText(s *scanner.Scanner, depth int) ([]ast, error) {
	var slice []ast
	for {
		switch s.Scan() {
		case '+', '-', '/', '%', '*', '=', '<', '>', '!':
			slice = append(slice, parseIdent(s.TokenText()))
		case scanner.Ident:
			ident := s.TokenText()
			// Periods are allowed in package names.
			for s.Peek() == '.' {
				s.Next()
				ident += "."
				if s.Scan() != scanner.Ident {
					return nil, stitchError{pos: s.Pos(),
						err: fmt.Errorf("bad ident name: %s",
							ident)}
				}
				ident += s.TokenText()
			}
			slice = append(slice, parseIdent(ident))
		case scanner.Float:
			x, _ := strconv.ParseFloat(s.TokenText(), 64)
			slice = append(slice, astFloat(x))
		case scanner.Int:
			x, _ := strconv.Atoi(s.TokenText())
			slice = append(slice, astInt(x))
		case scanner.String:
			str := strings.Trim(s.TokenText(), "\"")
			slice = append(slice, astString(str))
		case '(':
			// We need to save our position before recursing because the
			// scanner will have moved on by the time the recursive call
			// returns.
			pos := s.Pos()
			sexp, err := parseText(s, depth+1)
			if err != nil {
				return nil, err
			}

			slice = append(slice, astSexp{sexp: sexp, pos: pos})

		case ')':
			if depth == 0 {
				return nil, stitchError{s.Pos(), errUnbalancedParens}
			}
			return slice, nil
		case scanner.EOF:
			if depth != 0 {
				return nil, stitchError{s.Pos(), errUnbalancedParens}
			}
			return slice, nil

		default:
			return nil, stitchError{s.Pos(), fmt.Errorf("bad element: %s",
				s.TokenText())}
		}
	}
}
Esempio n. 18
0
func main() {
	s := new(scanner.Scanner)
	s.Init(os.Stdin)
	for {
		spew.Printf("> ")
		x := s.Scan()
		spew.Println(x, s.TokenText())
	}
}
Esempio n. 19
0
File: gen.go Progetto: streadway/zk
func main() {
	s := new(scanner.Scanner)
	s.Init(os.Stdin)
	s.Error = err

	parse := start
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		parse = parse(s, tok)
	}
}
Esempio n. 20
0
// generate cuda wrapper for file.
func cuda2go(fname string) {
	// open cuda file
	f, err := os.Open(fname)
	util.PanicErr(err)
	defer f.Close()

	// read tokens
	var token []string
	var s scanner.Scanner
	s.Init(f)
	tok := s.Scan()
	for tok != scanner.EOF {
		if !filter(s.TokenText()) {
			token = append(token, s.TokenText())
		}
		tok = s.Scan()
	}

	// find function name and arguments
	funcname := ""
	argstart, argstop := -1, -1
	for i := 0; i < len(token); i++ {
		if token[i] == "__global__" {
			funcname = token[i+2]
			argstart = i + 4
		}
		if argstart > 0 && token[i] == ")" {
			argstop = i + 1
			break
		}
	}
	argl := token[argstart:argstop]

	// isolate individual arguments
	var args [][]string
	start := 0
	for i, a := range argl {
		if a == "," || a == ")" {
			args = append(args, argl[start:i])
			start = i + 1
		}
	}

	// separate arg names/types and make pointers Go-style
	argn := make([]string, len(args))
	argt := make([]string, len(args))
	for i := range args {
		if args[i][1] == "*" {
			args[i] = []string{args[i][0] + "*", args[i][2]}
		}
		argt[i] = typemap(args[i][0])
		argn[i] = args[i][1]
	}
	wrapgen(fname, funcname, argt, argn)
}
Esempio n. 21
0
func main() {
	values := url.Values{}

	//リクエストパラメータ用のkey-valueを追加
	values.Add("city", "200010")

	fmt.Println("パラメータ:\n", values.Encode())

	/***
	  //時間制限ありGET
	  client := &http.Client{Timeout: time.Duration(10 * time.Second)}
	  get(client, values)
	****/

	//普通のGET
	get(values)
	//普通のPOST
	post(values)

	fmt.Println("GETリクエスト:\n", responseGet)
	fmt.Println("POSTリクエスト:\n", responsePost)

	//受け取るJSONファイルの解析
	dec := json.NewDecoder(bytes.NewBufferString(responseGet))
	var d data
	dec.Decode(&d)

	b, err := json.Marshal(d)
	if err != nil {
		fmt.Errorf("%s", err)
	}
	jsonData := string(b)

	fmt.Println("【date: 日付, title: 天気, url: URL】")
	fmt.Println(jsonData)

	//字句解析
	lexArray := make([]string, 1)
	lexical := strings.NewReader(jsonData)
	var scan scanner.Scanner
	scan.Init(lexical)
	for {
		x := scan.Scan()
		//fmt.Println(x, scan.TokenText())

		if x == -1 {
			break
		} else if x == -6 {
			lexArray = append(lexArray, scan.TokenText())
		}
	}

	fmt.Println("明日は", lexArray[10], "です. 天気は", lexArray[13], "です. URLは", lexArray[15], "です.")

}
Esempio n. 22
0
func main() {
	const src = "hello 1"
	var s scanner.Scanner
	s.Init(bytes.NewBufferString(src))
	tok := s.Scan()
	for tok != scanner.EOF {
		fmt.Println(scanner.TokenString(tok))
		// do something with tok
		tok = s.Scan()
	}
}
Esempio n. 23
0
func cc_Calc(r io.Reader) int {
	cc := 1 // default
	var s scanner.Scanner
	s.Init(r)
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		if tok == scanner.Ident && isBranch(s.TokenText()) {
			cc++
		}
	}
	return cc
}
Esempio n. 24
0
func (this delimiterSpan) scanString(s *scanner.Scanner) (string, error) {
	tok := s.Scan()
	if tok != scanner.EOF && tok != -2 && tok != 45 && tok != -3 {
		return s.TokenText(), nil
	} else {
		if debugLayoutDef {
			fmt.Printf("expected tok:=!(-2,-3,45), received:%d ('%s')\n", tok, s.TokenText())
		}
	}
	return "", NewInvalidTypeError()
}
Esempio n. 25
0
func scanBracketedKey(s *scanner.Scanner) (key string, err error) {
	s.Scan() // scan the '['
	key, err = scanKey(s)
	if err == nil {
		t := s.Scan()
		if t != ']' {
			err = fmt.Errorf("Unexpected token at %s. Expected ']', had %s", s.Pos(), scanner.TokenString(t))
		}
	}
	return
}
Esempio n. 26
0
func main() {
	var s scanner.Scanner
	s.Init(os.Stdin)
	for {
		switch s.Scan() {
		case scanner.EOF:
			return // all done
		case scanner.Ident:
			fmt.Println(s.TokenText())
		}
	}
}
Esempio n. 27
0
func scanKey(s *scanner.Scanner) (key string, err error) {
	t := s.Scan()
	switch t {
	case scanner.Ident, scanner.Int, scanner.Float:
		key = s.TokenText()
	case scanner.String:
		key = strings.Trim(s.TokenText(), "\"")
	default:
		err = fmt.Errorf("Unexpected token at %s. Expected ident, number or string, had %s", s.Pos(), scanner.TokenString(t))
	}
	return
}
Esempio n. 28
0
func Map(fileName string, intermediate chan Partial) {
	file, err := os.Open(fileName)
	if err == nil {
		var s scanner.Scanner
		s.Init(file)
		tok := s.Scan()
		for tok != scanner.EOF {
			intermediate <- Partial{s.TokenText(), fileName}
			tok = s.Scan()
		}
	}
	intermediate <- Partial{"", ""}
}
Esempio n. 29
0
func StringToMap(value string) (map[string]string, error) {
	var tokenizer scanner.Scanner
	tokenizer.Init(strings.NewReader(value))
	tokenizer.Error = func(*scanner.Scanner, string) {}

	result := make(map[string]string)
	next := func() string {
		tokenizer.Scan()
		return tokenizer.TokenText()
	}

	var lvalue, rvalue, expression string
	for {
		lvalue = next()
		if lvalue == "" {
			return result, errors.New(fmt.Sprintf("Expected key at pos '%d' but found none; "+
				"map values should be 'key=value' separated by commas", tokenizer.Pos().Offset))
		}
		if lvalue == "{" {
			// Assume this is JSON format and attempt to un-marshal
			return JSONToMap(value)
		}

		expression = next()
		if expression != "=" {
			return result, errors.New(fmt.Sprintf("Expected '=' after '%s' but found '%s'; "+
				"map values should be 'key=value' separated by commas", lvalue, expression))
		}
		rvalue = next()
		if rvalue == "" {
			return result, errors.New(fmt.Sprintf("Expected value after '%s' but found none; "+
				"map values should be 'key=value' separated by commas", expression))
		}
		// TODO: Handle quoted strings and escaped double quotes
		result[lvalue] = rvalue

		// Are there anymore tokens?
		delimiter := next()
		if delimiter == "" {
			break
		}

		// Should be a comma next
		if delimiter != "," {
			return result, errors.New(fmt.Sprintf("Expected ',' after '%s' but found '%s'; "+
				"map values should be 'key=value' separated by commas", rvalue, delimiter))
		}
	}

	return result, nil
}
Esempio n. 30
-1
func main() {

	//get a single integer
	var a int
	fmt.Scanf("%d", &a)
	fmt.Println(a)

	//get a string (no spaces)
	var b string
	fmt.Scanf("%s", &b)
	fmt.Println(b)

	//list of n ints
	var myList []int
	fmt.Print("Enter list: ")
	var input string
	fmt.Scanln(&input)
	fmt.Print(input)

	var s scanner.Scanner
	s.Init(input)
	tok := s.Scan()
	for tok != scanner.EOF {
		// do something with tok
		tok = s.Scan()
		asInt, _ := strconv.Atoi(tok)
		myList = append(myList, asInt)
	}

	fmt.Println(myList)

}