Esempio n. 1
0
File: cmd.go Progetto: udhos/nexthop
func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {

	var s scanner.Scanner
	s.Error = func(s *scanner.Scanner, msg string) {
		log.Printf("command scan error: %s [%s]", msg, path)
	}
	s.Init(strings.NewReader(path))

	parent := root
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		//log.Printf("cmdFind: token: [%s]", s.TokenText())
		label := s.TokenText()
		children := matchChildren(parent.Children, label)
		size := len(children)
		if size < 1 {
			return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
		}
		if size > 1 {
			return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
		}
		parent = children[0]
	}

	//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)

	return parent, nil
}
Esempio n. 2
0
func Example() {
	const src = `
	// This is scanned code.
	if a > 10 {
		someParsable = text
	}`
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()
		fmt.Println("At position", s.Pos(), ":", s.TokenText())
	}

	// Output:
	// At position 3:4 : if
	// At position 3:6 : a
	// At position 3:8 : >
	// At position 3:11 : 10
	// At position 3:13 : {
	// At position 4:15 : someParsable
	// At position 4:17 : =
	// At position 4:22 : text
	// At position 5:3 : }
	// At position 5:3 :
}
Esempio n. 3
0
func TestReduceScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	mapOut := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	redOut := Reduce(mapOut, func(key interface{}, values []interface{}) interface{} {
		return len(values)
	})
	for tok := range redOut {
		if tok.Value.(int) == 0 {
			t.Error("expect reduce should be positive")
		}
	}
}
Esempio n. 4
0
func (this limitedStringSpan) scanString(s *scanner.Scanner) (string, error) {
	tok := s.Scan()
	if tok != scanner.EOF && tok == -2 {
		return s.TokenText(), nil
	}
	return "", NewInvalidTypeError()
}
Esempio n. 5
0
func (t Tokenizer) CreateTokens(request *messaging.ParserRequest) (response messaging.ParserResponse) {

	response.Body = make(map[string]string)
	b := bytes.NewBufferString(request.Query)
	var s scanner.Scanner
	s.Init(b)

	index := 0
	for {
		tok := s.Scan()
		if tok != scanner.EOF {
			response.Body[strconv.Itoa(index)] = s.TokenText()
			index++
		} else {
			break
		}

	}

	if index == 0 {
		response.IsSuccess = false
		response.Message = "Error! Nil Query Allocated!"
	} else {
		response.IsSuccess = false
		response.Message = "Success! Query Tokenized Successfully!"
	}

	return
}
Esempio n. 6
0
/**
 * This function parses a value string parameter and returns Number value
 * embedded within the string. It returns nil if it doesn't find any
 * Number value in the value string.
 * Example: "some4.56more" would return 4.56
 */
func extractFloat32(value string) Number {
	var sc scanner.Scanner
	var tok rune
	var valFloat64 float64
	var valFloat32 Number
	var err error
	var isFound bool

	if len(value) > 0 {
		sc.Init(strings.NewReader(value))
		sc.Mode = scanner.ScanFloats

		for tok != scanner.EOF {
			tok = sc.Scan()
			// fmt.Println("At position", sc.Pos(), ":", sc.TokenText())
			valFloat64, err = strconv.ParseFloat(sc.TokenText(), 64)
			if err == nil {
				isFound = true
				break
			}
		}
	}

	if isFound {
		valFloat32 = Number(valFloat64)
	}

	return valFloat32
}
Esempio n. 7
0
// Read contents of a Go file up to the package declaration. This can be used
// to find the the build tags.
func readGoContents(fp string) ([]byte, error) {
	f, err := os.Open(fp)
	defer f.Close()
	if err != nil {
		return []byte{}, err
	}

	var s scanner.Scanner
	s.Init(f)
	var tok rune
	var pos scanner.Position
	for tok != scanner.EOF {
		tok = s.Scan()

		// Getting the token text will skip comments by default.
		tt := s.TokenText()
		// build tags will not be after the package declaration.
		if tt == "package" {
			pos = s.Position
			break
		}
	}

	buf := bytes.NewBufferString("")
	f.Seek(0, 0)
	_, err = io.CopyN(buf, f, int64(pos.Offset))
	if err != nil {
		return []byte{}, err
	}

	return buf.Bytes(), nil
}
Esempio n. 8
0
func TestMapScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	for tok := range out {
		for _, val := range tok.Values {
			if !strings.HasPrefix(val.(string), tok.Key.(string)) {
				t.Error(val.(string) + ` should has "` + tok.Key.(string) + `" prefix`)
			}
		}
	}
}
Esempio n. 9
0
// gofmtFlags looks for a comment of the form
//
//	//gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
	f, err := os.Open(filename)
	if err != nil {
		return "" // ignore errors - they will be found later
	}
	defer f.Close()

	// initialize scanner
	var s scanner.Scanner
	s.Init(f)
	s.Error = func(*scanner.Scanner, string) {}       // ignore errors
	s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments

	// look for //gofmt comment
	for s.Line <= maxLines {
		switch s.Scan() {
		case scanner.Comment:
			const prefix = "//gofmt "
			if t := s.TokenText(); strings.HasPrefix(t, prefix) {
				return strings.TrimSpace(t[len(prefix):])
			}
		case scanner.EOF:
			return ""
		}

	}

	return ""
}
Esempio n. 10
0
File: parse.go Progetto: simleb/span
// ParseKeySet parses a string of comma separated keys
// made of dot separated sections into a KeySet.
func ParseKeySet(s string) (KeySet, error) {
	sc := new(scanner.Scanner).Init(strings.NewReader(s))
	sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
	var k Key
	var ks KeySet
	for {
		// scan section
		switch sc.Scan() {
		case scanner.String:
			u, _ := strconv.Unquote(sc.TokenText())
			if len(u) == 0 {
				return nil, fmt.Errorf("part of key missing in %q", s)
			}
			k = append(k, u)
		case scanner.Ident, scanner.Int:
			k = append(k, sc.TokenText())
		default:
			return nil, fmt.Errorf("bad formatting in %q", s)
		}
		// scan separator
		switch sc.Scan() {
		case '.':
			continue
		case ',':
			ks = append(ks, k)
			k = nil
		case scanner.EOF:
			return append(ks, k), nil
		default:
			return nil, fmt.Errorf("invalid separator in %q", s)
		}
	}
}
Esempio n. 11
0
func Parser(s string) []string {
	var items []string
	var sc scanner.Scanner
	sc.Init(strings.NewReader(s))
	var tok rune
	for tok != scanner.EOF {
		tok = sc.Scan()
		item := sc.TokenText()
		if tok != scanner.EOF {
			i := len(items)
			if i > 0 {
				if item == "=" && (items[i-1] == "!" || items[i-1] == ">" || items[i-1] == "<" || items[i-1] == "=") {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "|" && (items[i-1]) == "|" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "&" && (items[i-1]) == "&" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
			}
			items = append(items, item)
		}
	}
	return items
}
// Parse updates an inverted index with the file's words
func (index *InvertedIndex) Parse(filename string) {
	if index.alreadyParsed(filename) {
		return
	}
	file, err := os.Open(filename)
	if err != nil {
		cwd, _ := os.Getwd()
		fmt.Printf("The file named %s doesn't exist, the current directory is %s\n",
			filename, cwd)
		return
	}
	defer file.Close()
	var s scanner.Scanner
	s.Init(file)
	var token rune
	for token != scanner.EOF {
		token = s.Scan()
		tokenText := s.TokenText()
		_, found := index.indexMap[tokenText]
		pos := s.Pos()
		result := Result{File: filename,
			Line:  pos.Line,
			Index: pos.Column - len(tokenText)}
		if !found {
			index.indexMap[tokenText] = []Result{result}
		} else {
			index.indexMap[tokenText] = append(index.indexMap[tokenText],
				result)
		}
	}
	index.files = append(index.files, filename)
}
Esempio n. 13
0
func TestUnwindScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(unwindInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Unwind(in, func(input interface{}, output chan<- interface{}) {
		output <- strings.ToLower(input.(string))
		output <- strings.ToUpper(input.(string))
	})
	index := 0
	for tok := range out {
		if index%2 == 0 {
			if strings.ToLower(tok.(string)) != tok.(string) {
				t.Error(tok, " should be lower cased")
			}
		} else {
			if strings.ToUpper(tok.(string)) != tok.(string) {
				t.Error(tok, " should be upper cased")
			}
		}
		index++
	}
}
Esempio n. 14
0
func main() {
	s := new(scanner.Scanner)
	s.Init(os.Stdin)
	for {
		spew.Printf("> ")
		x := s.Scan()
		spew.Println(x, s.TokenText())
	}
}
Esempio n. 15
0
// generate cuda wrapper for file.
func cuda2go(fname string) {
	// open cuda file
	f, err := os.Open(fname)
	util.PanicErr(err)
	defer f.Close()

	// read tokens
	var token []string
	var s scanner.Scanner
	s.Init(f)
	tok := s.Scan()
	for tok != scanner.EOF {
		if !filter(s.TokenText()) {
			token = append(token, s.TokenText())
		}
		tok = s.Scan()
	}

	// find function name and arguments
	funcname := ""
	argstart, argstop := -1, -1
	for i := 0; i < len(token); i++ {
		if token[i] == "__global__" {
			funcname = token[i+2]
			argstart = i + 4
		}
		if argstart > 0 && token[i] == ")" {
			argstop = i + 1
			break
		}
	}
	argl := token[argstart:argstop]

	// isolate individual arguments
	var args [][]string
	start := 0
	for i, a := range argl {
		if a == "," || a == ")" {
			args = append(args, argl[start:i])
			start = i + 1
		}
	}

	// separate arg names/types and make pointers Go-style
	argn := make([]string, len(args))
	argt := make([]string, len(args))
	for i := range args {
		if args[i][1] == "*" {
			args[i] = []string{args[i][0] + "*", args[i][2]}
		}
		argt[i] = typemap(args[i][0])
		argn[i] = args[i][1]
	}
	wrapgen(fname, funcname, argt, argn)
}
Esempio n. 16
0
func main() {
	values := url.Values{}

	//リクエストパラメータ用のkey-valueを追加
	values.Add("city", "200010")

	fmt.Println("パラメータ:\n", values.Encode())

	/***
	  //時間制限ありGET
	  client := &http.Client{Timeout: time.Duration(10 * time.Second)}
	  get(client, values)
	****/

	//普通のGET
	get(values)
	//普通のPOST
	post(values)

	fmt.Println("GETリクエスト:\n", responseGet)
	fmt.Println("POSTリクエスト:\n", responsePost)

	//受け取るJSONファイルの解析
	dec := json.NewDecoder(bytes.NewBufferString(responseGet))
	var d data
	dec.Decode(&d)

	b, err := json.Marshal(d)
	if err != nil {
		fmt.Errorf("%s", err)
	}
	jsonData := string(b)

	fmt.Println("【date: 日付, title: 天気, url: URL】")
	fmt.Println(jsonData)

	//字句解析
	lexArray := make([]string, 1)
	lexical := strings.NewReader(jsonData)
	var scan scanner.Scanner
	scan.Init(lexical)
	for {
		x := scan.Scan()
		//fmt.Println(x, scan.TokenText())

		if x == -1 {
			break
		} else if x == -6 {
			lexArray = append(lexArray, scan.TokenText())
		}
	}

	fmt.Println("明日は", lexArray[10], "です. 天気は", lexArray[13], "です. URLは", lexArray[15], "です.")

}
Esempio n. 17
0
// klexer scans the sourcefile and returns tokens on the channel c.
func klexer(s scanner.Scanner, c chan lex) {
	var l lex
	str := "" // Hold the current read text
	commt := false
	key := true
	tok := s.Scan()
	defer close(c)
	for tok != scanner.EOF {
		l.column = s.Position.Column
		l.line = s.Position.Line
		switch x := s.TokenText(); x {
		case ":":
			if commt {
				break
			}
			l.token = str
			if key {
				l.value = _KEY
				c <- l
				// Next token is a space, eat it
				s.Scan()
				key = false
				str = ""
			} else {
				l.value = _VALUE
			}
		case ";":
			commt = true
		case "\n":
			if commt {
				// Reset a comment
				commt = false
			}
			l.value = _VALUE
			l.token = str
			c <- l
			str = ""
			commt = false
			key = true
		default:
			if commt {
				break
			}
			str += x
		}
		tok = s.Scan()
	}
	if len(str) > 0 {
		// Send remainder
		l.token = str
		l.value = _VALUE
		c <- l
	}
}
Esempio n. 18
0
func (this delimiterSpan) scanString(s *scanner.Scanner) (string, error) {
	tok := s.Scan()
	if tok != scanner.EOF && tok != -2 && tok != 45 && tok != -3 {
		return s.TokenText(), nil
	} else {
		if debugLayoutDef {
			fmt.Printf("expected tok:=!(-2,-3,45), received:%d ('%s')\n", tok, s.TokenText())
		}
	}
	return "", NewInvalidTypeError()
}
Esempio n. 19
0
func cc_Calc(r io.Reader) int {
	cc := 1 // default
	var s scanner.Scanner
	s.Init(r)
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		if tok == scanner.Ident && isBranch(s.TokenText()) {
			cc++
		}
	}
	return cc
}
Esempio n. 20
0
func scanKey(s *scanner.Scanner) (key string, err error) {
	t := s.Scan()
	switch t {
	case scanner.Ident, scanner.Int, scanner.Float:
		key = s.TokenText()
	case scanner.String:
		key = strings.Trim(s.TokenText(), "\"")
	default:
		err = fmt.Errorf("Unexpected token at %s. Expected ident, number or string, had %s", s.Pos(), scanner.TokenString(t))
	}
	return
}
Esempio n. 21
0
func main() {
	var s scanner.Scanner
	s.Init(os.Stdin)
	for {
		switch s.Scan() {
		case scanner.EOF:
			return // all done
		case scanner.Ident:
			fmt.Println(s.TokenText())
		}
	}
}
Esempio n. 22
0
func Map(fileName string, intermediate chan Partial) {
	file, err := os.Open(fileName)
	if err == nil {
		var s scanner.Scanner
		s.Init(file)
		tok := s.Scan()
		for tok != scanner.EOF {
			intermediate <- Partial{s.TokenText(), fileName}
			tok = s.Scan()
		}
	}
	intermediate <- Partial{"", ""}
}
Esempio n. 23
0
func StringToMap(value string) (map[string]string, error) {
	var tokenizer scanner.Scanner
	tokenizer.Init(strings.NewReader(value))
	tokenizer.Error = func(*scanner.Scanner, string) {}

	result := make(map[string]string)
	next := func() string {
		tokenizer.Scan()
		return tokenizer.TokenText()
	}

	var lvalue, rvalue, expression string
	for {
		lvalue = next()
		if lvalue == "" {
			return result, errors.New(fmt.Sprintf("Expected key at pos '%d' but found none; "+
				"map values should be 'key=value' separated by commas", tokenizer.Pos().Offset))
		}
		if lvalue == "{" {
			// Assume this is JSON format and attempt to un-marshal
			return JSONToMap(value)
		}

		expression = next()
		if expression != "=" {
			return result, errors.New(fmt.Sprintf("Expected '=' after '%s' but found '%s'; "+
				"map values should be 'key=value' separated by commas", lvalue, expression))
		}
		rvalue = next()
		if rvalue == "" {
			return result, errors.New(fmt.Sprintf("Expected value after '%s' but found none; "+
				"map values should be 'key=value' separated by commas", expression))
		}
		// TODO: Handle quoted strings and escaped double quotes
		result[lvalue] = rvalue

		// Are there anymore tokens?
		delimiter := next()
		if delimiter == "" {
			break
		}

		// Should be a comma next
		if delimiter != "," {
			return result, errors.New(fmt.Sprintf("Expected ',' after '%s' but found '%s'; "+
				"map values should be 'key=value' separated by commas", rvalue, delimiter))
		}
	}

	return result, nil
}
Esempio n. 24
0
func readString(s *scanner.Scanner, tok *rune, err *error) string {
	val := ""
	*tok = s.Scan()
	var str string
	for *tok == scanner.String {
		str, *err = strconv.Unquote(s.TokenText())
		if *err != nil {
			break
		}
		val += str
		*tok = s.Scan()
	}
	return val
}
Esempio n. 25
0
// collect the next characters in s that are numbers
// assumes at least TokenText is a number
func Number(s *scanner.Scanner) string {
	var buffer bytes.Buffer

	for s.Scan() != scanner.EOF {
		buffer.WriteString(s.TokenText())

		if _, ok := strconv.Atoi(string(s.Peek())); ok != nil {
			return buffer.String()
		}
	}

	// reached the end of file, but that's ok potentially?
	return buffer.String()
}
Esempio n. 26
0
func Print(s *scanner.Scanner, w io.Writer, p Printer) error {
	tok := s.Scan()
	for tok != scanner.EOF {
		tokText := s.TokenText()
		err := p.Print(w, tokenKind(tok, tokText), tokText)
		if err != nil {
			return err
		}

		tok = s.Scan()
	}

	return nil
}
Esempio n. 27
0
func ReadNotes(src io.Reader) (ret []interface{}, err error) {

	var s scanner.Scanner
	s.Init(src)

	isGroup := false

	curSign := 0

	var lastGroup []Note

	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()

		v := s.TokenText()

		fmt.Println(v)

		switch v {
		case "(":
			isGroup = true
			continue
		case ")":
			isGroup = false
			continue
		case "+":
			curSing = 1
			continue
		case "-":
			curSign = -1
			continue
		}

		if !isGroup {

			note, err := Parse5(v, curSign)

			if err != nil {
				return ret, err
			}

			lastGroup = make([]Note, 1)
			lastGroup[0] = note
		}

	}
	return
}
Esempio n. 28
0
// Parse parses a SPARQL query from the reader
func (sparql *SPARQLUpdate) Parse(src io.Reader) error {
	b, _ := ioutil.ReadAll(src)
	s := new(scanner.Scanner).Init(bytes.NewReader(b))
	s.Mode = scanner.ScanIdents | scanner.ScanStrings

	start := 0
	level := 0
	verb := ""
	tok := s.Scan()
	for tok != scanner.EOF {
		switch tok {
		case -2:
			if level == 0 {
				if len(verb) > 0 {
					verb += " "
				}
				verb += s.TokenText()
			}

		case 123: // {
			if level == 0 {
				start = s.Position.Offset
			}
			level++

		case 125: // }
			level--
			if level == 0 {
				query := SPARQLUpdateQuery{
					body:  string(b[start+1 : s.Position.Offset]),
					graph: NewGraph(sparql.baseURI),
					verb:  verb,
				}
				query.graph.Parse(strings.NewReader(query.body), "text/turtle")
				sparql.queries = append(sparql.queries, query)
			}

		case 59: // ;
			if level == 0 {
				verb = ""
			}
		}

		tok = s.Scan()
	}

	return nil
}
Esempio n. 29
0
func TestSortScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(sortInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	Sort(in, func(a interface{}, b interface{}) bool { return a.(string) < b.(string) })
	// No valid way to assert the result
}
Esempio n. 30
0
// Text satifies the Extractor interface
//
//ReplaceGo is a specialized routine for correcting Golang source
// files.  Currently only checks comments, not identifiers for
// spelling.
//
// Other items:
//   - check strings, but need to ignore
//      * import "statements" blocks
//      * import ( "blocks" )
//   - skip first comment (line 0) if build comment
//
func (p *GolangText) Text(raw []byte) []byte {
	out := bytes.Buffer{}
	s := scanner.Scanner{}
	s.Init(bytes.NewReader(raw))
	s.Error = (func(s *scanner.Scanner, msg string) {})
	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
	for {
		switch s.Scan() {
		case scanner.Comment:
			out.WriteString(s.TokenText())
			out.WriteByte('\n')
		case scanner.EOF:
			return out.Bytes()
		}
	}
}