func main() {
	var str = "hello world\ni am jemy\r"

	var buf = []byte(str)
	fmt.Println("----------ScanBytes----------")
	for {
		advance, token, err := bufio.ScanBytes(buf, true)
		if advance == 0 {
			break
		}
		fmt.Println(advance, token, err)
		if advance <= len(buf) {
			buf = buf[advance:]
		}
	}

	fmt.Println("----------ScanLines----------")
	buf = []byte(str)
	for {
		advance, token, err := bufio.ScanLines(buf, true)
		if advance == 0 {
			break
		}
		fmt.Print(advance, string(token), err)
		fmt.Println()
		if advance <= len(buf) {
			buf = buf[advance:]
		}
	}

	fmt.Println("----------ScanRunes----------")
	buf = []byte(str)
	for {
		advance, token, err := bufio.ScanRunes(buf, true)
		if advance == 0 {
			break
		}
		fmt.Print(advance, string(token), len(token), err)
		fmt.Println()
		if advance <= len(buf) {
			buf = buf[advance:]
		}
	}

	fmt.Println("----------ScanWords----------")
	buf = []byte(str)
	for {
		advance, token, err := bufio.ScanWords(buf, true)
		if advance == 0 {
			break
		}
		fmt.Print(advance, string(token), len(token), err)
		fmt.Println()
		if advance <= len(buf) {
			buf = buf[advance:]
		}
	}
}
示例#2
0
// Use a Scanner with a custom split function (built by wrapping ScanWords) to validate
// 32-bit decimal input.
func ExampleScanner_custom() {
	// An artificial input source.
	const input = "1234 5678 1234567901234567890"
	scanner := bufio.NewScanner(strings.NewReader(input))
	// Create a custom split function by wrapping the existing ScanWords function.
	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		advance, token, err = bufio.ScanWords(data, atEOF)
		if err == nil && token != nil {
			_, err = strconv.ParseInt(string(token), 10, 32)
		}
		return
	}
	// Set the split function for the scanning operation.
	scanner.Split(split)
	// Validate the input
	for scanner.Scan() {
		fmt.Printf("%s\n", scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		fmt.Printf("Invalid input: %s", err)
	}
	// Output:
	// 1234
	// 5678
	// Invalid input: strconv.ParseInt: parsing "1234567901234567890": value out of range
}
示例#3
0
// tokenization function on the file
func split(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanWords(data, atEOF)
	if advance > 0 {
		token = []byte(strings.ToLower(strings.Trim(string(token), "_:-,!.?;\"''")))
	}
	return
}
示例#4
0
// splitTokens can be used with bufio.Scanner to tokenize CLDR plural rules.
func splitTokens(data []byte, atEOF bool) (advance int, token []byte, err error) {
	condTokens := [][]byte{
		[]byte(".."),
		[]byte(","),
		[]byte("!="),
		[]byte("="),
	}
	advance, token, err = bufio.ScanWords(data, atEOF)
	for _, t := range condTokens {
		if len(t) >= len(token) {
			continue
		}
		switch p := bytes.Index(token, t); {
		case p == -1:
		case p == 0:
			advance = len(t)
			token = token[:len(t)]
			return advance - len(token) + len(t), token[:len(t)], err
		case p < advance:
			// Don't split when "=" overlaps "!=".
			if t[0] == '=' && token[p-1] == '!' {
				continue
			}
			advance = p
			token = token[:p]
		}
	}
	return advance, token, err
}
示例#5
0
func (c *WordCounter) Write(p []byte) (int, error) {
	//*c += ByteCounter(len(p))
	//return len(p), nil
	var _, s, _ = bufio.ScanWords(p, true)
	*c += WordCounter(len(s))
	return len(s), nil
}
示例#6
0
文件: builder.go 项目: Tonkpils/snag
func splitFunc(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanWords(data, atEOF)
	if err != nil {
		return
	}

	if len(token) == 0 {
		return
	}

	b := token[0]
	if b != '"' && b != '\'' {
		return
	}

	if token[len(token)-1] == b {
		return
	}

	chunk := data[advance-1:]
	i := bytes.IndexByte(chunk, b)
	if i == -1 {
		advance = len(data)
		token = append(token, chunk...)
		return
	}

	advance += i
	token = append(token, chunk[:i+1]...)

	return
}
示例#7
0
func words(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanWords(data, atEOF)
	if err == nil && token != nil {
		token = append(token, ' ')
	}
	return
}
示例#8
0
文件: module.go 项目: apigee/henchman
// Split args from the cli that are of the form,
// "a=x b=y c=z" as a map of form { "a": "b", "b": "y", "c": "z" }
// These plan arguments override the variables that may be defined
// as part of the plan file.
func parseModuleArgs(args string) (map[string]string, error) {
	extraArgs := make(map[string]string)
	scanner := bufio.NewScanner(strings.NewReader(args))

	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		advance, nextToken, err := bufio.ScanWords(data, atEOF)
		tokenParts := strings.Split(string(nextToken), "=")
		seps := []byte{'"', '\''}
		for _, sep := range seps {
			if len(tokenParts) > 1 && tokenParts[1][0] == sep && tokenParts[1][len(tokenParts[1])-1] != sep {
				//get the remaining token
				remainingToken, err := getRemainingToken(data[(advance-1):], sep)
				if err == nil {
					token = append(nextToken, remainingToken...)
					break
				}
			} else {
				token = nextToken
			}
		}
		return
	}

	scanner.Split(split)
	// Validate the input
	for scanner.Scan() {
		text := scanner.Text()
		if extraArgsHasText(extraArgs, text) {
			continue
		} else if strings.Contains(text, "=") {
			splitValues := strings.Split(text, "=")
			//this may happen for cases where '=' is in the string
			if len(splitValues) > 2 {
				buffer := bytes.NewBufferString(splitValues[1])
				for i := 2; i < len(splitValues); i++ {
					buffer.WriteString("=")
					buffer.WriteString(splitValues[i])
				}
				splitValues[1] = buffer.String()
			}
			extraArgs[splitValues[0]] = splitValues[1]
		} else {
			// this check takes care of 2nd part of " def'" part of 'abc def'
			return nil, HenchErr(fmt.Errorf("Module args are invalid"), map[string]interface{}{
				"args":     args,
				"solution": "Refer to wiki on proper use of modules",
			}, "")
		}
	}
	// remove all quotes. Value for the respective key
	// should not have quotes
	extraArgs = stripQuotes(extraArgs)

	if err := scanner.Err(); err != nil {
		return extraArgs, HenchErr(err, nil, "Invalid input")
	}
	return extraArgs, nil
}
示例#9
0
文件: module.go 项目: rmenn/henchman
// Split args from the cli that are of the form,
// "a=x b=y c=z" as a map of form { "a": "b", "b": "y", "c": "z" }
// These plan arguments override the variables that may be defined
// as part of the plan file.
func parseModuleArgs(args string) (map[string]string, error) {
	extraArgs := make(map[string]string)
	scanner := bufio.NewScanner(strings.NewReader(args))

	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		advance, nextToken, err := bufio.ScanWords(data, atEOF)
		tokenParts := strings.Split(string(nextToken), "=")
		seps := []byte{'"', '\''}
		for _, sep := range seps {
			if len(tokenParts) > 1 && tokenParts[1][0] == sep && tokenParts[1][len(tokenParts[1])-1] != sep {
				remainingToken, err := getRemainingToken(data[(advance-1):], sep)
				//get the remaining token
				if err == nil {
					token = append(nextToken, remainingToken...)
					break
				}
			} else {
				token = nextToken
			}
		}
		return
	}

	scanner.Split(split)
	// Validate the input
	for scanner.Scan() {
		text := scanner.Text()
		if strings.Contains(text, "=") {
			splitValues := strings.Split(text, "=")
			extraArgs[splitValues[0]] = splitValues[1]
		} else if !extraArgsHasText(extraArgs, text) {
			// this check takes care of 2nd part of " def'" part of 'abc def'
			return nil, errors.New("Module args are invalid")
		}
	}
	// remove all quotes. Value for the respective key
	// should not have quotes
	extraArgs = stripQuotes(extraArgs)

	if err := scanner.Err(); err != nil {
		fmt.Printf("Invalid input: %s", err)
		return extraArgs, err
	}
	return extraArgs, nil
}
示例#10
0
func newToken(rawb []byte) (*token, error) {
	_, tkind, err := bufio.ScanWords(rawb, true)
	if err != nil {
		return nil, err
	}
	var ok bool
	t := token{rawkind: rawb}
	t.kind, ok = keywords[string(tkind)]
	if !ok {
		trimmed := strings.TrimSpace(string(tkind))
		if trimmed == "" {
			t.kind = tkWhitespace // whitespace-only, should happen only at EOF
			return &t, nil
		}
		if strings.HasPrefix(trimmed, "#") {
			t.kind = tkComment // this is a comment
			return &t, nil
		}
		return &t, nil
	}
	return &t, nil
}
示例#11
0
func main() {
	sc := bufio.NewScanner(os.Stdin)

	split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		advance, token, err = bufio.ScanWords(data, atEOF)
		if err == nil && token != nil {
			_, err = strconv.ParseInt(string(token), 10, 32)
		}
		return
	}

	sc.Split(split)

	for sc.Scan() {
		fmt.Printf("%s\n", sc.Text())
	}

	if err := sc.Err(); err != nil {
		fmt.Printf("Invalid input: %s", err)
	}

}
示例#12
0
func (c *WordCounter) Write(p []byte) (int, error) {
	advance, _, _ := bufio.ScanWords(p, true)
	*c += WordCounter(advance)
	return len(p), nil
}
示例#13
0
func LoadJournalFromReader(s interfaces.IState, r *bufio.Reader) {
	s.SetIsReplaying()
	defer s.SetIsDoneReplaying()

	fmt.Println("Replaying Journal")
	time.Sleep(time.Second * 5)
	fmt.Println("GO!")
	t := 0
	p := 0
	for {
		t++
		fmt.Println("total: ", t, " processed: ", p, "            \r")

		// line is empty if no more data
		line, err := r.ReadBytes('\n')
		if len(line) == 0 || err != nil {
			break
		}

		// Get the next word.  If not MsgHex:, then go to next line.
		adv, word, err := bufio.ScanWords(line, true)
		if string(word) != "MsgHex:" {
			continue // Go to next line.
		}
		line = line[adv:] // Remove "MsgHex:" from the line.

		// Remove spaces.
		adv, data, err := bufio.ScanWords(line, true)
		if err != nil {
			fmt.Println(err)
			return
		}

		// Decode the hex
		binary, err := hex.DecodeString(string(data))
		if err != nil {
			fmt.Println(err)
			return
		}

		// Unmarshal the message.
		msg, err := messages.UnmarshalMessage(binary)
		if err != nil {
			fmt.Println(err)
			return
		}

		// Process the message.
		s.InMsgQueue() <- msg
		p++
		if len(s.InMsgQueue()) > 200 {
			for len(s.InMsgQueue()) > 50 {
				time.Sleep(time.Millisecond * 10)
			}
			time.Sleep(time.Millisecond * 100)
		}
	}

	//Waiting for state to process the message queue
	//before we disable "IsDoneReplaying"
	for len(s.InMsgQueue()) > 0 {
		time.Sleep(time.Millisecond * 100)
	}
}
示例#14
0
文件: netrc.go 项目: shaunduncan/hk
func getToken(b []byte, pos *filePos) ([]byte, *token, error) {
	adv, wordb, err := bufio.ScanWords(b, true)
	if err != nil {
		return b, nil, err // should never happen
	}
	b = b[adv:]
	word := string(wordb)
	if word == "" {
		return b, nil, nil // EOF reached
	}

	t := new(token)
	var ok bool
	t.kind, ok = keywords[word]
	if !ok {
		return b, nil, &Error{pos.name, pos.line, "keyword expected; got " + word}
	}
	if t.kind == tkDefault {
		return b, t, nil
	}
	if t.kind == tkComment {
		t.value = word + " "
		adv, wordb, err = bufio.ScanLines(b, true)
		if err != nil {
			return b, nil, err // should never happen
		}
		t.value = t.value + string(wordb)
		return b[adv:], t, nil
	}

	if word == "" {
		return b, nil, &Error{pos.name, pos.line, "word expected"}
	}
	if t.kind == tkMacdef {
		adv, lineb, err := bufio.ScanLines(b, true)
		if err != nil {
			return b, nil, err // should never happen
		}
		b = b[adv:]
		adv, wordb, err = bufio.ScanWords(lineb, true)
		if err != nil {
			return b, nil, err // should never happen
		}
		word = string(wordb)
		t.macroName = word
		lineb = lineb[adv:]

		// Macro value starts on next line. The rest of current line
		// should contain nothing but whitespace
		i := 0
		for i < len(lineb) {
			r, size := utf8.DecodeRune(lineb[i:])
			if r == '\n' {
				i += size
				pos.line++
				break
			}
			if !unicode.IsSpace(r) {
				return b, nil, &Error{pos.name, pos.line, "unexpected word"}
			}
			i += size
		}

		// Find end of macro value
		i = bytes.Index(b, []byte("\n\n"))
		if i < 0 { // EOF reached
			i = len(b)
		}
		t.value = string(b[0:i])

		return b[i:], t, nil
	} else {
		adv, wordb, err = bufio.ScanWords(b, true)
		if err != nil {
			return b, nil, err // should never happen
		}
		word = string(wordb)
		b = b[adv:]
	}
	t.value = word
	return b, t, nil
}