func main() {
var str = "hello world\ni am jemy\r"
var buf = []byte(str)
fmt.Println("----------ScanBytes----------")
for {
advance, token, err := bufio.ScanBytes(buf, true)
if advance == 0 {
break
}
fmt.Println(advance, token, err)
if advance <= len(buf) {
buf = buf[advance:]
}
}
fmt.Println("----------ScanLines----------")
buf = []byte(str)
for {
advance, token, err := bufio.ScanLines(buf, true)
if advance == 0 {
break
}
fmt.Print(advance, string(token), err)
fmt.Println()
if advance <= len(buf) {
buf = buf[advance:]
}
}
fmt.Println("----------ScanRunes----------")
buf = []byte(str)
for {
advance, token, err := bufio.ScanRunes(buf, true)
if advance == 0 {
break
}
fmt.Print(advance, string(token), len(token), err)
fmt.Println()
if advance <= len(buf) {
buf = buf[advance:]
}
}
fmt.Println("----------ScanWords----------")
buf = []byte(str)
for {
advance, token, err := bufio.ScanWords(buf, true)
if advance == 0 {
break
}
fmt.Print(advance, string(token), len(token), err)
fmt.Println()
if advance <= len(buf) {
buf = buf[advance:]
}
}
}
// Use a Scanner with a custom split function (built by wrapping ScanWords) to validate
// 32-bit decimal input.
func ExampleScanner_custom() {
// An artificial input source.
const input = "1234 5678 1234567901234567890"
scanner := bufio.NewScanner(strings.NewReader(input))
// Create a custom split function by wrapping the existing ScanWords function.
split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
advance, token, err = bufio.ScanWords(data, atEOF)
if err == nil && token != nil {
_, err = strconv.ParseInt(string(token), 10, 32)
}
return
}
// Set the split function for the scanning operation.
scanner.Split(split)
// Validate the input
for scanner.Scan() {
fmt.Printf("%s\n", scanner.Text())
}
if err := scanner.Err(); err != nil {
fmt.Printf("Invalid input: %s", err)
}
// Output:
// 1234
// 5678
// Invalid input: strconv.ParseInt: parsing "1234567901234567890": value out of range
}
// tokenization function on the file
func split(data []byte, atEOF bool) (advance int, token []byte, err error) {
advance, token, err = bufio.ScanWords(data, atEOF)
if advance > 0 {
token = []byte(strings.ToLower(strings.Trim(string(token), "_:-,!.?;\"''")))
}
return
}
// splitTokens can be used with bufio.Scanner to tokenize CLDR plural rules.
func splitTokens(data []byte, atEOF bool) (advance int, token []byte, err error) {
condTokens := [][]byte{
[]byte(".."),
[]byte(","),
[]byte("!="),
[]byte("="),
}
advance, token, err = bufio.ScanWords(data, atEOF)
for _, t := range condTokens {
if len(t) >= len(token) {
continue
}
switch p := bytes.Index(token, t); {
case p == -1:
case p == 0:
advance = len(t)
token = token[:len(t)]
return advance - len(token) + len(t), token[:len(t)], err
case p < advance:
// Don't split when "=" overlaps "!=".
if t[0] == '=' && token[p-1] == '!' {
continue
}
advance = p
token = token[:p]
}
}
return advance, token, err
}
func (c *WordCounter) Write(p []byte) (int, error) {
//*c += ByteCounter(len(p))
//return len(p), nil
var _, s, _ = bufio.ScanWords(p, true)
*c += WordCounter(len(s))
return len(s), nil
}
func splitFunc(data []byte, atEOF bool) (advance int, token []byte, err error) {
advance, token, err = bufio.ScanWords(data, atEOF)
if err != nil {
return
}
if len(token) == 0 {
return
}
b := token[0]
if b != '"' && b != '\'' {
return
}
if token[len(token)-1] == b {
return
}
chunk := data[advance-1:]
i := bytes.IndexByte(chunk, b)
if i == -1 {
advance = len(data)
token = append(token, chunk...)
return
}
advance += i
token = append(token, chunk[:i+1]...)
return
}
func words(data []byte, atEOF bool) (advance int, token []byte, err error) {
advance, token, err = bufio.ScanWords(data, atEOF)
if err == nil && token != nil {
token = append(token, ' ')
}
return
}
// Split args from the cli that are of the form,
// "a=x b=y c=z" as a map of form { "a": "b", "b": "y", "c": "z" }
// These plan arguments override the variables that may be defined
// as part of the plan file.
func parseModuleArgs(args string) (map[string]string, error) {
extraArgs := make(map[string]string)
scanner := bufio.NewScanner(strings.NewReader(args))
split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
advance, nextToken, err := bufio.ScanWords(data, atEOF)
tokenParts := strings.Split(string(nextToken), "=")
seps := []byte{'"', '\''}
for _, sep := range seps {
if len(tokenParts) > 1 && tokenParts[1][0] == sep && tokenParts[1][len(tokenParts[1])-1] != sep {
//get the remaining token
remainingToken, err := getRemainingToken(data[(advance-1):], sep)
if err == nil {
token = append(nextToken, remainingToken...)
break
}
} else {
token = nextToken
}
}
return
}
scanner.Split(split)
// Validate the input
for scanner.Scan() {
text := scanner.Text()
if extraArgsHasText(extraArgs, text) {
continue
} else if strings.Contains(text, "=") {
splitValues := strings.Split(text, "=")
//this may happen for cases where '=' is in the string
if len(splitValues) > 2 {
buffer := bytes.NewBufferString(splitValues[1])
for i := 2; i < len(splitValues); i++ {
buffer.WriteString("=")
buffer.WriteString(splitValues[i])
}
splitValues[1] = buffer.String()
}
extraArgs[splitValues[0]] = splitValues[1]
} else {
// this check takes care of 2nd part of " def'" part of 'abc def'
return nil, HenchErr(fmt.Errorf("Module args are invalid"), map[string]interface{}{
"args": args,
"solution": "Refer to wiki on proper use of modules",
}, "")
}
}
// remove all quotes. Value for the respective key
// should not have quotes
extraArgs = stripQuotes(extraArgs)
if err := scanner.Err(); err != nil {
return extraArgs, HenchErr(err, nil, "Invalid input")
}
return extraArgs, nil
}
// Split args from the cli that are of the form,
// "a=x b=y c=z" as a map of form { "a": "b", "b": "y", "c": "z" }
// These plan arguments override the variables that may be defined
// as part of the plan file.
func parseModuleArgs(args string) (map[string]string, error) {
extraArgs := make(map[string]string)
scanner := bufio.NewScanner(strings.NewReader(args))
split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
advance, nextToken, err := bufio.ScanWords(data, atEOF)
tokenParts := strings.Split(string(nextToken), "=")
seps := []byte{'"', '\''}
for _, sep := range seps {
if len(tokenParts) > 1 && tokenParts[1][0] == sep && tokenParts[1][len(tokenParts[1])-1] != sep {
remainingToken, err := getRemainingToken(data[(advance-1):], sep)
//get the remaining token
if err == nil {
token = append(nextToken, remainingToken...)
break
}
} else {
token = nextToken
}
}
return
}
scanner.Split(split)
// Validate the input
for scanner.Scan() {
text := scanner.Text()
if strings.Contains(text, "=") {
splitValues := strings.Split(text, "=")
extraArgs[splitValues[0]] = splitValues[1]
} else if !extraArgsHasText(extraArgs, text) {
// this check takes care of 2nd part of " def'" part of 'abc def'
return nil, errors.New("Module args are invalid")
}
}
// remove all quotes. Value for the respective key
// should not have quotes
extraArgs = stripQuotes(extraArgs)
if err := scanner.Err(); err != nil {
fmt.Printf("Invalid input: %s", err)
return extraArgs, err
}
return extraArgs, nil
}
func newToken(rawb []byte) (*token, error) {
_, tkind, err := bufio.ScanWords(rawb, true)
if err != nil {
return nil, err
}
var ok bool
t := token{rawkind: rawb}
t.kind, ok = keywords[string(tkind)]
if !ok {
trimmed := strings.TrimSpace(string(tkind))
if trimmed == "" {
t.kind = tkWhitespace // whitespace-only, should happen only at EOF
return &t, nil
}
if strings.HasPrefix(trimmed, "#") {
t.kind = tkComment // this is a comment
return &t, nil
}
return &t, nil
}
return &t, nil
}
func main() {
sc := bufio.NewScanner(os.Stdin)
split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
advance, token, err = bufio.ScanWords(data, atEOF)
if err == nil && token != nil {
_, err = strconv.ParseInt(string(token), 10, 32)
}
return
}
sc.Split(split)
for sc.Scan() {
fmt.Printf("%s\n", sc.Text())
}
if err := sc.Err(); err != nil {
fmt.Printf("Invalid input: %s", err)
}
}
func (c *WordCounter) Write(p []byte) (int, error) {
advance, _, _ := bufio.ScanWords(p, true)
*c += WordCounter(advance)
return len(p), nil
}
func LoadJournalFromReader(s interfaces.IState, r *bufio.Reader) {
s.SetIsReplaying()
defer s.SetIsDoneReplaying()
fmt.Println("Replaying Journal")
time.Sleep(time.Second * 5)
fmt.Println("GO!")
t := 0
p := 0
for {
t++
fmt.Println("total: ", t, " processed: ", p, " \r")
// line is empty if no more data
line, err := r.ReadBytes('\n')
if len(line) == 0 || err != nil {
break
}
// Get the next word. If not MsgHex:, then go to next line.
adv, word, err := bufio.ScanWords(line, true)
if string(word) != "MsgHex:" {
continue // Go to next line.
}
line = line[adv:] // Remove "MsgHex:" from the line.
// Remove spaces.
adv, data, err := bufio.ScanWords(line, true)
if err != nil {
fmt.Println(err)
return
}
// Decode the hex
binary, err := hex.DecodeString(string(data))
if err != nil {
fmt.Println(err)
return
}
// Unmarshal the message.
msg, err := messages.UnmarshalMessage(binary)
if err != nil {
fmt.Println(err)
return
}
// Process the message.
s.InMsgQueue() <- msg
p++
if len(s.InMsgQueue()) > 200 {
for len(s.InMsgQueue()) > 50 {
time.Sleep(time.Millisecond * 10)
}
time.Sleep(time.Millisecond * 100)
}
}
//Waiting for state to process the message queue
//before we disable "IsDoneReplaying"
for len(s.InMsgQueue()) > 0 {
time.Sleep(time.Millisecond * 100)
}
}
func getToken(b []byte, pos *filePos) ([]byte, *token, error) {
adv, wordb, err := bufio.ScanWords(b, true)
if err != nil {
return b, nil, err // should never happen
}
b = b[adv:]
word := string(wordb)
if word == "" {
return b, nil, nil // EOF reached
}
t := new(token)
var ok bool
t.kind, ok = keywords[word]
if !ok {
return b, nil, &Error{pos.name, pos.line, "keyword expected; got " + word}
}
if t.kind == tkDefault {
return b, t, nil
}
if t.kind == tkComment {
t.value = word + " "
adv, wordb, err = bufio.ScanLines(b, true)
if err != nil {
return b, nil, err // should never happen
}
t.value = t.value + string(wordb)
return b[adv:], t, nil
}
if word == "" {
return b, nil, &Error{pos.name, pos.line, "word expected"}
}
if t.kind == tkMacdef {
adv, lineb, err := bufio.ScanLines(b, true)
if err != nil {
return b, nil, err // should never happen
}
b = b[adv:]
adv, wordb, err = bufio.ScanWords(lineb, true)
if err != nil {
return b, nil, err // should never happen
}
word = string(wordb)
t.macroName = word
lineb = lineb[adv:]
// Macro value starts on next line. The rest of current line
// should contain nothing but whitespace
i := 0
for i < len(lineb) {
r, size := utf8.DecodeRune(lineb[i:])
if r == '\n' {
i += size
pos.line++
break
}
if !unicode.IsSpace(r) {
return b, nil, &Error{pos.name, pos.line, "unexpected word"}
}
i += size
}
// Find end of macro value
i = bytes.Index(b, []byte("\n\n"))
if i < 0 { // EOF reached
i = len(b)
}
t.value = string(b[0:i])
return b[i:], t, nil
} else {
adv, wordb, err = bufio.ScanWords(b, true)
if err != nil {
return b, nil, err // should never happen
}
word = string(wordb)
b = b[adv:]
}
t.value = word
return b, t, nil
}
