func main() {
var Stdin = os.NewFile(uintptr(syscall.Stdin), "/dev/stdin")
var s scanner.Scanner
s.Init(Stdin)
s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
tok := s.Scan()
for tok != scanner.EOF {
switch tok {
case scanner.Ident:
fmt.Println("Ident")
case scanner.Int:
fmt.Println("Int")
case scanner.Float:
fmt.Println("Float")
case scanner.Char:
fmt.Println("Char")
case scanner.String:
fmt.Println("String")
case scanner.RawString:
fmt.Println("RawString")
case scanner.Comment:
fmt.Println("Comment")
}
tok = s.Scan()
}
}
func format(str, dbname string, tm time.Time) string {
buf := bytes.Buffer{}
var s scanner.Scanner
s.Init(strings.NewReader(str))
s.Mode = 0
s.Whitespace = 0
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
if tok != '%' {
buf.WriteRune(tok)
continue
}
switch s := s.Scan(); s {
case '%':
buf.WriteRune('%')
case 'n':
buf.WriteString(dbname)
case 'Y', 'y':
buf.WriteString(strconv.Itoa(tm.Year()))
case 'm':
buf.WriteString(strconv.Itoa(int(tm.Month())))
case 'd':
buf.WriteString(strconv.Itoa(tm.Day()))
case 'H':
buf.WriteString(twodig(strconv.Itoa(tm.Hour())))
case 'M':
buf.WriteString(twodig(strconv.Itoa(tm.Minute())))
case 'S':
buf.WriteString(twodig(strconv.Itoa(tm.Second())))
}
}
return buf.String()
}
func Tokenize(text string, stopwords map[string]int, words map[string]int) {
var s scanner.Scanner
s.Init(strings.NewReader(text))
tok := s.Scan()
for tok != scanner.EOF {
if tok == scanner.String {
Tokenize(strings.Trim(s.TokenText(), "\"`"), stopwords, words)
} else if tok == scanner.Char {
Tokenize(strings.Trim(s.TokenText(), "'"), stopwords, words)
} else if tok == scanner.Ident {
word := s.TokenText()
if _, ok := stopwords[word]; !ok && len(word) > 2 {
stem, err := snowball.Stem(word, "english", true)
if err != nil {
fmt.Errorf("Couldnt stem word: %s", word)
stem = word
}
if _, ok := stopwords[stem]; !ok {
words[stem] += 1
}
}
}
tok = s.Scan()
}
}
// gofmtFlags looks for a comment of the form
//
// //gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
f, err := os.Open(filename)
if err != nil {
return "" // ignore errors - they will be found later
}
defer f.Close()
// initialize scanner
var s scanner.Scanner
s.Init(f)
s.Error = func(*scanner.Scanner, string) {} // ignore errors
s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments
// look for //gofmt comment
for s.Line <= maxLines {
switch s.Scan() {
case scanner.Comment:
const prefix = "//gofmt "
if t := s.TokenText(); strings.HasPrefix(t, prefix) {
return strings.TrimSpace(t[len(prefix):])
}
case scanner.EOF:
return ""
}
}
return ""
}
func (w *wordsStruct) addString(str string) {
var scan scanner.Scanner
scan.Init(strings.NewReader(str))
for scan.Peek() != scanner.EOF {
w.addChar(scan.Next())
}
}
func split(expr string) (keys []string, err error) {
var msgs []string
var s scanner.Scanner
s.Init(strings.NewReader(expr))
s.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings
s.Error = func(s *scanner.Scanner, msg string) { msgs = append(msgs, fmt.Sprintf("%s %s", s.Pos(), msg)) }
key := ""
keys = []string{}
for err == nil {
t := s.Peek()
// fmt.Printf(">>> %s: %s %s\n", s.Pos(), scanner.TokenString(t), s.TokenText())
switch t {
case '[':
key, err = scanBracketedKey(&s)
case '.':
s.Scan()
continue
case scanner.EOF:
goto end
default:
key, err = scanKey(&s)
}
if len(msgs) > 0 {
err = errors.New(strings.Join(msgs, "\n"))
}
if err == nil {
keys = append(keys, key)
}
}
end:
return
}
// Parse updates an inverted index with the file's words
func (index *InvertedIndex) Parse(filename string) {
if index.alreadyParsed(filename) {
return
}
file, err := os.Open(filename)
if err != nil {
cwd, _ := os.Getwd()
fmt.Printf("The file named %s doesn't exist, the current directory is %s\n",
filename, cwd)
return
}
defer file.Close()
var s scanner.Scanner
s.Init(file)
var token rune
for token != scanner.EOF {
token = s.Scan()
tokenText := s.TokenText()
_, found := index.indexMap[tokenText]
pos := s.Pos()
result := Result{File: filename,
Line: pos.Line,
Index: pos.Column - len(tokenText)}
if !found {
index.indexMap[tokenText] = []Result{result}
} else {
index.indexMap[tokenText] = append(index.indexMap[tokenText],
result)
}
}
index.files = append(index.files, filename)
}
// parseKey reads a private key from r. It returns a map[string]string,
// with the key-value pairs, or an error when the file is not correct.
func parseKey(r io.Reader, file string) (map[string]string, error) {
var s scanner.Scanner
m := make(map[string]string)
c := make(chan lex)
k := ""
s.Init(r)
s.Mode = 0
s.Whitespace = 0
// Start the lexer
go klexer(s, c)
for l := range c {
// It should alternate
switch l.value {
case _KEY:
k = l.token
case _VALUE:
if k == "" {
return nil, &ParseError{file, "No key seen", l}
}
//println("Setting", strings.ToLower(k), "to", l.token, "b")
m[strings.ToLower(k)] = l.token
k = ""
}
}
return m, nil
}
func getLineToken(line string) []string {
var tokenScanner scanner.Scanner
tokenScanner.Init(strings.NewReader(line))
return tokenize(&tokenScanner)
}
/**
* This function parses a value string parameter and returns Number value
* embedded within the string. It returns nil if it doesn't find any
* Number value in the value string.
* Example: "some4.56more" would return 4.56
*/
func extractFloat32(value string) Number {
var sc scanner.Scanner
var tok rune
var valFloat64 float64
var valFloat32 Number
var err error
var isFound bool
if len(value) > 0 {
sc.Init(strings.NewReader(value))
sc.Mode = scanner.ScanFloats
for tok != scanner.EOF {
tok = sc.Scan()
// fmt.Println("At position", sc.Pos(), ":", sc.TokenText())
valFloat64, err = strconv.ParseFloat(sc.TokenText(), 64)
if err == nil {
isFound = true
break
}
}
}
if isFound {
valFloat32 = Number(valFloat64)
}
return valFloat32
}
func (s *PGNSuite) TestPGNParseInfiniteLoopF4(c *C) {
pgnstr := `[Event "BKL-Turnier"]
[Site "Leipzig"]
[Date "1984.??.??"]
[Round "5"]
[White "Polgar, Zsuzsa"]
[Black "Moehring, Guenther"]
[Result "1-0"]
[WhiteElo "2275"]
[BlackElo "2395"]
[ECO "A49"]
1.d4 Nf6 2.Nf3 d6 3.b3 g6 4.Bb2 Bg7 5.g3 c5 6.Bg2 cxd4 7.Nxd4 d5 8.O-O O-O
9.Na3 Re8 10.Nf3 Nc6 11.c4 dxc4 12.Nxc4 Be6 13.Rc1 Rc8 14.Nfe5 Nxe5 15.Bxe5 Bxc4
16.Rxc4 Rxc4 17.bxc4 Qa5 18.Bxf6 Bxf6 19.Bxb7 Rd8 20.Qb3 Rb8 21.e3 h5 22.Rb1 h4
23.Qb5 Qc7 24.a4 hxg3 25.hxg3 Be5 26.Kg2 Bd6 27.a5 Bc5 28.a6 Rd8 29.Qc6 Qxc6+
30.Bxc6 Rd2 31.Kf3 Rc2 32.Rb8+ Kg7 33.Bb5 Kf6 34.Rc8 Bb6 35.Ba4 Ra2 36.Bb5 Rc2
37.Ke4 e6 38.Kd3 Rc1 39.Kd2 Rb1 40.Kc2 Rb4 41.Rb8 Bc5 42.Rc8 Bb6 43.Rc6 Ba5
44.Rd6 g5 45.f4 gxf4 46.gxf4 Kf5 47.Rd7 Bb6 48.Rxf7+ Ke4 49.Rb7 Bc5 50.Kc3 Kxe3
51.Rc7 Bb6 52.Rc6 Ba5 53.Kc2 Kxf4 54.Rxe6 Bd8 55.Kc3 Rb1 56.Kd4 Rd1+ 57.Kc5 Kf5
58.Re8 Bb6+ 59.Kc6 Kf6 60.Kb7 Bg1 61.Ra8 Re1 62.Rf8+ Kg7 63.Rf5 Kg6 64.Rd5 Rc1
65.Ka8 Be3 66.Rd6+ Kf5 67.Rd3 Ke4 68.Rxe3+ Kxe3 69.Kxa7 Kd4 70.Kb6 Rg1 71.a7 Rg8
72.Kb7 Rg7+ 73.Kb6 1-0`
r := strings.NewReader(pgnstr)
sc := scanner.Scanner{}
sc.Init(r)
game, err := ParseGame(&sc)
c.Assert(err, IsNil)
// fmt.Println(game)
c.Assert(game.Tags["Site"], Equals, "Leipzig")
c.Assert(len(game.Moves), Equals, 145)
}
func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {
var s scanner.Scanner
s.Error = func(s *scanner.Scanner, msg string) {
log.Printf("command scan error: %s [%s]", msg, path)
}
s.Init(strings.NewReader(path))
parent := root
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
//log.Printf("cmdFind: token: [%s]", s.TokenText())
label := s.TokenText()
children := matchChildren(parent.Children, label)
size := len(children)
if size < 1 {
return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
}
if size > 1 {
return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
}
parent = children[0]
}
//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)
return parent, nil
}
func TestReduceScanner(t *testing.T) {
in := make(chan interface{})
go func() {
reader := strings.NewReader(mapInput)
s := scanner.Scanner{}
s.Init(reader)
tok := s.Scan()
for tok != scanner.EOF {
tok = s.Scan()
in <- s.TokenText()
}
close(in)
}()
mapOut := Map(in, func(input interface{}) interface{} {
key := input.(string)
if len(key) == 0 {
return ""
} else {
return string(key[0])
}
})
redOut := Reduce(mapOut, func(key interface{}, values []interface{}) interface{} {
return len(values)
})
for tok := range redOut {
if tok.Value.(int) == 0 {
t.Error("expect reduce should be positive")
}
}
}
func (t Tokenizer) CreateTokens(request *messaging.ParserRequest) (response messaging.ParserResponse) {
response.Body = make(map[string]string)
b := bytes.NewBufferString(request.Query)
var s scanner.Scanner
s.Init(b)
index := 0
for {
tok := s.Scan()
if tok != scanner.EOF {
response.Body[strconv.Itoa(index)] = s.TokenText()
index++
} else {
break
}
}
if index == 0 {
response.IsSuccess = false
response.Message = "Error! Nil Query Allocated!"
} else {
response.IsSuccess = false
response.Message = "Success! Query Tokenized Successfully!"
}
return
}
func Example() {
const src = `
// This is scanned code.
if a > 10 {
someParsable = text
}`
var s scanner.Scanner
s.Init(strings.NewReader(src))
var tok rune
for tok != scanner.EOF {
tok = s.Scan()
fmt.Println("At position", s.Pos(), ":", s.TokenText())
}
// Output:
// At position 3:4 : if
// At position 3:6 : a
// At position 3:8 : >
// At position 3:11 : 10
// At position 3:13 : {
// At position 4:15 : someParsable
// At position 4:17 : =
// At position 4:22 : text
// At position 5:3 : }
// At position 5:3 :
}
func NewDecoder(r io.Reader) *Decoder {
var scan scanner.Scanner
scan.Init(r)
dec := Decoder{scan: scan}
scan.Error = dec.setError
return &dec
}
func TestMapScanner(t *testing.T) {
in := make(chan interface{})
go func() {
reader := strings.NewReader(mapInput)
s := scanner.Scanner{}
s.Init(reader)
tok := s.Scan()
for tok != scanner.EOF {
tok = s.Scan()
in <- s.TokenText()
}
close(in)
}()
out := Map(in, func(input interface{}) interface{} {
key := input.(string)
if len(key) == 0 {
return ""
} else {
return string(key[0])
}
})
for tok := range out {
for _, val := range tok.Values {
if !strings.HasPrefix(val.(string), tok.Key.(string)) {
t.Error(val.(string) + ` should has "` + tok.Key.(string) + `" prefix`)
}
}
}
}
func Parser(s string) []string {
var items []string
var sc scanner.Scanner
sc.Init(strings.NewReader(s))
var tok rune
for tok != scanner.EOF {
tok = sc.Scan()
item := sc.TokenText()
if tok != scanner.EOF {
i := len(items)
if i > 0 {
if item == "=" && (items[i-1] == "!" || items[i-1] == ">" || items[i-1] == "<" || items[i-1] == "=") {
items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
continue
}
if item == "|" && (items[i-1]) == "|" {
items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
continue
}
if item == "&" && (items[i-1]) == "&" {
items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
continue
}
}
items = append(items, item)
}
}
return items
}
// ProcessConfig replaces references of environment varialbes for the given data
// Support variable syntax: $varname, ${varname}
func ProcessConfig(data []byte, e *env.Env, escapeChar rune) ([]byte, error) {
var result []byte
var sc scanner.Scanner
sc.Init(bytes.NewReader(data))
DONE:
for {
switch ch := sc.Peek(); ch {
default:
result = append(result, byte(sc.Next()))
case scanner.EOF:
break DONE
case escapeChar:
curr, next := sc.Next(), sc.Peek()
if next != '$' {
result = append(result, byte(curr))
}
if next != scanner.EOF {
result = append(result, byte(sc.Next()))
}
case '$':
name, err := parseVariable(&sc)
if err != nil {
pos := sc.Pos()
return result, fmt.Errorf(`parseError:%d:%d: %v %q`, pos.Line, pos.Offset, err, name)
}
result = append(result, e.Get(string(name))...)
}
}
return result, nil
}
// Read contents of a Go file up to the package declaration. This can be used
// to find the the build tags.
func readGoContents(fp string) ([]byte, error) {
f, err := os.Open(fp)
defer f.Close()
if err != nil {
return []byte{}, err
}
var s scanner.Scanner
s.Init(f)
var tok rune
var pos scanner.Position
for tok != scanner.EOF {
tok = s.Scan()
// Getting the token text will skip comments by default.
tt := s.TokenText()
// build tags will not be after the package declaration.
if tt == "package" {
pos = s.Position
break
}
}
buf := bytes.NewBufferString("")
f.Seek(0, 0)
_, err = io.CopyN(buf, f, int64(pos.Offset))
if err != nil {
return []byte{}, err
}
return buf.Bytes(), nil
}
func Tokenize(r io.Reader) []string {
var scr1 scanner.Scanner
scr1.Init(r)
list1 := make([]string, 0, 100)
for rune1 := scr1.Next(); rune1 != scanner.EOF; {
for rune1 != scanner.EOF && unicode.IsSpace(rune1) {
rune1 = scr1.Next()
}
if rune1 == '(' {
list1 = append(list1, "(")
rune1 = scr1.Next()
} else if rune1 == ')' {
list1 = append(list1, ")")
rune1 = scr1.Next()
} else {
var token string
token, rune1 = readtoken(&scr1, rune1)
if token != "" {
list1 = append(list1, token)
}
}
}
return list1
}
func TestUnwindScanner(t *testing.T) {
in := make(chan interface{})
go func() {
reader := strings.NewReader(unwindInput)
s := scanner.Scanner{}
s.Init(reader)
tok := s.Scan()
for tok != scanner.EOF {
tok = s.Scan()
in <- s.TokenText()
}
close(in)
}()
out := Unwind(in, func(input interface{}, output chan<- interface{}) {
output <- strings.ToLower(input.(string))
output <- strings.ToUpper(input.(string))
})
index := 0
for tok := range out {
if index%2 == 0 {
if strings.ToLower(tok.(string)) != tok.(string) {
t.Error(tok, " should be lower cased")
}
} else {
if strings.ToUpper(tok.(string)) != tok.(string) {
t.Error(tok, " should be upper cased")
}
}
index++
}
}
func NewScanner(src []byte) *scanner.Scanner {
var s scanner.Scanner
s.Init(bytes.NewReader(src))
s.Error = func(_ *scanner.Scanner, _ string) {}
s.Whitespace = 0
s.Mode = s.Mode ^ scanner.SkipComments
return &s
}
// NewScannerReader takes a reader src and creates a Scanner.
func NewScannerReader(src io.Reader) *scanner.Scanner {
var s scanner.Scanner
s.Init(src)
s.Error = func(_ *scanner.Scanner, _ string) {}
s.Whitespace = 0
s.Mode = s.Mode ^ scanner.SkipComments
return &s
}
func main() {
d := freenect.NewFreenectDevice(StartWithDevice)
var s scanner.Scanner
s.Init(os.Stdin)
var cmd string
var arg int = OutOfRange
fmt.Print("gonect> ")
tok := s.Scan()
for tok != scanner.EOF {
if scanner.TokenString(tok) == "Ident" {
cmd = s.TokenText()
}
switch {
case cmd == "help":
showHelpMessage()
break
case cmd == "tilt":
s.Scan()
arg, _ = strconv.Atoi(s.TokenText())
if arg == OutOfRange { break }
d.SetTiltDegs(arg)
cmd = ""
arg = OutOfRange
case cmd == "led":
s.Scan()
led_string := s.TokenText()
flash_led(d, led_string)
break
case cmd == "rgb_frame":
s.Scan()
filename := s.TokenText()
freenect.SaveRGBAFrame(d, filename)
break
case cmd == "ir_frame":
s.Scan()
filename := s.TokenText()
freenect.SaveIRFrame(d, filename)
break
case cmd == "depth_frame":
s.Scan()
filename := s.TokenText()
freenect.SaveDepthFrame(d, filename)
break
case cmd == "quit":
return
}
fmt.Printf("gonect> ")
s.Scan()
}
}
func resolveImportsRec(asts []ast, path string, imported []string,
download bool) ([]ast, error) {
var newAsts []ast
top := true // Imports are required to be at the top of the file.
for _, ast := range asts {
name := parseImport(ast)
if name == "" {
newAsts = append(newAsts, ast)
top = false
continue
}
if !top {
return nil, errors.New("import must be begin the module")
}
// Check for any import cycles.
for _, importedModule := range imported {
if name == importedModule {
return nil, fmt.Errorf("import cycle: %s",
append(imported, name))
}
}
modulePath := filepath.Join(path, name+".spec")
var sc scanner.Scanner
sc.Filename = modulePath
if _, err := os.Stat(modulePath); os.IsNotExist(err) && download {
GetSpec(name)
}
f, err := util.Open(modulePath)
if err != nil {
return nil, fmt.Errorf("unable to open import %s", name)
}
defer f.Close()
sc.Init(bufio.NewReader(f))
parsed, err := parse(sc)
if err != nil {
return nil, err
}
// Rename module name to last name in import path
name = filepath.Base(name)
parsed, err = resolveImportsRec(parsed, path, append(imported, name),
download)
if err != nil {
return nil, err
}
module := astModule{body: parsed, moduleName: astString(name)}
newAsts = append(newAsts, module)
}
return newAsts, nil
}
// New returns new lexer
func New(r io.Reader) *Lex {
var s scanner.Scanner
s.Init(r)
// only scan characters. implement lexer myself.
s.Mode &^= scanner.ScanChars | scanner.ScanRawStrings
return &Lex{
Scanner: &s,
}
}
func (c *chuControlButton) loadBamInfo() ([][]string, error) {
bamIn, err := os.Open(c.Bam.String() + ".bamd")
if err != nil {
return nil, fmt.Errorf("Error loading bamd: %s %s", c.Bam.String()+".bamd", err)
}
defer bamIn.Close()
var s scanner.Scanner
s.Init(bamIn)
s.Whitespace = 1<<'\t' | 1<<' '
frames := map[string]string{}
sequences := make([][]string, 0)
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
if strings.ToLower(s.TokenText()) == "frame" {
center_x, center_y := 0, 0
tok = s.Scan()
name := strings.TrimSpace(s.TokenText())
tok = s.Scan()
path := strings.Trim(s.TokenText(), "\"")
tok = s.Scan()
if s.TokenText() != "\n" {
neg_x := 1
neg_y := 1
if s.TokenText() == "-" {
neg_x = -1
s.Scan()
}
if center_x, err = strconv.Atoi(s.TokenText()); err != nil {
return nil, fmt.Errorf("Error converting %s to an int: %v", s.TokenText(), err)
}
center_x *= neg_x
tok = s.Scan()
if s.TokenText() == "-" {
neg_y = -1
s.Scan()
}
if center_y, err = strconv.Atoi(s.TokenText()); err != nil {
return nil, fmt.Errorf("Error converting %s to an int: %v", s.TokenText(), err)
}
center_y *= neg_y
}
frames[name] = path
} else if strings.ToLower(s.TokenText()) == "sequence" {
seqframes := make([]string, 0)
for tok = s.Scan(); !(s.TokenText() == "\n" || s.TokenText() == "\r" || tok == scanner.EOF); tok = s.Scan() {
frame := strings.TrimSpace(s.TokenText())
seqframes = append(seqframes, frames[frame])
}
sequences = append(sequences, seqframes)
}
}
return sequences, nil
}
func main() {
s := new(scanner.Scanner)
s.Init(os.Stdin)
for {
spew.Printf("> ")
x := s.Scan()
spew.Println(x, s.TokenText())
}
}
func main() {
//get a single integer
var a int
fmt.Scanf("%d", &a)
fmt.Println(a)
//get a string (no spaces)
var b string
fmt.Scanf("%s", &b)
fmt.Println(b)
//list of n ints
var myList []int
fmt.Print("Enter list: ")
var input string
fmt.Scanln(&input)
fmt.Print(input)
var s scanner.Scanner
s.Init(input)
tok := s.Scan()
for tok != scanner.EOF {
// do something with tok
tok = s.Scan()
asInt, _ := strconv.Atoi(tok)
myList = append(myList, asInt)
}
fmt.Println(myList)
}