// gofmtFlags looks for a comment of the form
//
// //gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
f, err := os.Open(filename)
if err != nil {
return "" // ignore errors - they will be found later
}
defer f.Close()
// initialize scanner
var s scanner.Scanner
s.Init(f)
s.Error = func(*scanner.Scanner, string) {} // ignore errors
s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments
// look for //gofmt comment
for s.Line <= maxLines {
switch s.Scan() {
case scanner.Comment:
const prefix = "//gofmt "
if t := s.TokenText(); strings.HasPrefix(t, prefix) {
return strings.TrimSpace(t[len(prefix):])
}
case scanner.EOF:
return ""
}
}
return ""
}
func split(expr string) (keys []string, err error) {
var msgs []string
var s scanner.Scanner
s.Init(strings.NewReader(expr))
s.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings
s.Error = func(s *scanner.Scanner, msg string) { msgs = append(msgs, fmt.Sprintf("%s %s", s.Pos(), msg)) }
key := ""
keys = []string{}
for err == nil {
t := s.Peek()
// fmt.Printf(">>> %s: %s %s\n", s.Pos(), scanner.TokenString(t), s.TokenText())
switch t {
case '[':
key, err = scanBracketedKey(&s)
case '.':
s.Scan()
continue
case scanner.EOF:
goto end
default:
key, err = scanKey(&s)
}
if len(msgs) > 0 {
err = errors.New(strings.Join(msgs, "\n"))
}
if err == nil {
keys = append(keys, key)
}
}
end:
return
}
func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {
var s scanner.Scanner
s.Error = func(s *scanner.Scanner, msg string) {
log.Printf("command scan error: %s [%s]", msg, path)
}
s.Init(strings.NewReader(path))
parent := root
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
//log.Printf("cmdFind: token: [%s]", s.TokenText())
label := s.TokenText()
children := matchChildren(parent.Children, label)
size := len(children)
if size < 1 {
return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
}
if size > 1 {
return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
}
parent = children[0]
}
//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)
return parent, nil
}
func NewDecoder(r io.Reader) *Decoder {
var scan scanner.Scanner
scan.Init(r)
dec := Decoder{scan: scan}
scan.Error = dec.setError
return &dec
}
func NewScanner(src []byte) *scanner.Scanner {
var s scanner.Scanner
s.Init(bytes.NewReader(src))
s.Error = func(_ *scanner.Scanner, _ string) {}
s.Whitespace = 0
s.Mode = s.Mode ^ scanner.SkipComments
return &s
}
// NewScannerReader takes a reader src and creates a Scanner.
func NewScannerReader(src io.Reader) *scanner.Scanner {
var s scanner.Scanner
s.Init(src)
s.Error = func(_ *scanner.Scanner, _ string) {}
s.Whitespace = 0
s.Mode = s.Mode ^ scanner.SkipComments
return &s
}
func main() {
s := new(scanner.Scanner)
s.Init(os.Stdin)
s.Error = err
parse := start
for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
parse = parse(s, tok)
}
}
func StringToMap(value string) (map[string]string, error) {
var tokenizer scanner.Scanner
tokenizer.Init(strings.NewReader(value))
tokenizer.Error = func(*scanner.Scanner, string) {}
result := make(map[string]string)
next := func() string {
tokenizer.Scan()
return tokenizer.TokenText()
}
var lvalue, rvalue, expression string
for {
lvalue = next()
if lvalue == "" {
return result, errors.New(fmt.Sprintf("Expected key at pos '%d' but found none; "+
"map values should be 'key=value' separated by commas", tokenizer.Pos().Offset))
}
if lvalue == "{" {
// Assume this is JSON format and attempt to un-marshal
return JSONToMap(value)
}
expression = next()
if expression != "=" {
return result, errors.New(fmt.Sprintf("Expected '=' after '%s' but found '%s'; "+
"map values should be 'key=value' separated by commas", lvalue, expression))
}
rvalue = next()
if rvalue == "" {
return result, errors.New(fmt.Sprintf("Expected value after '%s' but found none; "+
"map values should be 'key=value' separated by commas", expression))
}
// TODO: Handle quoted strings and escaped double quotes
result[lvalue] = rvalue
// Are there anymore tokens?
delimiter := next()
if delimiter == "" {
break
}
// Should be a comma next
if delimiter != "," {
return result, errors.New(fmt.Sprintf("Expected ',' after '%s' but found '%s'; "+
"map values should be 'key=value' separated by commas", rvalue, delimiter))
}
}
return result, nil
}
func parse(s scanner.Scanner) ([]ast, error) {
scanErrors := []string{}
s.Error = func(s *scanner.Scanner, msg string) {
scanErrors = append(scanErrors, msg)
}
pt, err := parseText(&s, 0)
if s.ErrorCount != 0 {
return nil, errors.New(strings.Join(scanErrors, "\n"))
} else if err != nil {
return nil, err
}
return astRoot(pt), nil
}
// Text satifies the Extractor interface
//
//ReplaceGo is a specialized routine for correcting Golang source
// files. Currently only checks comments, not identifiers for
// spelling.
//
// Other items:
// - check strings, but need to ignore
// * import "statements" blocks
// * import ( "blocks" )
// - skip first comment (line 0) if build comment
//
func (p *GolangText) Text(raw []byte) []byte {
out := bytes.Buffer{}
s := scanner.Scanner{}
s.Init(bytes.NewReader(raw))
s.Error = (func(s *scanner.Scanner, msg string) {})
s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
for {
switch s.Scan() {
case scanner.Comment:
out.WriteString(s.TokenText())
out.WriteByte('\n')
case scanner.EOF:
return out.Bytes()
}
}
}
// parseCommand reads command name and arguments from the given input.
func parseCommand(data string) (string, ParamList) {
var scan scanner.Scanner
var list []string
scan.Init(bytes.NewBufferString(strings.TrimSpace(data)))
scan.Error = func(s *scanner.Scanner, msg string) {}
tok := scan.Scan()
for tok != scanner.EOF {
list = append(list, unquote(scan.TokenText()))
tok = scan.Scan()
}
if len(list) == 0 || scan.ErrorCount > 0 {
return "", nil
}
return strings.ToLower(list[0]), ParamList(list[1:])
}
func funcFromFormula(form string) (string, error) {
f, _, err := extractFormula(form)
if err != nil {
return "", err
}
var s scanner.Scanner
s.Init(strings.NewReader(f))
s.Error = func(s *scanner.Scanner, msg string) {
err = fmt.Errorf("error parsing plural formula %s: %s", s.Pos(), msg)
}
s.Mode = scanner.ScanIdents | scanner.ScanInts
s.Whitespace = 0
tok := s.Scan()
var code []string
var buf bytes.Buffer
for tok != scanner.EOF && err == nil {
switch tok {
case scanner.Ident, scanner.Int:
buf.WriteString(s.TokenText())
case '?':
code = append(code, fmt.Sprintf("if %s {\n", buf.String()))
buf.Reset()
case ':':
code = append(code, fmt.Sprintf("return %s\n}\n", buf.String()))
buf.Reset()
default:
buf.WriteRune(tok)
}
tok = s.Scan()
}
if err != nil {
return "", err
}
if len(code) == 0 && buf.Len() > 0 && buf.String() != "0" {
code = append(code, fmt.Sprintf("if %s {\nreturn 1\n}\nreturn 0\n", buf.String()))
buf.Reset()
}
if buf.Len() > 0 {
code = append(code, fmt.Sprintf("\nreturn %s\n", buf.String()))
}
return strings.Join(code, ""), nil
}
func (cl *contLiner) countDepth() int {
reader := bytes.NewBufferString(cl.buffer)
sc := new(scanner.Scanner)
sc.Init(reader)
sc.Error = func(_ *scanner.Scanner, msg string) {
debugf("scanner: %s", msg)
}
depth := 0
for {
switch sc.Scan() {
case '{':
depth++
case '}':
depth--
case scanner.EOF:
return depth
}
}
}
func TokenizeQuery(q string) ([]token, error) {
tokens := make([]token, 0, 10)
s := scanner.Scanner{}
s.Init(strings.NewReader(q))
var err error
s.Error = func(s *scanner.Scanner, msg string) {
err = fmt.Errorf("%s", msg)
}
for tok := s.Scan(); tok != scanner.EOF && err == nil; tok = s.Scan() {
if strings.ContainsRune("():,", tok) {
tokens = append(tokens, token{int(tok), ""})
} else if s.TokenText() == "SELECT" {
tokens = append(tokens, token{SELECT, ""})
} else if s.TokenText() == "WHERE" {
tokens = append(tokens, token{WHERE, ""})
} else if s.TokenText() == "AND" {
tokens = append(tokens, token{AND, ""})
} else if s.TokenText() == "HOSTS" {
tokens = append(tokens, token{HOSTS, ""})
} else {
text := s.TokenText()
if strings.HasPrefix(text, "\"") && strings.HasSuffix(text, "\"") {
text = text[1 : len(text)-1]
}
tokens = append(tokens, token{GENERIC_IDENTIFIER, text})
}
}
return tokens, err
}
func parsePo(r io.Reader, filename string) (*Po, error) {
comment := false
s := new(scanner.Scanner)
var err error
s.Init(r)
s.Filename = filename
s.Error = func(s *scanner.Scanner, msg string) {
if !comment {
err = fmt.Errorf("error parsing %s: %s", s.Pos(), msg)
}
}
s.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
tok := s.Scan()
po := &Po{Attrs: make(map[string]string)}
var trans *Translation
for tok != scanner.EOF && err == nil {
if tok == '#' {
// Skip until EOL
comment = true
s.Whitespace = whitespace
for tok != '\n' && tok != scanner.EOF {
tok = s.Scan()
}
s.Whitespace = scanner.GoWhitespace
comment = false
tok = s.Scan()
continue
}
if tok != scanner.Ident {
err = unexpected(s, tok)
break
}
text := s.TokenText()
switch text {
case "msgctxt":
if trans != nil {
if len(trans.Translations) == 0 {
err = unexpected(s, tok)
break
}
po.addTranslation(trans)
}
trans = &Translation{Context: readString(s, &tok, &err)}
case "msgid":
if trans != nil {
if len(trans.Translations) > 0 || trans.Singular != "" {
po.addTranslation(trans)
} else if trans.Context != "" {
trans.Singular = readString(s, &tok, &err)
break
}
}
trans = &Translation{Singular: readString(s, &tok, &err)}
case "msgid_plural":
if trans == nil || trans.Plural != "" {
err = unexpected(s, tok)
break
}
trans.Plural = readString(s, &tok, &err)
case "msgstr":
str := readString(s, &tok, &err)
if tok == '[' {
tok = s.Scan()
if tok != scanner.Int {
err = unexpected(s, tok)
break
}
ii, _ := strconv.Atoi(s.TokenText())
if ii != len(trans.Translations) {
err = unexpected(s, tok)
break
}
if tok = s.Scan(); tok != ']' {
err = unexpected(s, tok)
break
}
str = readString(s, &tok, &err)
}
trans.Translations = append(trans.Translations, str)
default:
err = unexpected(s, tok)
}
}
if trans != nil {
po.addTranslation(trans)
}
if err != nil {
return nil, err
}
for _, v := range po.Messages {
if v.Context == "" && v.Singular == "" {
if len(v.Translations) > 0 {
meta := v.Translations[0]
for _, line := range strings.Split(meta, "\n") {
colon := strings.Index(line, ":")
if colon > 0 {
key := strings.TrimSpace(line[:colon])
value := strings.TrimSpace(line[colon+1:])
po.Attrs[key] = value
}
}
}
break
}
}
return po, nil
}
func fail(s *scanner.Scanner, tok rune) step {
s.Error(s, fmt.Sprintf("unhandled: %s %s", scanner.TokenString(tok), s.TokenText()))
return fail
}