コード例 #1
0
ファイル: trie_test.go プロジェクト: AlanQuatermain/go-trie
func loadPatterns(reader io.Reader) (*Trie, os.Error) {
	trie := NewTrie()
	var s scanner.Scanner
	s.Init(reader)
	s.Mode = scanner.ScanIdents | scanner.ScanRawStrings | scanner.SkipComments

	var which string

	tok := s.Scan()
	for tok != scanner.EOF {
		switch tok {
		case scanner.Ident:
			// we handle two identifiers: 'patterns' and 'exceptions'
			switch ident := s.TokenText(); ident {
			case `patterns`, `exceptions`:
				which = ident
			default:
				return nil, os.ErrorString(fmt.Sprintf("Unrecognized identifier '%s' at position %v",
					ident, s.Pos()))
			}
		case scanner.String, scanner.RawString:
			// trim the quotes from around the string
			tokstr := s.TokenText()
			str := tokstr[1 : len(tokstr)-1]

			switch which {
			case `patterns`:
				trie.AddPatternString(str)
			}
		}
		tok = s.Scan()
	}

	return trie, nil
}
コード例 #2
0
func (h *Hyphenator) Hyphenate(s, hyphen string) (string, bool) {
	var sc scanner.Scanner
	sc.Init(strings.NewReader(s))
	sc.Mode = scanner.ScanIdents
	sc.Whitespace = 0

	var outstr string

	tok := sc.Scan()
	for tok != scanner.EOF {
		switch tok {
		case scanner.Ident:
			// a word (or part thereof) to hyphenate
			t := sc.TokenText()

			// try the exceptions first
			exc := h.exceptions[t]
			if len(exc) != 0 {
				if hyphen != `-` {
					strings.Replace(exc, `-`, hyphen, -1)
				}
				return exc, true
			}

			// not an exception, hyphenate normally
			outstr += h.hyphenateWord(sc.TokenText(), hyphen)
		default:
			// A Unicode rune to append to the output
			p := make([]byte, utf8.UTFMax)
			l := utf8.EncodeRune(tok, p)
			outstr += string(p[0:l])
		}

		tok = sc.Scan()
	}

	return outstr, true
}
コード例 #3
0
func (h *Hyphenator) loadPatterns(reader io.Reader) os.Error {
	var s scanner.Scanner
	s.Init(reader)
	s.Mode = scanner.ScanIdents | scanner.ScanRawStrings | scanner.SkipComments

	var which string

	tok := s.Scan()
	for tok != scanner.EOF {
		switch tok {
		case scanner.Ident:
			// we handle two identifiers: 'patterns' and 'exceptions'
			switch ident := s.TokenText(); ident {
			case `patterns`, `exceptions`:
				which = ident
			default:
				return os.ErrorString(fmt.Sprintf("Unrecognized identifier '%s' at position %v",
					ident, s.Pos()))
			}
		case scanner.String, scanner.RawString:
			// trim the quotes from around the string
			tokstr := s.TokenText()
			str := tokstr[1 : len(tokstr)-1]

			switch which {
			case `patterns`:
				h.patterns.AddPatternString(str)
			case `exceptions`:
				key := strings.Replace(str, `-`, ``, -1)
				h.exceptions[key] = str
			}
		}
		tok = s.Scan()
	}
	return nil
}