Example #1
0
func BenchmarkHyphenation(b *testing.B) {
	b.StopTimer()
	trie := setupTrie()
	if trie == nil {
		return
	}
	testStr := `.hyphenation.`
	v := make([]int, utf8.RuneCountInString(testStr))
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		for i := 0; i < len(v); i++ {
			v[i] = 0
		}
		vIndex := 0
		for pos, _ := range testStr {
			t := testStr[pos:]
			strs, values := trie.AllSubstringsAndValues(t)
			for i := 0; i < values.Len(); i++ {
				str := strs.At(i)
				val := values.At(i).(*vector.IntVector)

				diff := val.Len() - len(str)
				vs := v[vIndex-diff:]

				for i := 0; i < val.Len(); i++ {
					if val.At(i) > vs[i] {
						vs[i] = val.At(i)
					}
				}
			}
			vIndex++
		}
	}
}
Example #2
0
func (v *verifier) verifyChar(x *Token) int {
	s := x.String
	if utf8.RuneCountInString(s) != 1 {
		v.error(x.Pos(), "single char expected, found "+s)
		return 0
	}
	ch, _ := utf8.DecodeRuneInString(s)
	return ch
}
Example #3
0
// Runes returns a slice of runes (Unicode code points) equivalent to the string s.
func Runes(s string) []int {
	t := make([]int, utf8.RuneCountInString(s))
	i := 0
	for _, r := range s {
		t[i] = r
		i++
	}
	return t
}
Example #4
0
func numLeadingSpaces(s string) int {
	for i, char := range s {
		if char != int(' ') {
			return i;
		}
	}
	
	return utf8.RuneCountInString(s);
}
Example #5
0
func TestCaseConsistency(t *testing.T) {
	// Make a string of all the runes.
	numRunes := unicode.MaxRune + 1
	if testing.Short() {
		numRunes = 1000
	}
	a := make([]int, numRunes)
	for i := range a {
		a[i] = i
	}
	s := string(a)
	// convert the cases.
	upper := ToUpper(s)
	lower := ToLower(s)

	// Consistency checks
	if n := utf8.RuneCountInString(upper); n != numRunes {
		t.Error("rune count wrong in upper:", n)
	}
	if n := utf8.RuneCountInString(lower); n != numRunes {
		t.Error("rune count wrong in lower:", n)
	}
	if !equal("ToUpper(upper)", ToUpper(upper), upper, t) {
		t.Error("ToUpper(upper) consistency fail")
	}
	if !equal("ToLower(lower)", ToLower(lower), lower, t) {
		t.Error("ToLower(lower) consistency fail")
	}
	/*
		  These fail because of non-one-to-oneness of the data, such as multiple
		  upper case 'I' mapping to 'i'.  We comment them out but keep them for
		  interest.
		  For instance: CAPITAL LETTER I WITH DOT ABOVE:
			unicode.ToUpper(unicode.ToLower('\u0130')) != '\u0130'

		if !equal("ToUpper(lower)", ToUpper(lower), upper, t) {
			t.Error("ToUpper(lower) consistency fail");
		}
		if !equal("ToLower(upper)", ToLower(upper), lower, t) {
			t.Error("ToLower(upper) consistency fail");
		}
	*/
}
Example #6
0
func trim(s string, num int) string { //trims num characters off the end of s
	dat := make([]int, utf8.RuneCountInString(s)-num);
	
	for i, char := range s {
		dat[i] = char;
		if i == len(dat)-1 { //get outa here before it segfaults because s has more values than dat has room for
			break
		}
	}
	return string(dat)
}
// Encode takes payload, encodes it and writes it to dst. Payload must be one
// of the following: a heartbeat, a handshake, []byte, string, int or anything
// than can be marshalled by the default json package. If payload can't be
// encoded or the writing fails, an error will be returned.
func (enc *sioStreamingEncoder) Encode(dst io.Writer, payload interface{}) (err os.Error) {
	enc.elem.Reset()

	switch t := payload.(type) {
	case heartbeat:
		s := strconv.Itoa(int(t))
		_, err = fmt.Fprintf(dst, "%d:%d:%s,", sioMessageTypeHeartbeat, len(s), s)

	case handshake:
		_, err = fmt.Fprintf(dst, "%d:%d:%s,", sioMessageTypeHandshake, len(t), t)

	case []byte:
		l := utf8.RuneCount(t)
		if l == 0 {
			break
		}
		_, err = fmt.Fprintf(dst, "%d:%d::%s,", sioMessageTypeMessage, 1+l, t)

	case string:
		l := utf8.RuneCountInString(t)
		if l == 0 {
			break
		}
		_, err = fmt.Fprintf(dst, "%d:%d::%s,", sioMessageTypeMessage, 1+l, t)

	case int:
		s := strconv.Itoa(t)
		if s == "" {
			break
		}
		_, err = fmt.Fprintf(dst, "%d:%d::%s,", sioMessageTypeMessage, 1+len(s), s)

	default:
		data, err := json.Marshal(payload)
		if len(data) == 0 || err != nil {
			break
		}
		err = json.Compact(&enc.elem, data)
		if err != nil {
			break
		}

		_, err = fmt.Fprintf(dst, "%d:%d:%s\n:", sioMessageTypeMessage, 2+len(SIOAnnotationJSON)+utf8.RuneCount(enc.elem.Bytes()), SIOAnnotationJSON)
		if err == nil {
			_, err = enc.elem.WriteTo(dst)
			if err == nil {
				_, err = dst.Write([]byte{','})
			}
		}
	}

	return err
}
Example #8
0
// truncate truncates the string to the specified precision, if present.
func (f *fmt) truncate(s string) string {
	if f.precPresent && f.prec < utf8.RuneCountInString(s) {
		n := f.prec
		for i := range s {
			if n == 0 {
				s = s[:i]
				break
			}
			n--
		}
	}
	return s
}
Example #9
0
// Encode takes payload, encodes it and writes it to dst. Payload must be one
// of the following: a heartbeat, a handshake, []byte, string, int or anything
// than can be marshalled by the default json package. If payload can't be
// encoded or the writing fails, an error will be returned.
func (enc *sioEncoder) Encode(dst io.Writer, payload interface{}) (err os.Error) {
	enc.elem.Reset()

	switch t := payload.(type) {
	case heartbeat:
		s := strconv.Itoa(int(t))
		_, err = fmt.Fprintf(dst, "%s%d%s%s%s", sioFrameDelim, len(s)+len(sioFrameDelimHeartbeat), sioFrameDelim, sioFrameDelimHeartbeat, s)

	case handshake:
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, len(t), sioFrameDelim, t)

	case []byte:
		l := utf8.RuneCount(t)
		if l == 0 {
			break
		}
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, l, sioFrameDelim, t)

	case string:
		l := utf8.RuneCountInString(t)
		if l == 0 {
			break
		}
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, l, sioFrameDelim, t)

	case int:
		s := strconv.Itoa(t)
		if s == "" {
			break
		}
		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, len(s), sioFrameDelim, s)

	default:
		data, err := json.Marshal(payload)
		if len(data) == 0 || err != nil {
			break
		}
		err = json.Compact(&enc.elem, data)
		if err != nil {
			break
		}

		_, err = fmt.Fprintf(dst, "%s%d%s%s", sioFrameDelim, utf8.RuneCount(enc.elem.Bytes())+len(sioFrameDelimJSON), sioFrameDelim, sioFrameDelimJSON)
		if err == nil {
			_, err = enc.elem.WriteTo(dst)
		}
	}

	return err
}
Example #10
0
// Count counts the number of non-overlapping instances of sep in s.
func Count(s, sep string) int {
	if sep == "" {
		return utf8.RuneCountInString(s) + 1
	}
	c := sep[0]
	n := 0
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			n++
			i += len(sep) - 1
		}
	}
	return n
}
// append s to buf, padded on left (w > 0) or right (w < 0 or f.minus).
// clear flags aftewards.
func (f *fmt) padString(s string) {
	var padding []byte
	var left, right int
	if f.widPresent && f.wid != 0 {
		padding, left, right = f.computePadding(utf8.RuneCountInString(s))
	}
	if left > 0 {
		f.writePadding(left, padding)
	}
	f.buf.WriteString(s)
	if right > 0 {
		f.writePadding(right, padding)
	}
}
Example #12
0
func doesMatch(s string) bool {
	for _, match := range matches {
		rx := regexp.MustCompile(match.exp);
		res := rx.ExecuteString(s);
		if len(res) == 0 {
			continue
		}
		for i := 0; i < len(res)/2; i += 2 {
			if res[i] == 0 && res[i + 1] == utf8.RuneCountInString(s) {
				return true
			}
		}
	}
	return false
}
Example #13
0
func maxOptionColsize(op *OptionParser, max, limit int) int {
    for _, opt := range op.options {
        length := utf8.RuneCountInString(opt.String());
        if length > max && length < limit {
            max = length
        }
    }
    for _, sub := range op.optGroups {
        submax := maxOptionColsize(sub, max, limit);
        if submax > max {
            max = submax
        }
    }
    return max
}
Example #14
0
// explode splits s into an array of UTF-8 sequences, one per Unicode character (still strings) up to a maximum of n (n <= 0 means no limit).
// Invalid UTF-8 sequences become correct encodings of U+FFF8.
func explode(s string, n int) []string {
	l := utf8.RuneCountInString(s)
	if n <= 0 || n > l {
		n = l
	}
	a := make([]string, n)
	var size, rune int
	i, cur := 0, 0
	for ; i+1 < n; i++ {
		rune, size = utf8.DecodeRuneInString(s[cur:])
		a[i] = string(rune)
		cur += size
	}
	// add the rest
	a[i] = s[cur:]
	return a
}
func (h *Hyphenator) hyphenateWord(s, hyphen string) string {
	testStr := `.` + s + `.`
	v := make([]int, utf8.RuneCountInString(testStr))
	vIndex := 0
	for pos, _ := range testStr {
		t := testStr[pos:]
		strs, values := h.patterns.AllSubstringsAndValues(t)
		for i := 0; i < values.Len(); i++ {
			str := strs.At(i)
			val := values.At(i).(*vector.IntVector)

			diff := val.Len() - len(str)
			vs := v[vIndex-diff:]

			for i := 0; i < val.Len(); i++ {
				if val.At(i) > vs[i] {
					vs[i] = val.At(i)
				}
			}
		}
		vIndex++
	}

	var outstr string

	// trim the values for the beginning and ending dots
	markers := v[1 : len(v)-1]
	mIndex := 0
	u := make([]byte, 4)
	for _, ch := range s {
		l := utf8.EncodeRune(ch, u)
		outstr += string(u[0:l])
		// don't hyphenate between (or after) the last two characters of a string
		if mIndex < len(markers)-2 {
			// hyphens are inserted on odd values, skipped on even ones
			if markers[mIndex]%2 != 0 {
				outstr += hyphen
			}
		}
		mIndex++
	}

	return outstr
}
Example #16
0
func tokenizeLexeme(s string) string {
	for _, match := range matches {
		rx := regexp.MustCompile(match.exp);
		res := rx.ExecuteString(s);
		if len(res) == 0 {
			continue
		}
		for i := 0; i < len(res)/2; i += 2 {
			if res[i] == 0 && res[i + 1] == utf8.RuneCountInString(s) {
				if match.needsString {
					return fmt.Sprintf(match.out, s);
				}
				else {
					return fmt.Sprintf(match.out);
				}
			}
		}
	}
	return s; //this is bad
}
Example #17
0
func (o *option) matches(opt string) bool {
	j := utf8.RuneCountInString(opt)
	if j < 2 || opt[0] != '-' {
		return false
	}
	if j == 2 {
		for _, s := range o.shortOpts {
			if s == opt {
				return true
			}
		}
	} else {
		for _, s := range o.longOpts {
			if s == opt {
				return true
			}
		}
	}
	return false
}
Example #18
0
// Splits the string s over a number of lines width characters wide.
func linewrap(s string, width int, firstblank bool) []string {
    start := 0;
    length := -1;
    words := splitWords(s);
    lines := make([]string, 0, 5);
    if firstblank {
        appendString(&lines, "");
    }
    for i, word := range words {
        wordLen := utf8.RuneCountInString(word);
        if length + wordLen + 1 > width {
            appendString(&lines, strings.Join(words[start:i], " "));
            start = i;
            length = wordLen;
        } else {
            length += wordLen + 1;
        }
    }
    appendString(&lines, strings.Join(words[start:len(words)], " "));
    return lines;
}
Example #19
0
// explode splits s into an array of UTF-8 sequences, one per Unicode character (still strings) up to a maximum of n (n < 0 means no limit).
// Invalid UTF-8 sequences become correct encodings of U+FFF8.
func explode(s string, n int) []string {
	if n == 0 {
		return nil
	}
	l := utf8.RuneCountInString(s)
	if n <= 0 || n > l {
		n = l
	}
	a := make([]string, n)
	var size, rune int
	i, cur := 0, 0
	for ; i+1 < n; i++ {
		rune, size = utf8.DecodeRuneInString(s[cur:])
		a[i] = string(rune)
		cur += size
	}
	// add the rest, if there is any
	if cur < len(s) {
		a[i] = s[cur:]
	}
	return a
}
Example #20
0
func (o *option) setOpts(opts []string) os.Error {
	i := len(opts)
	longOpts := make([]string, 0, i)
	shortOpts := make([]string, 0, i)
	for _, opt := range opts {
		j := utf8.RuneCountInString(opt)
		if strings.HasPrefix(opt, "-") {
			if j == 2 {
				shortOpts = shortOpts[0 : len(shortOpts)+1]
				shortOpts[len(shortOpts)-1] = opt
			} else {
				longOpts = longOpts[0 : len(longOpts)+1]
				longOpts[len(longOpts)-1] = opt
			}
		} else {
			return os.NewError(fmt.Sprintf("'%s' is not a valid option", opt))
		}
	}
	o.longOpts = longOpts
	o.shortOpts = shortOpts
	return nil
}
Example #21
0
// Count counts the number of non-overlapping instances of sep in s.
func Count(s, sep string) int {
	if sep == "" {
		return utf8.RuneCountInString(s) + 1
	}
	c := sep[0]
	l := len(sep)
	n := 0
	if l == 1 {
		// special case worth making fast
		for i := 0; i < len(s); i++ {
			if s[i] == c {
				n++
			}
		}
		return n
	}
	for i := 0; i+l <= len(s); i++ {
		if s[i] == c && s[i:i+l] == sep {
			n++
			i += l - 1
		}
	}
	return n
}
Example #22
0
func Gui() {
	//--------------------------------------------------------
	// Setting up the GTK-Foo
	//--------------------------------------------------------
	gdk.ThreadsInit()
	gtk.Init(&os.Args)
	window := gtk.Window(gtk.GTK_WINDOW_TOPLEVEL)
	window.SetTitle("Zwitscher!")
	window.Connect("destroy", func() {
		gtk.MainQuit()
	})

	vbox := gtk.VBox(false, 1)
	notebook := gtk.Notebook()

	//--------------------------------------------------------
	// Home View
	//--------------------------------------------------------
	vboxHome := gtk.VBox(false, 1)
	scrolledWinHome := gtk.ScrolledWindow(nil, nil)
	//Disable hscrollbar, enable vscrollbar
	scrolledWinHome.SetPolicy(gtk.GTK_POLICY_NEVER, gtk.GTK_POLICY_ALWAYS)
	vboxHome.Add(scrolledWinHome)
	vboxScrolledWinHome := gtk.VBox(false, 1)
	scrolledWinHome.AddWithViewPort(vboxScrolledWinHome)

	tweetwidgets := []*gtk.GtkFrame{}
	buttonUpdateTimeline := gtk.ButtonWithLabel("Update Timeline")
	buttonUpdateTimeline.Clicked(func() {
		var tweet gotter.Tweet
		tweets, err := gotter.GetTweets(accounts.Credentials, "https://api.twitter.com/1/statuses/home_timeline.json", map[string]string{})
		if err != nil {
			println("failed to get tweets:", err.String())
			return
		}
		for i := len(tweets) - 1; i >= 0; i-- {
			tweet = tweets[i]
			id, _ := strconv.Atoi64(tweet.Identifier)
			if accounts.Maxreadid < id {
				if len(tweetwidgets) > 20 {
					tweetwidgets[0].Destroy()
					tweetwidgets = tweetwidgets[1:]
				}
				tweetwidget := TweetWidget(tweet)
				vboxScrolledWinHome.PackEnd(tweetwidget, false, false, 0)
				tweetwidget.ShowAll()
				tweetwidgets = append(tweetwidgets, tweetwidget)
				accounts.Maxreadid = id
			}
		}
	})
	vboxHome.PackEnd(buttonUpdateTimeline, false, false, 0)
	notebook.AppendPage(vboxHome, gtk.Label("Home"))

	//--------------------------------------------------------
	// Mentions View
	//--------------------------------------------------------
	scrolledwin := gtk.ScrolledWindow(nil, nil)
	notebook.AppendPage(scrolledwin, gtk.Label("Mentions"))

	//--------------------------------------------------------
	// Messages View
	//--------------------------------------------------------
	scrolledwin = gtk.ScrolledWindow(nil, nil)
	notebook.AppendPage(scrolledwin, gtk.Label("Messages"))

	vbox.Add(notebook)

	//--------------------------------------------------------
	// Fild for Tweets
	//--------------------------------------------------------
	hbox := gtk.HBox(false, 1)

	dir, _ := filepath.Split(os.Args[0])
	imagefile := filepath.Join(dir, "Awesome Smiley Original.jpg")
	image := gtk.ImageFromFile(imagefile)
	hbox.PackStart(image, false, false, 0)

	buttonZwitscher := gtk.ButtonWithLabel("Zwitscher!")
	newTweetTextField := gtk.Entry()
	charCounterLabel := gtk.Label("140")

	buttonZwitscher.SetTooltipMarkup("Tweet")

	buttonZwitscher.Clicked(func() {
		charCounterLabel.SetLabel("140")
		SendTweet(newTweetTextField.GetText())
		newTweetTextField.SetText("")
	})

	newTweetTextField.Connect("key-release-event", func() {
		length := utf8.RuneCountInString(newTweetTextField.GetText())
		charCounterLabel.SetLabel((string)(strconv.Itoa(140 - length)))
	})

	newTweetTextField.Connect("activate", func() {
		if newTweetTextField.GetText() != "" { //pressed enter, and text is not empty
			charCounterLabel.SetLabel("140")
			SendTweet(newTweetTextField.GetText())
			newTweetTextField.SetText("")
		}
	})
	hbox.PackStartDefaults(newTweetTextField)
	hbox.PackStart(charCounterLabel, false, false, 0)
	hbox.PackEnd(buttonZwitscher, false, false, 0)

	vbox.PackEnd(hbox, false, false, 0)

	//--------------------------------------------------------
	// Event
	//--------------------------------------------------------
	window.Add(vbox)
	window.SetSizeRequest(400, 550)
	window.ShowAll()

	gdk.ThreadsEnter()
	gtk.Main()
	gdk.ThreadsLeave()
}