Example #1
0
func main() {
	rang, except := scan(0, 0xFFFF)
	range16 = to16(rang)
	except16 = to16(except)
	range32, except32 = scan(0x10000, unicode.MaxRune)

	for i := rune(0); i <= unicode.MaxRune; i++ {
		if isPrint(i) != unicode.IsPrint(i) {
			fmt.Fprintf(os.Stderr, "%U: isPrint=%v, want %v\n", i, isPrint(i), unicode.IsPrint(i))
			return
		}
	}

	fmt.Printf(`// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.` + "\n\n")
	fmt.Printf("// DO NOT EDIT.  GENERATED BY\n")
	fmt.Printf("//     go run makeisprint.go >x && mv x isprint.go\n\n")
	fmt.Printf("package strconv\n\n")

	fmt.Printf("// (%d+%d+%d)*2 + (%d)*4 = %d bytes\n\n",
		len(range16), len(except16), len(except32),
		len(range32),
		(len(range16)+len(except16)+len(except32))*2+
			(len(range32))*4)

	fmt.Printf("var isPrint16 = []uint16{\n")
	for i := 0; i < len(range16); i += 2 {
		fmt.Printf("\t%#04x, %#04x,\n", range16[i], range16[i+1])
	}
	fmt.Printf("}\n\n")

	fmt.Printf("var isNotPrint16 = []uint16{\n")
	for _, r := range except16 {
		fmt.Printf("\t%#04x,\n", r)
	}
	fmt.Printf("}\n\n")

	fmt.Printf("var isPrint32 = []uint32{\n")
	for i := 0; i < len(range32); i += 2 {
		fmt.Printf("\t%#06x, %#06x,\n", range32[i], range32[i+1])
	}
	fmt.Printf("}\n\n")

	fmt.Printf("var isNotPrint32 = []uint16{ // add 0x10000 to each entry\n")
	for _, r := range except32 {
		if r >= 0x20000 {
			fmt.Fprintf(os.Stderr, "%U too big for isNotPrint32\n", r)
			return
		}
		fmt.Printf("\t%#04x,\n", r-0x10000)
	}
	fmt.Printf("}\n")
}
Example #2
0
func main() {
	rang, except := scan(0, 0xFFFF)
	range16 = to16(rang)
	except16 = to16(except)
	range32, except32 = scan(0x10000, unicode.MaxRune)

	for i := rune(0); i <= unicode.MaxRune; i++ {
		if isPrint(i) != unicode.IsPrint(i) {
			fmt.Fprintf(os.Stderr, "%U: isPrint=%v, want %v\n", i, isPrint(i), unicode.IsPrint(i))
			return
		}
	}

	fmt.Printf("// DO NOT EDIT.  GENERATED BY\n")
	fmt.Printf("//     go run makeisprint.go >x && mv x isprint.go\n\n")
	fmt.Printf("package strconv\n\n")

	fmt.Printf("// (%d+%d+%d)*2 + (%d)*4 = %d bytes\n\n",
		len(range16), len(except16), len(except32),
		len(range32),
		(len(range16)+len(except16)+len(except32))*2+
			(len(range32))*4)

	fmt.Printf("var isPrint16 = []uint16{\n")
	for i := 0; i < len(range16); i += 2 {
		fmt.Printf("\t%#04x, %#04x,\n", range16[i], range16[i+1])
	}
	fmt.Printf("}\n\n")

	fmt.Printf("var isNotPrint16 = []uint16{\n")
	for _, r := range except16 {
		fmt.Printf("\t%#04x,\n", r)
	}
	fmt.Printf("}\n\n")

	fmt.Printf("var isPrint32 = []uint32{\n")
	for i := 0; i < len(range32); i += 2 {
		fmt.Printf("\t%#06x, %#06x,\n", range32[i], range32[i+1])
	}
	fmt.Printf("}\n\n")

	fmt.Printf("var isNotPrint32 = []uint16{ // add 0x10000 to each entry\n")
	for _, r := range except32 {
		if r >= 0x20000 {
			fmt.Fprintf(os.Stderr, "%U too big for isNotPrint32\n", r)
			return
		}
		fmt.Printf("\t%#04x,\n", r-0x10000)
	}
	fmt.Printf("}\n")
}
Example #3
0
func quoteDouble(s string) string {
	var buf bytes.Buffer
	buf.WriteByte('"')
	for _, r := range s {
		if r == '\\' || r == '"' {
			buf.WriteByte('\\')
			buf.WriteRune(r)
		} else if !unicode.IsPrint(r) {
			buf.WriteByte('\\')
			if r <= 0xff {
				buf.WriteByte('x')
				buf.Write(rtohex(r, 2))
			} else if r <= 0xffff {
				buf.WriteByte('u')
				buf.Write(rtohex(r, 4))
			} else {
				buf.WriteByte('U')
				buf.Write(rtohex(r, 8))
			}
		} else {
			buf.WriteRune(r)
		}
	}
	buf.WriteByte('"')
	return buf.String()
}
Example #4
0
func incrementCount(r rune, counts map[int]int) {
	switch {
	case unicode.IsControl(r):
		counts[isControl]++

	case unicode.IsNumber(r):
		counts[isNumber]++

	case unicode.IsDigit(r):
		counts[isDigit]++

	case unicode.IsLetter(r):
		counts[isLetter]++

	case unicode.IsMark(r):
		counts[isMark]++

	case unicode.IsPunct(r):
		counts[isPunct]++

	case unicode.IsSpace(r):
		counts[isSpace]++

	case unicode.IsSymbol(r):
		counts[isSymbol]++

	case unicode.IsPrint(r):
		counts[isPrint]++

	case unicode.IsGraphic(r):
		counts[isGraphic]++
	}

}
Example #5
0
func valid(id string) error {
	if utf8.RuneCountInString(id) == 0 {
		return ErrBadID{"Empty ID is not allowed"}
	}

	if !utf8.ValidString(id) {
		return ErrBadID{fmt.Sprintf("Invalid utf-8: %v", id)}
	}

	for idx, rn := range id {
		if unicode.IsSpace(rn) {
			return ErrBadID{
				fmt.Sprintf("Space not allowed: %s (at %d)", id, idx),
			}
		}

		if !unicode.IsPrint(rn) {
			return ErrBadID{
				fmt.Sprintf("Only printable runes allowed: %s (at %d)", id, idx),
			}
		}
	}

	return nil
}
Example #6
0
// CharType returns a string representing the unicode type of a rune
func CharType(r rune) string {
	switch {
	case unicode.IsLetter(r):
		return "letter"
	case unicode.IsSpace(r):
		return "space"
	case unicode.IsPunct(r):
		return "punct"
	case unicode.IsNumber(r):
		return "number"
	case unicode.IsSymbol(r):
		return "symbol"
	case unicode.IsMark(r):
		return "mark"
	case unicode.IsDigit(r):
		return "digit"
	case unicode.IsPrint(r):
		return "print"
	case unicode.IsControl(r):
		return "control"
	case unicode.IsGraphic(r):
		return "graphic"
	default:
		return "invalid"
	}
}
Example #7
0
// The following are allowed in variable names:
// * Anything beyond ASCII that is printable
// * Letters and numbers
// * The symbols "-_:"
func allowedInVariableName(r rune) bool {
	return (r >= 0x80 && unicode.IsPrint(r)) ||
		('0' <= r && r <= '9') ||
		('a' <= r && r <= 'z') ||
		('A' <= r && r <= 'Z') ||
		r == '-' || r == '_' || r == ':'
}
Example #8
0
// QuoteAs returns a representation of s in elvish syntax, using the syntax
// specified by q, which must be one of Bareword, SingleQuoted, or
// DoubleQuoted. It returns the quoted string and the actual quoting.
func QuoteAs(s string, q PrimaryType) (string, PrimaryType) {
	if q == DoubleQuoted {
		// Everything can be quoted using double quotes, return directly.
		return quoteDouble(s), DoubleQuoted
	}
	if s == "" {
		return "''", SingleQuoted
	}

	// Keep track of whether it is a valid bareword.
	bare := s[0] != '~'
	for _, r := range s {
		if !unicode.IsPrint(r) {
			// Contains unprintable character; force double quote.
			return quoteDouble(s), DoubleQuoted
		}
		if !allowedInBareword(r, false) {
			bare = false
		}
	}

	if q == Bareword && bare {
		return s, Bareword
	}
	return quoteSingle(s), SingleQuoted
}
Example #9
0
func quoteDouble(s string) string {
	var buf bytes.Buffer
	buf.WriteByte('"')
	for _, r := range s {
		if e, ok := doubleUnescape[r]; ok {
			// Takes care of " and \ as well.
			buf.WriteByte('\\')
			buf.WriteRune(e)
		} else if !unicode.IsPrint(r) {
			buf.WriteByte('\\')
			if r <= 0xff {
				buf.WriteByte('x')
				buf.Write(rtohex(r, 2))
			} else if r <= 0xffff {
				buf.WriteByte('u')
				buf.Write(rtohex(r, 4))
			} else {
				buf.WriteByte('U')
				buf.Write(rtohex(r, 8))
			}
		} else {
			buf.WriteRune(r)
		}
	}
	buf.WriteByte('"')
	return buf.String()
}
Example #10
0
File: sx.go Project: hlandau/sx
func writeQuotedString(s string, b *bufio.Writer, f *Format) {
  b.WriteRune('"')
  for i := range s {
    c := s[i] // don't decode as runes
    switch c {
    case '\r':
      b.WriteString(`\r`)
    case '\n':
      b.WriteString(`\n`)
    case '\t':
      b.WriteString(`\t`)
    case '"':
      b.WriteString(`\"`)
    case '\\':
      b.WriteString(`\\`)
    default:
      if c < 0x80 && unicode.IsPrint(rune(c)) {
        b.WriteRune(rune(c))
      } else {
        b.WriteString(`\x`)
        b.WriteRune(enchex((c >> 4) & 0x0F))
        b.WriteRune(enchex(c & 0x0F))
      }
    }
  }
  b.WriteRune('"')
}
Example #11
0
// Stat calculates statistics for all runes read from r.
func (m *Main) Stat(r io.RuneReader) (Stats, error) {
	var stats Stats

	for {
		// Read next character.
		ch, sz, err := r.ReadRune()
		if err == io.EOF {
			break
		} else if err != nil {
			return stats, err
		}

		// Calculate stats.
		stats.TotalN++
		if unicode.IsControl(ch) {
			stats.ControlN++
		}
		if unicode.IsDigit(ch) {
			stats.DigitN++
		}
		if unicode.IsGraphic(ch) {
			stats.GraphicN++
		}
		if unicode.IsLetter(ch) {
			stats.LetterN++
		}
		if unicode.IsLower(ch) {
			stats.LowerN++
		}
		if unicode.IsMark(ch) {
			stats.MarkN++
		}
		if unicode.IsNumber(ch) {
			stats.NumberN++
		}
		if unicode.IsPrint(ch) {
			stats.PrintN++
		}
		if unicode.IsPunct(ch) {
			stats.PunctN++
		}
		if unicode.IsSpace(ch) {
			stats.SpaceN++
		}
		if unicode.IsSymbol(ch) {
			stats.SymbolN++
		}
		if unicode.IsTitle(ch) {
			stats.TitleN++
		}
		if unicode.IsUpper(ch) {
			stats.UpperN++
		}
		if sz > 1 {
			stats.MultiByteN++
		}
	}

	return stats, nil
}
Example #12
0
func (q *quotedwriter) Write(data []byte) (n int, err error) {
	b := q.b
	if !q.add {
		b = append(b, '"')
		q.add = true
	}
	for _, c := range data {
		switch {
		case q.esc, c > unicode.MaxASCII:
			fallthrough
		default:
			x := []byte{'\\', 'x', 0, c}
			hex.Encode(x[2:], x[3:])
			b = append(b, x...)
		case c == '"':
			b = append(b, '\\', '"')
		case c == '\\':
			b = append(b, '\\', '\\')
		case unicode.IsPrint(rune(c)):
			b = append(b, c)
		}
	}
	n, err = q.w.Write(b)
	q.b = b[:0]
	if err == nil {
		if n != len(b) {
			err = io.ErrShortWrite
		} else {
			n = len(data)
		}
	}
	return
}
Example #13
0
func (field_editor *FieldEditor) handleKeyEvent(event termbox.Event) (string, bool) {
	is_done := false
	if event.Key == termbox.KeyEnter {
		is_done = true
	} else if event.Key == termbox.KeyEsc {
		is_done = true
		field_editor.value = nil
	} else if event.Key == termbox.KeyArrowLeft {
		if field_editor.cursor_pos > 0 {
			field_editor.cursor_pos--
		}
	} else if event.Key == termbox.KeyArrowUp || event.Key == termbox.KeyCtrlA {
		field_editor.cursor_pos = 0
	} else if event.Key == termbox.KeyArrowRight {
		if field_editor.cursor_pos < utf8.RuneCount(field_editor.value) {
			field_editor.cursor_pos++
		}
	} else if event.Key == termbox.KeyArrowDown || event.Key == termbox.KeyCtrlE {
		field_editor.cursor_pos = utf8.RuneCount(field_editor.value)
	} else if event.Key == termbox.KeyCtrlH || event.Key == termbox.KeyBackspace {
		if field_editor.cursor_pos > 0 {
			field_editor.value = removeRuneAtIndex(field_editor.value, field_editor.cursor_pos-1)
			field_editor.cursor_pos--
		}
	} else if unicode.IsPrint(event.Ch) {
		field_editor.value = insertRuneAtIndex(field_editor.value, field_editor.cursor_pos, event.Ch)
		field_editor.cursor_pos++
	} else if event.Key == termbox.KeySpace {
		field_editor.value = insertRuneAtIndex(field_editor.value, field_editor.cursor_pos, ' ')
		field_editor.cursor_pos++
	}
	return string(field_editor.value), is_done
}
Example #14
0
File: util.go Project: Safe3/oz
func isPrintableASCII(s string) bool {
	for _, x := range s {
		if unicode.IsPrint(x) == false {
			return false
		}
	}
	return true
}
Example #15
0
// checks if s is ascii and printable, aka doesn't include tab, backspace, etc.
func IsAsciiPrintable(s string) bool {
	for _, r := range s {
		if r > unicode.MaxASCII || !unicode.IsPrint(r) {
			return false
		}
	}
	return true
}
Example #16
0
func isPrintable(s string) bool {
	for _, c := range s {
		if c == 0 || c > unicode.MaxASCII || !unicode.IsPrint(c) {
			return false
		}
	}
	return true
}
Example #17
0
// checks if s is ascii and printable, aka doesn't include tab, backspace, etc.
func (i *HttpOutput) IsPrintable(s string) bool {
	for _, r := range s {
		if !unicode.IsPrint(r) && r != rune('\t') && r != rune('\r') && r != rune('\n') {
			return false
		}
	}
	return true
}
Example #18
0
func IsPrintable(input string) bool {
	for _, r := range input {
		if !unicode.IsPrint(r) {
			return false
		}
	}
	return true
}
Example #19
0
func isPrintable(s string) bool {
	for _, r := range s {
		if !unicode.IsPrint(r) {
			return false
		}
	}
	return true
}
Example #20
0
func needQuote(str string) bool {
	for _, char := range str {
		if unicode.IsSpace(char) || !unicode.IsPrint(char) || unicode.Is(unicode.Quotation_Mark, char) {
			return true
		}
	}

	return false
}
Example #21
0
func quote_rune(buf *bytes.Buffer, r rune, size int) {
	const lowerhex = "0123456789abcdef"
	if size == 1 && r == utf8.RuneError {
		// invalid rune, write the byte as is
		buf.WriteString(`\x`)
		buf.WriteByte(lowerhex[r>>4])
		buf.WriteByte(lowerhex[r&0xF])
		return
	}

	// first check for special TCL escaping cases
	switch r {
	case '{', '}', '[', ']', '"', '$', '\\':
		buf.WriteString("\\")
		buf.WriteRune(r)
		return
	}

	// other printable characters
	if unicode.IsPrint(r) {
		buf.WriteRune(r)
		return
	}

	// non-printable characters
	switch r {
	case '\a':
		buf.WriteString(`\a`)
	case '\b':
		buf.WriteString(`\b`)
	case '\f':
		buf.WriteString(`\f`)
	case '\n':
		buf.WriteString(`\n`)
	case '\r':
		buf.WriteString(`\r`)
	case '\t':
		buf.WriteString(`\t`)
	case '\v':
		buf.WriteString(`\v`)
	default:
		switch {
		case r < ' ':
			buf.WriteString(`\x`)
			buf.WriteByte(lowerhex[r>>4])
			buf.WriteByte(lowerhex[r&0xF])
		case r >= 0x10000:
			r = 0xFFFD
			fallthrough
		case r < 0x10000:
			buf.WriteString(`\u`)
			for s := 12; s >= 0; s -= 4 {
				buf.WriteByte(lowerhex[r>>uint(s)&0xF])
			}
		}
	}
}
Example #22
0
// Version return the string of the splice version
// with only printable characters
func (h Header) Version() string {
	var buff bytes.Buffer
	for _, b := range h.Vers {
		if unicode.IsPrint(rune(b)) {
			buff.WriteByte(b)
		}
	}
	return buff.String()
}
Example #23
0
func addToName(r rune) {
	// Ignore non-printable characters or if the name is to long.
	if !unicode.IsPrint(r) || len(name) > 30 {
		log.Printf("%v\n", r)
		return
	}
	name += string(r)
	printName()
}
Example #24
0
func getPrintableString(dirty string) string {
	var b bytes.Buffer
	for _, by := range dirty {
		if unicode.IsPrint(rune(by)) {
			b.WriteString(string(by))
		}
	}
	return b.String()
}
Example #25
0
// String returns a string representation for the literal.
func (l lit) String() string {
	var c byte
	if unicode.IsPrint(rune(l.b)) {
		c = l.b
	} else {
		c = '.'
	}
	return fmt.Sprintf("lit{%02x %c}", l.b, c)
}
Example #26
0
// Print a byte as ASCII, using escape sequences where necessary.
func printAsciiByte(b uint8) {
	r := rune(b)
	if unicode.IsPrint(r) || unicode.IsSpace(r) {
		fmt.Print(string(r))
	} else {
		charStr := strconv.QuoteRuneToASCII(r)
		fmt.Print(charStr[1 : len(charStr)-1])
	}
}
func (tz *BadXMLTokenizer) Next() (*Token, error) {

	for {
		tok := tz.scanner.Peek()
		log.Tracef("Scanner found: %v", tok)

		if tok == scanner.EOF {
			return nil, io.EOF
		}

		switch {
		case unicode.IsPrint(tok) == false:
			log.Tracef("Skipping unprintable character")
			fallthrough
		case unicode.IsSpace(tok):
			tz.scanner.Scan()
			continue

		case tok == '<':
			log.Tracef("parsing XML")
			token, ok := parseXML(tz.scanner)
			// We actually bump the phrase no matter what. It's
			// either a comment, an xml token, or something weird
			tz.current_phrase_id = rand.Intn(1000)
			if ok {
				log.Tracef("Returning XML Token: %s", token)
				return token, nil
			}

		case tok == '&':
			log.Tracef("parsing HTML")
			if token := parseHTMLEntity(tz.scanner); token != nil {
				tz.current_phrase_id = rand.Intn(1000)
				return token, nil
			}

		case tok == '`': // Handle this speciallly - technically it's a 'grave accent'
			fallthrough
		case unicode.Is(unicode.Punct, tok):
			log.Tracef("Ignoring punctuation: %v", tok)
			tz.current_phrase_id = rand.Intn(1000)
			tok = tz.scanner.Scan()

		default:
			/* Catch special things in words */
			log.Tracef("Found '%s' . Parsing Text", string(tok))
			token, ok := tz.parseCompound()
			if ok {
				log.Debugf("Returing Text Token: %s", token)
				return token, nil
			} else {
				tz.scanner.Scan()
			}
		}
	}
}
func isSafeToPrintInCharClass(ch rune) bool {
	switch ch {
	case '[', '^', '-', '\\', ']':
		return false
	}
	if !unicode.IsPrint(ch) {
		return false
	}
	return true
}
Example #29
0
func bytesToUTF16String(in []byte) string {
	var out bytes.Buffer
	out.WriteString(`u"`)
	for i := 0; i < len(in)/2; i++ {
		u := rune(in[2*i])<<8 | rune(in[2*i+1])
		if utf16.IsSurrogate(u) && i+1 < len(in)/2 {
			u2 := rune(in[2*i+2])<<8 | rune(in[2*i+3])
			r := utf16.DecodeRune(u, u2)
			if r != unicode.ReplacementChar {
				if unicode.IsPrint(r) {
					out.WriteRune(r)
				} else {
					fmt.Fprintf(&out, `\U%08x`, r)
				}
				i++
				continue
			}
		}

		if u == '\n' {
			out.WriteString(`\n`)
		} else if u == '"' {
			out.WriteString(`\"`)
		} else if u == '\\' {
			out.WriteString(`\\`)
		} else if !utf16.IsSurrogate(u) && unicode.IsPrint(u) {
			out.WriteRune(u)
		} else if u <= 0xff {
			fmt.Fprintf(&out, `\x%02x`, u)
		} else {
			fmt.Fprintf(&out, `\u%04x`, u)
		}
	}
	out.WriteString(`"`)

	// Print the trailing byte if needed.
	if len(in)&1 == 1 {
		fmt.Fprintf(&out, " `\\x%02x`", in[len(in)-1])
	}

	return out.String()
}
Example #30
0
File: derived.go Project: llir/llvm
// escape replaces any characters which are not printable with corresponding
// hexadecimal escape sequence (\XX).
func escape(s string) string {
	// Check if a replacement is required.
	extra := 0
	for i := 0; i < len(s); {
		r, size := utf8.DecodeRuneInString(s[i:])
		if utf8.ValidRune(r) && unicode.IsPrint(r) {
			i += size
			continue
		}
		// Two extra bytes are required for each non-printable byte; e.g.
		//    "\n" -> `\0A`
		//    "\x00" -> `\00`
		extra += 2
		i++
	}
	if extra == 0 {
		return s
	}

	// Replace non-printable bytes.
	const hextable = "0123456789ABCDEF"
	buf := make([]byte, len(s)+extra)
	j := 0
	for i := 0; i < len(s); {
		r, size := utf8.DecodeRuneInString(s[i:])
		if utf8.ValidRune(r) && unicode.IsPrint(r) {
			for k := 0; k < size; k++ {
				buf[j+k] = s[i+k]
			}
			i += size
			j += size
			continue
		}
		b := s[i]
		buf[j] = '\\'
		buf[j+1] = hextable[b>>4]
		buf[j+2] = hextable[b&0x0F]
		i++
		j += 3
	}
	return string(buf)
}