func main() {
rang, except := scan(0, 0xFFFF)
range16 = to16(rang)
except16 = to16(except)
range32, except32 = scan(0x10000, unicode.MaxRune)
for i := rune(0); i <= unicode.MaxRune; i++ {
if isPrint(i) != unicode.IsPrint(i) {
fmt.Fprintf(os.Stderr, "%U: isPrint=%v, want %v\n", i, isPrint(i), unicode.IsPrint(i))
return
}
}
fmt.Printf(`// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.` + "\n\n")
fmt.Printf("// DO NOT EDIT. GENERATED BY\n")
fmt.Printf("// go run makeisprint.go >x && mv x isprint.go\n\n")
fmt.Printf("package strconv\n\n")
fmt.Printf("// (%d+%d+%d)*2 + (%d)*4 = %d bytes\n\n",
len(range16), len(except16), len(except32),
len(range32),
(len(range16)+len(except16)+len(except32))*2+
(len(range32))*4)
fmt.Printf("var isPrint16 = []uint16{\n")
for i := 0; i < len(range16); i += 2 {
fmt.Printf("\t%#04x, %#04x,\n", range16[i], range16[i+1])
}
fmt.Printf("}\n\n")
fmt.Printf("var isNotPrint16 = []uint16{\n")
for _, r := range except16 {
fmt.Printf("\t%#04x,\n", r)
}
fmt.Printf("}\n\n")
fmt.Printf("var isPrint32 = []uint32{\n")
for i := 0; i < len(range32); i += 2 {
fmt.Printf("\t%#06x, %#06x,\n", range32[i], range32[i+1])
}
fmt.Printf("}\n\n")
fmt.Printf("var isNotPrint32 = []uint16{ // add 0x10000 to each entry\n")
for _, r := range except32 {
if r >= 0x20000 {
fmt.Fprintf(os.Stderr, "%U too big for isNotPrint32\n", r)
return
}
fmt.Printf("\t%#04x,\n", r-0x10000)
}
fmt.Printf("}\n")
}
func main() {
rang, except := scan(0, 0xFFFF)
range16 = to16(rang)
except16 = to16(except)
range32, except32 = scan(0x10000, unicode.MaxRune)
for i := rune(0); i <= unicode.MaxRune; i++ {
if isPrint(i) != unicode.IsPrint(i) {
fmt.Fprintf(os.Stderr, "%U: isPrint=%v, want %v\n", i, isPrint(i), unicode.IsPrint(i))
return
}
}
fmt.Printf("// DO NOT EDIT. GENERATED BY\n")
fmt.Printf("// go run makeisprint.go >x && mv x isprint.go\n\n")
fmt.Printf("package strconv\n\n")
fmt.Printf("// (%d+%d+%d)*2 + (%d)*4 = %d bytes\n\n",
len(range16), len(except16), len(except32),
len(range32),
(len(range16)+len(except16)+len(except32))*2+
(len(range32))*4)
fmt.Printf("var isPrint16 = []uint16{\n")
for i := 0; i < len(range16); i += 2 {
fmt.Printf("\t%#04x, %#04x,\n", range16[i], range16[i+1])
}
fmt.Printf("}\n\n")
fmt.Printf("var isNotPrint16 = []uint16{\n")
for _, r := range except16 {
fmt.Printf("\t%#04x,\n", r)
}
fmt.Printf("}\n\n")
fmt.Printf("var isPrint32 = []uint32{\n")
for i := 0; i < len(range32); i += 2 {
fmt.Printf("\t%#06x, %#06x,\n", range32[i], range32[i+1])
}
fmt.Printf("}\n\n")
fmt.Printf("var isNotPrint32 = []uint16{ // add 0x10000 to each entry\n")
for _, r := range except32 {
if r >= 0x20000 {
fmt.Fprintf(os.Stderr, "%U too big for isNotPrint32\n", r)
return
}
fmt.Printf("\t%#04x,\n", r-0x10000)
}
fmt.Printf("}\n")
}
func quoteDouble(s string) string {
var buf bytes.Buffer
buf.WriteByte('"')
for _, r := range s {
if r == '\\' || r == '"' {
buf.WriteByte('\\')
buf.WriteRune(r)
} else if !unicode.IsPrint(r) {
buf.WriteByte('\\')
if r <= 0xff {
buf.WriteByte('x')
buf.Write(rtohex(r, 2))
} else if r <= 0xffff {
buf.WriteByte('u')
buf.Write(rtohex(r, 4))
} else {
buf.WriteByte('U')
buf.Write(rtohex(r, 8))
}
} else {
buf.WriteRune(r)
}
}
buf.WriteByte('"')
return buf.String()
}
func incrementCount(r rune, counts map[int]int) {
switch {
case unicode.IsControl(r):
counts[isControl]++
case unicode.IsNumber(r):
counts[isNumber]++
case unicode.IsDigit(r):
counts[isDigit]++
case unicode.IsLetter(r):
counts[isLetter]++
case unicode.IsMark(r):
counts[isMark]++
case unicode.IsPunct(r):
counts[isPunct]++
case unicode.IsSpace(r):
counts[isSpace]++
case unicode.IsSymbol(r):
counts[isSymbol]++
case unicode.IsPrint(r):
counts[isPrint]++
case unicode.IsGraphic(r):
counts[isGraphic]++
}
}
func valid(id string) error {
if utf8.RuneCountInString(id) == 0 {
return ErrBadID{"Empty ID is not allowed"}
}
if !utf8.ValidString(id) {
return ErrBadID{fmt.Sprintf("Invalid utf-8: %v", id)}
}
for idx, rn := range id {
if unicode.IsSpace(rn) {
return ErrBadID{
fmt.Sprintf("Space not allowed: %s (at %d)", id, idx),
}
}
if !unicode.IsPrint(rn) {
return ErrBadID{
fmt.Sprintf("Only printable runes allowed: %s (at %d)", id, idx),
}
}
}
return nil
}
// CharType returns a string representing the unicode type of a rune
func CharType(r rune) string {
switch {
case unicode.IsLetter(r):
return "letter"
case unicode.IsSpace(r):
return "space"
case unicode.IsPunct(r):
return "punct"
case unicode.IsNumber(r):
return "number"
case unicode.IsSymbol(r):
return "symbol"
case unicode.IsMark(r):
return "mark"
case unicode.IsDigit(r):
return "digit"
case unicode.IsPrint(r):
return "print"
case unicode.IsControl(r):
return "control"
case unicode.IsGraphic(r):
return "graphic"
default:
return "invalid"
}
}
// The following are allowed in variable names:
// * Anything beyond ASCII that is printable
// * Letters and numbers
// * The symbols "-_:"
func allowedInVariableName(r rune) bool {
return (r >= 0x80 && unicode.IsPrint(r)) ||
('0' <= r && r <= '9') ||
('a' <= r && r <= 'z') ||
('A' <= r && r <= 'Z') ||
r == '-' || r == '_' || r == ':'
}
// QuoteAs returns a representation of s in elvish syntax, using the syntax
// specified by q, which must be one of Bareword, SingleQuoted, or
// DoubleQuoted. It returns the quoted string and the actual quoting.
func QuoteAs(s string, q PrimaryType) (string, PrimaryType) {
if q == DoubleQuoted {
// Everything can be quoted using double quotes, return directly.
return quoteDouble(s), DoubleQuoted
}
if s == "" {
return "''", SingleQuoted
}
// Keep track of whether it is a valid bareword.
bare := s[0] != '~'
for _, r := range s {
if !unicode.IsPrint(r) {
// Contains unprintable character; force double quote.
return quoteDouble(s), DoubleQuoted
}
if !allowedInBareword(r, false) {
bare = false
}
}
if q == Bareword && bare {
return s, Bareword
}
return quoteSingle(s), SingleQuoted
}
func quoteDouble(s string) string {
var buf bytes.Buffer
buf.WriteByte('"')
for _, r := range s {
if e, ok := doubleUnescape[r]; ok {
// Takes care of " and \ as well.
buf.WriteByte('\\')
buf.WriteRune(e)
} else if !unicode.IsPrint(r) {
buf.WriteByte('\\')
if r <= 0xff {
buf.WriteByte('x')
buf.Write(rtohex(r, 2))
} else if r <= 0xffff {
buf.WriteByte('u')
buf.Write(rtohex(r, 4))
} else {
buf.WriteByte('U')
buf.Write(rtohex(r, 8))
}
} else {
buf.WriteRune(r)
}
}
buf.WriteByte('"')
return buf.String()
}
func writeQuotedString(s string, b *bufio.Writer, f *Format) {
b.WriteRune('"')
for i := range s {
c := s[i] // don't decode as runes
switch c {
case '\r':
b.WriteString(`\r`)
case '\n':
b.WriteString(`\n`)
case '\t':
b.WriteString(`\t`)
case '"':
b.WriteString(`\"`)
case '\\':
b.WriteString(`\\`)
default:
if c < 0x80 && unicode.IsPrint(rune(c)) {
b.WriteRune(rune(c))
} else {
b.WriteString(`\x`)
b.WriteRune(enchex((c >> 4) & 0x0F))
b.WriteRune(enchex(c & 0x0F))
}
}
}
b.WriteRune('"')
}
// Stat calculates statistics for all runes read from r.
func (m *Main) Stat(r io.RuneReader) (Stats, error) {
var stats Stats
for {
// Read next character.
ch, sz, err := r.ReadRune()
if err == io.EOF {
break
} else if err != nil {
return stats, err
}
// Calculate stats.
stats.TotalN++
if unicode.IsControl(ch) {
stats.ControlN++
}
if unicode.IsDigit(ch) {
stats.DigitN++
}
if unicode.IsGraphic(ch) {
stats.GraphicN++
}
if unicode.IsLetter(ch) {
stats.LetterN++
}
if unicode.IsLower(ch) {
stats.LowerN++
}
if unicode.IsMark(ch) {
stats.MarkN++
}
if unicode.IsNumber(ch) {
stats.NumberN++
}
if unicode.IsPrint(ch) {
stats.PrintN++
}
if unicode.IsPunct(ch) {
stats.PunctN++
}
if unicode.IsSpace(ch) {
stats.SpaceN++
}
if unicode.IsSymbol(ch) {
stats.SymbolN++
}
if unicode.IsTitle(ch) {
stats.TitleN++
}
if unicode.IsUpper(ch) {
stats.UpperN++
}
if sz > 1 {
stats.MultiByteN++
}
}
return stats, nil
}
func (q *quotedwriter) Write(data []byte) (n int, err error) {
b := q.b
if !q.add {
b = append(b, '"')
q.add = true
}
for _, c := range data {
switch {
case q.esc, c > unicode.MaxASCII:
fallthrough
default:
x := []byte{'\\', 'x', 0, c}
hex.Encode(x[2:], x[3:])
b = append(b, x...)
case c == '"':
b = append(b, '\\', '"')
case c == '\\':
b = append(b, '\\', '\\')
case unicode.IsPrint(rune(c)):
b = append(b, c)
}
}
n, err = q.w.Write(b)
q.b = b[:0]
if err == nil {
if n != len(b) {
err = io.ErrShortWrite
} else {
n = len(data)
}
}
return
}
func (field_editor *FieldEditor) handleKeyEvent(event termbox.Event) (string, bool) {
is_done := false
if event.Key == termbox.KeyEnter {
is_done = true
} else if event.Key == termbox.KeyEsc {
is_done = true
field_editor.value = nil
} else if event.Key == termbox.KeyArrowLeft {
if field_editor.cursor_pos > 0 {
field_editor.cursor_pos--
}
} else if event.Key == termbox.KeyArrowUp || event.Key == termbox.KeyCtrlA {
field_editor.cursor_pos = 0
} else if event.Key == termbox.KeyArrowRight {
if field_editor.cursor_pos < utf8.RuneCount(field_editor.value) {
field_editor.cursor_pos++
}
} else if event.Key == termbox.KeyArrowDown || event.Key == termbox.KeyCtrlE {
field_editor.cursor_pos = utf8.RuneCount(field_editor.value)
} else if event.Key == termbox.KeyCtrlH || event.Key == termbox.KeyBackspace {
if field_editor.cursor_pos > 0 {
field_editor.value = removeRuneAtIndex(field_editor.value, field_editor.cursor_pos-1)
field_editor.cursor_pos--
}
} else if unicode.IsPrint(event.Ch) {
field_editor.value = insertRuneAtIndex(field_editor.value, field_editor.cursor_pos, event.Ch)
field_editor.cursor_pos++
} else if event.Key == termbox.KeySpace {
field_editor.value = insertRuneAtIndex(field_editor.value, field_editor.cursor_pos, ' ')
field_editor.cursor_pos++
}
return string(field_editor.value), is_done
}
func isPrintableASCII(s string) bool {
for _, x := range s {
if unicode.IsPrint(x) == false {
return false
}
}
return true
}
// checks if s is ascii and printable, aka doesn't include tab, backspace, etc.
func IsAsciiPrintable(s string) bool {
for _, r := range s {
if r > unicode.MaxASCII || !unicode.IsPrint(r) {
return false
}
}
return true
}
func isPrintable(s string) bool {
for _, c := range s {
if c == 0 || c > unicode.MaxASCII || !unicode.IsPrint(c) {
return false
}
}
return true
}
// checks if s is ascii and printable, aka doesn't include tab, backspace, etc.
func (i *HttpOutput) IsPrintable(s string) bool {
for _, r := range s {
if !unicode.IsPrint(r) && r != rune('\t') && r != rune('\r') && r != rune('\n') {
return false
}
}
return true
}
func IsPrintable(input string) bool {
for _, r := range input {
if !unicode.IsPrint(r) {
return false
}
}
return true
}
func isPrintable(s string) bool {
for _, r := range s {
if !unicode.IsPrint(r) {
return false
}
}
return true
}
func needQuote(str string) bool {
for _, char := range str {
if unicode.IsSpace(char) || !unicode.IsPrint(char) || unicode.Is(unicode.Quotation_Mark, char) {
return true
}
}
return false
}
func quote_rune(buf *bytes.Buffer, r rune, size int) {
const lowerhex = "0123456789abcdef"
if size == 1 && r == utf8.RuneError {
// invalid rune, write the byte as is
buf.WriteString(`\x`)
buf.WriteByte(lowerhex[r>>4])
buf.WriteByte(lowerhex[r&0xF])
return
}
// first check for special TCL escaping cases
switch r {
case '{', '}', '[', ']', '"', '$', '\\':
buf.WriteString("\\")
buf.WriteRune(r)
return
}
// other printable characters
if unicode.IsPrint(r) {
buf.WriteRune(r)
return
}
// non-printable characters
switch r {
case '\a':
buf.WriteString(`\a`)
case '\b':
buf.WriteString(`\b`)
case '\f':
buf.WriteString(`\f`)
case '\n':
buf.WriteString(`\n`)
case '\r':
buf.WriteString(`\r`)
case '\t':
buf.WriteString(`\t`)
case '\v':
buf.WriteString(`\v`)
default:
switch {
case r < ' ':
buf.WriteString(`\x`)
buf.WriteByte(lowerhex[r>>4])
buf.WriteByte(lowerhex[r&0xF])
case r >= 0x10000:
r = 0xFFFD
fallthrough
case r < 0x10000:
buf.WriteString(`\u`)
for s := 12; s >= 0; s -= 4 {
buf.WriteByte(lowerhex[r>>uint(s)&0xF])
}
}
}
}
// Version return the string of the splice version
// with only printable characters
func (h Header) Version() string {
var buff bytes.Buffer
for _, b := range h.Vers {
if unicode.IsPrint(rune(b)) {
buff.WriteByte(b)
}
}
return buff.String()
}
func addToName(r rune) {
// Ignore non-printable characters or if the name is to long.
if !unicode.IsPrint(r) || len(name) > 30 {
log.Printf("%v\n", r)
return
}
name += string(r)
printName()
}
func getPrintableString(dirty string) string {
var b bytes.Buffer
for _, by := range dirty {
if unicode.IsPrint(rune(by)) {
b.WriteString(string(by))
}
}
return b.String()
}
// String returns a string representation for the literal.
func (l lit) String() string {
var c byte
if unicode.IsPrint(rune(l.b)) {
c = l.b
} else {
c = '.'
}
return fmt.Sprintf("lit{%02x %c}", l.b, c)
}
// Print a byte as ASCII, using escape sequences where necessary.
func printAsciiByte(b uint8) {
r := rune(b)
if unicode.IsPrint(r) || unicode.IsSpace(r) {
fmt.Print(string(r))
} else {
charStr := strconv.QuoteRuneToASCII(r)
fmt.Print(charStr[1 : len(charStr)-1])
}
}
func (tz *BadXMLTokenizer) Next() (*Token, error) {
for {
tok := tz.scanner.Peek()
log.Tracef("Scanner found: %v", tok)
if tok == scanner.EOF {
return nil, io.EOF
}
switch {
case unicode.IsPrint(tok) == false:
log.Tracef("Skipping unprintable character")
fallthrough
case unicode.IsSpace(tok):
tz.scanner.Scan()
continue
case tok == '<':
log.Tracef("parsing XML")
token, ok := parseXML(tz.scanner)
// We actually bump the phrase no matter what. It's
// either a comment, an xml token, or something weird
tz.current_phrase_id = rand.Intn(1000)
if ok {
log.Tracef("Returning XML Token: %s", token)
return token, nil
}
case tok == '&':
log.Tracef("parsing HTML")
if token := parseHTMLEntity(tz.scanner); token != nil {
tz.current_phrase_id = rand.Intn(1000)
return token, nil
}
case tok == '`': // Handle this speciallly - technically it's a 'grave accent'
fallthrough
case unicode.Is(unicode.Punct, tok):
log.Tracef("Ignoring punctuation: %v", tok)
tz.current_phrase_id = rand.Intn(1000)
tok = tz.scanner.Scan()
default:
/* Catch special things in words */
log.Tracef("Found '%s' . Parsing Text", string(tok))
token, ok := tz.parseCompound()
if ok {
log.Debugf("Returing Text Token: %s", token)
return token, nil
} else {
tz.scanner.Scan()
}
}
}
}
func isSafeToPrintInCharClass(ch rune) bool {
switch ch {
case '[', '^', '-', '\\', ']':
return false
}
if !unicode.IsPrint(ch) {
return false
}
return true
}
func bytesToUTF16String(in []byte) string {
var out bytes.Buffer
out.WriteString(`u"`)
for i := 0; i < len(in)/2; i++ {
u := rune(in[2*i])<<8 | rune(in[2*i+1])
if utf16.IsSurrogate(u) && i+1 < len(in)/2 {
u2 := rune(in[2*i+2])<<8 | rune(in[2*i+3])
r := utf16.DecodeRune(u, u2)
if r != unicode.ReplacementChar {
if unicode.IsPrint(r) {
out.WriteRune(r)
} else {
fmt.Fprintf(&out, `\U%08x`, r)
}
i++
continue
}
}
if u == '\n' {
out.WriteString(`\n`)
} else if u == '"' {
out.WriteString(`\"`)
} else if u == '\\' {
out.WriteString(`\\`)
} else if !utf16.IsSurrogate(u) && unicode.IsPrint(u) {
out.WriteRune(u)
} else if u <= 0xff {
fmt.Fprintf(&out, `\x%02x`, u)
} else {
fmt.Fprintf(&out, `\u%04x`, u)
}
}
out.WriteString(`"`)
// Print the trailing byte if needed.
if len(in)&1 == 1 {
fmt.Fprintf(&out, " `\\x%02x`", in[len(in)-1])
}
return out.String()
}
// escape replaces any characters which are not printable with corresponding
// hexadecimal escape sequence (\XX).
func escape(s string) string {
// Check if a replacement is required.
extra := 0
for i := 0; i < len(s); {
r, size := utf8.DecodeRuneInString(s[i:])
if utf8.ValidRune(r) && unicode.IsPrint(r) {
i += size
continue
}
// Two extra bytes are required for each non-printable byte; e.g.
// "\n" -> `\0A`
// "\x00" -> `\00`
extra += 2
i++
}
if extra == 0 {
return s
}
// Replace non-printable bytes.
const hextable = "0123456789ABCDEF"
buf := make([]byte, len(s)+extra)
j := 0
for i := 0; i < len(s); {
r, size := utf8.DecodeRuneInString(s[i:])
if utf8.ValidRune(r) && unicode.IsPrint(r) {
for k := 0; k < size; k++ {
buf[j+k] = s[i+k]
}
i += size
j += size
continue
}
b := s[i]
buf[j] = '\\'
buf[j+1] = hextable[b>>4]
buf[j+2] = hextable[b&0x0F]
i++
j += 3
}
return string(buf)
}