func ExampleValid() {
valid := []byte("Hello, 世界")
invalid := []byte{0xff, 0xfe, 0xfd}
fmt.Println(utf8.Valid(valid))
fmt.Println(utf8.Valid(invalid))
// Output:
// true
// false
}
// IsUTF8 reports whether the blob is entirely UTF-8.
func (b *Blob) IsUTF8() bool {
if b.mem != nil {
return utf8.Valid(b.mem)
}
rc := b.Open()
defer rc.Close()
slurp, err := ioutil.ReadAll(rc)
if err != nil {
return false
}
return utf8.Valid(slurp)
}
func makestatic() error {
f, err := os.Create("static.go")
if err != nil {
return err
}
defer f.Close()
w := bufio.NewWriter(f)
fmt.Fprintf(w, "%v\npackage static\n\n", warning)
fmt.Fprintf(w, "var Files = map[string]string{\n")
for _, fn := range files {
b, err := ioutil.ReadFile(fn)
if err != nil {
return err
}
fmt.Fprintf(w, "\t%q: ", fn)
if utf8.Valid(b) {
fmt.Fprintf(w, "`%s`", sanitize(b))
} else {
fmt.Fprintf(w, "%q", b)
}
fmt.Fprintln(w, ",\n")
}
fmt.Fprintln(w, "}")
if err := w.Flush(); err != nil {
return err
}
return f.Close()
}
func backup(c *cli.Context) (err error) {
// Get KV client
client, backupResult, err := getConnectionFromFlags(c)
if err != nil {
return
}
kv := client.KV()
// Dump all
pairs, _, err := kv.List(c.GlobalString("prefix"), &api.QueryOptions{})
if err != nil {
return
}
bkup := map[string]valueEnc{}
for _, p := range pairs {
validUtf8 := utf8.Valid(p.Value)
if validUtf8 {
bkup[p.Key] = valueEnc{"", string(p.Value)}
} else {
sEnc := base64.StdEncoding.EncodeToString(p.Value)
bkup[p.Key] = valueEnc{"base64", sEnc}
}
}
backupResult.Values = bkup
// Send results to outfile (if defined) or stdout
dumpOutput(c.String("outfile"), backupResult)
return
}
func makestatic() error {
f, err := os.Create("static.go")
if err != nil {
return err
}
defer f.Close()
buf := new(bytes.Buffer)
fmt.Fprintf(buf, "%v\npackage static\n\n", warning)
fmt.Fprintf(buf, "var Files = map[string]string{\n")
for _, fn := range files {
b, err := ioutil.ReadFile(fn)
if err != nil {
return err
}
fmt.Fprintf(buf, "\t%q: ", fn)
if utf8.Valid(b) {
fmt.Fprintf(buf, "`%s`", sanitize(b))
} else {
fmt.Fprintf(buf, "%q", b)
}
fmt.Fprintln(buf, ",\n")
}
fmt.Fprintln(buf, "}")
fmtbuf, err := format.Source(buf.Bytes())
if err != nil {
return err
}
return ioutil.WriteFile("static.go", fmtbuf, 0666)
}
// DecryptTags looks for any tagged data of the form [gosecret|authtext|ciphertext|initvector|keyname] in the
// input content byte array and replaces each with a decrypted version of the ciphertext. Note that the
// input content must be valid UTF-8. The second parameter is the path to the directory in which keyfiles
// live. For each |keyname| in a gosecret block, there must be a corresponding file of the same name in the
// keystore directory.
// DecryptTags returns a []byte with all [gosecret] blocks replaced by plaintext.
func DecryptTags(content []byte, keyroot string) ([]byte, error) {
if !utf8.Valid(content) {
return nil, errors.New("File is not valid UTF-8")
}
content = gosecretRegex.ReplaceAllFunc(content, func(match []byte) []byte {
matchString := string(match)
matchString = matchString[:len(matchString)-1]
parts := strings.Split(matchString, "|")
if len(parts) < 5 {
// Block is not encrypted. Noop.
return match
} else {
plaintext, err := decryptTag(parts, keyroot)
if err != nil {
fmt.Println("Unable to decrypt tag", err)
return nil
}
return plaintext
}
})
return content, nil
}
func (e *echoHandler) extractBody(r io.ReadCloser, fileName, mimeType string) (body, error) {
defer r.Close()
bytes, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
//either display it as a string or store as file
if utf8.Valid(bytes) {
return body(string(bytes)), nil
}
//calc mime
if (mimeType == "" || mimeType == defaultmtype) && fileName != "" {
mimeType = mime.TypeByExtension(filepath.Ext(fileName))
}
//hash bytes
hash := md5.New()
hash.Write([]byte(mimeType + "|"))
hash.Write(bytes)
md5 := hex.EncodeToString(hash.Sum(nil))
b := &bodyValues{
Length: len(bytes),
Type: mimeType,
MD5: md5,
URL: "/file/" + md5,
}
e.cache.Add(b.MD5, fileName, mimeType, bytes)
return b, nil
}
// Copy everything from the pty master to the socket.
func handleOutputSock(ptym *os.File, conn *net.TCPConn) {
buf := make([]byte, 512)
var payload, overflow []byte
// TODO: more graceful exit on socket close / process exit.
for {
n, err := ptym.Read(buf)
if err != nil {
fmt.Println("failed to read from pty master: ", err)
return
}
// Empty the overflow from the last read into the payload first.
payload = append(payload[0:], overflow...)
overflow = nil
// Then empty the new buf read into the payload.
payload = append(payload, buf[:n]...)
// Strip out any incomplete utf-8 from current payload into overflow.
for !utf8.Valid(payload) {
overflow = append(overflow[:0], append(payload[len(payload)-1:], overflow[0:]...)...)
payload = payload[:len(payload)-1]
}
// Send out the finished payload as long as it's not empty.
if len(payload) >= 1 {
_, err = conn.Write(payload)
if err != nil {
fmt.Println("Write: ", err)
}
}
// Empty the payload.
payload = nil
}
}
/**
Encode string to CBOR binary string
*/
func (encoder *cborEncode) encodeString(variable string) (bool, error) {
byteBuf := []byte(variable)
majorType := majorTypeUtf8String
if !utf8.Valid(byteBuf) {
majorType = majorTypeByteString
}
initByte, err := packNumber(majorType, uint64(len(byteBuf)))
if err != nil {
return false, err
}
_, err = encoder.buff.Write(initByte)
if err != nil {
return false, err
}
_, err = encoder.buff.Write(byteBuf)
if err != nil {
return false, err
}
return true, nil
}
func script(name string) {
file, err := os.Open(name)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
bytes, err := ioutil.ReadAll(file)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if !utf8.Valid(bytes) {
fmt.Fprintf(os.Stderr, "source %v is not valid UTF-8\n", name)
os.Exit(1)
}
src := string(bytes)
ev := eval.NewEvaluator()
n, pe := parse.Parse(name, src)
if pe != nil {
fmt.Print(pe.(*util.ContextualError).Pprint())
os.Exit(1)
}
ee := ev.Eval(name, src, n)
if ee != nil {
fmt.Print(ee.(*util.ContextualError).Pprint())
os.Exit(1)
}
}
func main() {
var b1 []byte = []byte("안녕하세요")
fmt.Println(utf8.Valid(b1)) // true: "안녕하세요"는 UTF-8이 맞으므로 true
var b2 []byte = []byte{0xff, 0xf1, 0xc1}
fmt.Println(utf8.Valid(b2)) // false: 0xff 0xf1 0xc1은 UTF-8이 아니므로 false
var r1 rune = '한'
fmt.Println(utf8.ValidRune(r1)) // true: '한'은 UTF-8이 맞으므로 true
var r2 rune = 0x11111111
fmt.Println(utf8.ValidRune(r2)) // false: 0x11111111은 UTF-8이 아니므로 false
var s1 string = "한글"
fmt.Println(utf8.ValidString(s1)) // true: "한글"은 UTF-8이 맞으므로 true
var s2 string = string([]byte{0xff, 0xf1, 0xc1})
fmt.Println(utf8.ValidString(s2)) // false: 0xff 0xf1 0xc1은 UTF-8이 아니므로 false
}
// ReadHistory reads scrollback history from r. Returns the number of lines
// read, and any read error (except io.EOF).
func (s *State) ReadHistory(r io.Reader) (num int, err error) {
s.historyMutex.Lock()
defer s.historyMutex.Unlock()
in := bufio.NewReader(r)
num = 0
for {
line, part, err := in.ReadLine()
if err == io.EOF {
break
}
if err != nil {
return num, err
}
if part {
return num, fmt.Errorf("line %d is too long", num+1)
}
if !utf8.Valid(line) {
return num, fmt.Errorf("invalid string at line %d", num+1)
}
num++
s.history = append(s.history, string(line))
if len(s.history) > HistoryLimit {
s.history = s.history[1:]
}
}
return num, nil
}
func uncompressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error {
_, err := fmt.Fprintf(w, `var _%s = []byte(`, asset.Func)
if err != nil {
return err
}
b, err := ioutil.ReadAll(r)
if err != nil {
return err
}
if utf8.Valid(b) {
fmt.Fprintf(w, "`%s`", sanitize(b))
} else {
fmt.Fprintf(w, "%q", b)
}
_, err = fmt.Fprintf(w, `)
func %s_bytes() ([]byte, error) {
return _%s, nil
}
`, asset.Func, asset.Func)
return err
}
// ReadHistory reads scrollback history from r. Returns the number of lines
// read, and any read error (except io.EOF).
func (s *State) ReadHistory(r io.Reader) (num int, err error) {
in := bufio.NewReader(r)
num = 0
for {
line, part, err := in.ReadLine()
if err == io.EOF {
break
}
if err != nil {
return num, err
}
if part {
return num, errors.New("Line too long")
}
if !utf8.Valid(line) {
return num, errors.New("Invalid string")
}
num++
s.history = append(s.history, string(line))
if len(s.history) > HistoryLimit {
s.history = s.history[1:]
}
}
return num, nil
}
func parseHeaders(head []byte) (map[string]interface{}, error) {
if !utf8.Valid(head) {
return nil, fmt.Errorf("header is not utf8")
}
headers := make(map[string]interface{})
lines := strings.Split(string(head), "\n")
for i := 0; i < len(lines); {
entry := lines[i]
nameValueSplit := strings.Index(entry, ":")
if nameValueSplit == -1 {
return nil, fmt.Errorf("header entry missing ':' separator: %q", entry)
}
name := entry[:nameValueSplit]
if !headerNameSanity.MatchString(name) {
return nil, fmt.Errorf("invalid header name: %q", name)
}
consumed := nameValueSplit + 1
var value interface{}
var err error
value, i, err = parseEntry(consumed, i, lines, 0)
if err != nil {
return nil, err
}
if _, ok := headers[name]; ok {
return nil, fmt.Errorf("repeated header: %q", name)
}
headers[name] = value
}
return headers, nil
}
func convertFile(path string) (changed bool) {
abspath, err := filepath.Abs(path)
if err != nil {
log.Fatal("convertFile: filepath.Abs:", err)
}
oldData, err := ioutil.ReadFile(abspath)
if err != nil {
log.Fatal("convertFile: ioutil.ReadFile:", err)
}
if !utf8.Valid(oldData) {
return false
}
newData := reDataRevision.ReplaceAll(oldData, []byte(goldenDataRevision))
if string(newData) == string(oldData) {
return false
}
err = ioutil.WriteFile(abspath, newData, 0666)
if err != nil {
log.Fatal("convertFile: ioutil.WriteFile:", err)
}
return true
}
func addToMap(hashmap JournalEntry, name string, value []byte) {
v, ok := hashmap[name]
if !ok {
// if the field does not exist, simply add the value
if utf8.Valid(value) {
hashmap[name] = string(value)
} else {
hashmap[name] = value
}
} else {
// if the field does exist, make it a slice and append
switch t := v.(type) {
default:
fmt.Printf("Unexpected type: %T\n", t)
case string:
// NOTE: it is assumed here that consecutive fields with the same name are also UTF-8 strings
hashmap[name] = []string{t, string(value)}
case []byte:
hashmap[name] = [][]byte{t, value}
case []string:
// NOTE: it is assumed here that consecutive fields with the same name are also UTF-8 strings
hashmap[name] = append(t, string(value))
case [][]byte:
hashmap[name] = append(t, value)
}
}
}
func testEncodeSQL(t *testing.T, encode func(*bytes.Buffer, string), forceUTF8 bool) {
type entry struct{ i, j int }
seen := make(map[string]entry)
for i := 0; i < 256; i++ {
for j := 0; j < 256; j++ {
bytepair := []byte{byte(i), byte(j)}
if forceUTF8 && !utf8.Valid(bytepair) {
continue
}
s := string(bytepair)
var buf bytes.Buffer
encode(&buf, s)
sql := fmt.Sprintf("SELECT %s", buf.String())
for n := 0; n < len(sql); n++ {
ch := sql[n]
if ch < 0x20 || ch >= 0x7F {
t.Fatalf("unprintable character: %v (%v, %v): %s %v", ch, i, j, sql, []byte(sql))
}
}
stmts, err := parseTraditional(sql)
if err != nil {
t.Fatalf("%s: expected success, but found %s", sql, err)
}
stmt := stmts.String()
if e, ok := seen[stmt]; ok {
t.Fatalf("duplicate entry: %s, from %v, currently at %v, %v", stmt, e, i, j)
}
seen[stmt] = entry{i, j}
if sql != stmt {
t.Fatalf("expected %s, but found %s", sql, stmt)
}
}
}
}
// PutObject uploads a Google Cloud Storage object.
// shouldRetry will be true if the put failed due to authorization, but
// credentials have been refreshed and another attempt is likely to succeed.
// In this case, content will have been consumed.
func (gsa *Client) PutObject(obj *Object, content io.Reader) error {
if err := obj.valid(); err != nil {
return err
}
const maxSlurp = 2 << 20
var buf bytes.Buffer
n, err := io.CopyN(&buf, content, maxSlurp)
if err != nil && err != io.EOF {
return err
}
contentType := http.DetectContentType(buf.Bytes())
if contentType == "application/octet-stream" && n < maxSlurp && utf8.Valid(buf.Bytes()) {
contentType = "text/plain; charset=utf-8"
}
objURL := gsAccessURL + "/" + obj.Bucket + "/" + obj.Key
var req *http.Request
if req, err = http.NewRequest("PUT", objURL, ioutil.NopCloser(io.MultiReader(&buf, content))); err != nil {
return err
}
req.Header.Set("x-goog-api-version", "2")
req.Header.Set("Content-Type", contentType)
var resp *http.Response
if resp, err = gsa.client.Do(req); err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Bad put response code: %v", resp.Status)
}
return nil
}
func main() {
in := os.Stdin
b := make([]byte, 1)
for {
r := make([]byte, 0, utf8.UTFMax)
for {
n, err := in.Read(b)
if err != nil {
if err.Error() == "EOF" {
os.Exit(0)
}
panic(err)
}
if n < 1 {
panic("read less than 1 byte")
}
r = append(r, b[0])
if utf8.Valid(r) {
ru := bytes.Runes(r)[0]
fmt.Printf("%U %c\n", ru, ru)
break
}
}
}
}
// SetNameBytes will check name for valid UTF-8 string, and set the appropriate
// field
func (e *Entry) SetNameBytes(name []byte) {
if utf8.Valid(name) {
e.Name = string(name)
} else {
e.NameRaw = name
}
}
func SafeBytesToStr(output []byte) (outputStr string, isBase64 bool) {
if utf8.Valid(output) {
return string(output), false
} else {
encoded := base64.StdEncoding.EncodeToString(output)
return encoded, true
}
}
func unmarshalMetadata(metadataXML []byte) (ContentRef, error, bool) {
metadata := ContentRef{}
err := xml.Unmarshal(metadataXML, &metadata)
if err == nil {
return metadata, nil, false
}
return metadata, err, !utf8.Valid(metadataXML)
}
func parseHeaders(head []byte) (map[string]string, error) {
if !utf8.Valid(head) {
return nil, fmt.Errorf("header is not utf8")
}
headers := make(map[string]string)
lines := strings.Split(string(head), "\n")
for i := 0; i < len(lines); {
entry := lines[i]
i++
nameValueSplit := strings.Index(entry, ":")
if nameValueSplit == -1 {
return nil, fmt.Errorf("header entry missing ':' separator: %q", entry)
}
name := entry[:nameValueSplit]
if !headerNameSanity.MatchString(name) {
return nil, fmt.Errorf("invalid header name: %q", name)
}
afterSplit := nameValueSplit + 1
if afterSplit == len(entry) {
// multiline value
size := 0
j := i
for j < len(lines) {
iline := lines[j]
if len(iline) == 0 || iline[0] != ' ' {
break
}
size += len(iline)
j++
}
if j == i {
return nil, fmt.Errorf("empty multiline header value: %q", entry)
}
valueBuf := bytes.NewBuffer(make([]byte, 0, size-1))
valueBuf.WriteString(lines[i][1:])
i++
for i < j {
valueBuf.WriteByte('\n')
valueBuf.WriteString(lines[i][1:])
i++
}
headers[name] = valueBuf.String()
continue
}
if entry[afterSplit] != ' ' {
return nil, fmt.Errorf("header entry should have a space or newline (multiline) before value: %q", entry)
}
headers[name] = entry[afterSplit+1:]
}
return headers, nil
}
func readFileUTF8(fname string) (string, error) {
bytes, err := ioutil.ReadFile(fname)
if err != nil {
return "", err
}
if !utf8.Valid(bytes) {
return "", fmt.Errorf("%s: source is not valid UTF-8", fname)
}
return string(bytes), nil
}
// MarshalJSON returns a JSON encoded value. If the value is valid UTF-8 it will be encoded as a string, otherwise it will be encoded as a list of hex values.
func (i *Value) MarshalJSON() ([]byte, error) {
if utf8.Valid(*i) {
b, err := json.Marshal(string(*i))
if err != nil {
return nil, err
}
return []byte(fmt.Sprintf("%s", b)), nil /* output as string if valid utf8 */
}
return []byte(fmt.Sprintf("{ \"hex\" : \"%x\" }", *i)), nil /* if not valid, output in hex format */
}
func encodeBytes(input []byte) (value string, encoding string) {
if utf8.Valid(input) {
value = string(input)
encoding = "utf8"
} else {
value = base64.StdEncoding.EncodeToString(input)
encoding = "base64"
}
return value, encoding
}
// ReadString reads a value as string. The value must contain a valid UTF-8 encoded string.
func (rest *Conn) ReadString(path Path) (string, error) {
res, err := rest.Read(path)
if err != nil {
return "", err
}
if utf8.Valid(res) {
return string(res), nil
}
return "", fmt.Errorf("path %s does not contain a valid utf8 string", path.String())
}
func TestIssue5880(t *testing.T) {
type T []byte
data, err := Marshal(T{192, 168, 0, 1})
if err != nil {
t.Errorf("Marshal error: %v", err)
}
if !utf8.Valid(data) {
t.Errorf("Marshal generated invalid UTF-8: %x", data)
}
}
func TestZipName(t *testing.T) {
buf, err := ioutil.ReadFile("examples/twilight.txt")
if err != nil {
t.Fatalf("failed to open twilight.txt, got: %v", err)
}
nm := ZipName(string(buf))
if !utf8.Valid([]byte(nm)) {
t.Fatalf("not valid: %s", nm)
}
}
