func load(direction string, enc encoding.Encoding) ([]byte, []byte, Transcoder, error) { basename, ext, count := "", "", 0 for _, tf := range testdataFiles { if tf.enc == enc { basename, ext = tf.basename, tf.ext count++ } } if count != 1 { if count == 0 { return nil, nil, nil, fmt.Errorf("no testdataFiles for %s", enc) } return nil, nil, nil, fmt.Errorf("too many testdataFiles for %s", enc) } dstFile := fmt.Sprintf("testdata/%s-%s.txt", basename, ext) srcFile := fmt.Sprintf("testdata/%s-utf-8.txt", basename) var coder Transcoder = encoding.ReplaceUnsupported(enc.NewEncoder()) if direction == "Decode" { dstFile, srcFile = srcFile, dstFile coder = enc.NewDecoder() } dst, err := ioutil.ReadFile(dstFile) if err != nil { return nil, nil, nil, err } src, err := ioutil.ReadFile(srcFile) if err != nil { return nil, nil, nil, err } return dst, src, coder, nil }
// convertSJIS converts an string (a valid UTF-8 string) to/from a ShiftJIS string func convertSJIS(b string, toSJIS bool) string { var t transform.Transformer t = japanese.ShiftJIS.NewDecoder() if toSJIS { tt := japanese.ShiftJIS.NewEncoder() t = encoding.ReplaceUnsupported(tt) } r, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader([]byte(b)), t)) if err != nil { log.Println(err) } return string(r) }
func TestEncodeInvalidUTF8(t *testing.T) { inputs := []string{ "hello.", "wo\ufffdld.", "ABC\xff\x80\x80", // Invalid UTF-8. "\x80\x80\x80\x80\x80", "\x80\x80D\x80\x80", // Valid rune at "D". "E\xed\xa0\x80\xed\xbf\xbfF", // Two invalid UTF-8 runes (surrogates). "G", "H\xe2\x82", // U+20AC in UTF-8 is "\xe2\x82\xac", which we split over two "\xacI\xe2\x82", // input lines. It maps to 0x80 in the Windows-1252 encoding. } // Each invalid source byte becomes '\x1a'. want := strings.Replace("hello.wo?ld.ABC??????????D??E??????FGH\x80I??", "?", "\x1a", -1) transformer := encoding.ReplaceUnsupported(charmap.Windows1252.NewEncoder()) gotBuf := make([]byte, 0, 1024) src := make([]byte, 0, 1024) for i, input := range inputs { dst := make([]byte, 1024) src = append(src, input...) atEOF := i == len(inputs)-1 nDst, nSrc, err := transformer.Transform(dst, src, atEOF) gotBuf = append(gotBuf, dst[:nDst]...) src = src[nSrc:] if err != nil && err != transform.ErrShortSrc { t.Fatalf("i=%d: %v", i, err) } if atEOF && err != nil { t.Fatalf("i=%d: atEOF: %v", i, err) } } if got := string(gotBuf); got != want { t.Fatalf("\ngot %+q\nwant %+q", got, want) } }