Ejemplo n.º 1
0
func load(direction string, enc encoding.Encoding) ([]byte, []byte, Transcoder, error) {
	basename, ext, count := "", "", 0
	for _, tf := range testdataFiles {
		if tf.enc == enc {
			basename, ext = tf.basename, tf.ext
			count++
		}
	}
	if count != 1 {
		if count == 0 {
			return nil, nil, nil, fmt.Errorf("no testdataFiles for %s", enc)
		}
		return nil, nil, nil, fmt.Errorf("too many testdataFiles for %s", enc)
	}
	dstFile := fmt.Sprintf("testdata/%s-%s.txt", basename, ext)
	srcFile := fmt.Sprintf("testdata/%s-utf-8.txt", basename)
	var coder Transcoder = encoding.ReplaceUnsupported(enc.NewEncoder())
	if direction == "Decode" {
		dstFile, srcFile = srcFile, dstFile
		coder = enc.NewDecoder()
	}
	dst, err := ioutil.ReadFile(dstFile)
	if err != nil {
		return nil, nil, nil, err
	}
	src, err := ioutil.ReadFile(srcFile)
	if err != nil {
		return nil, nil, nil, err
	}
	return dst, src, coder, nil
}
Ejemplo n.º 2
0
// convertSJIS converts an string (a valid UTF-8 string) to/from a ShiftJIS string
func convertSJIS(b string, toSJIS bool) string {
	var t transform.Transformer
	t = japanese.ShiftJIS.NewDecoder()
	if toSJIS {
		tt := japanese.ShiftJIS.NewEncoder()
		t = encoding.ReplaceUnsupported(tt)
	}
	r, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader([]byte(b)), t))
	if err != nil {
		log.Println(err)
	}
	return string(r)
}
Ejemplo n.º 3
0
func TestEncodeInvalidUTF8(t *testing.T) {
	inputs := []string{
		"hello.",
		"wo\ufffdld.",
		"ABC\xff\x80\x80", // Invalid UTF-8.
		"\x80\x80\x80\x80\x80",
		"\x80\x80D\x80\x80",          // Valid rune at "D".
		"E\xed\xa0\x80\xed\xbf\xbfF", // Two invalid UTF-8 runes (surrogates).
		"G",
		"H\xe2\x82",     // U+20AC in UTF-8 is "\xe2\x82\xac", which we split over two
		"\xacI\xe2\x82", // input lines. It maps to 0x80 in the Windows-1252 encoding.
	}
	// Each invalid source byte becomes '\x1a'.
	want := strings.Replace("hello.wo?ld.ABC??????????D??E??????FGH\x80I??", "?", "\x1a", -1)

	transformer := encoding.ReplaceUnsupported(charmap.Windows1252.NewEncoder())
	gotBuf := make([]byte, 0, 1024)
	src := make([]byte, 0, 1024)
	for i, input := range inputs {
		dst := make([]byte, 1024)
		src = append(src, input...)
		atEOF := i == len(inputs)-1
		nDst, nSrc, err := transformer.Transform(dst, src, atEOF)
		gotBuf = append(gotBuf, dst[:nDst]...)
		src = src[nSrc:]
		if err != nil && err != transform.ErrShortSrc {
			t.Fatalf("i=%d: %v", i, err)
		}
		if atEOF && err != nil {
			t.Fatalf("i=%d: atEOF: %v", i, err)
		}
	}
	if got := string(gotBuf); got != want {
		t.Fatalf("\ngot  %+q\nwant %+q", got, want)
	}
}