// Convert unicode text to ASCII text
// using specific codepage mapping.
func convertUnicodeToAscii(text string,
codepage encoding.Encoding) []byte {
b := []byte(text)
// fmt.Printf("Text length: %d\n", len(b))
var buf bytes.Buffer
if codepage == nil {
codepage = charmap.Windows1252
}
w := transform.NewWriter(&buf, codepage.NewEncoder())
defer w.Close()
w.Write(b)
// fmt.Printf("Buffer length: %d\n", len(buf.Bytes()))
return buf.Bytes()
}
func load(direction string, enc encoding.Encoding) ([]byte, []byte, Transcoder, error) {
basename, ext, count := "", "", 0
for _, tf := range testdataFiles {
if tf.enc == enc {
basename, ext = tf.basename, tf.ext
count++
}
}
if count != 1 {
if count == 0 {
return nil, nil, nil, fmt.Errorf("no testdataFiles for %s", enc)
}
return nil, nil, nil, fmt.Errorf("too many testdataFiles for %s", enc)
}
dstFile := fmt.Sprintf("testdata/%s-%s.txt", basename, ext)
srcFile := fmt.Sprintf("testdata/%s-utf-8.txt", basename)
var coder Transcoder = encoding.ReplaceUnsupported(enc.NewEncoder())
if direction == "Decode" {
dstFile, srcFile = srcFile, dstFile
coder = enc.NewDecoder()
}
dst, err := ioutil.ReadFile(dstFile)
if err != nil {
return nil, nil, nil, err
}
src, err := ioutil.ReadFile(srcFile)
if err != nil {
return nil, nil, nil, err
}
return dst, src, coder, nil
}
func encodeText(e encoding.Encoding, text string) []byte {
res, _, err := transform.Bytes(e.NewEncoder(), []byte(text))
if err != nil {
panic(err)
}
return res
}
func ConvTo(b []byte, e encoding.Encoding) (result []byte, err error) {
w := new(bytes.Buffer)
writer := transform.NewWriter(w, e.NewEncoder())
defer writer.Close()
if _, err = writer.Write(b); err != nil {
return
}
return w.Bytes(), nil
}
func verifyFromUTF(enc encoding.Encoding, b byte, r rune) {
encoder := enc.NewEncoder()
out := make([]byte, 6)
utf := make([]byte, utf8.RuneLen(r))
utf8.EncodeRune(utf, r)
ndst, nsrc, err := encoder.Transform(out, utf, true)
So(err, ShouldBeNil)
So(nsrc, ShouldEqual, len(utf))
So(ndst, ShouldEqual, 1)
So(b, ShouldEqual, out[0])
}
// NewLine creates a new Line reader object
func NewLine(input io.Reader, codec encoding.Encoding, bufferSize int) (*Line, error) {
encoder := codec.NewEncoder()
// Create newline char based on encoding
nl, _, err := transform.Bytes(encoder, []byte{'\n'})
if err != nil {
return nil, err
}
return &Line{
reader: input,
codec: codec,
bufferSize: bufferSize,
nl: nl,
decoder: codec.NewDecoder(),
inBuffer: streambuf.New(nil),
outBuffer: streambuf.New(nil),
}, nil
}
func enc(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
return "Encode", e.NewEncoder(), internal.ErrASCIIReplacement
}
// NewWriter returns a writer which encodes to the given encoding, utf8.
//
// If enc is nil, then only an utf8-enforcing replacement writer
// (see http://godoc.org/code.google.com/p/go.text/encoding#pkg-variables)
// is used.
func NewWriter(w io.Writer, enc encoding.Encoding) io.WriteCloser {
if enc == nil || enc == encoding.Replacement {
return transform.NewWriter(w, encoding.Replacement.NewEncoder())
}
return transform.NewWriter(w, transform.Chain(enc.NewEncoder()))
}
