Example #1
0
func main() {
	data, err := utfutil.ReadFile("inputfile.txt", utfutil.HTML5)
	if err != nil {
		log.Fatal(err)
	}
	final := strings.Replace(string(data), "\r\n", "\n", -1)
	fmt.Println(final)
}
Example #2
0
func TestReadFile(t *testing.T) {

	expected, err := ioutil.ReadFile(filepath.Join("testdata", "calblur8.htm"))
	if err != nil {
		log.Fatal(err)
	}

	// The test files were generate with:
	//    for i in $(iconv  -l|grep UTF)  ; do
	//        iconv -f UTF-8 -t $i calblur8.htm > calblur8.htm.$i
	//    done
	for _, tst := range []struct {
		works bool // This combination is expected to work.
		ume   utfutil.EncodingHint
		name  string
	}{
		// Assume missing BOM means UTF8
		{true, utfutil.UTF8, "calblur8.htm.UTF-8"},     // No BOM
		{true, utfutil.UTF8, "calblur8.htm.UTF-16"},    // BOM=fffe
		{false, utfutil.UTF8, "calblur8.htm.UTF-16LE"}, // no BOM
		{false, utfutil.UTF8, "calblur8.htm.UTF-16BE"}, // no BOM
		// Assume missing BOM means UFT16LE
		{false, utfutil.UTF16LE, "calblur8.htm.UTF-8"},    // No BOM
		{true, utfutil.UTF16LE, "calblur8.htm.UTF-16"},    // BOM=fffe
		{true, utfutil.UTF16LE, "calblur8.htm.UTF-16LE"},  // no BOM
		{false, utfutil.UTF16LE, "calblur8.htm.UTF-16BE"}, // no BOM
		// Assume missing BOM means UFT16BE
		{false, utfutil.UTF16BE, "calblur8.htm.UTF-8"},    // No BOM
		{true, utfutil.UTF16BE, "calblur8.htm.UTF-16"},    // BOM=fffe
		{false, utfutil.UTF16BE, "calblur8.htm.UTF-16LE"}, // no BOM
		{true, utfutil.UTF16BE, "calblur8.htm.UTF-16BE"},  // no BOM
	} {

		actual, err := utfutil.ReadFile(filepath.Join("testdata", tst.name), tst.ume)
		if err != nil {
			log.Fatal(err)
		}

		if tst.works {
			if string(expected) == string(actual) {
				t.Log("SUCCESS:", tst.ume, tst.name)
			} else {
				t.Errorf("FAIL: %v/%v: expected %#v got %#v\n", tst.ume, tst.name, string(expected)[:4], actual[:4])
			}
		} else {
			if string(expected) != string(actual) {
				t.Logf("SUCCESS: %v/%v: failed as expected.", tst.ume, tst.name)
			} else {
				t.Errorf("FAILUREish: %v/%v: unexpected success!", tst.ume, tst.name)
			}
		}
	}

}