// GetRawTTYEncoding returns the TTY encoding, or nil if not found. func GetRawTTYEncoding() encoding.Encoding { lang := GetTTYEncodingName() if lang == "" { return nil } return text.GetEncoding(lang) }
func DecodeRFC2047Word(s string) (string, error) { fields := strings.Split(s, "?") if len(fields) != 5 || fields[0] != "=" || fields[4] != "=" { return "", errors.New("mail: address not RFC 2047 encoded") } charset, encMark := strings.ToLower(fields[1]), strings.ToLower(fields[2]) enc := text.GetEncoding(charset) if enc == nil { return "", fmt.Errorf("mail: charset not supported: %q", charset) } in := bytes.NewBufferString(fields[3]) var r io.Reader switch encMark { case "b": r = base64.NewDecoder(base64.StdEncoding, in) case "q": r = qDecoder{r: in} default: return "", fmt.Errorf("mail: RFC 2047 encoding not supported: %q", encMark) } dec, err := ioutil.ReadAll(text.NewReader(r, enc)) if err != nil { return "", err } return string(dec), err }
// GetTTYEncoding returns the TTY encoding, or UTF-8 if not found. func GetTTYEncoding() encoding.Encoding { enc := GetRawTTYEncoding() if enc != nil { return enc } return text.GetEncoding("UTF-8") }
func TestText(t *testing.T) { out, err := os.Create("/tmp/a.pdf") if err != nil { t.Errorf("error creating out file /tmp/a.pdf: %s", err) return } defer out.Close() in := bytes.NewBuffer(nil) in.WriteString("UTF-8: ") in.WriteString(accented) sep := []byte("\n\n---\n\n") in.Write(sep) in.WriteString("UTF-16: ") u := utf16.Encode([]rune(accented)) for i := 0; i < len(u); i++ { in.Write([]byte{byte(u[i] >> 8), byte(u[i] & 0xff)}) } in.Write(sep) in.WriteString("ISO-8859-15: ") i15, _ := text.Encode(accented, text.GetEncoding("ISO-8859-15")) t.Logf("i15: %q", i15) in.Write(i15) in.Write(sep) in.WriteString("windows-1252: ") w1252, _ := text.Encode(accented, text.GetEncoding("windows-1252")) t.Logf("w1252: %q", w1252) in.Write(w1252) in.Write(sep) in.WriteString("macroman: ") mr, _ := text.Encode(accented, text.GetEncoding("macroman")) t.Logf("mr: %q", mr) in.Write(mr) in.Write(sep) err = WriteTextAsPdf(out, bytes.NewReader(in.Bytes())) if err != nil { t.Errorf("error writing %q: %s", accented, err) } t.Logf("out: %q", out) }
func main() { Log.SetHandler(log15.StderrHandler) flagConnect := flag.String("connect", os.Getenv("BRUNO_ID"), "database connection string") flagCharset := flag.String("charset", term.GetTTYEncodingName(), "input charset of the csv") flagTruncate := flag.Bool("truncate", false, "truncate table?") flagSep := flag.String("sep", ";", "csv field separator") flag.Parse() var enc encoding.Encoding if *flagCharset != "" { enc = text.GetEncoding(*flagCharset) if enc == nil { Log.Error("unknown charset " + *flagCharset) os.Exit(1) } } db, err := sql.Open("ora", *flagConnect) if err != nil { Log.Crit("connect to db", "dsn", *flagConnect, "error", err) os.Exit(1) } defer db.Close() fh, err := os.Open(flag.Arg(0)) if err != nil { Log.Crit("open csv", "file", flag.Arg(0), "error", err) os.Exit(1) } defer fh.Close() r := io.Reader(fh) if enc != nil { Log.Debug("NewReader", "encoding", enc) r = text.NewReader(bufio.NewReaderSize(r, 1<<20), enc) } if *flagTruncate { if _, err = db.Exec("TRUNCATE TABLE " + flag.Arg(1)); err != nil { Log.Error("TRUNCATE", "table", flag.Arg(1), "error", err) os.Exit(1) } } if os.Getenv("GOMAXPROCS") == "" { Log.Info("Setting GOMAXPROCS", "numCPU", runtime.NumCPU()) runtime.GOMAXPROCS(runtime.NumCPU()) } cr := csv.NewReader(bufio.NewReaderSize(r, 16<<20)) cr.Comma = ([]rune(*flagSep))[0] cr.TrimLeadingSpace = true cr.LazyQuotes = true if err := load(db, flag.Arg(1), cr); err != nil { Log.Error("load", "error", err) os.Exit(2) } }
// NewTextReader wraps a reader with a proper charset converter func NewTextReader(r io.Reader, charset string) io.Reader { if charset == "" || charset == "utf-8" { return text.NewReader(r, nil) } enc := text.GetEncoding(charset) if enc == nil { logger.Warn("msg", "no decoder for charset", "charset", charset) return r } return text.NewReader(r, enc) }
func downloadResults(ctx context.Context, result []Result, page string, client *http.Client) ([]Result, error) { //http://80.249.172.60/cgi-bin/afaalany/ktmp/2015090310172791EC26C5_catv_pool_telekom_hutempfile.txt: Log.Debug("get", "url", page) resp, err := client.Get(page) if err != nil { Log.Error("get", "url", page, "error", err) return result, errgo.Notef(err, "GET "+page) } defer resp.Body.Close() select { case <-ctx.Done(): return result, ctx.Err() default: } Log.Debug("response", "header", resp.Header) var enc string if _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")); err == nil { enc = params["charset"] } if enc == "" { enc = "iso-8859-2" } scanner := bufio.NewScanner(text.NewReader(resp.Body, text.GetEncoding(enc))) for scanner.Scan() { line := bytes.TrimSpace(scanner.Bytes()) i := bytes.IndexByte(line, ';') if i < 0 { continue } taxNo := bytes.TrimRight(line[:i], " ") line = line[i+1:] if i = bytes.IndexByte(line, ';'); i >= 0 { line = line[:i] } result = append(result, Result{ TaxNo: string(taxNo), Owner: string(line), Valid: true, Exist: len(taxNo) == 11, }) } Log.Info("Download finished", "result.count", len(result)) return result, err }
// Extract hosts value from config and store it on the plugin instance. func (fr *FileReadFilter) Init(config interface{}) (err error) { var ( value interface{} str string ok bool ) conf := config.(pipeline.PluginConfig) if value, ok = conf["newType"]; !ok { return errors.New("FileReadFilter: No 'newType' setting specified.") } if fr.newType, ok = value.(string); !ok { return errors.New("FileReadFilter: 'newType' setting not a string value.") } if value, ok = conf["sourceEncoding"]; !ok { log.Printf("FileReadFilter: no source encoding specified - source should be UTF-8!") } else { if str, ok = value.(string); !ok { return errors.New("FileReadFilter: 'sourceEncoding' setting not a string value.") } e := text.GetEncoding(str) if e == nil { return fmt.Errorf("FileReadFilter: 'sourceEncoding' with value %q is unknown.", str) } fr.decoder = e.NewDecoder() } if value, ok = conf["pattern"]; !ok { return errors.New("FileReadFilter: No 'pattern' setting specified.") } if str, ok = value.(string); !ok { return errors.New("FileReadFilter: 'pattern' setting not a string value.") } if fr.tmpl, err = template.New("filereader").Parse(str); err == nil && fr.tmpl == nil { return fmt.Errorf("FileReadFilter: empty template (%q)", str) } return err }