// GetRawTTYEncoding returns the TTY encoding, or nil if not found.
func GetRawTTYEncoding() encoding.Encoding {
lang := GetTTYEncodingName()
if lang == "" {
return nil
}
return text.GetEncoding(lang)
}
func DecodeRFC2047Word(s string) (string, error) {
fields := strings.Split(s, "?")
if len(fields) != 5 || fields[0] != "=" || fields[4] != "=" {
return "", errors.New("mail: address not RFC 2047 encoded")
}
charset, encMark := strings.ToLower(fields[1]), strings.ToLower(fields[2])
enc := text.GetEncoding(charset)
if enc == nil {
return "", fmt.Errorf("mail: charset not supported: %q", charset)
}
in := bytes.NewBufferString(fields[3])
var r io.Reader
switch encMark {
case "b":
r = base64.NewDecoder(base64.StdEncoding, in)
case "q":
r = qDecoder{r: in}
default:
return "", fmt.Errorf("mail: RFC 2047 encoding not supported: %q", encMark)
}
dec, err := ioutil.ReadAll(text.NewReader(r, enc))
if err != nil {
return "", err
}
return string(dec), err
}
// GetTTYEncoding returns the TTY encoding, or UTF-8 if not found.
func GetTTYEncoding() encoding.Encoding {
enc := GetRawTTYEncoding()
if enc != nil {
return enc
}
return text.GetEncoding("UTF-8")
}
func TestText(t *testing.T) {
out, err := os.Create("/tmp/a.pdf")
if err != nil {
t.Errorf("error creating out file /tmp/a.pdf: %s", err)
return
}
defer out.Close()
in := bytes.NewBuffer(nil)
in.WriteString("UTF-8: ")
in.WriteString(accented)
sep := []byte("\n\n---\n\n")
in.Write(sep)
in.WriteString("UTF-16: ")
u := utf16.Encode([]rune(accented))
for i := 0; i < len(u); i++ {
in.Write([]byte{byte(u[i] >> 8), byte(u[i] & 0xff)})
}
in.Write(sep)
in.WriteString("ISO-8859-15: ")
i15, _ := text.Encode(accented, text.GetEncoding("ISO-8859-15"))
t.Logf("i15: %q", i15)
in.Write(i15)
in.Write(sep)
in.WriteString("windows-1252: ")
w1252, _ := text.Encode(accented, text.GetEncoding("windows-1252"))
t.Logf("w1252: %q", w1252)
in.Write(w1252)
in.Write(sep)
in.WriteString("macroman: ")
mr, _ := text.Encode(accented, text.GetEncoding("macroman"))
t.Logf("mr: %q", mr)
in.Write(mr)
in.Write(sep)
err = WriteTextAsPdf(out, bytes.NewReader(in.Bytes()))
if err != nil {
t.Errorf("error writing %q: %s", accented, err)
}
t.Logf("out: %q", out)
}
func main() {
Log.SetHandler(log15.StderrHandler)
flagConnect := flag.String("connect", os.Getenv("BRUNO_ID"), "database connection string")
flagCharset := flag.String("charset", term.GetTTYEncodingName(), "input charset of the csv")
flagTruncate := flag.Bool("truncate", false, "truncate table?")
flagSep := flag.String("sep", ";", "csv field separator")
flag.Parse()
var enc encoding.Encoding
if *flagCharset != "" {
enc = text.GetEncoding(*flagCharset)
if enc == nil {
Log.Error("unknown charset " + *flagCharset)
os.Exit(1)
}
}
db, err := sql.Open("ora", *flagConnect)
if err != nil {
Log.Crit("connect to db", "dsn", *flagConnect, "error", err)
os.Exit(1)
}
defer db.Close()
fh, err := os.Open(flag.Arg(0))
if err != nil {
Log.Crit("open csv", "file", flag.Arg(0), "error", err)
os.Exit(1)
}
defer fh.Close()
r := io.Reader(fh)
if enc != nil {
Log.Debug("NewReader", "encoding", enc)
r = text.NewReader(bufio.NewReaderSize(r, 1<<20), enc)
}
if *flagTruncate {
if _, err = db.Exec("TRUNCATE TABLE " + flag.Arg(1)); err != nil {
Log.Error("TRUNCATE", "table", flag.Arg(1), "error", err)
os.Exit(1)
}
}
if os.Getenv("GOMAXPROCS") == "" {
Log.Info("Setting GOMAXPROCS", "numCPU", runtime.NumCPU())
runtime.GOMAXPROCS(runtime.NumCPU())
}
cr := csv.NewReader(bufio.NewReaderSize(r, 16<<20))
cr.Comma = ([]rune(*flagSep))[0]
cr.TrimLeadingSpace = true
cr.LazyQuotes = true
if err := load(db, flag.Arg(1), cr); err != nil {
Log.Error("load", "error", err)
os.Exit(2)
}
}
// NewTextReader wraps a reader with a proper charset converter
func NewTextReader(r io.Reader, charset string) io.Reader {
if charset == "" || charset == "utf-8" {
return text.NewReader(r, nil)
}
enc := text.GetEncoding(charset)
if enc == nil {
logger.Warn("msg", "no decoder for charset", "charset", charset)
return r
}
return text.NewReader(r, enc)
}
func downloadResults(ctx context.Context, result []Result, page string, client *http.Client) ([]Result, error) {
//http://80.249.172.60/cgi-bin/afaalany/ktmp/2015090310172791EC26C5_catv_pool_telekom_hutempfile.txt:
Log.Debug("get", "url", page)
resp, err := client.Get(page)
if err != nil {
Log.Error("get", "url", page, "error", err)
return result, errgo.Notef(err, "GET "+page)
}
defer resp.Body.Close()
select {
case <-ctx.Done():
return result, ctx.Err()
default:
}
Log.Debug("response", "header", resp.Header)
var enc string
if _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")); err == nil {
enc = params["charset"]
}
if enc == "" {
enc = "iso-8859-2"
}
scanner := bufio.NewScanner(text.NewReader(resp.Body, text.GetEncoding(enc)))
for scanner.Scan() {
line := bytes.TrimSpace(scanner.Bytes())
i := bytes.IndexByte(line, ';')
if i < 0 {
continue
}
taxNo := bytes.TrimRight(line[:i], " ")
line = line[i+1:]
if i = bytes.IndexByte(line, ';'); i >= 0 {
line = line[:i]
}
result = append(result, Result{
TaxNo: string(taxNo),
Owner: string(line),
Valid: true,
Exist: len(taxNo) == 11,
})
}
Log.Info("Download finished", "result.count", len(result))
return result, err
}
// Extract hosts value from config and store it on the plugin instance.
func (fr *FileReadFilter) Init(config interface{}) (err error) {
var (
value interface{}
str string
ok bool
)
conf := config.(pipeline.PluginConfig)
if value, ok = conf["newType"]; !ok {
return errors.New("FileReadFilter: No 'newType' setting specified.")
}
if fr.newType, ok = value.(string); !ok {
return errors.New("FileReadFilter: 'newType' setting not a string value.")
}
if value, ok = conf["sourceEncoding"]; !ok {
log.Printf("FileReadFilter: no source encoding specified - source should be UTF-8!")
} else {
if str, ok = value.(string); !ok {
return errors.New("FileReadFilter: 'sourceEncoding' setting not a string value.")
}
e := text.GetEncoding(str)
if e == nil {
return fmt.Errorf("FileReadFilter: 'sourceEncoding' with value %q is unknown.", str)
}
fr.decoder = e.NewDecoder()
}
if value, ok = conf["pattern"]; !ok {
return errors.New("FileReadFilter: No 'pattern' setting specified.")
}
if str, ok = value.(string); !ok {
return errors.New("FileReadFilter: 'pattern' setting not a string value.")
}
if fr.tmpl, err = template.New("filereader").Parse(str); err == nil && fr.tmpl == nil {
return fmt.Errorf("FileReadFilter: empty template (%q)", str)
}
return err
}