// NewTextReader wraps a reader with a proper charset converter
func NewTextReader(r io.Reader, charset string) io.Reader {
if charset == "" || charset == "utf-8" {
return text.NewReader(r, nil)
}
enc := text.GetEncoding(charset)
if enc == nil {
logger.Warn("msg", "no decoder for charset", "charset", charset)
return r
}
return text.NewReader(r, enc)
}
func DecodeRFC2047Word(s string) (string, error) {
fields := strings.Split(s, "?")
if len(fields) != 5 || fields[0] != "=" || fields[4] != "=" {
return "", errors.New("mail: address not RFC 2047 encoded")
}
charset, encMark := strings.ToLower(fields[1]), strings.ToLower(fields[2])
enc := text.GetEncoding(charset)
if enc == nil {
return "", fmt.Errorf("mail: charset not supported: %q", charset)
}
in := bytes.NewBufferString(fields[3])
var r io.Reader
switch encMark {
case "b":
r = base64.NewDecoder(base64.StdEncoding, in)
case "q":
r = qDecoder{r: in}
default:
return "", fmt.Errorf("mail: RFC 2047 encoding not supported: %q", encMark)
}
dec, err := ioutil.ReadAll(text.NewReader(r, enc))
if err != nil {
return "", err
}
return string(dec), err
}
func main() {
Log.SetHandler(log15.StderrHandler)
flagConnect := flag.String("connect", os.Getenv("BRUNO_ID"), "database connection string")
flagCharset := flag.String("charset", term.GetTTYEncodingName(), "input charset of the csv")
flagTruncate := flag.Bool("truncate", false, "truncate table?")
flagSep := flag.String("sep", ";", "csv field separator")
flag.Parse()
var enc encoding.Encoding
if *flagCharset != "" {
enc = text.GetEncoding(*flagCharset)
if enc == nil {
Log.Error("unknown charset " + *flagCharset)
os.Exit(1)
}
}
db, err := sql.Open("ora", *flagConnect)
if err != nil {
Log.Crit("connect to db", "dsn", *flagConnect, "error", err)
os.Exit(1)
}
defer db.Close()
fh, err := os.Open(flag.Arg(0))
if err != nil {
Log.Crit("open csv", "file", flag.Arg(0), "error", err)
os.Exit(1)
}
defer fh.Close()
r := io.Reader(fh)
if enc != nil {
Log.Debug("NewReader", "encoding", enc)
r = text.NewReader(bufio.NewReaderSize(r, 1<<20), enc)
}
if *flagTruncate {
if _, err = db.Exec("TRUNCATE TABLE " + flag.Arg(1)); err != nil {
Log.Error("TRUNCATE", "table", flag.Arg(1), "error", err)
os.Exit(1)
}
}
if os.Getenv("GOMAXPROCS") == "" {
Log.Info("Setting GOMAXPROCS", "numCPU", runtime.NumCPU())
runtime.GOMAXPROCS(runtime.NumCPU())
}
cr := csv.NewReader(bufio.NewReaderSize(r, 16<<20))
cr.Comma = ([]rune(*flagSep))[0]
cr.TrimLeadingSpace = true
cr.LazyQuotes = true
if err := load(db, flag.Arg(1), cr); err != nil {
Log.Error("load", "error", err)
os.Exit(2)
}
}
func downloadResults(ctx context.Context, result []Result, page string, client *http.Client) ([]Result, error) {
//http://80.249.172.60/cgi-bin/afaalany/ktmp/2015090310172791EC26C5_catv_pool_telekom_hutempfile.txt:
Log.Debug("get", "url", page)
resp, err := client.Get(page)
if err != nil {
Log.Error("get", "url", page, "error", err)
return result, errgo.Notef(err, "GET "+page)
}
defer resp.Body.Close()
select {
case <-ctx.Done():
return result, ctx.Err()
default:
}
Log.Debug("response", "header", resp.Header)
var enc string
if _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")); err == nil {
enc = params["charset"]
}
if enc == "" {
enc = "iso-8859-2"
}
scanner := bufio.NewScanner(text.NewReader(resp.Body, text.GetEncoding(enc)))
for scanner.Scan() {
line := bytes.TrimSpace(scanner.Bytes())
i := bytes.IndexByte(line, ';')
if i < 0 {
continue
}
taxNo := bytes.TrimRight(line[:i], " ")
line = line[i+1:]
if i = bytes.IndexByte(line, ';'); i >= 0 {
line = line[:i]
}
result = append(result, Result{
TaxNo: string(taxNo),
Owner: string(line),
Valid: true,
Exist: len(taxNo) == 11,
})
}
Log.Info("Download finished", "result.count", len(result))
return result, err
}
// MaskIn masks the input stream for Reads.
func MaskIn(in *os.File, enc encoding.Encoding) (*os.File, error) {
pr, pw, err := os.Pipe()
if err != nil {
return in, err
}
iohlp.SetDirect(pr)
iohlp.SetDirect(pw)
// in -> pw -> pr
go func() {
defer in.Close()
io.Copy(pw, text.NewReader(in, enc))
}()
return pr, nil
}