func main() {
showVersion := flag.Bool("v", false, "prints current program version")
size := flag.Int("b", 20000, "batch size")
numWorkers := flag.Int("w", runtime.NumCPU(), "number of workers")
flag.Parse()
if *showVersion {
fmt.Println(span.AppVersion)
os.Exit(0)
}
var readers []io.Reader
if flag.NArg() == 0 {
readers = append(readers, os.Stdin)
} else {
for _, filename := range flag.Args() {
file, err := os.Open(filename)
if err != nil {
log.Fatal(err)
}
defer file.Close()
readers = append(readers, file)
}
}
for _, r := range readers {
p := bytebatch.NewLineProcessor(r, os.Stdout, func(b []byte) ([]byte, error) {
is := finc.IntermediateSchema{}
if err := json.Unmarshal(b, &is); err != nil {
log.Printf("failed to unmarshal: %s", string(b))
return b, err
}
// Redact full text.
is.Fulltext = ""
bb, err := json.Marshal(is)
if err != nil {
return bb, err
}
bb = append(bb, '\n')
return bb, nil
})
p.NumWorkers = *numWorkers
p.BatchSize = *size
if err := p.Run(); err != nil {
log.Fatal(err)
}
}
}
