func readCsv(ch chan []string) {
var reader *csv.Reader
if inputFn == "" {
reader = csv.NewReader(os.Stdin)
} else {
file, err := os.Open(inputFn)
if err != nil {
fmt.Println("Error:", err)
os.Exit(1)
}
defer file.Close()
reader = csv.NewReader(file)
}
if !strictLen {
reader.FieldsPerRecord = -1
}
r, _ := utf8.DecodeRuneInString(inputSep)
reader.Comma = r
reader.LazyQuotes = lazyQuotes
for {
record, err := reader.Read()
if err == io.EOF {
close(ch)
break
} else if err != nil {
fmt.Println("Error:", err)
close(ch)
break
}
ch <- record
}
}
func (con Sif2Cx) readSIF(reader *csv.Reader, w *bufio.Writer) {
// Set delimiter
var netName string
if con.Name == "" {
netName = "CX from SIF file"
} else {
netName = con.Name
}
reader.Comma = con.Delimiter
reader.LazyQuotes = true
// nodes already serialized
nodesExists := make(map[string]int64)
nodeCounter := int64(0)
w.Write([]byte("["))
for {
record, err := reader.Read()
if err == io.EOF {
// Add network attributes at the end of doc.
netAttr := cx.NetworkAttribute{N: "name", V: netName}
attrList := []cx.NetworkAttribute{netAttr}
netAttrs := make(map[string][]cx.NetworkAttribute)
netAttrs["networkAttributes"] = attrList
json.NewEncoder(w).Encode(netAttrs)
w.Write([]byte("]"))
w.Flush()
break
}
if err != nil {
log.Fatal(err)
}
if len(record) == 3 {
toJson(record, nodesExists, &nodeCounter, w)
}
w.Flush()
}
}
func importCSV(filename string, connStr string, schema string, tableName string, ignoreErrors bool, skipHeader bool, fields string, delimiter string) error {
db, err := connect(connStr, schema)
if err != nil {
return err
}
defer db.Close()
var reader *csv.Reader
var bar *pb.ProgressBar
if filename != "" {
file, err := os.Open(filename)
if err != nil {
return err
}
defer file.Close()
bar = NewProgressBar(file)
reader = csv.NewReader(io.TeeReader(file, bar))
} else {
reader = csv.NewReader(os.Stdin)
}
reader.Comma, _ = utf8.DecodeRuneInString(delimiter)
reader.LazyQuotes = true
columns, err := parseColumns(reader, skipHeader, fields)
if err != nil {
return err
}
reader.FieldsPerRecord = len(columns)
i, err := NewCSVImport(db, schema, tableName, columns)
if err != nil {
return err
}
var success, failed int
if filename != "" {
bar.Start()
err, success, failed = copyCSVRows(i, reader, ignoreErrors, delimiter, columns)
bar.Finish()
} else {
err, success, failed = copyCSVRows(i, reader, ignoreErrors, delimiter, columns)
}
if err != nil {
lineNumber := success + failed
if !skipHeader {
lineNumber++
}
return errors.New(fmt.Sprintf("line %d: %s", lineNumber, err))
} else {
fmt.Println(fmt.Sprintf("%d rows imported into %s.%s", success, schema, tableName))
if ignoreErrors && failed > 0 {
fmt.Println(fmt.Sprintf("%d rows could not be imported into %s.%s and have been written to stderr.", failed, schema, tableName))
}
return i.Commit()
}
}
func ImportDictionaries() map[string][]*models.SuggestItem {
var itemMap = make(map[string][]*models.SuggestItem)
fileInfo, err := ioutil.ReadDir(DataDirectory)
if err != nil {
log.Print(err)
return itemMap
}
numberOfDictionaries := 0
for _, file := range fileInfo {
if !file.IsDir() && (strings.HasSuffix(file.Name(), ".txt") || strings.HasSuffix(file.Name(), ".txt.gz")) {
dictionaryFile := fmt.Sprintf("%s%s%s", DataDirectory, string(os.PathSeparator), file.Name())
dictionaryName := strings.TrimSuffix(strings.TrimSuffix(file.Name(), ".gz"), ".txt")
log.Printf("Importing dictionary %s from file %s", dictionaryName, dictionaryFile)
csvFile, err := os.Open(dictionaryFile)
if err != nil {
log.Print(err)
continue
}
defer csvFile.Close()
var csvReader *csv.Reader
if strings.HasSuffix(file.Name(), ".txt.gz") {
gzipReader, gzerr := gzip.NewReader(csvFile)
if gzerr == nil {
defer gzipReader.Close()
csvReader = csv.NewReader(gzipReader)
} else {
log.Print(gzerr)
continue
}
} else {
csvReader = csv.NewReader(csvFile)
}
csvReader.FieldsPerRecord = 2
csvReader.Comma = '|'
csvReader.LazyQuotes = true
csvReader.TrimLeadingSpace = true
rawCSVdata, err := csvReader.ReadAll()
if err != nil {
log.Print(err)
continue
}
for _, each := range rawCSVdata {
var suggestItem = new(models.SuggestItem)
suggestItem.Term = each[0]
weight, err := strconv.Atoi(each[1])
if err == nil {
suggestItem.Weight = weight
itemMap[dictionaryName] = append(itemMap[dictionaryName], suggestItem)
}
}
numberOfDictionaries++
}
}
log.Printf("Imported %d dictionaries", numberOfDictionaries)
return itemMap
}