func (p *Decoder) Decode() (*Chunk, error) {
if p.cc == nil {
p.cc = make(chan cce)
p.cq = make(chan struct{})
go func() {
// java standard is ISO 8859-1 for properties files.
// it's dumb but whatever.
r := latinx.NewReader(latinx.ISO_8859_1, p.r)
buf := make([]byte, 1024)
for {
n, err := r.Read(buf)
if n > 0 {
for _, b := range buf[:n] {
p.consume(b)
}
} else if err != nil {
p.cc <- cce{nil, err}
return
}
}
}()
}
c, ok := <-p.cc
if !ok {
return nil, errDecoderClosed
}
if c.err == io.EOF {
return nil, nil
}
return c.chunk, c.err
}
// Search the most recently added zip file in config.ZipPath,
// extracts any image (keeping the path) with config.ImagesExtension in config.ImagesPath
// then read CSV files and replaces database content with the CSV content.
func updateDatabase() error {
// Search for zip file
infos, err := ioutil.ReadDir(config.ZipPath)
if err != nil {
return err
}
// Early return if no file present
if len(infos) == 0 {
return nil
}
// Select most recent valid archive infos
sort.Sort(sort.Reverse(byModTime(infos)))
var info os.FileInfo = nil
for _, i := range infos {
if strings.HasSuffix(i.Name(), ".zip") && !strings.HasPrefix(i.Name(), ".") {
info = i
break
}
}
// Early exit if no zip file found
if info == nil {
return nil
}
// Unzip archive
archive, err := zip.OpenReader(config.ZipPath + info.Name())
if err != nil {
return err
}
defer archive.Close()
defer func() {
if err != nil {
return // do not delete anything if import failed
}
// Delete all files listed earlier
for _, info = range infos {
os.Remove(config.ZipPath + info.Name())
}
}()
// Open database
db, err := sql.Open("mysql", config.DatabaseConnection)
if err != nil {
return err
}
defer db.Close()
// Drop old table
_, err = db.Exec("TRUNCATE TABLE " + config.TableName)
if err != nil {
return err
}
// Reading files in archive
for _, f := range archive.File {
if strings.HasPrefix(filepath.Base(f.Name), ".") {
continue
}
if strings.HasSuffix(f.Name, ".csv") {
// Reading CSV file
ff, err := f.Open()
if err != nil {
return err
}
// Reader from ISO-8859-1
ffutf8 := latinx.NewReader(latinx.ISO_8859_1, ff)
// Open CSV reader
c := csv.NewReader(ffutf8)
c.Comma = rune(config.CsvComma[0])
c.FieldsPerRecord = -1
c.TrimLeadingSpace = false
rs, err := c.ReadAll()
if err != nil {
return err
}
// Convert csv to slice of csvElement
elements := make([]csvElement, 0, 50)
var columns []string
for i, r := range rs {
if i == 0 {
// Columns
columns = r
} else {
// Content
element := make(csvElement)
for c, cv := range r {
element[columns[c]] = cv
}
elements = append(elements, element)
}
}
ff.Close()
// Build insert prepared query
columnsCount := len(config.TableMapping)
insertQueryString := "INSERT INTO " + config.TableName + " ("
for i, column := range config.TableMapping {
insertQueryString += column.TableColumn
if i+1 < columnsCount {
insertQueryString += ","
}
}
insertQueryString += ") VALUES ("
for i := 0; i < columnsCount; i++ {
insertQueryString += "?"
if i+1 < columnsCount {
insertQueryString += ","
}
}
insertQueryString += ")"
insertQuery, err := db.Prepare(insertQueryString)
if err != nil {
return err
}
// Put csvElements in database
for _, element := range elements {
values := make([]interface{}, 0, columnsCount)
for _, column := range config.TableMapping {
if column.Transformer != "" && transformers[column.Transformer] != nil {
element[column.Alias] = transformers[column.Transformer](element[column.Alias])
}
values = append(values, element[column.Alias])
}
_, err = insertQuery.Exec(values...)
if err != nil {
return err
}
}
insertQuery.Close()
} else if strings.HasSuffix(strings.ToLower(f.Name), config.ImagesExtension) {
// Create direcotry
dir := filepath.Dir(f.Name)
if len(dir) > 0 {
err = os.MkdirAll(config.ImagesPath+filepath.Dir(f.Name), os.FileMode(0777))
if err != nil {
return err
}
}
// Extract image
ff, err := f.Open()
if err != nil {
return err
}
dest, err := os.Create(config.ImagesPath + f.Name)
if err != nil {
return err
}
_, err = io.Copy(dest, ff)
if err != nil {
return err
}
ff.Close()
}
}
return nil
}