Example #1
0
func (p *Decoder) Decode() (*Chunk, error) {
	if p.cc == nil {
		p.cc = make(chan cce)
		p.cq = make(chan struct{})
		go func() {
			// java standard is ISO 8859-1 for properties files.
			// it's dumb but whatever.
			r := latinx.NewReader(latinx.ISO_8859_1, p.r)

			buf := make([]byte, 1024)

			for {
				n, err := r.Read(buf)
				if n > 0 {
					for _, b := range buf[:n] {
						p.consume(b)
					}
				} else if err != nil {
					p.cc <- cce{nil, err}
					return
				}
			}
		}()
	}
	c, ok := <-p.cc
	if !ok {
		return nil, errDecoderClosed
	}
	if c.err == io.EOF {
		return nil, nil
	}
	return c.chunk, c.err
}
Example #2
0
// Search the most recently added zip file in config.ZipPath,
// extracts any image (keeping the path) with config.ImagesExtension in config.ImagesPath
// then read CSV files and replaces database content with the CSV content.
func updateDatabase() error {

	// Search for zip file
	infos, err := ioutil.ReadDir(config.ZipPath)
	if err != nil {
		return err
	}

	// Early return if no file present
	if len(infos) == 0 {
		return nil
	}

	// Select most recent valid archive infos
	sort.Sort(sort.Reverse(byModTime(infos)))
	var info os.FileInfo = nil
	for _, i := range infos {
		if strings.HasSuffix(i.Name(), ".zip") && !strings.HasPrefix(i.Name(), ".") {
			info = i
			break
		}
	}

	// Early exit if no zip file found
	if info == nil {
		return nil
	}

	// Unzip archive
	archive, err := zip.OpenReader(config.ZipPath + info.Name())
	if err != nil {
		return err
	}
	defer archive.Close()
	defer func() {
		if err != nil {
			return // do not delete anything if import failed
		}
		// Delete all files listed earlier
		for _, info = range infos {
			os.Remove(config.ZipPath + info.Name())
		}
	}()

	// Open database
	db, err := sql.Open("mysql", config.DatabaseConnection)
	if err != nil {
		return err
	}
	defer db.Close()

	// Drop old table
	_, err = db.Exec("TRUNCATE TABLE " + config.TableName)
	if err != nil {
		return err
	}

	// Reading files in archive
	for _, f := range archive.File {
		if strings.HasPrefix(filepath.Base(f.Name), ".") {
			continue
		}

		if strings.HasSuffix(f.Name, ".csv") {

			// Reading CSV file
			ff, err := f.Open()
			if err != nil {
				return err
			}

			// Reader from ISO-8859-1
			ffutf8 := latinx.NewReader(latinx.ISO_8859_1, ff)

			// Open CSV reader
			c := csv.NewReader(ffutf8)
			c.Comma = rune(config.CsvComma[0])
			c.FieldsPerRecord = -1
			c.TrimLeadingSpace = false
			rs, err := c.ReadAll()
			if err != nil {
				return err
			}

			// Convert csv to slice of csvElement
			elements := make([]csvElement, 0, 50)
			var columns []string
			for i, r := range rs {
				if i == 0 {
					// Columns
					columns = r
				} else {
					// Content
					element := make(csvElement)
					for c, cv := range r {
						element[columns[c]] = cv
					}
					elements = append(elements, element)
				}
			}
			ff.Close()

			// Build insert prepared query
			columnsCount := len(config.TableMapping)
			insertQueryString := "INSERT INTO " + config.TableName + " ("
			for i, column := range config.TableMapping {
				insertQueryString += column.TableColumn
				if i+1 < columnsCount {
					insertQueryString += ","
				}
			}
			insertQueryString += ") VALUES ("
			for i := 0; i < columnsCount; i++ {
				insertQueryString += "?"
				if i+1 < columnsCount {
					insertQueryString += ","
				}
			}
			insertQueryString += ")"
			insertQuery, err := db.Prepare(insertQueryString)
			if err != nil {
				return err
			}
			// Put csvElements in database
			for _, element := range elements {
				values := make([]interface{}, 0, columnsCount)
				for _, column := range config.TableMapping {
					if column.Transformer != "" && transformers[column.Transformer] != nil {
						element[column.Alias] = transformers[column.Transformer](element[column.Alias])
					}
					values = append(values, element[column.Alias])
				}
				_, err = insertQuery.Exec(values...)
				if err != nil {
					return err
				}
			}
			insertQuery.Close()
		} else if strings.HasSuffix(strings.ToLower(f.Name), config.ImagesExtension) {

			// Create direcotry
			dir := filepath.Dir(f.Name)
			if len(dir) > 0 {
				err = os.MkdirAll(config.ImagesPath+filepath.Dir(f.Name), os.FileMode(0777))
				if err != nil {
					return err
				}
			}

			// Extract image
			ff, err := f.Open()
			if err != nil {
				return err
			}
			dest, err := os.Create(config.ImagesPath + f.Name)
			if err != nil {
				return err
			}
			_, err = io.Copy(dest, ff)
			if err != nil {
				return err
			}
			ff.Close()
		}
	}

	return nil
}