Ejemplo n.º 1
0
func MakeZip(zipname string, fnames []string) error {
	log.Printf("zipping %d entrires to %s ...", len(fnames), zipname)
	zipfile, err := os.Create(zipname)
	if err != nil {
		return err
	}
	defer zipfile.Close()
	zipwriter := zip.NewWriter(zipfile)
	defer zipwriter.Close()

	for _, fname := range fnames {
		realm, ts, good := util.Parse_FName(fname)
		if !good {
			log.Printf("warning: skip ill-named file '%s'", fname)
			continue // skip
		}
		data, err := util.Load(fname)
		if err != nil {
			return err
		}
		name := util.Make_FName(realm, ts, false)
		f, err := zipwriter.Create(name)
		if err != nil {
			return err
		}
		log.Printf("zip %d bytes for file %s", len(data), name)
		_, err = f.Write(data)
		if err != nil {
			return err
		}
	}
	log.Printf("%s zipped without errors", zipname)
	return nil
}
Ejemplo n.º 2
0
func DoFetch(cf *config.Config) {
	log.Println("=== FETCH BEGIN ===")
	s := new(fetcher.Session)
	s.Config = cf
	for _, realm := range cf.RealmsList {
		for _, locale := range cf.LocalesList {
			file_url, file_ts := s.Fetch_FileURL(realm, locale)
			log.Printf("FILE URL: %s", file_url)
			log.Printf("FILE PIT: %s / %s", file_ts, util.TSStr(file_ts.UTC()))
			fname := util.Make_FName(realm, file_ts, true)
			json_fname := cf.DownloadDirectory + fname
			if !util.CheckFile(json_fname) {
				log.Printf("downloading from %s ...", file_url)
				data := s.Get(file_url)
				log.Printf("... got %d octets", len(data))
				zdata := util.Zip(data)
				log.Printf("... zipped to %d octets (%d%%)",
					len(zdata), len(zdata)*100/len(data))
				util.Store(json_fname, zdata)
				log.Printf("stored to %s .", json_fname)
			} else {
				log.Println("... already downloaded")
			}
		}
	}
	log.Println("=== FETCH END ===")
}
Ejemplo n.º 3
0
func MakeTarball(tarname string, fnames []string) error {
	log.Printf("tarring %d entrires to %s ...", len(fnames), tarname)
	tarfile, err := os.Create(tarname)
	if err != nil {
		return err
	}
	defer tarfile.Close()
	var tarwriter *tar.Writer
	if strings.HasSuffix(tarname, ".gz") {
		zipper := gzip.NewWriter(tarfile)
		defer zipper.Close()
		tarwriter = tar.NewWriter(zipper)
		/*
			} else if strings.HasSuffix(tarname, ".xz") {
				p := xz.WriterDefaults
				p.DictCap = 1 << 24
				zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile)
				if err != nil {
					return err
				}
				defer zipper.Close()
				tarwriter = tar.NewWriter(zipper)
		*/
	} else {
		tarwriter = tar.NewWriter(tarfile)
	}
	defer tarwriter.Close()

	for _, fname := range fnames {
		realm, ts, good := util.Parse_FName(fname)
		if !good {
			log.Printf("warning: skip ill-named file '%s'", fname)
			continue // skip
		}
		data, err := util.Load(fname)
		if err != nil {
			return err
		}

		hdr := new(tar.Header)
		hdr.Name = util.Make_FName(realm, ts, false)
		hdr.Size = int64(len(data))
		hdr.ModTime = ts
		hdr.Mode = 0644
		err = tarwriter.WriteHeader(hdr)
		if err != nil {
			return err
		}
		log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name)
		_, err = tarwriter.Write(data)
		if err != nil {
			return err
		}
	}
	log.Printf("%s tarred without errors", tarname)
	return nil
}