func MakeZip(zipname string, fnames []string) error {
log.Printf("zipping %d entrires to %s ...", len(fnames), zipname)
zipfile, err := os.Create(zipname)
if err != nil {
return err
}
defer zipfile.Close()
zipwriter := zip.NewWriter(zipfile)
defer zipwriter.Close()
for _, fname := range fnames {
realm, ts, good := util.Parse_FName(fname)
if !good {
log.Printf("warning: skip ill-named file '%s'", fname)
continue // skip
}
data, err := util.Load(fname)
if err != nil {
return err
}
name := util.Make_FName(realm, ts, false)
f, err := zipwriter.Create(name)
if err != nil {
return err
}
log.Printf("zip %d bytes for file %s", len(data), name)
_, err = f.Write(data)
if err != nil {
return err
}
}
log.Printf("%s zipped without errors", zipname)
return nil
}
func DoFetch(cf *config.Config) {
log.Println("=== FETCH BEGIN ===")
s := new(fetcher.Session)
s.Config = cf
for _, realm := range cf.RealmsList {
for _, locale := range cf.LocalesList {
file_url, file_ts := s.Fetch_FileURL(realm, locale)
log.Printf("FILE URL: %s", file_url)
log.Printf("FILE PIT: %s / %s", file_ts, util.TSStr(file_ts.UTC()))
fname := util.Make_FName(realm, file_ts, true)
json_fname := cf.DownloadDirectory + fname
if !util.CheckFile(json_fname) {
log.Printf("downloading from %s ...", file_url)
data := s.Get(file_url)
log.Printf("... got %d octets", len(data))
zdata := util.Zip(data)
log.Printf("... zipped to %d octets (%d%%)",
len(zdata), len(zdata)*100/len(data))
util.Store(json_fname, zdata)
log.Printf("stored to %s .", json_fname)
} else {
log.Println("... already downloaded")
}
}
}
log.Println("=== FETCH END ===")
}
func MakeTarball(tarname string, fnames []string) error {
log.Printf("tarring %d entrires to %s ...", len(fnames), tarname)
tarfile, err := os.Create(tarname)
if err != nil {
return err
}
defer tarfile.Close()
var tarwriter *tar.Writer
if strings.HasSuffix(tarname, ".gz") {
zipper := gzip.NewWriter(tarfile)
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
/*
} else if strings.HasSuffix(tarname, ".xz") {
p := xz.WriterDefaults
p.DictCap = 1 << 24
zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile)
if err != nil {
return err
}
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
*/
} else {
tarwriter = tar.NewWriter(tarfile)
}
defer tarwriter.Close()
for _, fname := range fnames {
realm, ts, good := util.Parse_FName(fname)
if !good {
log.Printf("warning: skip ill-named file '%s'", fname)
continue // skip
}
data, err := util.Load(fname)
if err != nil {
return err
}
hdr := new(tar.Header)
hdr.Name = util.Make_FName(realm, ts, false)
hdr.Size = int64(len(data))
hdr.ModTime = ts
hdr.Mode = 0644
err = tarwriter.WriteHeader(hdr)
if err != nil {
return err
}
log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name)
_, err = tarwriter.Write(data)
if err != nil {
return err
}
}
log.Printf("%s tarred without errors", tarname)
return nil
}