コード例 #1
0
ファイル: backup.go プロジェクト: gourytch/gowowuction
func MakeZip(zipname string, fnames []string) error {
	log.Printf("zipping %d entrires to %s ...", len(fnames), zipname)
	zipfile, err := os.Create(zipname)
	if err != nil {
		return err
	}
	defer zipfile.Close()
	zipwriter := zip.NewWriter(zipfile)
	defer zipwriter.Close()

	for _, fname := range fnames {
		realm, ts, good := util.Parse_FName(fname)
		if !good {
			log.Printf("warning: skip ill-named file '%s'", fname)
			continue // skip
		}
		data, err := util.Load(fname)
		if err != nil {
			return err
		}
		name := util.Make_FName(realm, ts, false)
		f, err := zipwriter.Create(name)
		if err != nil {
			return err
		}
		log.Printf("zip %d bytes for file %s", len(data), name)
		_, err = f.Write(data)
		if err != nil {
			return err
		}
	}
	log.Printf("%s zipped without errors", zipname)
	return nil
}
コード例 #2
0
ファイル: backup.go プロジェクト: gourytch/gowowuction
func MakeTarball(tarname string, fnames []string) error {
	log.Printf("tarring %d entrires to %s ...", len(fnames), tarname)
	tarfile, err := os.Create(tarname)
	if err != nil {
		return err
	}
	defer tarfile.Close()
	var tarwriter *tar.Writer
	if strings.HasSuffix(tarname, ".gz") {
		zipper := gzip.NewWriter(tarfile)
		defer zipper.Close()
		tarwriter = tar.NewWriter(zipper)
		/*
			} else if strings.HasSuffix(tarname, ".xz") {
				p := xz.WriterDefaults
				p.DictCap = 1 << 24
				zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile)
				if err != nil {
					return err
				}
				defer zipper.Close()
				tarwriter = tar.NewWriter(zipper)
		*/
	} else {
		tarwriter = tar.NewWriter(tarfile)
	}
	defer tarwriter.Close()

	for _, fname := range fnames {
		realm, ts, good := util.Parse_FName(fname)
		if !good {
			log.Printf("warning: skip ill-named file '%s'", fname)
			continue // skip
		}
		data, err := util.Load(fname)
		if err != nil {
			return err
		}

		hdr := new(tar.Header)
		hdr.Name = util.Make_FName(realm, ts, false)
		hdr.Size = int64(len(data))
		hdr.ModTime = ts
		hdr.Mode = 0644
		err = tarwriter.WriteHeader(hdr)
		if err != nil {
			return err
		}
		log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name)
		_, err = tarwriter.Write(data)
		if err != nil {
			return err
		}
	}
	log.Printf("%s tarred without errors", tarname)
	return nil
}
コード例 #3
0
ファイル: processor.go プロジェクト: gourytch/gowowuction
func (prc *AuctionProcessor) LoadState() {
	if prc.Started {
		log.Panic("LoadState inside snapshot session")
	}
	if util.CheckFile(prc.StateFName) {
		log.Printf("AuctionProcessor loading state from %s ...", prc.StateFName)
		data, _ := util.Load(prc.StateFName)
		if err := json.Unmarshal(data, &prc.State); err != nil {
			log.Panicf("... failed: %s", prc.StateFName, err)
		}
		log.Printf("... loaded with %d list enties", len(prc.State.WorkList))
		prc.State.WorkSet = make(WorkSetType)
		for _, e := range prc.State.WorkList {
			prc.State.WorkSet[e.Entry.Auc] = e
		}
	} else {
		log.Printf("AuctionProcessor has no state named %s ...", prc.StateFName)
	}
}
コード例 #4
0
ファイル: batcher.go プロジェクト: gourytch/gowowuction
func ParseDir(cf *config.Config, realm string, safe bool) {
	mask := cf.DownloadDirectory +
		strings.Replace(realm, ":", "-", -1) + "-*.json.gz"
	log.Printf("scan by mask %s ...", mask)
	fnames, err := filepath.Glob(mask)
	if err != nil {
		log.Fatalln("glob failed:", err)
	}
	log.Printf("... %d entries collected", len(fnames))

	var goodfnames []string

	for _, fname := range fnames {
		// realm, ts, good := util.Parse_FName(fname)
		_, _, good := util.Parse_FName(fname)
		if good {
			// log.Printf("fname %s -> %s, %v", fname, realm, ts)
			goodfnames = append(goodfnames, fname)
		} else {
			// log.Printf("skip fname %s", fname)
		}
	}
	sort.Sort(util.ByBasename(goodfnames))
	prc := new(AuctionProcessor)
	prc.Init(cf, realm)
	prc.LoadState()
	badfiles := make(map[string]string)

	for _, fname := range fnames {
		//log.Println(fname)
		f_realm, f_time, ok := util.Parse_FName(fname)
		if !ok {
			log.Fatalf("not parsed correctly: %s", fname)
			continue
		}
		if f_realm != realm {
			log.Fatalf("not my realm (%s != %s)", f_realm, realm)
			continue
		}
		if !prc.SnapshotNeeded(f_time) {
			log.Printf("snapshot not needed: %s", util.TSStr(f_time))
			continue
		}
		data, err := util.Load(fname)
		if err != nil {
			//log.Fatalf("load error: %s", err)
			log.Printf("%s LOAD ERROR: %s", fname, err)
			badfiles[fname] = fmt.Sprint(err)
			continue
		}
		ss, err := ParseSnapshot(data)
		if err != nil {
			//log.Fatalf("load error: %s", err)
			log.Printf("%s PARSE ERROR: %s", fname, err)
			badfiles[fname] = fmt.Sprint(err)
			continue
		}

		prc.StartSnapshot(f_time)
		for _, auc := range ss.Auctions {
			prc.AddAuctionEntry(&auc)
		}
		prc.FinishSnapshot()
		if safe {
			prc.SaveState()
		}
	}
	if !safe {
		prc.SaveState()
	}
	if len(badfiles) == 0 {
		log.Printf("all files loaded without errors")
	} else {
		log.Printf("%d files with errors", len(badfiles))
		for fname, err := range badfiles {
			log.Printf("%s: %s", fname, err)
		}
	}
}