Esempio n. 1
0
func (self *UpRPut) doTestGet() (msg string, err error) {

	begin := time.Now()
	entryURI := self.Bucket + ":" + self.Key
	ret, code, err := self.Rscli.Get(entryURI, "", "", 3600)
	end := time.Now()
	duration := end.Sub(begin)
	msg = util.GenLog("UP    "+self.Env.Id+"_"+self.Name+"_doTestGet", begin, end, duration)
	if err != nil || code/100 != 2 {
		if err == nil {
			err = errors.New("Invalid response code")
		}
		err = errors.Info(err, "download failed", entryURI)
		return
	}
	resp, err := http.Get(ret.URL)
	if err != nil {
		err = errors.Info(err, "download failed", entryURI, ret.URL)
		return
	}
	defer resp.Body.Close()
	h := sha1.New()
	io.Copy(h, resp.Body)
	hash := hex.EncodeToString(h.Sum(nil))
	if hash != self.DataSha1 {
		err = errors.Info(errors.New("Invalid data sha1"), self.DataSha1, hash)
		return
	}
	return
}
Esempio n. 2
0
File: publish.go Progetto: shtlee/qa
func (p *Pub) Init(conf, env, path string) (err error) {

	if err = config.LoadEx(p, conf); err != nil {
		return
	}
	if err = config.LoadEx(&p.Env, env); err != nil {
		return
	}
	dt := da.NewTransport(p.Env.AccessKey, p.Env.SecretKey, nil)
	p.rsCli, err = rs.New(p.Env.Hosts, p.Env.Ips, dt)
	if err != nil {
		err = errors.Info(err, "Pub init failed")
		return
	}
	p.pubCli, err = pub.New(p.Env.Hosts["pu"], p.Env.Ips["pu"], dt)
	if err != nil {
		err = errors.Info(err, "Pub init failed")
		return
	}
	p.DataFile = filepath.Join(path, p.DataFile)
	domainRegexp, err := regexp.Compile(p.NormalDomainRegexp)
	if err != nil {
		err = errors.Info(err, "Pub init failed")
		return
	}
	p.isNormalDomain = domainRegexp.Match([]byte(p.Domain))

	return
}
Esempio n. 3
0
func (self *UpRPut) Init(conf, env, path string) (err error) {

	if err = config.LoadEx(self, conf); err != nil {
		err = errors.Info(err, "UpResuPut init failed")
		return
	}
	if err = config.LoadEx(&self.Env, env); err != nil {
		err = errors.Info(err, "UpEnv init failed")
		return
	}
	self.DataFile = filepath.Join(path, self.DataFile)
	dt := digest.NewTransport(self.Env.AccessKey, self.Env.SecretKey, nil)
	host := self.Env.Hosts["up"]
	ip := self.Env.Ips["up"]

	self.Rscli, err = rs.New(self.Env.Hosts, self.Env.Ips, dt)
	if err != nil {
		err = errors.Info(err, "Rscli init failed")
		return
	}
	self.Up2cli, err = up2.New(host, ip, self.BlockBits, self.ChunkSize, self.PutRetryTimes, dt)
	if err != nil {
		err = errors.Info(err, "Up2cli init failed")
		return
	}
	return
}
Esempio n. 4
0
File: publish.go Progetto: shtlee/qa
func (p *Pub) doTestDownload() (msg string, err error) {

	var (
		url string
	)
	if p.isNormalDomain {
		url = "http://" + p.Domain + "/" + p.Key
	} else {
		url = "http://" + p.DomainIp + "/" + p.Key
	}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		err = errors.Info(err, "Download failed:", url)
		return
	}
	if !p.isNormalDomain {
		req.Host = p.Domain
	}
	begin := time.Now()
	resp, err := http.DefaultClient.Do(req)
	end := time.Now()
	duration := end.Sub(begin)
	msg = util.GenLogEx("Fp    "+p.Env.Id+"_"+p.Name+"_doTestDownload", begin, end, duration)
	if err != nil {
		err = errors.Info(err, "Download failed:", url)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode/100 != 2 {
		err = errors.New("download status code is not 20x!")
		err = errors.Info(err, url, resp.StatusCode)
		return
	}
	if ct, ok := resp.Header["Content-Type"]; !ok || len(ct) != 1 || ct[0] != p.dataType {
		err = errors.New("Invalid content type")
		return
	}

	h := sha1.New()
	if _, err = io.Copy(h, resp.Body); err != nil {
		err = errors.Info(err, "check sha1 failed")
		return
	}
	if p.DataSha1 != hex.EncodeToString(h.Sum(nil)) {
		err = errors.New("Invalid data sha1")
		err = errors.Info(err, p.DataSha1, hex.EncodeToString(h.Sum(nil)))
		return
	}

	return
}
Esempio n. 5
0
func testTimeRange() {
	sql := "select vlaue from req where time > now() - 1h"

	st, err := influxql.NewParser(strings.NewReader(sql)).ParseStatement()
	if err != nil {
		err = errors.Info(err).Detail(err)
		log.Debug(err)
		return
	}

	//ensure the sql statement is a select statement
	stmt, ok := st.(*influxql.SelectStatement)
	if !ok {
		log.Info("not a valid select statement")
		return
	}

	//get min and max of sql
	log.Info(stmt.Condition)
	min, max := influxql.TimeRange(stmt.Condition)
	log.Printf("Min:%v, Max:%v", min, max)

	log.Printf("Min:%d, Max:%d", min.Unix()+offsetSeconds, max.Unix()+offsetSeconds)

}
Esempio n. 6
0
func (p *PubImage) Init(conf, env, path string) (err error) {

	if err = config.LoadEx(p, conf); err != nil {
		err = errors.Info(err, "pub_image load conf failed", conf)
		return
	}
	if err = config.LoadEx(&p.Env, env); err != nil {
		err = errors.Info(err, "pub_image load env failed", env)
		return
	}
	dt := da.NewTransport(p.Env.AccessKey, p.Env.SecretKey, nil)
	p.Pubcli, err = pub.New(p.Env.Hosts["pu"], p.Env.Ips["pu"], dt)
	if err != nil {
		err = errors.Info(err, "pub_image init failed")
		return
	}
	return
}
Esempio n. 7
0
func (p *PubImage) doTestImage() (msg string, err error) {

	from := []string{p.FromDomain}
	code, err := p.Pubcli.Image(p.Bucket, from, p.SrcHost, 0)
	if err != nil || code/100 != 2 {
		if err == nil {
			err = errors.New("doTestImage failed")
		}
		err = errors.Info(err, code, p.SrcHost, p.FromDomain)
		return
	}
	url := "http://" + p.Env.Hosts["io"] + "/" + p.SrcKey
	_, err = httputil.DownloadEx(url, p.FromDomain)
	if err != nil {
		err = errors.Info(err, "doTestImage failed", url)
		return
	}
	return
}
Esempio n. 8
0
func main() {
	sql := "select value from xxx where host='server01' group by  host, region"

	st, err := influxql.NewParser(strings.NewReader(sql)).ParseStatement()
	if err != nil {
		err = errors.Info(err).Detail(err)
		log.Debug(err)
		return
	}

	//ensure the sql statement is a select statement
	stmt, ok := st.(*influxql.SelectStatement)
	if !ok {
		log.Info("not a valid select statement")
		return
	}

	//get min and max of sql
	log.Info(stmt.Condition)
	min, max := influxql.TimeRange(stmt.Condition)
	log.Printf("Min:%v, Max:%v", min, max)

	log.Printf("Min:%d, Max:%d", min.Unix()+offsetSeconds, max.Unix()+offsetSeconds)

	loc, err := time.LoadLocation("")

	acientTime := time.Date(1970, 1, 1, 0, 0, 0, 0, loc)
	log.Println(acientTime.Unix(), acientTime.UnixNano(), acientTime)
	log.Info("...", min.Unix(), max.Unix(), acientTime.Unix())
	start, end := true, true
	if err != nil {
		return
	}

	if !min.Round(time.Microsecond).Equal(acientTime.Round(time.Microsecond)) {
		log.Println("<<<<<<<<<<<<")
		start = true
	}
	if !max.Equal(acientTime) {
		end = true
	}

	stmt.SetPartialTimeRange(start, end, min.Add(time.Second*offsetSeconds), max.Add(time.Second*offsetSeconds))

	log.Println(stmt.String())

	//get group by
	log.Println(stmt.Dimensions)

	//host := "127.0.0.1:9092"
	log.Println(time.Now().UTC())

	log.Println(".............................")
	testTimeRange()
}
Esempio n. 9
0
File: publish.go Progetto: shtlee/qa
func (p *Pub) doTestUnpublish() (msg string, err error) {
	begin := time.Now()
	if _, err = p.rsCli.Unpublish(p.Domain); err != nil {
		err = errors.Info(err, "unpublish domain failed", p.Domain)
		return
	}
	end := time.Now()
	duration := end.Sub(begin)
	msg = util.GenLogEx("Pb    "+p.Env.Id+"_"+p.Name+"_doTestUnpublish", begin, end, duration)
	return
}
Esempio n. 10
0
File: publish.go Progetto: shtlee/qa
func (p *Pub) doTestUpload() (msg string, err error) {

	p.dataType = "application/qbox-mon"
	entryName := p.Bucket + ":" + p.Key
	f, err := os.Open(p.DataFile)
	if err != nil {
		err = errors.Info(err, "Upload failed: ", p.DataFile)
		return
	}
	defer f.Close()
	fi, _ := f.Stat()
	begin := time.Now()
	_, _, err = p.rsCli.Put(entryName, p.dataType, f, fi.Size())
	end := time.Now()
	duration := end.Sub(begin)
	msg = util.GenLogEx("Pb    "+p.Env.Id+"_"+p.Name+"_doTestUpload", begin, end, duration)
	if err != nil {
		err = errors.Info(err, "upload failed:", entryName)
	}
	return
}
Esempio n. 11
0
func (p *PubImage) doTestUnimage() (msg string, err error) {

	code, err := p.Pubcli.Unimage(p.Bucket)
	if err != nil || code/100 != 2 {
		if err == nil {
			err = errors.New("doTestUnimage failed")
		}
		err = errors.Info(err, code, p.Bucket)
		return
	}
	return
}
Esempio n. 12
0
func (self *UpRPut) doTestRPut() (msg string, err error) {

	f, err := os.Open(self.DataFile)
	if err != nil {
		err = errors.Info(err, "Resumable put failed")
		return
	}
	defer f.Close()
	fi, _ := f.Stat()
	entryURI := self.Bucket + ":" + self.Key
	blockcnt := self.Up2cli.BlockCount(fi.Size())
	progs := make([]up2.BlockputProgress, blockcnt)

	chunkNotify := func(idx int, p *up2.BlockputProgress) {
		if rand.Intn(blockcnt)/3 == 0 {
			p1 := *p
			progs[idx] = p1
		}
	}
	blockNotify := func(idx int, p *up2.BlockputProgress) {
	}
	t1 := self.Up2cli.NewRPtask(entryURI, "", "", "", "", f, fi.Size(), nil)
	t1.ChunkNotify = chunkNotify
	t1.BlockNotify = blockNotify

	begin := time.Now()
	for i := 0; i < blockcnt; i++ {
		t1.PutBlock(i)
	}
	t1.Progress = progs
	code, err := t1.Run(10, 10, nil, nil)
	end := time.Now()
	duration := end.Sub(begin)
	msg = util.GenLog("UP    "+self.Env.Id+"_"+self.Name+"_doTestRPut", begin, end, duration)
	if err != nil || code/100 != 2 {
		err = errors.Info(errors.New("Resumable put failed"), entryURI, err, code)
		return
	}
	return
}
Esempio n. 13
0
File: publish.go Progetto: shtlee/qa
func (p *Pub) doTestPublish() (msg string, err error) {

	if p.isNormalDomain {
		p.Domain = p.Domain + "/" + strconv.FormatInt(rand.Int63(), 10)
	} else {
		p.Domain = strconv.FormatInt(rand.Int63(), 10) + "." + p.Domain
	}
	begin := time.Now()
	if _, err = p.rsCli.Publish(p.Domain, p.Bucket); err != nil {
		err = errors.Info(err, "Publish failed: ", p.Bucket, p.Domain)
		return
	}
	end := time.Now()
	duration := end.Sub(begin)
	msg = util.GenLogEx("Pb    "+p.Env.Id+"_"+p.Name+"_doTestPublish", begin, end, duration)
	return
}