Esempio n. 1
0
func (this *TaobaoESStore) saveShopTrace(cd *CombinationData) {
	var date = timestamp.GetTimestamp(fmt.Sprintf("%s %s:%s:%s", cd.Date, cd.Clock, "00", "00"))
	var db = this.prefix + "_tb_shop_trace"
	var id = encrypt.DefaultMd5.Encode(date + cd.Ad + cd.Ua)
	var table = "shop"
	var shopids = make([]string, 0, len(cd.Ginfos))
	for _, v := range cd.Ginfos {
		shopids = append(shopids, v.Shop_id)
	}

	//查询是否存在
	res, err := this.client.Search().Index(db).Type(table).Query(elastic.NewIdsQuery(table).Ids(id)).Fields("shop").Do()
	if err != nil {
		log.Error(err)
	}

	if res == nil || res.TotalHits() == 0 {
		//log.Info(this.client.Index().Index(db).Type(table).Id(id).BodyJson(map[string]interface{}{
		//	"ad":        cd.Ad,
		//	"ua":        cd.Ua,
		//	"timestamp": date,
		//	"shop":      shopids,
		//}).Do())
		this.bulk.Add(elastic.NewBulkIndexRequest().Index(db).Type(table).Id(id).Doc(
			map[string]interface{}{
				"ad":        cd.Ad,
				"ua":        cd.Ua,
				"timestamp": date,
				"shop":      shopids,
			}))
	} else {
		oshopids := res.Hits.Hits[0].Fields["shop"].([]interface{})
		var tmpMap = make(map[string]byte)
		for _, vv := range oshopids {
			tmpMap[vv.(string)] = 1
		}
		for _, vv := range shopids {
			tmpMap[vv] = 1
		}

		nshopids := make([]string, 0, len(tmpMap))
		for k, _ := range tmpMap {
			nshopids = append(nshopids, k)
		}

		//log.Info(this.client.Update().Index(db).Type(table).Doc(map[string]interface{}{
		//	"shop": nshopids,
		//}).Id(id).Do())
		this.bulk.Add(elastic.NewBulkUpdateRequest().Index(db).Type(table).Doc(map[string]interface{}{
			"shop": nshopids,
		}).Id(id))
	}
}
Esempio n. 2
0
func (this *TaobaoESStore) ParseData(data interface{}) interface{} {
	cdata := &CombinationData{}
	err := json.Unmarshal(data.([]byte), cdata)
	if err != nil {
		log.Error("数据解析出错,错误信息为:", err)
		return nil
	}

	if cdata.Date != time.Now().Format("2006-01-02") {
		return nil
	}
	return cdata
}
Esempio n. 3
0
func Repo(ctx *middleware.Context, params martini.Params, req *http.Request) {
	reponame := params["_1"]
	var repo *models.Repository
	var err error
	var tasks []models.Task
	var recentTask *models.Task
	if reponame != "" {
		repo, err = models.GetRepositoryByName(reponame)
		if err == models.ErrRepositoryNotExists {
			r, er := models.AddRepository(reponame)
			if er != nil {
				err = er
				ctx.Data["Error"] = err.Error()
				ctx.HTML(200, "repo")
				return
			}
			TriggerBuildRepositoryById(r.Id)
			ctx.Redirect(302, "/"+r.Uri)
			return
		}
		if err != nil {
			log.Errorf("get single repo from db error: %v", err)
		}
	} else {
		id, _ := strconv.Atoi(req.FormValue("id"))
		rid := int64(id)
		repo, err = models.GetRepositoryById(rid)
		if err != nil {
			log.Errorf("get single repo from db error: %v", err)
		}
	}
	tasks, err = models.GetTasksByRid(repo.Id)
	if err != nil {
		log.Errorf("get tasks by id, error: %v", err)
	}
	recentTask, _ = models.GetTaskById(1)
	ctx.Data["Repo"] = repo
	ctx.Data["RecentTask"] = recentTask
	ctx.Data["Tasks"] = tasks
	ctx.Data["DownCnt"] = models.RefreshPageView("/d/"+base.ToStr(repo.Id), 0)
	ctx.Data["RootUrl"] = config.Config.Server.RootUrl
	rus, err := models.GetAllLastRepoUpdate(repo.Id)
	if err != nil {
		log.Error("get last repo error: %v", err)
	}
	ctx.Data["Last"] = rus
	ctx.HTML(200, "repo")
}
Esempio n. 4
0
func setUp() error {
	var err error
	var binDir = filepath.Join(SELFDIR, "bin")
	var tmpDir = filepath.Join(SELFDIR, "tmp/tmp-gopath")
	os.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
	if _, err := exec.LookPath("go"); err != nil {
		log.Error("require go tool installed")
		return err
	}
	sess := sh.NewSession()
	sess.SetEnv("GOBIN", binDir)
	sess.SetEnv("GOPATH", tmpDir)
	if !sh.Test("x", GOPM) {
		os.RemoveAll(filepath.Join(binDir, "gopm"))
		defer os.RemoveAll(tmpDir)
		err = sess.Command("go", "get", "-u", "-v", "github.com/gpmgo/gopm").Run()
		if err != nil {
			log.Errorf("install gopm error: %v", err)
			return err
		}
	}
	return nil
}