func (this *TaobaoESStore) saveShopTrace(cd *CombinationData) {
var date = timestamp.GetTimestamp(fmt.Sprintf("%s %s:%s:%s", cd.Date, cd.Clock, "00", "00"))
var db = this.prefix + "_tb_shop_trace"
var id = encrypt.DefaultMd5.Encode(date + cd.Ad + cd.Ua)
var table = "shop"
var shopids = make([]string, 0, len(cd.Ginfos))
for _, v := range cd.Ginfos {
shopids = append(shopids, v.Shop_id)
}
//查询是否存在
res, err := this.client.Search().Index(db).Type(table).Query(elastic.NewIdsQuery(table).Ids(id)).Fields("shop").Do()
if err != nil {
log.Error(err)
}
if res == nil || res.TotalHits() == 0 {
//log.Info(this.client.Index().Index(db).Type(table).Id(id).BodyJson(map[string]interface{}{
// "ad": cd.Ad,
// "ua": cd.Ua,
// "timestamp": date,
// "shop": shopids,
//}).Do())
this.bulk.Add(elastic.NewBulkIndexRequest().Index(db).Type(table).Id(id).Doc(
map[string]interface{}{
"ad": cd.Ad,
"ua": cd.Ua,
"timestamp": date,
"shop": shopids,
}))
} else {
oshopids := res.Hits.Hits[0].Fields["shop"].([]interface{})
var tmpMap = make(map[string]byte)
for _, vv := range oshopids {
tmpMap[vv.(string)] = 1
}
for _, vv := range shopids {
tmpMap[vv] = 1
}
nshopids := make([]string, 0, len(tmpMap))
for k, _ := range tmpMap {
nshopids = append(nshopids, k)
}
//log.Info(this.client.Update().Index(db).Type(table).Doc(map[string]interface{}{
// "shop": nshopids,
//}).Id(id).Do())
this.bulk.Add(elastic.NewBulkUpdateRequest().Index(db).Type(table).Doc(map[string]interface{}{
"shop": nshopids,
}).Id(id))
}
}
func (this *TaobaoESStore) ParseData(data interface{}) interface{} {
cdata := &CombinationData{}
err := json.Unmarshal(data.([]byte), cdata)
if err != nil {
log.Error("数据解析出错,错误信息为:", err)
return nil
}
if cdata.Date != time.Now().Format("2006-01-02") {
return nil
}
return cdata
}
func Repo(ctx *middleware.Context, params martini.Params, req *http.Request) {
reponame := params["_1"]
var repo *models.Repository
var err error
var tasks []models.Task
var recentTask *models.Task
if reponame != "" {
repo, err = models.GetRepositoryByName(reponame)
if err == models.ErrRepositoryNotExists {
r, er := models.AddRepository(reponame)
if er != nil {
err = er
ctx.Data["Error"] = err.Error()
ctx.HTML(200, "repo")
return
}
TriggerBuildRepositoryById(r.Id)
ctx.Redirect(302, "/"+r.Uri)
return
}
if err != nil {
log.Errorf("get single repo from db error: %v", err)
}
} else {
id, _ := strconv.Atoi(req.FormValue("id"))
rid := int64(id)
repo, err = models.GetRepositoryById(rid)
if err != nil {
log.Errorf("get single repo from db error: %v", err)
}
}
tasks, err = models.GetTasksByRid(repo.Id)
if err != nil {
log.Errorf("get tasks by id, error: %v", err)
}
recentTask, _ = models.GetTaskById(1)
ctx.Data["Repo"] = repo
ctx.Data["RecentTask"] = recentTask
ctx.Data["Tasks"] = tasks
ctx.Data["DownCnt"] = models.RefreshPageView("/d/"+base.ToStr(repo.Id), 0)
ctx.Data["RootUrl"] = config.Config.Server.RootUrl
rus, err := models.GetAllLastRepoUpdate(repo.Id)
if err != nil {
log.Error("get last repo error: %v", err)
}
ctx.Data["Last"] = rus
ctx.HTML(200, "repo")
}
func setUp() error {
var err error
var binDir = filepath.Join(SELFDIR, "bin")
var tmpDir = filepath.Join(SELFDIR, "tmp/tmp-gopath")
os.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
if _, err := exec.LookPath("go"); err != nil {
log.Error("require go tool installed")
return err
}
sess := sh.NewSession()
sess.SetEnv("GOBIN", binDir)
sess.SetEnv("GOPATH", tmpDir)
if !sh.Test("x", GOPM) {
os.RemoveAll(filepath.Join(binDir, "gopm"))
defer os.RemoveAll(tmpDir)
err = sess.Command("go", "get", "-u", "-v", "github.com/gpmgo/gopm").Run()
if err != nil {
log.Errorf("install gopm error: %v", err)
return err
}
}
return nil
}