Beispiel #1
0
// Open opens or creates a storage with specific format for a local engine Driver.
func (d Driver) Open(schema string) (kv.Storage, error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	if store, ok := mc.cache[schema]; ok {
		// TODO: check the cache store has the same engine with this Driver.
		log.Info("cache store", schema)
		return store, nil
	}

	db, err := d.Driver.Open(schema)
	if err != nil {
		return nil, errors.Trace(err)
	}

	log.Info("New store", schema)
	s := &dbStore{
		txns:       make(map[uint64]*dbTxn),
		keysLocked: make(map[string]uint64),
		uuid:       uuid.NewV4().String(),
		path:       schema,
		db:         db,
		compactor:  newLocalCompactor(localCompactDefaultPolicy, db),
	}
	mc.cache[schema] = s
	s.compactor.Start()
	return s, nil
}
Beispiel #2
0
func (s *HBasePutTestSuit) TestGetPut(c *C) {
	log.Info(codec.EncodeKey(170))

	p := NewPut([]byte("1_\xff\xff"))
	p2 := NewPut([]byte("1_\xff\xfe"))
	p3 := NewPut([]byte("1_\xff\xee"))
	p.AddValue([]byte("cf"), []byte("q"), []byte("!"))
	p2.AddValue([]byte("cf"), []byte("q"), []byte("!"))
	p3.AddValue([]byte("cf"), []byte("q"), []byte("!"))

	cli, err := NewClient(getTestZkHosts(), "/hbase")
	c.Assert(err, Equals, nil)

	cli.Put("t2", p)
	cli.Put("t2", p2)
	cli.Put("t2", p3)

	scan := NewScan([]byte("t2"), 100, cli)
	scan.StartRow = []byte("1_")
	for {
		r := scan.Next()
		if r == nil {
			break
		}
		log.Info(r.SortedColumns[0].Row)
	}

	cli.Delete("t2", NewDelete([]byte("1_\xff\xff")))
	cli.Delete("t2", NewDelete([]byte("1_\xff\xfe")))
	cli.Delete("t2", NewDelete([]byte("1_\xff\xee")))

}
Beispiel #3
0
func (s *testModelSuite) TestNewAction(c *C) {
	log.Info("[TestNewAction][start]")
	fakeCoordConn := zkhelper.NewConn()

	err := NewAction(fakeCoordConn, productName, ACTION_TYPE_SLOT_CHANGED, nil, "desc", false)
	c.Assert(err, IsNil)

	prefix := GetWatchActionPath(productName)
	exist, _, err := fakeCoordConn.Exists(prefix)
	c.Assert(exist, Equals, true)
	c.Assert(err, IsNil)

	// test if response node exists
	d, _, err := fakeCoordConn.Get(prefix + "/0000000001")
	c.Assert(err, IsNil)

	// test get action data
	d, _, err = fakeCoordConn.Get(GetActionResponsePath(productName) + "/0000000001")
	c.Assert(err, IsNil)

	var action Action
	json.Unmarshal(d, &action)
	c.Assert(action.Desc, Equals, "desc")
	c.Assert(action.Type, Equals, ACTION_TYPE_SLOT_CHANGED)

	fakeCoordConn.Close()
	log.Info("[TestNewAction][end]")
}
Beispiel #4
0
func BulkInsertData(reader *bufio.Reader, ktype string) {
	bk := es.Bulk()
	num := 0
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		}
		info := strings.Split(line, "\t#\t")
		log.Info(len(info))
		if len(info) != 2 {
			continue
		}
		ad, ua := getAdUa(info[0], "\t")
		keyword := getKeyWord(info[1], "\t")
		lonlat := GetLonLat(ad)
		if lonlat == "" {
			continue
		}
		num++
		id := encrypt.DefaultMd5.Encode("1456185600" + ad + ua)
		pinfo := map[string]interface{}{
			"ad":  ad,
			"ua":  ua,
			ktype: keyword,
			"geo": lonlat,
		}
		bk.Add(elastic.NewBulkUpdateRequest().Index("map_trace").Type("map").Doc(pinfo).Id(id).DocAsUpsert(true))
		bk.Add(elastic.NewBulkUpdateRequest().Index("map_trace_search").Type("map").Doc(pinfo).Id(id).DocAsUpsert(true))
		if num%10000 == 0 {
			log.Error(bk.Do())
		}
	}
	log.Info(bk.Do())
}
Beispiel #5
0
func (gc *localstoreCompactor) checkExpiredKeysWorker() {
	gc.workerWaitGroup.Add(1)
	defer gc.workerWaitGroup.Done()
	for {
		select {
		case <-gc.stopCh:
			log.Info("GC stopped")
			return
		case <-gc.ticker.C:
			gc.mu.Lock()
			m := gc.recentKeys
			if len(m) == 0 {
				gc.mu.Unlock()
				continue
			}
			gc.recentKeys = make(map[string]struct{})
			gc.mu.Unlock()
			log.Info("GC trigger")
			for k := range m {
				err := gc.Compact(nil, []byte(k))
				if err != nil {
					log.Error(err)
				}
			}
		}
	}
}
Beispiel #6
0
func (c *Cluster) DB(isread bool) (*mysql.DB, error) {
	if isread {
		log.Info("return a master conn")
		return c.Master()
	}

	log.Info("return a slave conn")
	return c.Slave()
}
Beispiel #7
0
func (e *etcdImpl) Create(wholekey string, value []byte, flags int32, aclv []zk.ACL) (keyCreated string, err error) {
	seq := (flags & zk.FlagSequence) != 0
	tmp := (flags & zk.FlagEphemeral) != 0
	ttl := uint64(MAX_TTL)
	if tmp {
		ttl = 5
	}

	var resp *etcd.Response

	conn, err := e.pool.Get()
	if err != nil {
		return "", err
	}

	defer e.pool.Put(conn)
	c := conn.(*PooledEtcdClient).c

	fn := c.Create
	log.Info("create", wholekey)

	if seq {
		wholekey = path.Dir(wholekey)
		fn = c.CreateInOrder
	} else {
		for _, v := range aclv {
			if v.Perms == PERM_DIRECTORY {
				log.Info("etcdImpl:create directory", wholekey)
				fn = nil
				resp, err = c.CreateDir(wholekey, uint64(ttl))
				if err != nil {
					return "", convertToZkError(err)
				}
			}
		}
	}

	if fn == nil {
		if tmp {
			e.keepAlive(wholekey, ttl)
		}
		return resp.Node.Key, nil
	}

	resp, err = fn(wholekey, string(value), uint64(ttl))
	if err != nil {
		return "", convertToZkError(err)
	}

	if tmp {
		e.keepAlive(resp.Node.Key, ttl)
	}

	return resp.Node.Key, nil
}
Beispiel #8
0
// Push metircs in background.
func Push(cfg *MetricConfig) {
	if cfg.PushInterval.Duration == zeroDuration || len(cfg.PushAddress) == 0 {
		log.Info("disable Prometheus push client")
		return
	}

	log.Info("start Prometheus push client")

	interval := cfg.PushInterval.Duration
	go prometheusPushClient(cfg.PushJob, cfg.PushAddress, interval)
}
Beispiel #9
0
Datei: ck.go Projekt: qgweb/new
func NewCookiePut() *CookiePut {
	var zj = &CookiePut{}
	zj.kf = dbfactory.NewKVFile(fmt.Sprintf("./%s.txt", convert.ToString(time.Now().Unix())))
	zj.putTags = make(map[string]map[string]int)
	zj.Timestamp = timestamp.GetHourTimestamp(-1)
	zj.initPutAdverts()
	zj.initPutTags("TAGS_5*", "cookie_", "")
	log.Info(zj.putAdverts)
	log.Info(zj.putTags)
	return zj
}
Beispiel #10
0
// experimental simple auto rebalance :)
func Rebalance(coordConn zkhelper.Conn, delay int) error {
	targetQuota, err := getQuotaMap(coordConn)
	if err != nil {
		return errors.Trace(err)
	}
	livingNodes, err := getLivingNodeInfos(coordConn)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info("start rebalance")
	for _, node := range livingNodes {
		for len(node.CurSlots) > targetQuota[node.GroupId] {
			for _, dest := range livingNodes {
				if dest.GroupId != node.GroupId && len(dest.CurSlots) < targetQuota[dest.GroupId] {
					slot := node.CurSlots[len(node.CurSlots)-1]
					// create a migration task
					t := NewMigrateTask(MigrateTaskInfo{
						Delay:      delay,
						FromSlot:   slot,
						ToSlot:     slot,
						NewGroupId: dest.GroupId,
						Status:     MIGRATE_TASK_MIGRATING,
						CreateAt:   strconv.FormatInt(time.Now().Unix(), 10),
					})
					u, err := uuid.NewV4()
					if err != nil {
						return errors.Trace(err)
					}
					t.Id = u.String()

					if ok, err := preMigrateCheck(t); ok {
						// do migrate
						err := t.run()
						if err != nil {
							log.Warning(err)
							return errors.Trace(err)
						}
					} else {
						log.Warning(err)
						return errors.Trace(err)
					}
					node.CurSlots = node.CurSlots[0 : len(node.CurSlots)-1]
					dest.CurSlots = append(dest.CurSlots, slot)
				}
			}
		}
	}
	log.Info("rebalance finish")
	return nil
}
Beispiel #11
0
// Fix https://github.com/pingcap/go-themis/issues/19
func (s *TransactionTestSuit) TestPrewriteSecondaryMissingRows(c *C) {
	conf := defaultTxnConf
	hook := createChoosePrimaryRowHook("A")
	hook.beforePrewriteSecondary = func(txn *themisTxn, ctx interface{}) (bool, interface{}, error) {
		go func() {
			hook2 := createChoosePrimaryRowHook("B")
			hook2.onSecondaryOccursLock = func(txn *themisTxn, ctx interface{}) (bool, interface{}, error) {
				log.Info("tx2 occurs secondary lock", ctx)
				return true, nil, nil
			}
			tx2 := newTxn(s.cli, conf)
			tx2.(*themisTxn).setHook(hook2)
			tx2.Put(themisTestTableName, hbase.NewPut([]byte("A")).AddValue(cf, q, []byte("A")))
			tx2.Put(themisTestTableName, hbase.NewPut([]byte("B")).AddValue(cf, q, []byte("B")))
			tx2.Put(themisTestTableName, hbase.NewPut([]byte("C")).AddValue(cf, q, []byte("C")))
			tx2.Commit()
		}()
		time.Sleep(500 * time.Millisecond)
		return true, nil, nil
	}

	hook.onSecondaryOccursLock = func(txn *themisTxn, ctx interface{}) (bool, interface{}, error) {
		log.Info("tx1", ctx)
		return true, nil, nil
	}

	hook.onPrewriteRow = func(txn *themisTxn, ctx interface{}) (bool, interface{}, error) {
		containPrimary := ctx.([]interface{})[1].(bool)
		if !containPrimary {
			rm := ctx.([]interface{})[0].(*rowMutation)
			log.Info(string(rm.row))
		}

		return true, nil, nil
	}

	tx1 := newTxn(s.cli, conf)
	tx1.(*themisTxn).setHook(hook)
	tx1.Put(themisTestTableName, hbase.NewPut([]byte("A")).AddValue(cf, q, []byte("A")))
	tx1.Put(themisTestTableName, hbase.NewPut([]byte("B")).AddValue(cf, q, []byte("B")))
	tx1.Put(themisTestTableName, hbase.NewPut([]byte("C")).AddValue(cf, q, []byte("C")))
	err := tx1.Commit()
	c.Assert(err, IsNil)

	tx3 := newTxn(s.cli, conf)
	rs, err := tx3.Get(themisTestTableName, hbase.NewGet([]byte("C")).AddColumn(cf, q))
	c.Assert(rs, NotNil)
	c.Assert(err, IsNil)
	tx3.Commit()
}
Beispiel #12
0
func (s *testModelSuite) TestAddSlaveToEmptyGroup(c *C) {
	log.Info("[TestAddSlaveToEmptyGroup][start]")
	fakeCoordConn := zkhelper.NewConn()

	g := NewServerGroup(productName, 1)
	g.Create(fakeCoordConn)

	s1 := NewServer(SERVER_TYPE_SLAVE, s.s1.addr)
	err := g.AddServer(fakeCoordConn, s1, auth)
	c.Assert(err, IsNil)
	c.Assert(g.Servers[0].Type, Equals, SERVER_TYPE_MASTER)

	fakeCoordConn.Close()
	log.Info("[TestAddSlaveToEmptyGroup][end]")
}
Beispiel #13
0
func (s *testModelSuite) TestServerGroup(c *C) {
	log.Info("[TestServerGroup][start]")
	fakeCoordConn := zkhelper.NewConn()

	g := NewServerGroup(productName, 1)
	g.Create(fakeCoordConn)

	// test create new group
	groups, err := ServerGroups(fakeCoordConn, productName)
	c.Assert(err, IsNil)
	c.Assert(len(groups), Not(Equals), 0)

	ok, err := g.Exists(fakeCoordConn)
	c.Assert(err, IsNil)
	c.Assert(ok, Equals, true)

	gg, err := GetGroup(fakeCoordConn, productName, 1)
	c.Assert(err, IsNil)
	c.Assert(gg, NotNil)
	c.Assert(gg.Id, Equals, g.Id)

	s1 := NewServer(SERVER_TYPE_MASTER, s.s1.addr)
	s2 := NewServer(SERVER_TYPE_MASTER, s.s2.addr)

	err = g.AddServer(fakeCoordConn, s1, auth)
	c.Assert(err, IsNil)

	servers, err := g.GetServers(fakeCoordConn)
	c.Assert(err, IsNil)
	c.Assert(len(servers), Equals, 1)

	g.AddServer(fakeCoordConn, s2, auth)
	c.Assert(len(g.Servers), Equals, 1)

	s2.Type = SERVER_TYPE_SLAVE
	g.AddServer(fakeCoordConn, s2, auth)
	c.Assert(len(g.Servers), Equals, 2)

	err = g.Promote(fakeCoordConn, s2.Addr, auth)
	c.Assert(err, IsNil)

	m, err := g.Master(fakeCoordConn)
	c.Assert(err, IsNil)
	c.Assert(m.Addr, Equals, s2.Addr)

	fakeCoordConn.Close()
	log.Info("[TestServerGroup][stop]")
}
Beispiel #14
0
func (s *ParallelTestSuit) TestParallelHbaseCall(c *C) {
	runtime.GOMAXPROCS(runtime.NumCPU() / 2)
	cli, err := createHBaseClient()
	c.Assert(err, Equals, nil)

	err = createNewTableAndDropOldTable(cli, themisTestTableName, "cf", nil)
	c.Assert(err, Equals, nil)

	wg := sync.WaitGroup{}
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			tx := newTxn(cli, defaultTxnConf)
			p := hbase.NewPut(getTestRowKey(c))
			p.AddValue(cf, q, []byte(strconv.Itoa(i)))
			tx.Put(themisTestTableName, p)
			tx.Commit()
		}(i)
	}
	wg.Wait()

	g := hbase.NewGet(getTestRowKey(c)).AddColumn(cf, q)
	rs, err := cli.Get(themisTestTableName, g)
	if err != nil {
		log.Fatal(err)
	}
	log.Info(string(rs.SortedColumns[0].Value))
}
Beispiel #15
0
func (s *Server) responseAction(seq int64) {
	log.Info("send response", seq)
	err := s.top.DoResponse(int(seq), &s.pi)
	if err != nil {
		log.Error(errors.ErrorStack(err))
	}
}
Beispiel #16
0
Datei: js.go Projekt: qgweb/new
// 初始化cox对应区域
func (this *JsPut) initArea() {
	this.areamap = make(map[string]string)
	f, err := os.Open(lib.GetConfVal("jiangsu::areapath"))
	if err != nil {
		log.Error(err)
		return
	}
	defer f.Close()

	bi := bufio.NewReader(f)
	for {
		line, err := bi.ReadString('\n')
		if err == io.EOF {
			break
		}

		//0006f21d119b032d59acc3c2b90f10624eeaebe8,511
		info := strings.Split(line, ",")
		if len(info) != 2 {
			continue
		}

		this.areamap[info[0]] = strings.TrimSpace(info[1])
	}
	log.Info("江苏区域数量", len(this.areamap))
}
Beispiel #17
0
Datei: js.go Projekt: qgweb/new
// 保存投放轨迹到电信ftp
func (this *JsPut) saveTraceToDianxin() {
	var (
		ftp     = lib.GetConfVal("jiangsu::ftp_path")
		ppath   = lib.GetConfVal("jiangsu::put_path")
		rk      = "account.10046.sha1." + time.Now().Add(-time.Hour).Format("200601021504")
		fname   = ppath + "/" + rk
		adcount = 0
	)

	f, err := os.Create(fname)
	if err != nil {
		log.Error("创建文件失败", err)
		return
	}
	defer f.Close()

	this.kf.AdSet(func(ad string) {
		if v, ok := this.areamap[ad]; ok {
			f.WriteString(ad + "," + v + "\n")
			adcount++
		}
	})
	cmd := exec.Command(ftp, rk)
	str, err := cmd.Output()
	log.Info(string(str), err)

	// 广告数量统计数据 dx_stats , js_1461016800, 11111
	lib.StatisticsData("dx_stats", fmt.Sprintf("js_%s", timestamp.GetHourTimestamp(-1)),
		convert.ToString(adcount), "")
}
Beispiel #18
0
func NewServer(cfg *config.Conf) (*Server, error) {
	s := new(Server)

	s.cfg = cfg

	var err error

	s.fingerprints = make(map[string]*LimitReqNode)
	// s.users = make(map[string]*User)
	// s.qpsOnServer = &LimitReqNode{}
	s.mu = &sync.Mutex{}
	s.restart = false
	port := s.cfg.GetConfig().Global.Port

	// get listenfd from file when restart
	if os.Getenv("_GRACEFUL_RESTART") == "true" {
		log.Info("graceful restart with previous listenfd")

		//get the linstenfd
		file := os.NewFile(3, "")
		s.listener, err = net.FileListener(file)
		if err != nil {
			log.Warn("get linstener err ")
		}

	} else {
		s.listener, err = net.Listen("tcp4", fmt.Sprintf(":%d", port))
	}
	if err != nil {
		return nil, err
	}

	log.Infof("Dbatman Listen(tcp4) at [%d]", port)
	return s, nil
}
Beispiel #19
0
func (sg *ServerGroup) RemoveServer(coordConn zkhelper.Conn, addr string) error {
	coordPath := fmt.Sprintf("/zk/reborn/db_%s/servers/group_%d/%s", sg.ProductName, sg.Id, addr)
	data, _, err := coordConn.Get(coordPath)
	if err != nil {
		return errors.Trace(err)
	}

	var s Server
	err = json.Unmarshal(data, &s)
	if err != nil {
		return errors.Trace(err)
	}
	log.Info(s)
	if s.Type == SERVER_TYPE_MASTER {
		return errors.New("cannot remove master, use promote first")
	}

	err = coordConn.Delete(coordPath, -1)
	if err != nil {
		return errors.Trace(err)
	}

	// update server list
	for i := 0; i < len(sg.Servers); i++ {
		if sg.Servers[i].Addr == s.Addr {
			sg.Servers = append(sg.Servers[:i], sg.Servers[i+1:]...)
			break
		}
	}

	// remove slave won't need proxy confirm
	err = NewAction(coordConn, sg.ProductName, ACTION_TYPE_SERVER_GROUP_CHANGED, sg, "", false)
	return errors.Trace(err)
}
Beispiel #20
0
func (s *HBaseDelTestSuit) TestWithClient(c *C) {
	// create new
	p := NewPut([]byte("test"))
	p.AddValue([]byte("cf"), []byte("q"), []byte("val"))
	s.cli.Put("t2", p)
	// check it

	g := NewGet([]byte("test"))
	g.AddStringFamily("cf")
	r, err := s.cli.Get("t2", g)
	c.Assert(err, Equals, nil)
	c.Assert(string(r.Columns["cf:q"].Value), Equals, "val")
	log.Info(string(r.Columns["cf:q"].Value))
	// delete it

	d := NewDelete([]byte("test"))
	d.AddColumn([]byte("cf"), []byte("q"))
	b, err := s.cli.Delete("t2", d)
	c.Assert(err, Equals, nil)
	c.Assert(b, Equals, true)

	// check it
	r, err = s.cli.Get("t2", g)
	c.Assert(err, Equals, nil)
	c.Assert(r == nil, Equals, true)
}
Beispiel #21
0
Datei: zj.go Projekt: qgweb/new
// 域名找回信息获取
func (this *ZjPut) VisitorData(out chan interface{}, in chan int8) {
	var datacount = 0
	defer func() {
		// 统计数据 zhejiang_put , other_1461016800, 11111
		lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_visitor",
			convert.ToString(datacount), "")
	}()
	m, err := lib.GetMongoObj()
	if err != nil {
		log.Error(err)
		in <- 1
		return
	}
	defer m.Close()

	qconf := mongodb.MongodbQueryConf{}
	qconf.Db = "data_source"
	qconf.Table = "zhejiang_visitor"
	qconf.Query = mongodb.MM{}
	m.Query(qconf, func(info map[string]interface{}) {
		ad := convert.ToString(info["ad"])
		ua := encrypt.DefaultMd5.Encode(encrypt.DefaultBase64.Decode(convert.ToString(info["ua"])))
		aids := convert.ToString(info["aids"])
		datacount++
		out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, aids)
	})
	log.Info("访客ok")
	in <- 1
}
Beispiel #22
0
func runStmt(ctx context.Context, s ast.Statement, args ...interface{}) (ast.RecordSet, error) {
	var err error
	var rs ast.RecordSet
	// before every execution, we must clear affectedrows.
	variable.GetSessionVars(ctx).SetAffectedRows(0)
	if s.IsDDL() {
		err = ctx.CommitTxn()
		if err != nil {
			return nil, errors.Trace(err)
		}
	}
	rs, err = s.Exec(ctx)
	// All the history should be added here.
	se := ctx.(*session)
	se.history.add(0, s)
	// MySQL DDL should be auto-commit.
	ac, err1 := autocommit.ShouldAutocommit(ctx)
	if err1 != nil {
		return nil, errors.Trace(err1)
	}
	if s.IsDDL() || ac {
		if err != nil {
			log.Info("RollbackTxn for ddl/autocommit error.")
			ctx.RollbackTxn()
		} else {
			err = ctx.CommitTxn()
		}
	}
	return rs, errors.Trace(err)
}
Beispiel #23
0
Datei: log.go Projekt: qgweb/new
func sscript(buf string) []byte {
	cmd := exec.Command("php", "a.php", encrypt.DefaultBase64.Encode(buf))
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Start()
	if err != nil {
		log.Info(err)
		return nil
	}
	err = cmd.Wait()
	if err != nil {
		log.Info(err)
		return nil
	}
	return out.Bytes()
}
Beispiel #24
0
func ForceRemoveDeadFence(coordConn zkhelper.Conn, productName string) error {
	proxies, err := ProxyList(coordConn, productName, func(p *ProxyInfo) bool {
		return p.State == PROXY_STATE_ONLINE
	})
	if err != nil {
		return errors.Trace(err)
	}

	fenceProxies, err := GetFenceProxyMap(coordConn, productName)
	if err != nil {
		return errors.Trace(err)
	}

	// remove online proxies's fence
	for _, proxy := range proxies {
		delete(fenceProxies, proxy.Addr)
	}

	// delete dead fence in zookeeper
	path := GetProxyFencePath(productName)
	for remainFence, _ := range fenceProxies {
		fencePath := filepath.Join(path, remainFence)
		log.Info("removing fence: ", fencePath)
		if err := zkhelper.DeleteRecursive(coordConn, fencePath, -1); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}
Beispiel #25
0
func (s *ScanTestSuit) SetUpSuite(c *C) {
	var (
		ok  bool
		err error
	)

	s.cli, err = NewClient(getTestZkHosts(), "/hbase")
	c.Assert(err, Equals, nil)

	log.Info("create table")
	table := NewTableNameWithDefaultNS("scan_test")
	s.cli.DisableTable(table)
	s.cli.DropTable(table)

	tblDesc := NewTableDesciptor(table)
	cf := newColumnFamilyDescriptor("cf", 3)
	tblDesc.AddColumnDesc(cf)
	err = s.cli.CreateTable(tblDesc, nil)
	c.Assert(err, IsNil)

	for i := 1; i <= 10000; i++ {
		p := NewPut([]byte(strconv.Itoa(i)))
		p.AddValue([]byte("cf"), []byte("q"), []byte(strconv.Itoa(i)))
		ok, err = s.cli.Put("scan_test", p)
		c.Assert(ok, IsTrue)
		c.Assert(err, IsNil)
	}
}
Beispiel #26
0
func (c *Cluster) HeartBeat() (*CrashDb, error) {
	if c.masterNode == nil {
		log.Info("master node did not exists")
		err := fmt.Errorf("config is nil")
		return nil, err
	}
	ret := &CrashDb{crashNum: 0, masterNode: nil}

	//get all the cluster cfg
	masterDb := c.masterNode
	slaveDbs := c.slaveNodes

	err := masterDb.HeartBeatPing() //HeartBeatPing or Ping use single conn or user conn from conn pools
	if err != nil {
		// log.Warn("db ping error ,", err.Error())
		ret.crashNum++
		ret.masterNode = masterDb
	}

	for _, slavedb := range slaveDbs {
		// check the alive status of db if has been cut down don't need to detect again
		if slavedb.GetDbAliveStatus() == true {
			err := slavedb.HeartBeatPing()
			if err != nil {
				ret.crashNum++
				// log.Warn("db ping error ,", err.Error())
				ret.slaveNode = append(ret.slaveNode, slavedb)
			}
		}
	}

	return ret, nil
}
Beispiel #27
0
func (th *TailHandler) Click(info []string) {
	var exstr = th.Script([]byte(strings.Join(info, "\t")))
	var exmap map[string]string
	if err := json.Unmarshal(exstr, &exmap); err != nil {
		return
	}
	if len(exmap) == 0 {
		return
	}
	log.Info(exmap)
	var ml MLog
	p, c := th.getIp(info)
	ml.Province = p
	ml.City = c
	ml.AdwId = convert.ToInt(th.getmap(exmap, "pd"))
	ml.AdId = convert.ToInt(th.getmap(exmap, "hd"))
	ml.UA = th.getmap(exmap, "ua")
	ml.AD = th.getmap(exmap, "ad")
	ml.Click = 1
	ml.Clock = convert.ToInt(time.Now().Format("15"))
	ml.Date = convert.ToInt(timestamp.GetDayTimestamp(0))
	ml.Url = strings.TrimSpace(th.get_url(th.getmap(exmap, "ltu"), th.getmap(exmap, "lftu")))
	ml.Domain = th.get_domain(ml.Url)
	ml.Cookie = th.get_cookie(info[9])
	ml.CusId = encrypt.DefaultMd5.Encode(ml.AD + ml.UA + ml.Cookie)
	ml.CusId2 = ml.CusId
	ml.Money, ml.Fmoney = th.click_money(ml.AdwId, ml.AdId)
	if _, ok := th.SouceMap[th.getmap(exmap, "pd")]; ok {
		ml.Source = 1
	}
	th.SaveData(ml)
}
Beispiel #28
0
func (s *MutationCacheTestSuit) TestMutationCache(c *C) {
	cache := newColumnMutationCache()
	row := []byte("r1")
	col := &hbase.Column{[]byte("f1"), []byte("q1")}
	cache.addMutation([]byte("tbl"), row, col, hbase.TypePut, []byte("test"), false)
	cache.addMutation([]byte("tbl"), row, col, hbase.TypeDeleteColumn, []byte("test"), false)
	cache.addMutation([]byte("tbl"), row, col, hbase.TypePut, []byte("test"), false)

	cc := &hbase.ColumnCoordinate{
		Table: []byte("tbl"),
		Row:   []byte("r1"),
		Column: hbase.Column{
			Family: []byte("f1"),
			Qual:   []byte("q1"),
		},
	}
	mutation := cache.getMutation(cc)
	if mutation == nil || mutation.typ != hbase.TypePut || bytes.Compare(mutation.value, []byte("test")) != 0 {
		c.Error("cache error")
	} else {
		log.Info(mutation)
	}

	p := hbase.NewPut([]byte("row"))
	p.AddStringValue("cf", "q", "v")
	p.AddStringValue("cf", "q1", "v")
	p.AddStringValue("cf", "q2", "v")
	p.AddStringValue("cf", "q3", "v")

	entries := getEntriesFromPut(p)
	c.Assert(len(entries), Equals, 4)
}
Beispiel #29
0
func (s *Server) Serve() error {
	log.Debug("this is ddbatman v4")
	s.running = true
	var sessionId int64 = 0
	for s.running {
		select {
		case sessionChan <- sessionId:
			//do nothing
		default:
			//warnning!
			log.Warnf("TASK_CHANNEL is full!")
		}

		conn, err := s.Accept()
		if err != nil {
			log.Warning("accept error %s", err.Error())
			continue
		}
		//allocate a sessionId for a session
		go s.onConn(conn)
		sessionId += 1
	}
	if s.restart == true {
		log.Debug("Begin to restart graceful")
		listenerFile, err := s.listener.(*net.TCPListener).File()
		if err != nil {
			log.Fatal("Fail to get socket file descriptor:", err)
		}
		listenerFd := listenerFile.Fd()

		os.Setenv("_GRACEFUL_RESTART", "true")
		execSpec := &syscall.ProcAttr{
			Env:   os.Environ(),
			Files: []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd(), listenerFd},
		}
		fork, err := syscall.ForkExec(os.Args[0], os.Args, execSpec)
		if err != nil {
			return fmt.Errorf("failed to forkexec: %v", err)
		}

		log.Infof("start new process success, pid %d.", fork)
	}
	timeout := time.NewTimer(time.Minute)
	wait := make(chan struct{})
	go func() {
		s.wg.Wait()
		wait <- struct{}{}
	}()

	select {
	case <-timeout.C:
		log.Error("server : Waittimeout error when close the service")
		return nil
	case <-wait:
		log.Info("server : all goroutine has been done")
		return nil
	}
	return nil
}
Beispiel #30
0
// PushMetric pushs metircs in background.
func pushMetric(addr string, interval time.Duration) {
	if interval == zeroDuration || len(addr) == 0 {
		log.Info("disable Prometheus push client")
		return
	}
	log.Infof("start Prometheus push client with server addr %s and interval %d", addr, interval)
	go prometheusPushClient(addr, interval)
}