예제 #1
0
// FetchURL fetches url and saves as a file, it includes retry and decoding bson.
func (p *BsonProxy) FetchURL(ref string) (string, error) {
	for tryCount := 0; tryCount < maxRetry; tryCount++ {
		if tryCount > 0 {
			log.Warnf("Retry after %d times", tryCount)
			time.Sleep(retryInterval)
		}
		resp, err := p.c.Get(ref)
		if err != nil {
			log.Error(err)
			continue
		}
		f, err := ioutil.TempFile(os.TempDir(), "duokan")
		if err != nil {
			log.Error(err)
			continue
		}
		// Why use ioutil.ReadAll doesn't work? It seems resp.Body can't Close,
		// so reader is blocking.
		_, err = io.Copy(f, resp.Body)
		if err != nil {
			log.Errorf("Get error url[%s], err[%s]", ref, err)
			continue
		}
		err = f.Close()
		if err != nil {
			log.Error(err)
			continue
		}
		return f.Name(), nil
	}
	return "", errors.Errorf("reach max retry time [%d]", maxRetry)
}
예제 #2
0
파일: jd.go 프로젝트: qgweb/new
//抓取京东商品页面
func GrabJDHTML(url string) string {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Error(err)
		return ""
	}

	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Pragma", "no-cache")

	if req.UserAgent() == "" {
		l := len(agent.UserAgents["common"])
		r := rand.New(rand.NewSource(time.Now().UnixNano()))
		req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
	}
	//	client.Lock()
	//	defer client.Unlock()
	resp, err := client.Do(req)
	if err != nil {
		log.Error(err)
		return ""
	}

	if resp.Body != nil {
		defer resp.Body.Close()
		return changeCharsetEncodingAuto(resp.Body, resp.Header.Get("Content-Type"))
	}
	return ""
}
예제 #3
0
파일: storage.go 프로젝트: qgweb/new
func (this *DispathWriter) uniq(fn string) {
	generator := exec.Command("sort", fn)
	consumer := exec.Command("uniq")

	p, err := generator.StdoutPipe()
	if err != nil {
		log.Error(err)
		return
	}
	generator.Start()
	consumer.Stdin = p
	pp, err := consumer.StdoutPipe()
	if err != nil {
		log.Error(err)
		return
	}
	consumer.Start()
	f, err := os.Create(fn + ".bak")
	if err != nil {
		log.Error(err)
		return
	}
	io.Copy(f, pp)
	f.Close()
	os.Rename(fn+".bak", fn)
}
예제 #4
0
파일: url.go 프로젝트: qgweb/new
func (this *Domain) run() {
	defer func() {
		if err := recover(); err != nil {
			log.Error(err)
			debug.PrintStack()
		}
	}()

	this.kvf.AddFun(this.domainData)
	log.Error(this.kvf.WriteFile())
	this.cleanPutTable()
	log.Error(this.kvf.Origin(this.saveData))
	this.mgb.Flush()
}
예제 #5
0
파일: other.go 프로젝트: qgweb/new
func (this *UserTrack) run() {
	defer func() {
		if err := recover(); err != nil {
			log.Error(err)
			debug.PrintStack()
		}
	}()
	this.kvf.AddFun(this.businessData)
	this.kvf.AddFun(this.otherData)
	log.Error(this.kvf.WriteFile())
	this.cleanPutTable()
	log.Error(this.kvf.Origin(this.saveData))
	this.mgb.Flush()
	this.mgb_big.Flush()
}
예제 #6
0
// CurrentVersion implements the VersionProvider's GetCurrentVer interface.
func (l *LocalVersionProvider) CurrentVersion() (kv.Version, error) {
	l.mu.Lock()
	defer l.mu.Unlock()

	for {
		var ts uint64
		ts = uint64((time.Now().UnixNano() / int64(time.Millisecond)) << timePrecisionOffset)

		if l.lastTimestamp > ts {
			log.Error("invalid physical time stamp")
			continue
		}

		if l.lastTimestamp == uint64(ts) {
			l.logical++
			if l.logical >= 1<<timePrecisionOffset {
				return kv.Version{}, ErrOverflow
			}
			return kv.Version{Ver: ts + l.logical}, nil
		}
		l.lastTimestamp = ts
		l.logical = 0
		return kv.Version{Ver: ts}, nil
	}
}
예제 #7
0
파일: server.go 프로젝트: pingcap/tidb
func (s *Server) startStatusHTTP() {
	once.Do(func() {
		go func() {
			http.HandleFunc("/status", func(w http.ResponseWriter, req *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				s := status{
					Connections: s.ConnectionCount(),
					Version:     mysql.ServerVersion,
					GitHash:     printer.TiDBGitHash,
				}
				js, err := json.Marshal(s)
				if err != nil {
					w.WriteHeader(http.StatusInternalServerError)
					log.Error("Encode json error", err)
				} else {
					w.Write(js)
				}

			})
			// HTTP path for prometheus.
			http.Handle("/metrics", prometheus.Handler())
			addr := s.cfg.StatusAddr
			if len(addr) == 0 {
				addr = defaultStatusAddr
			}
			log.Infof("Listening on %v for status and metrics report.", addr)
			err := http.ListenAndServe(addr, nil)
			if err != nil {
				log.Fatal(err)
			}
		}()
	})
}
예제 #8
0
func (t *testOrderBySuit) TestOrderBy(c *C) {
	tblPlan := &testTablePlan{t.data, []string{"id", "name"}}

	pln := &OrderByDefaultPlan{
		SelectList: &SelectList{
			HiddenFieldOffset: len(tblPlan.GetFields()),
			ResultFields:      tblPlan.GetFields(),
		},
		Src: tblPlan,
		By: []expression.Expression{
			&expressions.Ident{
				CIStr: model.NewCIStr("id"),
			},
		},
		Ascs: []bool{false},
	}

	prev := 10000
	err := pln.Do(nil, func(id interface{}, data []interface{}) (bool, error) {
		// DESC
		if data[0].(int) > prev {
			c.Error("should no be here", data[0], prev)
		}
		prev = data[0].(int)
		return true, nil
	})
	if err != nil {
		log.Error(err)
	}
}
예제 #9
0
func GetFeedById(channelName string, id int) *Feed {
	var ret *Feed
	err := db.Update(func(tx *bolt.Tx) error {
		bucketName := append(channelBucketNamePrefix, []byte(channelName)...)
		b, err := tx.CreateBucketIfNotExists(bucketName)
		if err != nil {
			return err
		}
		key := fmt.Sprintf("item:id:%d", id)
		bb := b.Get([]byte(key))
		if bb != nil {
			var t Feed
			err := json.Unmarshal(bb, &t)
			if err != nil {
				return err
			}
			ret = &t
		}
		return nil
	})
	if err != nil {
		log.Error(err)
	}
	return ret
}
예제 #10
0
파일: js.go 프로젝트: qgweb/new
// 保存投放轨迹到电信ftp
func (this *JsPut) saveTraceToDianxin() {
	var (
		ftp     = lib.GetConfVal("jiangsu::ftp_path")
		ppath   = lib.GetConfVal("jiangsu::put_path")
		rk      = "account.10046.sha1." + time.Now().Add(-time.Hour).Format("200601021504")
		fname   = ppath + "/" + rk
		adcount = 0
	)

	f, err := os.Create(fname)
	if err != nil {
		log.Error("创建文件失败", err)
		return
	}
	defer f.Close()

	this.kf.AdSet(func(ad string) {
		if v, ok := this.areamap[ad]; ok {
			f.WriteString(ad + "," + v + "\n")
			adcount++
		}
	})
	cmd := exec.Command(ftp, rk)
	str, err := cmd.Output()
	log.Info(string(str), err)

	// 广告数量统计数据 dx_stats , js_1461016800, 11111
	lib.StatisticsData("dx_stats", fmt.Sprintf("js_%s", timestamp.GetHourTimestamp(-1)),
		convert.ToString(adcount), "")
}
예제 #11
0
func (t *Table) unflatten(rec interface{}, col *column.Col) (interface{}, error) {
	if rec == nil {
		return nil, nil
	}
	switch col.Tp {
	case mysql.TypeFloat:
		return float32(rec.(float64)), nil
	case mysql.TypeTiny, mysql.TypeShort, mysql.TypeYear, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong,
		mysql.TypeDouble, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob,
		mysql.TypeVarchar, mysql.TypeString:
		return rec, nil
	case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp:
		var t mysql.Time
		t.Type = col.Tp
		t.Fsp = col.Decimal
		err := t.Unmarshal(rec.([]byte))
		if err != nil {
			return nil, errors.Trace(err)
		}
		return t, nil
	case mysql.TypeDuration:
		return mysql.Duration{Duration: time.Duration(rec.(int64)), Fsp: col.Decimal}, nil
	case mysql.TypeNewDecimal, mysql.TypeDecimal:
		return mysql.ParseDecimal(rec.(string))
	case mysql.TypeEnum:
		return mysql.ParseEnumValue(col.Elems, rec.(uint64))
	case mysql.TypeSet:
		return mysql.ParseSetValue(col.Elems, rec.(uint64))
	case mysql.TypeBit:
		return mysql.Bit{Value: rec.(uint64), Width: col.Flen}, nil
	}
	log.Error(col.Tp, rec, reflect.TypeOf(rec))
	return nil, nil
}
예제 #12
0
// GetSchemaVersion gets schema version in the context.
func GetSchemaVersion(ctx context.Context) int64 {
	v, ok := ctx.Value(schemaVersionKey).(int64)
	if !ok {
		log.Error("get schema version failed")
	}
	return v
}
예제 #13
0
파일: etcd.go 프로젝트: Goyoo/codis-docker
func (e *etcdImpl) doKeepAlive(key string, ttl uint64) error {
	conn, err := e.pool.Get()
	if err != nil {
		return err
	}

	defer e.pool.Put(conn)
	c := conn.(*PooledEtcdClient).c

	resp, err := c.Get(key, false, false)
	if err != nil {
		log.Error(err)
		return err
	}

	if resp.Node.Dir {
		return fmt.Errorf("can not set ttl to directory", key)
	}

	//log.Info("keep alive ", key)
	resp, err = c.CompareAndSwap(key, resp.Node.Value, ttl, resp.Node.Value, resp.Node.ModifiedIndex)
	if err == nil {
		return nil
	}

	if ec, ok := err.(*etcd.EtcdError); ok && ec.ErrorCode == etcderr.EcodeTestFailed {
		return nil
	}

	return err
}
예제 #14
0
파일: zj.go 프로젝트: qgweb/new
// 保存投放轨迹到投放系统
func (this *ZjPut) saveTraceToPutSys() {
	rdb, err := lib.GetPutRedisObj("put_redis_proxy_url")
	if err != nil {
		log.Error("redis连接失败", err)
		return
	}
	go func() {
		for {
			rdb.Receive()
		}
	}()
	//rdb.SelectDb("1")
	adcount := 0
	this.kf.AdUaIdsSet(func(ad string, ua string, aids map[string]int8) {
		key := ad
		if ua != "ua" {
			key = encrypt.DefaultMd5.Encode(ad + "_" + ua)
		}
		for aid, _ := range aids {
			rdb.HSet(key, "advert:"+aid, aid)
		}
		rdb.Expire(key, 5400)
		adcount++
	})
	rdb.Flush()
	rdb.Close()
	// 广告数量统计数据 put_stats , Zj_1461016800, 11111
	lib.StatisticsData("put_stats", fmt.Sprintf("zj_%s", this.Timestamp),
		convert.ToString(adcount), "")
}
예제 #15
0
파일: dashboard.go 프로젝트: vebin/reborn
func cmdDashboard(argv []string) (err error) {
	usage := `usage: reborn-config dashboard [--addr=<address>] [--http-log=<log_file>]

options:
	--addr		listen ip:port, e.g. localhost:12345, :8086, [default: :8086]
	--http-log	http request log [default: request.log ]
`

	args, err := docopt.Parse(usage, argv, true, "", false)
	if err != nil {
		log.Error(err)
		return errors.Trace(err)
	}
	log.Debug(args)

	logFileName := "request.log"
	if args["--http-log"] != nil {
		logFileName = args["--http-log"].(string)
	}

	addr := ":8086"
	if args["--addr"] != nil {
		addr = args["--addr"].(string)
	}

	runDashboard(addr, logFileName)
	return nil
}
예제 #16
0
파일: statement.go 프로젝트: XuHuaiyu/tidb
func (ps *perfSchema) EndStatement(state *StatementState) {
	if !enablePerfSchema {
		return
	}
	if state == nil {
		return
	}

	switch state.timerName {
	case timerNameNanosec:
		state.timerEnd = time.Now().UnixNano()
	case timerNameMicrosec:
		state.timerEnd = time.Now().UnixNano() / int64(time.Microsecond)
	case timerNameMillisec:
		state.timerEnd = time.Now().UnixNano() / int64(time.Millisecond)
	default:
		return
	}

	log.Debugf("EndStatement: sql %s, connection id %d, type %s", state.sqlText, state.connID, state.stmtType)

	record := state2Record(state)
	err := ps.updateEventsStmtsCurrent(state.connID, record)
	if err != nil {
		log.Error("Unable to update events_statements_current table")
	}
	err = ps.appendEventsStmtsHistory(record)
	if err != nil {
		log.Errorf("Unable to append to events_statements_history table %v", errors.ErrorStack(err))
	}
}
예제 #17
0
파일: txn.go 프로젝트: ninefive/tidb
// RunInNewTxn will run the f in a new transaction evnironment.
func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error {
	for {
		txn, err := store.Begin()
		if err != nil {
			log.Error(err)
			return errors.Trace(err)
		}

		err = f(txn)
		if retryable && IsRetryableError(err) {
			log.Warnf("Retry txn %v", txn)
			txn.Rollback()
			continue
		}
		if err != nil {
			return errors.Trace(err)
		}

		err = txn.Commit()
		if retryable && IsRetryableError(err) {
			log.Warnf("Retry txn %v", txn)
			txn.Rollback()
			continue
		}
		if err != nil {
			return errors.Trace(err)
		}
		break
	}

	return nil
}
예제 #18
0
파일: router.go 프로젝트: GavinHwa/reborn
func (s *Server) responseAction(seq int64) {
	log.Info("send response", seq)
	err := s.top.DoResponse(int(seq), &s.pi)
	if err != nil {
		log.Error(errors.ErrorStack(err))
	}
}
예제 #19
0
파일: conn.go 프로젝트: netroby/tidb
func (cc *clientConn) Run() {
	defer func() {
		r := recover()
		if r != nil {
			const size = 4096
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Errorf("lastCmd %s, %v, %s", cc.lastCmd, r, buf)
		}
		cc.Close()
	}()

	for {
		cc.alloc.Reset()
		data, err := cc.readPacket()
		if err != nil {
			if terror.ErrorNotEqual(err, io.EOF) {
				log.Error(err)
			}
			return
		}

		if err := cc.dispatch(data); err != nil {
			if terror.ErrorEqual(err, io.EOF) {
				return
			}
			log.Errorf("dispatch error %s, %s", errors.ErrorStack(err), cc)
			log.Errorf("cmd: %s", string(data[1:]))
			cc.writeError(err)
		}

		cc.pkg.sequence = 0
	}
}
예제 #20
0
파일: zj.go 프로젝트: qgweb/new
// 域名找回信息获取
func (this *ZjPut) VisitorData(out chan interface{}, in chan int8) {
	var datacount = 0
	defer func() {
		// 统计数据 zhejiang_put , other_1461016800, 11111
		lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_visitor",
			convert.ToString(datacount), "")
	}()
	m, err := lib.GetMongoObj()
	if err != nil {
		log.Error(err)
		in <- 1
		return
	}
	defer m.Close()

	qconf := mongodb.MongodbQueryConf{}
	qconf.Db = "data_source"
	qconf.Table = "zhejiang_visitor"
	qconf.Query = mongodb.MM{}
	m.Query(qconf, func(info map[string]interface{}) {
		ad := convert.ToString(info["ad"])
		ua := encrypt.DefaultMd5.Encode(encrypt.DefaultBase64.Decode(convert.ToString(info["ua"])))
		aids := convert.ToString(info["aids"])
		datacount++
		out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, aids)
	})
	log.Info("访客ok")
	in <- 1
}
예제 #21
0
파일: kv.go 프로젝트: jmptrader/tidb
func parsePath(path string) (etcdAddrs []string, clusterID uint64, disableGC bool, err error) {
	var u *url.URL
	u, err = url.Parse(path)
	if err != nil {
		err = errors.Trace(err)
		return
	}
	if strings.ToLower(u.Scheme) != "tikv" {
		err = errors.Errorf("Uri scheme expected[tikv] but found [%s]", u.Scheme)
		log.Error(err)
		return
	}
	clusterID, err = strconv.ParseUint(u.Query().Get("cluster"), 10, 64)
	if err != nil {
		log.Errorf("Parse clusterID error [%s]", err)
		err = errors.Trace(err)
		return
	}
	switch strings.ToLower(u.Query().Get("disableGC")) {
	case "true":
		disableGC = true
	case "false", "":
	default:
		err = errors.New("disableGC flag should be true/false")
		return
	}
	etcdAddrs = strings.Split(u.Host, ",")
	return
}
예제 #22
0
파일: zj.go 프로젝트: qgweb/new
// 保存投放轨迹到电信redis
func (this *ZjPut) saveTraceToDianxin() {
	var (
		db      = lib.GetConfVal("zhejiang::dx_redis_db")
		pwd     = lib.GetConfVal("zhejiang::dx_redis_pwd")
		adcount = 0
	)

	rdb, err := lib.GetZJDxRedisObj()
	if err != nil {
		log.Error("redis连接失败", err)
		return
	}
	rdb.Auth(pwd)
	rdb.SelectDb(db)
	rdb.FlushDb()
	// ua默认md5加密
	go func() {
		for {
			rdb.Receive()
		}
	}()
	this.kf.AdUaIdsSet(func(ad string, ua string, ids map[string]int8) {
		var key = ad + "|" + strings.ToUpper(ua)
		rdb.Set(key, "1")
		adcount++
	})
	rdb.Flush()
	rdb.Close()

	// 广告数量统计数据 dx_stats , Zj_1461016800, 11111
	lib.StatisticsData("dx_stats", fmt.Sprintf("zj_%s", this.Timestamp),
		convert.ToString(adcount), "")
}
예제 #23
0
파일: main.go 프로젝트: yzl11/vessel
func insert(rowCount int) {
	ct := time.Now()
	wg := sync.WaitGroup{}
	for i := 0; i < rowCount; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()

			tx := themis.NewTxn(c)

			put := hbase.NewPut([]byte(fmt.Sprintf("Row_%d", i)))
			put.AddValue([]byte("cf"), []byte("q"), []byte(strconv.Itoa(i)))

			put2 := hbase.NewPut([]byte(fmt.Sprintf("SRow_%d", i)))
			put2.AddValue([]byte("cf"), []byte("q"), []byte(strconv.Itoa(i)))

			tx.Put(tblName1, put)
			tx.Put(tblName2, put2)

			err := tx.Commit()
			if err != nil {
				log.Error(err)
			}
		}(i)
	}

	wg.Wait()
	log.Errorf("insert %d row data, consum time %s", rowCount, time.Since(ct))
}
예제 #24
0
파일: import.go 프로젝트: qgweb/new
func BulkInsertData(reader *bufio.Reader, ktype string) {
	bk := es.Bulk()
	num := 0
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			break
		}
		info := strings.Split(line, "\t#\t")
		log.Info(len(info))
		if len(info) != 2 {
			continue
		}
		ad, ua := getAdUa(info[0], "\t")
		keyword := getKeyWord(info[1], "\t")
		lonlat := GetLonLat(ad)
		if lonlat == "" {
			continue
		}
		num++
		id := encrypt.DefaultMd5.Encode("1456185600" + ad + ua)
		pinfo := map[string]interface{}{
			"ad":  ad,
			"ua":  ua,
			ktype: keyword,
			"geo": lonlat,
		}
		bk.Add(elastic.NewBulkUpdateRequest().Index("map_trace").Type("map").Doc(pinfo).Id(id).DocAsUpsert(true))
		bk.Add(elastic.NewBulkUpdateRequest().Index("map_trace_search").Type("map").Doc(pinfo).Id(id).DocAsUpsert(true))
		if num%10000 == 0 {
			log.Error(bk.Do())
		}
	}
	log.Info(bk.Do())
}
예제 #25
0
파일: proxy.go 프로젝트: superwood/reborn
func cmdProxy(argv []string) (err error) {
	usage := `usage:
	reborn-config proxy list
	reborn-config proxy offline <proxy_name>
	reborn-config proxy online <proxy_name>
`
	args, err := docopt.Parse(usage, argv, true, "", false)
	if err != nil {
		log.Error(err)
		return err
	}
	log.Debug(args)

	if args["list"].(bool) {
		return runProxyList()
	}

	proxyName := args["<proxy_name>"].(string)
	if args["online"].(bool) {
		return runSetProxyStatus(proxyName, models.PROXY_STATE_ONLINE)
	}
	if args["offline"].(bool) {
		return runSetProxyStatus(proxyName, models.PROXY_STATE_MARK_OFFLINE)
	}
	return nil
}
예제 #26
0
파일: etc.go 프로젝트: kfandhl/tidb
// Clone copies a interface to another interface.
// It does a deep copy.
func Clone(from interface{}) (interface{}, error) {
	if from == nil {
		return nil, nil
	}
	switch x := from.(type) {
	case uint8, uint16, uint32, uint64, float32, float64,
		int16, int8, bool, string, int, int64, int32,
		mysql.Time, mysql.Duration, mysql.Decimal,
		mysql.Hex, mysql.Bit, mysql.Enum, mysql.Set:
		return x, nil
	case []byte:
		target := make([]byte, len(from.([]byte)))
		copy(target, from.([]byte))
		return target, nil
	case []interface{}:
		var r []interface{}
		for _, v := range from.([]interface{}) {
			vv, err := Clone(v)
			if err != nil {
				return nil, err
			}
			r = append(r, vv)
		}
		return r, nil
	default:
		log.Error(reflect.TypeOf(from))
		return nil, errors.Errorf("Clone invalid type %T", from)
	}
}
예제 #27
0
파일: compactor.go 프로젝트: netroby/tidb
func (gc *localstoreCompactor) checkExpiredKeysWorker() {
	gc.workerWaitGroup.Add(1)
	defer gc.workerWaitGroup.Done()
	for {
		select {
		case <-gc.stopCh:
			log.Info("GC stopped")
			return
		case <-gc.ticker.C:
			gc.mu.Lock()
			m := gc.recentKeys
			if len(m) == 0 {
				gc.mu.Unlock()
				continue
			}
			gc.recentKeys = make(map[string]struct{})
			gc.mu.Unlock()
			log.Info("GC trigger")
			for k := range m {
				err := gc.Compact(nil, []byte(k))
				if err != nil {
					log.Error(err)
				}
			}
		}
	}
}
예제 #28
0
파일: js.go 프로젝트: qgweb/new
// 初始化cox对应区域
func (this *JsPut) initArea() {
	this.areamap = make(map[string]string)
	f, err := os.Open(lib.GetConfVal("jiangsu::areapath"))
	if err != nil {
		log.Error(err)
		return
	}
	defer f.Close()

	bi := bufio.NewReader(f)
	for {
		line, err := bi.ReadString('\n')
		if err == io.EOF {
			break
		}

		//0006f21d119b032d59acc3c2b90f10624eeaebe8,511
		info := strings.Split(line, ",")
		if len(info) != 2 {
			continue
		}

		this.areamap[info[0]] = strings.TrimSpace(info[1])
	}
	log.Info("江苏区域数量", len(this.areamap))
}
예제 #29
0
func (gc *localstoreCompactor) deleteWorker() {
	gc.workerWaitGroup.Add(1)
	defer gc.workerWaitGroup.Done()
	cnt := 0
	batch := gc.db.NewBatch()
	for {
		select {
		case <-gc.stopCh:
			return
		case key := <-gc.delCh:
			{
				cnt++
				batch.Delete(key)
				// Batch delete.
				if cnt == gc.policy.BatchDeleteCnt {
					err := gc.db.Commit(batch)
					if err != nil {
						log.Error(err)
					}
					batch = gc.db.NewBatch()
					cnt = 0
				}
			}
		}
	}
}
예제 #30
0
파일: main.go 프로젝트: anywhy/tidb
func main() {
	tidb.RegisterLocalStore("boltdb", boltdb.Driver{})
	tidb.RegisterStore("tikv", tikv.Driver{})

	metric.RunMetric(3 * time.Second)
	printer.PrintTiDBInfo()
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()

	if *lease < 0 {
		log.Fatalf("invalid lease seconds %d", *lease)
	}

	tidb.SetSchemaLease(time.Duration(*lease) * time.Second)

	cfg := &server.Config{
		Addr:       fmt.Sprintf(":%s", *port),
		LogLevel:   *logLevel,
		StatusAddr: fmt.Sprintf(":%s", *statusPort),
		Socket:     *socket,
	}

	log.SetLevelByString(cfg.LogLevel)
	store, err := tidb.NewStore(fmt.Sprintf("%s://%s", *store, *storePath))
	if err != nil {
		log.Fatal(errors.ErrorStack(err))
	}
	// Create a session to load information schema.
	se, err := tidb.CreateSession(store)
	if err != nil {
		log.Fatal(errors.ErrorStack(err))
	}
	se.Close()

	var driver server.IDriver
	driver = server.NewTiDBDriver(store)
	var svr *server.Server
	svr, err = server.NewServer(cfg, driver)
	if err != nil {
		log.Fatal(errors.ErrorStack(err))
	}

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		sig := <-sc
		log.Infof("Got signal [%d] to exit.", sig)
		svr.Close()
		os.Exit(0)
	}()

	log.Error(svr.Run())
}