// FetchURL fetches url and saves as a file, it includes retry and decoding bson.
func (p *BsonProxy) FetchURL(ref string) (string, error) {
for tryCount := 0; tryCount < maxRetry; tryCount++ {
if tryCount > 0 {
log.Warnf("Retry after %d times", tryCount)
time.Sleep(retryInterval)
}
resp, err := p.c.Get(ref)
if err != nil {
log.Error(err)
continue
}
f, err := ioutil.TempFile(os.TempDir(), "duokan")
if err != nil {
log.Error(err)
continue
}
// Why use ioutil.ReadAll doesn't work? It seems resp.Body can't Close,
// so reader is blocking.
_, err = io.Copy(f, resp.Body)
if err != nil {
log.Errorf("Get error url[%s], err[%s]", ref, err)
continue
}
err = f.Close()
if err != nil {
log.Error(err)
continue
}
return f.Name(), nil
}
return "", errors.Errorf("reach max retry time [%d]", maxRetry)
}
//抓取京东商品页面
func GrabJDHTML(url string) string {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Error(err)
return ""
}
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.8")
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Connection", "keep-alive")
req.Header.Set("Pragma", "no-cache")
if req.UserAgent() == "" {
l := len(agent.UserAgents["common"])
r := rand.New(rand.NewSource(time.Now().UnixNano()))
req.Header.Set("User-Agent", agent.UserAgents["common"][r.Intn(l)])
}
// client.Lock()
// defer client.Unlock()
resp, err := client.Do(req)
if err != nil {
log.Error(err)
return ""
}
if resp.Body != nil {
defer resp.Body.Close()
return changeCharsetEncodingAuto(resp.Body, resp.Header.Get("Content-Type"))
}
return ""
}
func (this *DispathWriter) uniq(fn string) {
generator := exec.Command("sort", fn)
consumer := exec.Command("uniq")
p, err := generator.StdoutPipe()
if err != nil {
log.Error(err)
return
}
generator.Start()
consumer.Stdin = p
pp, err := consumer.StdoutPipe()
if err != nil {
log.Error(err)
return
}
consumer.Start()
f, err := os.Create(fn + ".bak")
if err != nil {
log.Error(err)
return
}
io.Copy(f, pp)
f.Close()
os.Rename(fn+".bak", fn)
}
func (this *Domain) run() {
defer func() {
if err := recover(); err != nil {
log.Error(err)
debug.PrintStack()
}
}()
this.kvf.AddFun(this.domainData)
log.Error(this.kvf.WriteFile())
this.cleanPutTable()
log.Error(this.kvf.Origin(this.saveData))
this.mgb.Flush()
}
func (this *UserTrack) run() {
defer func() {
if err := recover(); err != nil {
log.Error(err)
debug.PrintStack()
}
}()
this.kvf.AddFun(this.businessData)
this.kvf.AddFun(this.otherData)
log.Error(this.kvf.WriteFile())
this.cleanPutTable()
log.Error(this.kvf.Origin(this.saveData))
this.mgb.Flush()
this.mgb_big.Flush()
}
// CurrentVersion implements the VersionProvider's GetCurrentVer interface.
func (l *LocalVersionProvider) CurrentVersion() (kv.Version, error) {
l.mu.Lock()
defer l.mu.Unlock()
for {
var ts uint64
ts = uint64((time.Now().UnixNano() / int64(time.Millisecond)) << timePrecisionOffset)
if l.lastTimestamp > ts {
log.Error("invalid physical time stamp")
continue
}
if l.lastTimestamp == uint64(ts) {
l.logical++
if l.logical >= 1<<timePrecisionOffset {
return kv.Version{}, ErrOverflow
}
return kv.Version{Ver: ts + l.logical}, nil
}
l.lastTimestamp = ts
l.logical = 0
return kv.Version{Ver: ts}, nil
}
}
func (s *Server) startStatusHTTP() {
once.Do(func() {
go func() {
http.HandleFunc("/status", func(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
s := status{
Connections: s.ConnectionCount(),
Version: mysql.ServerVersion,
GitHash: printer.TiDBGitHash,
}
js, err := json.Marshal(s)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Error("Encode json error", err)
} else {
w.Write(js)
}
})
// HTTP path for prometheus.
http.Handle("/metrics", prometheus.Handler())
addr := s.cfg.StatusAddr
if len(addr) == 0 {
addr = defaultStatusAddr
}
log.Infof("Listening on %v for status and metrics report.", addr)
err := http.ListenAndServe(addr, nil)
if err != nil {
log.Fatal(err)
}
}()
})
}
func (t *testOrderBySuit) TestOrderBy(c *C) {
tblPlan := &testTablePlan{t.data, []string{"id", "name"}}
pln := &OrderByDefaultPlan{
SelectList: &SelectList{
HiddenFieldOffset: len(tblPlan.GetFields()),
ResultFields: tblPlan.GetFields(),
},
Src: tblPlan,
By: []expression.Expression{
&expressions.Ident{
CIStr: model.NewCIStr("id"),
},
},
Ascs: []bool{false},
}
prev := 10000
err := pln.Do(nil, func(id interface{}, data []interface{}) (bool, error) {
// DESC
if data[0].(int) > prev {
c.Error("should no be here", data[0], prev)
}
prev = data[0].(int)
return true, nil
})
if err != nil {
log.Error(err)
}
}
func GetFeedById(channelName string, id int) *Feed {
var ret *Feed
err := db.Update(func(tx *bolt.Tx) error {
bucketName := append(channelBucketNamePrefix, []byte(channelName)...)
b, err := tx.CreateBucketIfNotExists(bucketName)
if err != nil {
return err
}
key := fmt.Sprintf("item:id:%d", id)
bb := b.Get([]byte(key))
if bb != nil {
var t Feed
err := json.Unmarshal(bb, &t)
if err != nil {
return err
}
ret = &t
}
return nil
})
if err != nil {
log.Error(err)
}
return ret
}
// 保存投放轨迹到电信ftp
func (this *JsPut) saveTraceToDianxin() {
var (
ftp = lib.GetConfVal("jiangsu::ftp_path")
ppath = lib.GetConfVal("jiangsu::put_path")
rk = "account.10046.sha1." + time.Now().Add(-time.Hour).Format("200601021504")
fname = ppath + "/" + rk
adcount = 0
)
f, err := os.Create(fname)
if err != nil {
log.Error("创建文件失败", err)
return
}
defer f.Close()
this.kf.AdSet(func(ad string) {
if v, ok := this.areamap[ad]; ok {
f.WriteString(ad + "," + v + "\n")
adcount++
}
})
cmd := exec.Command(ftp, rk)
str, err := cmd.Output()
log.Info(string(str), err)
// 广告数量统计数据 dx_stats , js_1461016800, 11111
lib.StatisticsData("dx_stats", fmt.Sprintf("js_%s", timestamp.GetHourTimestamp(-1)),
convert.ToString(adcount), "")
}
func (t *Table) unflatten(rec interface{}, col *column.Col) (interface{}, error) {
if rec == nil {
return nil, nil
}
switch col.Tp {
case mysql.TypeFloat:
return float32(rec.(float64)), nil
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeYear, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong,
mysql.TypeDouble, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob,
mysql.TypeVarchar, mysql.TypeString:
return rec, nil
case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp:
var t mysql.Time
t.Type = col.Tp
t.Fsp = col.Decimal
err := t.Unmarshal(rec.([]byte))
if err != nil {
return nil, errors.Trace(err)
}
return t, nil
case mysql.TypeDuration:
return mysql.Duration{Duration: time.Duration(rec.(int64)), Fsp: col.Decimal}, nil
case mysql.TypeNewDecimal, mysql.TypeDecimal:
return mysql.ParseDecimal(rec.(string))
case mysql.TypeEnum:
return mysql.ParseEnumValue(col.Elems, rec.(uint64))
case mysql.TypeSet:
return mysql.ParseSetValue(col.Elems, rec.(uint64))
case mysql.TypeBit:
return mysql.Bit{Value: rec.(uint64), Width: col.Flen}, nil
}
log.Error(col.Tp, rec, reflect.TypeOf(rec))
return nil, nil
}
// GetSchemaVersion gets schema version in the context.
func GetSchemaVersion(ctx context.Context) int64 {
v, ok := ctx.Value(schemaVersionKey).(int64)
if !ok {
log.Error("get schema version failed")
}
return v
}
func (e *etcdImpl) doKeepAlive(key string, ttl uint64) error {
conn, err := e.pool.Get()
if err != nil {
return err
}
defer e.pool.Put(conn)
c := conn.(*PooledEtcdClient).c
resp, err := c.Get(key, false, false)
if err != nil {
log.Error(err)
return err
}
if resp.Node.Dir {
return fmt.Errorf("can not set ttl to directory", key)
}
//log.Info("keep alive ", key)
resp, err = c.CompareAndSwap(key, resp.Node.Value, ttl, resp.Node.Value, resp.Node.ModifiedIndex)
if err == nil {
return nil
}
if ec, ok := err.(*etcd.EtcdError); ok && ec.ErrorCode == etcderr.EcodeTestFailed {
return nil
}
return err
}
// 保存投放轨迹到投放系统
func (this *ZjPut) saveTraceToPutSys() {
rdb, err := lib.GetPutRedisObj("put_redis_proxy_url")
if err != nil {
log.Error("redis连接失败", err)
return
}
go func() {
for {
rdb.Receive()
}
}()
//rdb.SelectDb("1")
adcount := 0
this.kf.AdUaIdsSet(func(ad string, ua string, aids map[string]int8) {
key := ad
if ua != "ua" {
key = encrypt.DefaultMd5.Encode(ad + "_" + ua)
}
for aid, _ := range aids {
rdb.HSet(key, "advert:"+aid, aid)
}
rdb.Expire(key, 5400)
adcount++
})
rdb.Flush()
rdb.Close()
// 广告数量统计数据 put_stats , Zj_1461016800, 11111
lib.StatisticsData("put_stats", fmt.Sprintf("zj_%s", this.Timestamp),
convert.ToString(adcount), "")
}
func cmdDashboard(argv []string) (err error) {
usage := `usage: reborn-config dashboard [--addr=<address>] [--http-log=<log_file>]
options:
--addr listen ip:port, e.g. localhost:12345, :8086, [default: :8086]
--http-log http request log [default: request.log ]
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.Error(err)
return errors.Trace(err)
}
log.Debug(args)
logFileName := "request.log"
if args["--http-log"] != nil {
logFileName = args["--http-log"].(string)
}
addr := ":8086"
if args["--addr"] != nil {
addr = args["--addr"].(string)
}
runDashboard(addr, logFileName)
return nil
}
func (ps *perfSchema) EndStatement(state *StatementState) {
if !enablePerfSchema {
return
}
if state == nil {
return
}
switch state.timerName {
case timerNameNanosec:
state.timerEnd = time.Now().UnixNano()
case timerNameMicrosec:
state.timerEnd = time.Now().UnixNano() / int64(time.Microsecond)
case timerNameMillisec:
state.timerEnd = time.Now().UnixNano() / int64(time.Millisecond)
default:
return
}
log.Debugf("EndStatement: sql %s, connection id %d, type %s", state.sqlText, state.connID, state.stmtType)
record := state2Record(state)
err := ps.updateEventsStmtsCurrent(state.connID, record)
if err != nil {
log.Error("Unable to update events_statements_current table")
}
err = ps.appendEventsStmtsHistory(record)
if err != nil {
log.Errorf("Unable to append to events_statements_history table %v", errors.ErrorStack(err))
}
}
// RunInNewTxn will run the f in a new transaction evnironment.
func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error {
for {
txn, err := store.Begin()
if err != nil {
log.Error(err)
return errors.Trace(err)
}
err = f(txn)
if retryable && IsRetryableError(err) {
log.Warnf("Retry txn %v", txn)
txn.Rollback()
continue
}
if err != nil {
return errors.Trace(err)
}
err = txn.Commit()
if retryable && IsRetryableError(err) {
log.Warnf("Retry txn %v", txn)
txn.Rollback()
continue
}
if err != nil {
return errors.Trace(err)
}
break
}
return nil
}
func (s *Server) responseAction(seq int64) {
log.Info("send response", seq)
err := s.top.DoResponse(int(seq), &s.pi)
if err != nil {
log.Error(errors.ErrorStack(err))
}
}
func (cc *clientConn) Run() {
defer func() {
r := recover()
if r != nil {
const size = 4096
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
log.Errorf("lastCmd %s, %v, %s", cc.lastCmd, r, buf)
}
cc.Close()
}()
for {
cc.alloc.Reset()
data, err := cc.readPacket()
if err != nil {
if terror.ErrorNotEqual(err, io.EOF) {
log.Error(err)
}
return
}
if err := cc.dispatch(data); err != nil {
if terror.ErrorEqual(err, io.EOF) {
return
}
log.Errorf("dispatch error %s, %s", errors.ErrorStack(err), cc)
log.Errorf("cmd: %s", string(data[1:]))
cc.writeError(err)
}
cc.pkg.sequence = 0
}
}
// 域名找回信息获取
func (this *ZjPut) VisitorData(out chan interface{}, in chan int8) {
var datacount = 0
defer func() {
// 统计数据 zhejiang_put , other_1461016800, 11111
lib.StatisticsData("dsource_stats", "zj_"+this.Timestamp+"_visitor",
convert.ToString(datacount), "")
}()
m, err := lib.GetMongoObj()
if err != nil {
log.Error(err)
in <- 1
return
}
defer m.Close()
qconf := mongodb.MongodbQueryConf{}
qconf.Db = "data_source"
qconf.Table = "zhejiang_visitor"
qconf.Query = mongodb.MM{}
m.Query(qconf, func(info map[string]interface{}) {
ad := convert.ToString(info["ad"])
ua := encrypt.DefaultMd5.Encode(encrypt.DefaultBase64.Decode(convert.ToString(info["ua"])))
aids := convert.ToString(info["aids"])
datacount++
out <- fmt.Sprintf("%s\t%s\t%s", ad, ua, aids)
})
log.Info("访客ok")
in <- 1
}
func parsePath(path string) (etcdAddrs []string, clusterID uint64, disableGC bool, err error) {
var u *url.URL
u, err = url.Parse(path)
if err != nil {
err = errors.Trace(err)
return
}
if strings.ToLower(u.Scheme) != "tikv" {
err = errors.Errorf("Uri scheme expected[tikv] but found [%s]", u.Scheme)
log.Error(err)
return
}
clusterID, err = strconv.ParseUint(u.Query().Get("cluster"), 10, 64)
if err != nil {
log.Errorf("Parse clusterID error [%s]", err)
err = errors.Trace(err)
return
}
switch strings.ToLower(u.Query().Get("disableGC")) {
case "true":
disableGC = true
case "false", "":
default:
err = errors.New("disableGC flag should be true/false")
return
}
etcdAddrs = strings.Split(u.Host, ",")
return
}
// 保存投放轨迹到电信redis
func (this *ZjPut) saveTraceToDianxin() {
var (
db = lib.GetConfVal("zhejiang::dx_redis_db")
pwd = lib.GetConfVal("zhejiang::dx_redis_pwd")
adcount = 0
)
rdb, err := lib.GetZJDxRedisObj()
if err != nil {
log.Error("redis连接失败", err)
return
}
rdb.Auth(pwd)
rdb.SelectDb(db)
rdb.FlushDb()
// ua默认md5加密
go func() {
for {
rdb.Receive()
}
}()
this.kf.AdUaIdsSet(func(ad string, ua string, ids map[string]int8) {
var key = ad + "|" + strings.ToUpper(ua)
rdb.Set(key, "1")
adcount++
})
rdb.Flush()
rdb.Close()
// 广告数量统计数据 dx_stats , Zj_1461016800, 11111
lib.StatisticsData("dx_stats", fmt.Sprintf("zj_%s", this.Timestamp),
convert.ToString(adcount), "")
}
func insert(rowCount int) {
ct := time.Now()
wg := sync.WaitGroup{}
for i := 0; i < rowCount; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
tx := themis.NewTxn(c)
put := hbase.NewPut([]byte(fmt.Sprintf("Row_%d", i)))
put.AddValue([]byte("cf"), []byte("q"), []byte(strconv.Itoa(i)))
put2 := hbase.NewPut([]byte(fmt.Sprintf("SRow_%d", i)))
put2.AddValue([]byte("cf"), []byte("q"), []byte(strconv.Itoa(i)))
tx.Put(tblName1, put)
tx.Put(tblName2, put2)
err := tx.Commit()
if err != nil {
log.Error(err)
}
}(i)
}
wg.Wait()
log.Errorf("insert %d row data, consum time %s", rowCount, time.Since(ct))
}
func BulkInsertData(reader *bufio.Reader, ktype string) {
bk := es.Bulk()
num := 0
for {
line, err := reader.ReadString('\n')
if err == io.EOF {
break
}
info := strings.Split(line, "\t#\t")
log.Info(len(info))
if len(info) != 2 {
continue
}
ad, ua := getAdUa(info[0], "\t")
keyword := getKeyWord(info[1], "\t")
lonlat := GetLonLat(ad)
if lonlat == "" {
continue
}
num++
id := encrypt.DefaultMd5.Encode("1456185600" + ad + ua)
pinfo := map[string]interface{}{
"ad": ad,
"ua": ua,
ktype: keyword,
"geo": lonlat,
}
bk.Add(elastic.NewBulkUpdateRequest().Index("map_trace").Type("map").Doc(pinfo).Id(id).DocAsUpsert(true))
bk.Add(elastic.NewBulkUpdateRequest().Index("map_trace_search").Type("map").Doc(pinfo).Id(id).DocAsUpsert(true))
if num%10000 == 0 {
log.Error(bk.Do())
}
}
log.Info(bk.Do())
}
func cmdProxy(argv []string) (err error) {
usage := `usage:
reborn-config proxy list
reborn-config proxy offline <proxy_name>
reborn-config proxy online <proxy_name>
`
args, err := docopt.Parse(usage, argv, true, "", false)
if err != nil {
log.Error(err)
return err
}
log.Debug(args)
if args["list"].(bool) {
return runProxyList()
}
proxyName := args["<proxy_name>"].(string)
if args["online"].(bool) {
return runSetProxyStatus(proxyName, models.PROXY_STATE_ONLINE)
}
if args["offline"].(bool) {
return runSetProxyStatus(proxyName, models.PROXY_STATE_MARK_OFFLINE)
}
return nil
}
// Clone copies a interface to another interface.
// It does a deep copy.
func Clone(from interface{}) (interface{}, error) {
if from == nil {
return nil, nil
}
switch x := from.(type) {
case uint8, uint16, uint32, uint64, float32, float64,
int16, int8, bool, string, int, int64, int32,
mysql.Time, mysql.Duration, mysql.Decimal,
mysql.Hex, mysql.Bit, mysql.Enum, mysql.Set:
return x, nil
case []byte:
target := make([]byte, len(from.([]byte)))
copy(target, from.([]byte))
return target, nil
case []interface{}:
var r []interface{}
for _, v := range from.([]interface{}) {
vv, err := Clone(v)
if err != nil {
return nil, err
}
r = append(r, vv)
}
return r, nil
default:
log.Error(reflect.TypeOf(from))
return nil, errors.Errorf("Clone invalid type %T", from)
}
}
func (gc *localstoreCompactor) checkExpiredKeysWorker() {
gc.workerWaitGroup.Add(1)
defer gc.workerWaitGroup.Done()
for {
select {
case <-gc.stopCh:
log.Info("GC stopped")
return
case <-gc.ticker.C:
gc.mu.Lock()
m := gc.recentKeys
if len(m) == 0 {
gc.mu.Unlock()
continue
}
gc.recentKeys = make(map[string]struct{})
gc.mu.Unlock()
log.Info("GC trigger")
for k := range m {
err := gc.Compact(nil, []byte(k))
if err != nil {
log.Error(err)
}
}
}
}
}
// 初始化cox对应区域
func (this *JsPut) initArea() {
this.areamap = make(map[string]string)
f, err := os.Open(lib.GetConfVal("jiangsu::areapath"))
if err != nil {
log.Error(err)
return
}
defer f.Close()
bi := bufio.NewReader(f)
for {
line, err := bi.ReadString('\n')
if err == io.EOF {
break
}
//0006f21d119b032d59acc3c2b90f10624eeaebe8,511
info := strings.Split(line, ",")
if len(info) != 2 {
continue
}
this.areamap[info[0]] = strings.TrimSpace(info[1])
}
log.Info("江苏区域数量", len(this.areamap))
}
func (gc *localstoreCompactor) deleteWorker() {
gc.workerWaitGroup.Add(1)
defer gc.workerWaitGroup.Done()
cnt := 0
batch := gc.db.NewBatch()
for {
select {
case <-gc.stopCh:
return
case key := <-gc.delCh:
{
cnt++
batch.Delete(key)
// Batch delete.
if cnt == gc.policy.BatchDeleteCnt {
err := gc.db.Commit(batch)
if err != nil {
log.Error(err)
}
batch = gc.db.NewBatch()
cnt = 0
}
}
}
}
}
func main() {
tidb.RegisterLocalStore("boltdb", boltdb.Driver{})
tidb.RegisterStore("tikv", tikv.Driver{})
metric.RunMetric(3 * time.Second)
printer.PrintTiDBInfo()
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Parse()
if *lease < 0 {
log.Fatalf("invalid lease seconds %d", *lease)
}
tidb.SetSchemaLease(time.Duration(*lease) * time.Second)
cfg := &server.Config{
Addr: fmt.Sprintf(":%s", *port),
LogLevel: *logLevel,
StatusAddr: fmt.Sprintf(":%s", *statusPort),
Socket: *socket,
}
log.SetLevelByString(cfg.LogLevel)
store, err := tidb.NewStore(fmt.Sprintf("%s://%s", *store, *storePath))
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
// Create a session to load information schema.
se, err := tidb.CreateSession(store)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
se.Close()
var driver server.IDriver
driver = server.NewTiDBDriver(store)
var svr *server.Server
svr, err = server.NewServer(cfg, driver)
if err != nil {
log.Fatal(errors.ErrorStack(err))
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
go func() {
sig := <-sc
log.Infof("Got signal [%d] to exit.", sig)
svr.Close()
os.Exit(0)
}()
log.Error(svr.Run())
}