Exemplo n.º 1
0
func Start() {
	if !g.Config().Http.Enabled {
		return
	}

	addr := g.Config().Http.Listen
	if addr == "" {
		return
	}
	s := &http.Server{
		Addr:           addr,
		MaxHeaderBytes: 1 << 30,
	}
	log.Println("http listening", addr)

	ln, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalln(err)
		return
	}

	l := ln.(*net.TCPListener)

	go s.Serve(TcpKeepAliveListener{l})

	select {
	case <-Close_chan:
		log.Println("http recv sigout and exit...")
		l.Close()
		Close_done_chan <- 1
		return
	}

}
Exemplo n.º 2
0
Arquivo: rpc.go Projeto: peteryj/graph
func Start() {
	if !g.Config().Rpc.Enabled {
		return
	}
	addr := g.Config().Rpc.Listen
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		log.Fatalf("net.ResolveTCPAddr fail: %s", err)
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		log.Fatalf("listen %s fail: %s", addr, err)
	} else {
		log.Println("rpc listening", addr)
	}

	rpc.Register(new(Graph))

	go func() {
		var tempDelay time.Duration // how long to sleep on accept failure
		for {
			conn, err := listener.Accept()
			if err != nil {
				if tempDelay == 0 {
					tempDelay = 5 * time.Millisecond
				} else {
					tempDelay *= 2
				}
				if max := 1 * time.Second; tempDelay > max {
					tempDelay = max
				}
				time.Sleep(tempDelay)
				continue
			}
			tempDelay = 0
			go func() {
				e := connects.insert(conn)
				defer connects.remove(e)
				rpc.ServeConn(conn)
			}()
		}
	}()

	select {
	case <-Close_chan:
		log.Println("api recv sigout and exit...")
		listener.Close()
		Close_done_chan <- 1

		connects.Lock()
		for e := connects.list.Front(); e != nil; e = e.Next() {
			e.Value.(net.Conn).Close()
		}
		connects.Unlock()

		return
	}

}
Exemplo n.º 3
0
func FlushRRD(idx int) {
	var debug_checksum string
	var debug bool

	storageDir := g.Config().RRD.Storage
	if g.Config().Debug {
		debug = true
		debug_checksum = g.Config().DebugChecksum
	} else {
		debug = false
	}

	keys := store.GraphItems.KeysByIndex(idx)
	if len(keys) == 0 {
		return
	}

	for _, checksum := range keys {

		items := store.GraphItems.PopAll(checksum)
		size := len(items)
		if size == 0 {
			continue
		}

		first := items[0]
		filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", storageDir, checksum[0:2], checksum, first.DsType, first.Step)
		if debug && debug_checksum == checksum {
			for _, item := range items {
				log.Printf(
					"2-flush:%d:%s:%lf",
					item.Timestamp,
					time.Unix(item.Timestamp, 0).Format("2006-01-02 15:04:05"),
					item.Value,
				)
			}
		}

		err := Flush(filename, items)
		if err != nil && debug && debug_checksum == checksum {
			log.Println("flush fail:", err, "filename:", filename)
		}
		Counter += 1
	}
	if debug {
		log.Println("flushrrd counter:", Counter)
	}
}
Exemplo n.º 4
0
func FlushRRD(idx int) {
	storageDir := g.Config().RRD.Storage

	keys := store.GraphItems.KeysByIndex(idx)
	if len(keys) == 0 {
		return
	}

	for _, ckey := range keys {
		// get md5, dstype, step
		checksum, dsType, step, err := g.SplitRrdCacheKey(ckey)
		if err != nil {
			continue
		}

		items := store.GraphItems.PopAll(ckey)
		size := len(items)
		if size == 0 {
			continue
		}

		filename := g.RrdFileName(storageDir, checksum, dsType, step)
		Flush(filename, items)
		Counter += 1
	}
}
Exemplo n.º 5
0
func configCommonRoutes() {
	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("ok"))
	})

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(g.VERSION))
	})

	http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, file.SelfDir())
	})

	http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, g.Config())
	})

	http.HandleFunc("/config/reload", func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.RemoteAddr, "127.0.0.1") {
			g.ParseConfig(g.ConfigFile)
			RenderDataJson(w, "ok")
		} else {
			RenderDataJson(w, "no privilege")
		}
	})
}
Exemplo n.º 6
0
func main() {
	cfg := flag.String("c", "cfg.json", "specify config file")
	version := flag.Bool("v", false, "show version")
	versionGit := flag.Bool("vg", false, "show version and git commit log")
	flag.Parse()

	if *version {
		fmt.Println(g.VERSION)
		os.Exit(0)
	}
	if *versionGit {
		fmt.Println(g.VERSION, g.COMMIT)
		os.Exit(0)
	}

	// global config
	g.ParseConfig(*cfg)
	// init db
	g.InitDB()
	// rrdtool before api for disable loopback connection
	rrdtool.Start()
	// start api
	go api.Start()
	// start indexing
	index.Start()
	// start http server
	go http.Start()

	start_signal(os.Getpid(), g.Config())
}
Exemplo n.º 7
0
// index收到一条新上报的监控数据,尝试用于增量更新索引
func ReceiveItem(item *cmodel.GraphItem, md5 string) {
	if item == nil {
		return
	}

	uuid := item.UUID()

	// 已上报过的数据
	if indexedItemCache.ContainsKey(md5) {
		old := indexedItemCache.Get(md5).(*IndexCacheItem)
		if uuid == old.UUID { // dsType+step没有发生变化,只更新缓存 TODO 存在线程安全的问题
			old.Item = item
		} else { // dsType+step变化了,当成一个新的增量来处理(甚至,不用rrd文件来过滤)
			//indexedItemCache.Remove(md5)
			unIndexedItemCache.Put(md5, NewIndexCacheItem(uuid, item))
		}
		return
	}

	// 是否有rrdtool文件存在,如果有 认为已建立索引
	// 针对 索引缓存重建场景 做的优化, 结合索引全量更新 来保证一致性
	rrdFileName := g.RrdFileName(g.Config().RRD.Storage, md5, item.DsType, item.Step)
	if g.IsRrdFileExist(rrdFileName) {
		indexedItemCache.Put(md5, NewIndexCacheItem(uuid, item))
		return
	}

	// 缓存未命中, 放入增量更新队列
	unIndexedItemCache.Put(md5, NewIndexCacheItem(uuid, item))
}
Exemplo n.º 8
0
func Start() {
	// check data dir
	dataDir := g.Config().RRD.Storage
	if err := file.EnsureDirRW(dataDir); err != nil {
		log.Fatalln("rrdtool.Start error, bad data dir", dataDir+",", err)
	}
	log.Println("rrdtool.Start, ok")
}
Exemplo n.º 9
0
func fetch_rrd(client **rpc.Client, key string, addr string) error {
	var (
		err      error
		flag     uint32
		md5      string
		dsType   string
		filename string
		step, i  int
		rrdfile  g.File
	)

	cfg := g.Config()

	if flag, err = store.GraphItems.GetFlag(key); err != nil {
		return err
	}

	store.GraphItems.SetFlag(key, flag|g.GRAPH_F_FETCHING)

	md5, dsType, step, _ = g.SplitRrdCacheKey(key)
	filename = g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.GetRrd", key, &rrdfile,
			time.Duration(cfg.CallTimeout)*time.Millisecond)

		if err == nil {
			done := make(chan error, 1)
			io_task_chan <- &io_task_t{
				method: IO_TASK_M_WRITE,
				args: &g.File{
					Filename: filename,
					Body:     rrdfile.Body[:],
				},
				done: done,
			}
			if err = <-done; err != nil {
				goto out
			} else {
				flag &= ^g.GRAPH_F_MISS
				goto out
			}
		} else {
			log.Println(err)
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
out:
	flag &= ^g.GRAPH_F_FETCHING
	store.GraphItems.SetFlag(key, flag)
	return err
}
Exemplo n.º 10
0
func CommitByKey(key string) {

	md5, dsType, step, err := g.SplitRrdCacheKey(key)
	if err != nil {
		return
	}
	filename := g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)

	items := store.GraphItems.PopAll(key)
	if len(items) == 0 {
		return
	}
	FlushFile(filename, items)
}
Exemplo n.º 11
0
func (this *Graph) GetRrd(key string, rrdfile *g.File) (err error) {
	if md5, dsType, step, err := g.SplitRrdCacheKey(key); err != nil {
		return err
	} else {
		rrdfile.Filename = g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
	}

	items := store.GraphItems.PopAll(key)
	if len(items) > 0 {
		rrdtool.FlushFile(rrdfile.Filename, items)
	}

	rrdfile.Body, err = rrdtool.ReadFile(rrdfile.Filename)
	return
}
Exemplo n.º 12
0
func Start() {
	cfg := g.Config()
	var err error
	// check data dir
	if err = file.EnsureDirRW(cfg.RRD.Storage); err != nil {
		log.Fatalln("rrdtool.Start error, bad data dir "+cfg.RRD.Storage+",", err)
	}

	migrate_start(cfg)

	// sync disk
	go syncDisk()
	go ioWorker()
	log.Println("rrdtool.Start ok")
}
Exemplo n.º 13
0
func (this *Graph) Info(param cmodel.GraphInfoParam, resp *cmodel.GraphInfoResp) error {
	// statistics
	proc.GraphInfoCnt.Incr()

	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
	if !exists {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)

	resp.ConsolFun = dsType
	resp.Step = step
	resp.Filename = filename

	return nil
}
Exemplo n.º 14
0
func send_data(client **rpc.Client, key string, addr string) error {
	var (
		err  error
		flag uint32
		resp *cmodel.SimpleRpcResponse
		i    int
	)

	//remote
	if flag, err = store.GraphItems.GetFlag(key); err != nil {
		return err
	}
	cfg := g.Config()

	store.GraphItems.SetFlag(key, flag|g.GRAPH_F_SENDING)

	items := store.GraphItems.PopAll(key)
	items_size := len(items)
	if items_size == 0 {
		goto out
	}
	resp = &cmodel.SimpleRpcResponse{}

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.Send", items, resp,
			time.Duration(cfg.CallTimeout)*time.Millisecond)

		if err == nil {
			goto out
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
	// err
	store.GraphItems.PushAll(key, items)
	//flag |= g.GRAPH_F_ERR
out:
	flag &= ^g.GRAPH_F_SENDING
	store.GraphItems.SetFlag(key, flag)
	return err

}
Exemplo n.º 15
0
func query_data(client **rpc.Client, addr string,
	args interface{}, resp interface{}) error {
	var (
		err error
		i   int
	)

	for i = 0; i < 3; i++ {
		err = rpc_call(*client, "Graph.Query", args, resp,
			time.Duration(g.Config().CallTimeout)*time.Millisecond)

		if err == nil {
			break
		}
		if err == rpc.ErrShutdown {
			reconnection(client, addr)
		}
	}
	return err
}
Exemplo n.º 16
0
func main() {

	cfg := flag.String("c", "cfg.json", "configuration file")
	version := flag.Bool("v", false, "show version")
	versionGit := flag.Bool("vg", false, "show version")
	flag.Parse()

	if *version {
		fmt.Println(g.VERSION)
		os.Exit(0)
	}

	if *versionGit {
		fmt.Println(g.VERSION, g.COMMIT)
		os.Exit(0)
	}

	// global config
	g.ParseConfig(*cfg)

	// init db
	g.InitDB()
	// start rrdtool
	rrdtool.Start()

	go api.Start()

	// 刷硬盘
	go cron.SyncDisk()

	// 索引更新2.0
	index.Start()

	// http
	go http.Start()

	start_signal(os.Getpid(), *g.Config())

}
Exemplo n.º 17
0
func handleItems(items []*cmodel.GraphItem) {
	if items == nil {
		return
	}

	count := len(items)
	if count == 0 {
		return
	}

	cfg := g.Config()

	for i := 0; i < count; i++ {
		if items[i] == nil {
			continue
		}
		dsType := items[i].DsType
		step := items[i].Step
		checksum := items[i].Checksum()
		key := g.FormRrdCacheKey(checksum, dsType, step)

		//statistics
		proc.GraphRpcRecvCnt.Incr()

		// To Graph
		first := store.GraphItems.First(key)
		if first != nil && items[i].Timestamp <= first.Timestamp {
			continue
		}
		store.GraphItems.PushFront(key, items[i], checksum, cfg)

		// To Index
		index.ReceiveItem(items[i], checksum)

		// To History
		store.AddItem(checksum, items[i])
	}
}
Exemplo n.º 18
0
func FlushRRD(idx int, force bool) {
	begin := time.Now()
	atomic.StoreInt32(&flushrrd_timeout, 0)

	keys := store.GraphItems.KeysByIndex(idx)
	if len(keys) == 0 {
		return
	}

	for _, key := range keys {
		flag, _ := store.GraphItems.GetFlag(key)

		//write err data to local filename
		if force == false && g.Config().Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
			if time.Since(begin) > time.Millisecond*g.FLUSH_DISK_STEP {
				atomic.StoreInt32(&flushrrd_timeout, 1)
			}
			PullByKey(key)
		} else {
			CommitByKey(key)
		}
	}
}
Exemplo n.º 19
0
func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
	// statistics
	proc.GraphQueryCnt.Incr()

	resp.Values = []*cmodel.RRDData{}
	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
	if !exists {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)
	datas, err := rrdtool.Fetch(filename, param.ConsolFun, param.Start, param.End, step)
	if err != nil {
		if store.GraphItems.LenOf(md5) <= 2 {
			return nil
		}
		// TODO not atomic, fix me
		items := store.GraphItems.PopAll(md5)
		size := len(items)
		if size > 2 {
			filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2],
				md5, items[0].DsType, items[0].Step)
			err := rrdtool.Flush(filename, items)
			if err != nil && g.Config().Debug && g.Config().DebugChecksum == md5 {
				log.Println("flush fail:", err, "filename:", filename)
			}
		} else {
			return nil
		}
	}
	items := store.GraphItems.FetchAll(md5)

	// merge
	items_size := len(items)
	datas_size := len(datas)
	if items_size > 1 && datas_size > 2 &&
		int(datas[1].Timestamp-datas[0].Timestamp) == step &&
		items[items_size-1].Timestamp > datas[0].Timestamp {

		var val cmodel.JsonFloat
		cache_size := int(items[items_size-1].Timestamp-items[0].Timestamp)/step + 1
		cache := make([]*cmodel.RRDData, cache_size, cache_size)

		//fix items
		items_idx := 0
		ts := items[0].Timestamp
		if dsType == g.DERIVE || dsType == g.COUNTER {
			for i := 0; i < cache_size; i++ {
				if items_idx < items_size-1 &&
					ts == items[items_idx].Timestamp &&
					ts != items[items_idx+1].Timestamp {
					val = cmodel.JsonFloat(items[items_idx+1].Value-items[items_idx].Value) /
						cmodel.JsonFloat(items[items_idx+1].Timestamp-items[items_idx].Timestamp)
					if val < 0 {
						val = cmodel.JsonFloat(math.NaN())
					}
					items_idx++
				} else {
					// miss
					val = cmodel.JsonFloat(math.NaN())
				}
				cache[i] = &cmodel.RRDData{
					Timestamp: ts,
					Value:     val,
				}
				ts = ts + int64(step)
			}
		} else if dsType == g.GAUGE {
			for i := 0; i < cache_size; i++ {
				if items_idx < items_size && ts == items[items_idx].Timestamp {
					val = cmodel.JsonFloat(items[items_idx].Value)
					items_idx++
				} else {
					// miss
					val = cmodel.JsonFloat(math.NaN())
				}
				cache[i] = &cmodel.RRDData{
					Timestamp: ts,
					Value:     val,
				}
				ts = ts + int64(step)
			}
		} else {
			log.Println("not support dstype")
			return nil
		}

		size := int(items[items_size-1].Timestamp-datas[0].Timestamp)/step + 1
		ret := make([]*cmodel.RRDData, size, size)
		cache_idx := 0
		ts = datas[0].Timestamp

		if g.Config().Debug && g.Config().DebugChecksum == md5 {
			log.Println("param.start", param.Start, "param.End:", param.End,
				"items:", items, "datas:", datas)
		}

		for i := 0; i < size; i++ {
			if g.Config().Debug && g.Config().DebugChecksum == md5 {
				log.Println("i", i, "size:", size, "items_idx:", items_idx, "ts:", ts)
			}
			if i < datas_size {
				if ts == cache[cache_idx].Timestamp {
					if math.IsNaN(float64(cache[cache_idx].Value)) {
						val = datas[i].Value
					} else {
						val = cache[cache_idx].Value
					}
					cache_idx++
				} else {
					val = datas[i].Value
				}
			} else {
				if cache_idx < cache_size && ts == cache[cache_idx].Timestamp {
					val = cache[cache_idx].Value
					cache_idx++
				} else {
					//miss
					val = cmodel.JsonFloat(math.NaN())
				}
			}
			ret[i] = &cmodel.RRDData{
				Timestamp: ts,
				Value:     val,
			}
			ts = ts + int64(step)
		}
		resp.Values = ret
	} else {
		resp.Values = datas
	}

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.DsType = dsType
	resp.Step = step

	// statistics
	proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
	return nil
}
Exemplo n.º 20
0
func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
	// statistics
	proc.GraphQueryCnt.Incr()

	// form empty response
	resp.Values = []*cmodel.RRDData{}
	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter) // complete dsType and step
	if !exists {
		return nil
	}
	resp.DsType = dsType
	resp.Step = step

	start_ts := param.Start - param.Start%int64(step)
	end_ts := param.End - param.End%int64(step) + int64(step)
	if end_ts-start_ts-int64(step) < 1 {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	ckey := g.FormRrdCacheKey(md5, dsType, step)
	filename := g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
	// read data from rrd file
	datas, _ := rrdtool.Fetch(filename, param.ConsolFun, start_ts, end_ts, step)
	datas_size := len(datas)
	// read cached items
	items := store.GraphItems.FetchAll(ckey)
	items_size := len(items)

	nowTs := time.Now().Unix()
	lastUpTs := nowTs - nowTs%int64(step)
	rra1StartTs := lastUpTs - int64(rrdtool.RRA1PointCnt*step)
	// consolidated, do not merge
	if start_ts < rra1StartTs {
		resp.Values = datas
		goto _RETURN_OK
	}

	// no cached items, do not merge
	if items_size < 1 {
		resp.Values = datas
		goto _RETURN_OK
	}

	// merge
	{
		// fmt cached items
		var val cmodel.JsonFloat
		cache := make([]*cmodel.RRDData, 0)

		ts := items[0].Timestamp
		itemEndTs := items[items_size-1].Timestamp
		itemIdx := 0
		if dsType == g.DERIVE || dsType == g.COUNTER {
			for ts < itemEndTs {
				if itemIdx < items_size-1 && ts == items[itemIdx].Timestamp &&
					ts == items[itemIdx+1].Timestamp-int64(step) {
					val = cmodel.JsonFloat(items[itemIdx+1].Value-items[itemIdx].Value) / cmodel.JsonFloat(step)
					if val < 0 {
						val = cmodel.JsonFloat(math.NaN())
					}
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		} else if dsType == g.GAUGE {
			for ts <= itemEndTs {
				if itemIdx < items_size && ts == items[itemIdx].Timestamp {
					val = cmodel.JsonFloat(items[itemIdx].Value)
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		}
		cache_size := len(cache)

		// do merging
		merged := make([]*cmodel.RRDData, 0)
		if datas_size > 0 {
			for _, val := range datas {
				if val.Timestamp >= start_ts && val.Timestamp <= end_ts {
					merged = append(merged, val) //rrdtool返回的数据,时间戳是连续的、不会有跳点的情况
				}
			}
		}

		if cache_size > 0 {
			rrdDataSize := len(merged)
			lastTs := cache[0].Timestamp

			// find junction
			rrdDataIdx := 0
			for rrdDataIdx = rrdDataSize - 1; rrdDataIdx >= 0; rrdDataIdx-- {
				if merged[rrdDataIdx].Timestamp < cache[0].Timestamp {
					lastTs = merged[rrdDataIdx].Timestamp
					break
				}
			}

			// fix missing
			for ts := lastTs + int64(step); ts < cache[0].Timestamp; ts += int64(step) {
				merged = append(merged, &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())})
			}

			// merge cached items to result
			rrdDataIdx += 1
			for cacheIdx := 0; cacheIdx < cache_size; cacheIdx++ {
				if rrdDataIdx < rrdDataSize {
					if !math.IsNaN(float64(cache[cacheIdx].Value)) {
						merged[rrdDataIdx] = cache[cacheIdx]
					}
				} else {
					merged = append(merged, cache[cacheIdx])
				}
				rrdDataIdx++
			}
		}
		mergedSize := len(merged)

		// fmt result
		ret_size := int((end_ts - start_ts) / int64(step))
		ret := make([]*cmodel.RRDData, ret_size, ret_size)
		mergedIdx := 0
		ts = start_ts
		for i := 0; i < ret_size; i++ {
			if mergedIdx < mergedSize && ts == merged[mergedIdx].Timestamp {
				ret[i] = merged[mergedIdx]
				mergedIdx++
			} else {
				ret[i] = &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())}
			}
			ts += int64(step)
		}
		resp.Values = ret
	}

_RETURN_OK:
	// statistics
	proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
	return nil
}