Exemplo n.º 1
0
func init() {
    file, _ := os.Open("conf/app.json")
    defer file.Close()
    
    decoder := json.NewDecoder(file)
    AppConf = App{}
    err := decoder.Decode(&AppConf)
    if err != nil {
        ara.Logger().Debug("error: %v", err)
    }
    ara.Logger().Debug("%v", AppConf)
}
Exemplo n.º 2
0
func (c *Controller) initKc() (err error) {
	ara.Logger().Debug("init kafka client")
	conn := c.getKafkaConn()
	c.kc, err = kfk.NewClient(conn, kfk.NewConfig())
	if err != nil {
		ara.Logger().Debug(err.Error())
		return
	}

	c.kc.Config().ClientID = "jackdaw"
	ara.Logger().Debug("got a kafka client, closed: %v", c.kc.Closed())
	return
}
Exemplo n.º 3
0
// init zookeeper connection
func (c *Controller) initZc() (err error) {
	ara.Logger().Debug("init zookeeper client")
	if c.zc != nil && c.zc.State() != zk.StateHasSession {
		c.zc.Close()
	}

	c.zc, _, err = zk.Connect(utils.AppConf.Zookeepers, time.Second, c.loggerOption)
	if err != nil {
		return
	}

	ara.Logger().Debug("new zookeeper client state: %v", c.zc.State())
	return
}
Exemplo n.º 4
0
func (c *Controller) getLogSize(topic string, pid int32) int64 {
	// from kafka
	if c.kc == nil || c.kc.Closed() {
		err := c.initKc()
		if err != nil {
			// TODO send a err message
			ara.Logger().Debug(err.Error())
			panic(err)
		}
	}

	latestOffset, err := c.kc.GetOffset(topic, pid, kfk.OffsetNewest)
	if err != nil {
		// TODO send a err message
		ara.Logger().Debug(err.Error())
		panic(err)
	}

	ara.Logger().Debug("last offset is: %d", latestOffset)
	return latestOffset
}
Exemplo n.º 5
0
// close zookeeper and kafka connection while shutdown
func (c *Controller) Release() {

	if c.kc != nil {
		err := c.kc.Close()
		if err != nil {
			ara.Logger().Debug("close kafka connection failed: %s", err.Error())
		}
	}

	if c.zc != nil {
		c.zc.Close()
	}
}
Exemplo n.º 6
0
func (c *Controller) ListTopics(w http.ResponseWriter, r *http.Request) {

	path := "/brokers/topics"
	topics, err := c.lsChildren(path)
	if err != nil {
		utils.WriteError(w, err)
		return
	}

	resp := make(map[string]string, len(topics))
	for _, topic := range topics {
		resp[topic] = "/uri/here?abc=123"
	}

	encoder := json.NewEncoder(w)
	err = encoder.Encode(resp)
	//    b, err := json.Marshal(topics)
	//    if err != nil {
	//        fmt.Println("error:", err)
	//    }
	//    w.Write(b)

	// from kafka
	client, err := kfk.NewClient([]string{"127.0.0.1:9092"}, kfk.NewConfig())
	if err != nil {
		panic(err)
	}
	defer client.Close()

	client.Config().ClientID = "jackdaw"
	latestOffset, err := client.GetOffset("tpk001", 0, kfk.OffsetNewest)
	if err != nil {
		panic(err)
	}

	ara.Logger().Debug("$$$$$$: %d", latestOffset)

	//    bts, stat, ch, err := zc.GetW(path)
	//    if err != nil {
	//        panic(err)
	//    }
	//    fmt.Printf("%s *** %+v\n", string(bts), stat)

	//    e := <-ch
	//    fmt.Printf("--- %+v\n", e)
	//    if e.Type == zk.EventNodeDataChanged {
	//        watchData(zc)
	//    }
}
Exemplo n.º 7
0
// request url: /brokers
// json from zk looks like:
//get /brokers/ids/0
//{"jmx_port":-1,"timestamp":"1446347718036","host":"U","version":1,"port":9092}
func (c *Controller) ListBrokers(w http.ResponseWriter, r *http.Request) {

	resp, err := c.getBrokers()
	if err != nil {
		utils.WriteError(w, err)
		return
	}

	encoder := json.NewEncoder(w)
	err = encoder.Encode(resp)

	ara.Logger().Debug("%v", resp)

	//    b, err := json.Marshal(topics)
	//    if err != nil {
	//        fmt.Println("error:", err)
	//    }
	//    w.Write(b)
}
Exemplo n.º 8
0
func (c *Controller) getKafkaConn() []string {
	brokerMap, err := c.getBrokers()
	if err != nil {
		return nil
	}

	conn := make([]string, len(brokerMap))

	for _, brokerJson := range brokerMap {

		var b Broker
		err := json.Unmarshal([]byte(brokerJson), &b)
		if err != nil {
			ara.Logger().Debug("error: %v", err)
			continue
		}
		conn = append(conn, b.Host+":"+strconv.Itoa(b.Port))
	}
	return conn
}
Exemplo n.º 9
0
// implement interface zk.Conn.Logger
// TODO output not right: /controller.go:69: Authenticated: id=[94863294880088115 6000], timeout=%!d(MISSING)
func (l appLogger) Printf(s string, v ...interface{}) {
	ara.Logger().Debug(s, v)
}
Exemplo n.º 10
0
func (c *Controller) ListGroups(w http.ResponseWriter, r *http.Request) {

	// ls /consumers
	//[console-consumer-40820]
	gPath := "/consumers"
	groups, err := c.lsChildren(gPath)
	if err != nil {
		utils.WriteError(w, err)
		return
	}

	ara.Logger().Debug("groups: %v", groups)

	// return 404 if no group found(that means no consumer exists)
	if len(groups) <= 0 {
		ara.Logger().Debug("return 404")
		utils.Write404(w)
		return
	}

	// map[groupName]map[topicName]map[partitionId]map[string]string (last map is offset, log size and lag etc.)
	resp := make(map[string]map[string]map[string]map[string]string, len(groups))
	for _, g := range groups {

		//ls /consumers/console-consumer-40820/offsets
		//[mytopic]
		topicsPath := path.Join(gPath, g, "offsets")
		topics, err := c.lsChildren(topicsPath)
		if err != nil {
			utils.WriteError(w, err)
			return
		}

		// init topic map
		topicMap := make(map[string]map[string]map[string]string)
		for _, topic := range topics {
			//ls /consumers/console-consumer-40820/offsets/mytopic
			//[0, 1, 2]
			partitionPath := path.Join(topicsPath, topic)
			partitions, err := c.lsChildren(partitionPath)
			if err != nil {
				utils.WriteError(w, err)
				return
			}

			// init partition map
			pMap := make(map[string]map[string]string)
			for _, pidStr := range partitions {
				//get /consumers/console-consumer-40820/offsets/mytopic/0
				//2
				offsetPath := path.Join(partitionPath, pidStr)
				offsetStr, err := c.getChildren(offsetPath)
				if err != nil {
					utils.WriteError(w, err)
					return
				}

				pid64, err := strconv.ParseInt(pidStr, 10, 32)
				if err != nil {
					utils.WriteError(w, err)
					return
				}
				pid := int32(pid64)
				offset, err := strconv.ParseInt(offsetStr, 10, 64)

				logSize := c.getLogSize(topic, pid)
				lag := logSize - offset

				pDataMap := make(map[string]string)
				pDataMap["offset"] = offsetStr
				pDataMap["logSize"] = strconv.FormatInt(logSize, 10)
				pDataMap["lag"] = strconv.FormatInt(lag, 10)
				pMap[pidStr] = pDataMap
			}

			topicMap[topic] = pMap
		}
		resp[g] = topicMap
	}
	encoder := json.NewEncoder(w)
	err = encoder.Encode(resp)
	if err != nil {
		ara.Logger().Debug(err.Error())
	}

	//    b, err := json.Marshal(topics)
	//    if err != nil {
	//        fmt.Println("error:", err)
	//    }
	//    w.Write(b)

	//    bts, stat, ch, err := zc.GetW(path)
	//    if err != nil {
	//        panic(err)
	//    }
	//    fmt.Printf("%s *** %+v\n", string(bts), stat)

	//    e := <-ch
	//    fmt.Printf("--- %+v\n", e)
	//    if e.Type == zk.EventNodeDataChanged {
	//        watchData(zc)
	//    }
}