func main() {
	flag.Parse()
	var rollingSize int
	var rollingNum int
	var err error
	if rollingSize, err = strconv.Atoi(*RollingSize); err != nil {
		log.Printf("rolling size is not valid! \n")
		return
	}
	if rollingNum, err = strconv.Atoi(*RollingNum); err != nil {
		log.Printf("rolling num is not valid! \n")
		return
	}

	leak_service.Port = *ListenPort
	gzlog.InitGZLogger(*LogFile, rollingSize*1000*1000, rollingNum)

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@(%s)/jwlwl?charset=utf8&parseTime=True", MysqlUser, MysqlPasswd, *MysqlHost))
	defer db.Close()
	if err != nil {
		log.Printf("mysql db connect failed !errMessage:%s \n", err)
		return
	}
	leak_service.Db = db
	log.Printf("Start to ServeRequest!")
	leak_service.ServeRequest()
}
Beispiel #2
0
func main() {
	flag.Parse()
	gzlog.InitGZLogger(*LogFile, 50*1000*1000, 5)
	if *kafkaTopic == "" {
		log.Printf("topicShould not be null!\n")
		return
	}
	if *zookeeper == "" {
		log.Printf("zookeeper should not be null! \n")
		return
	}
	if *brokerList == "" {
		log.Printf("kafka brokers must not be null\n")
	}
	config := consumergroup.NewConfig()
	config.Offsets.Initial = sarama.OffsetNewest
	config.Offsets.ProcessingTimeout = 2 * time.Second
	config.Consumer.MaxProcessingTime = 2 * time.Second
	kafkaTopics := strings.Split(*kafkaTopic, ",")
	zookeeperNodes, config.Zookeeper.Chroot = kazoo.ParseConnectionString(*zookeeper)
	consumer, consumerErr := consumergroup.JoinConsumerGroup(*consumerGroup, kafkaTopics, zookeeperNodes, config)
	if consumerErr != nil {
		log.Fatalln(consumerErr)
	}
	log.Printf("start to get mysl connection!\n")
	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@(%s)/jwlwl?charset=utf8&parseTime=True", MysqlUser, MysqlPasswd, MysqlHost))
	defer db.Close()
	if err != nil {
		log.Printf("mysql db connect failed !errMessage:%s \n", err)
		return
	}
	log.Printf("start to get kafka producer\n")
	producer, err := kafka_tool.GetKafkaProducer(*brokerList)
	if err != nil {
		log.Printf("Kafka  get producer failed !err: %s \n", err)
		return
	}

	log.Printf("Start to call consummer messages method !\n")
	for message := range consumer.Messages() {
		log.Printf("Start to call Run method with message:%s \n", message.Value)
		latestLeakEventArg := &u_leak_merge.LatestEventArg{Properties: string(message.Value)}
		err := u_leak_merge.Run("0", latestLeakEventArg, db, producer)
		if err != nil {
			log.Printf("message failed!:%s, errMessage:%s \n", message.Value, err)
			continue
		}
		log.Printf("Start to commit message! \n")
		consumer.CommitUpto(message)
		time.Sleep(100 * time.Millisecond)
	}
}
Beispiel #3
0
func main() {
	waiting := make(chan bool)
	var help = flag.String("h", "", "tools to split data,please go to github address get ")
	var configPath = flag.String("config", "", "config service path")
	var logFile = flag.String("log_file", "sc_producer.log", "config log file path")
	var registedGroupIds = flag.String("group_ids", "", "regitsted groupids!eg:a,b,c")
	var mysqlUser = flag.String("muser", "", "mysql user")
	var mysqlPassword = flag.String("mpass", "", "mysql password")
	var mysqlHost = flag.String("mhost", "", "mysqlhost")

	flag.Parse()
	if *help == "h" {
		fmt.Println("flag.Args")
		return
	}
	if *registedGroupIds == "" {
		fmt.Println("Please pre set groupIds to regist to master!")
		return
	}
	if *mysqlUser == "" || *mysqlPassword == "" || *mysqlHost == "" {
		fmt.Println("mysql config have to be given !")
		return
	}
	gzlog.InitGZLogger(*logFile, 50*1000*1000, 5)
	nps := spread_compute.NewProducerServer(*configPath)
	log.Printf("Start to start rpc service!")
	go nps.StartRpc()
	spread_compute.RegistProducerService(2,
		&ProducerServiceUser{MysqlHost: *mysqlHost, MysqlUser: *mysqlUser, MysqlPasswd: *mysqlPassword})
	log.Printf("Start to regist producer service!")
	for _, v := range strings.Split(*registedGroupIds, ",") {
		gId, err := strconv.Atoi(v)
		if err != nil {
			fmt.Printf("invalid groupid err:%s \n", err)
			return
		}
		err = nps.StartRegist(int64(gId))
		if err != nil {
			log.Printf("groupId %d regis failed !err:%s\n", gId, err)
			return
		}
	}
	fmt.Println("producer start rpc and regist suceess!")
	<-waiting
}
Beispiel #4
0
func main() {
	waiting := make(chan bool)
	var help = flag.String("h", "", "tools to split data,please go to github address get ")
	var configPath = flag.String("config", "", "config service path")
	var logFile = flag.String("log_file", "sc_consumer.log", "config log file path")
	var registedGroupIds = flag.String("group_ids", "", "regitsted groupids!eg:a,b,c,it must be integers!")
	var commandStr = flag.String("cmd", "", "system command to execute!")
	flag.Parse()
	if *help == "h" {
		fmt.Println("flag.Args")
		return
	}
	if *registedGroupIds == "" {
		fmt.Println("Please pre set groupIds to regist to master!")
		return
	}
	if *commandStr == "" {
		fmt.Println("Pleast set command str,command str cannot be nil")
		return
	}

	gzlog.InitGZLogger(*logFile, 50*1000*1000, 5)
	consumerServer := spread_compute.NewConsumer(*configPath)
	err := consumerServer.Init()
	if err != nil {
		log.Printf("init config failed! \n")
		return
	}
	log.Printf("Now we are starting consumer service!")
	go consumerServer.StartRpc()
	spread_compute.RegistConsumerService(2, &ConsumerServiceClient{*commandStr})
	for _, v := range strings.Split(*registedGroupIds, ",") {
		gId, err := strconv.Atoi(v)
		if err != nil {
			fmt.Printf("invalid groupid err:%s \n", err)
			return
		}
		err = consumerServer.StartRegist(int64(gId))
		if err != nil {
			log.Printf("consumer regist failed!\n")
		}
	}
	<-waiting
}
Beispiel #5
0
func main() {
	waiting := make(chan bool)
	var help = flag.String("h", "", "tools to split data,please go to github address get ")
	var configPath = flag.String("config", "", "config service path")
	var logFile = flag.String("log_file", "sc_master.log", "config log file path")
	flag.Parse()
	if *help == "h" {
		fmt.Println("flag.Args")
		return
	}
	gzlog.InitGZLogger(*logFile, 50*1000*1000, 5)
	master := &spread_compute.MasterServer{ConfigPath: *configPath}
	log.Printf("Start to init Master Server")
	err := master.Init()
	if err != nil {
		log.Printf("Start master failed! \n")
		return
	}
	log.Printf("Starting master server!")
	master.Start()
	<-waiting
}