コード例 #1
0
ファイル: application.go プロジェクト: JKolios/elasticgin
func main() {

	log.Println("Starting elasticgin")

	//Config fetch
	config := config.GetConfFromJSONFile("config.json")

	//ES init
	esClient := initESClient(config.ElasticURL, config.Indices, config.SniffCluster)
	defer esClient.Stop()

	//Rabbitmq init
	var amqpChannel *amqp.Channel

	if config.UseAMQP {

		amqpConnection, amqpChannel := initAMQPClient(config)
		defer amqpConnection.Close()
		defer amqpChannel.Close()

		rabbitmq.StartSubscribers(amqpChannel, esClient, config)
	} else {
		amqpChannel = nil
	}

	api := api.SetupAPI(esClient, amqpChannel, config)
	api.Run(config.ApiURL)
}
コード例 #2
0
func monitoring(uri string, queueName string, mgmtUri string, prefix string) {
	var (
		queueConn *amqp.Connection
		queueChan *amqp.Channel
		err       error
	)
	queueConn, queueChan, err = rabbitmqConnect(uri, queueName)
	if err != nil {
		return
	}
	for {
		log.Printf("fetch rabbitmq stats")
		var metrics []graphite.Metric
		for _, metric := range fetchQueueMetrics(mgmtUri, prefix) {
			metrics = append(metrics, metric)
		}
		for _, metric := range fetchExchangeMetrics(mgmtUri, prefix) {
			metrics = append(metrics, metric)
		}
		for _, metric := range metrics {
			body := []byte(metric.Name + "\t" + metric.Value + "\t" + strconv.FormatInt(metric.Timestamp, 10))
			msg := amqp.Publishing{ContentType: "text/plain", Body: body}
			err = queueChan.Publish("", queueName, false, false, msg)
			if err != nil {
				log.Printf("publish err: %s", err)
				return
			}
			//log.Printf("metric\t%s\t\t%s", metric.Name, metric.Value)
		}
		time.Sleep(time.Second * 5)
	}
	queueChan.Close()
	queueConn.Close()
}
コード例 #3
0
ファイル: amqp.go プロジェクト: johnkewforks/machinery
// Closes the connection
func (amqpBackend *AMQPBackend) close(channel *amqp.Channel, conn *amqp.Connection) error {
	if err := channel.Close(); err != nil {
		return fmt.Errorf("Channel Close: %s", err)
	}

	if err := conn.Close(); err != nil {
		return fmt.Errorf("Connection Close: %s", err)
	}

	return nil
}
コード例 #4
0
ファイル: main.go プロジェクト: denniselite/tooltips
func publish(ch *amqp.Channel, q amqp.Queue, c <-chan string) {
	for {
		connectionStatus := <-c
		if connectionStatus == "lost" {
			for i := 0; i < 40; i++ {
				conn, err := amqp.Dial("amqp://*****:*****@localhost:5672/")
				if err != nil {
					fmt.Println("Failed to connect to RabbitMQ")
					time.Sleep(1 * time.Second)
					continue
				} else {
					defer conn.Close()
				}
			}
		}
		if connectionStatus == "ok" {
			conn, err := amqp.Dial("amqp://*****:*****@localhost:5672/")
			if err != nil {
				fmt.Println("Failed to connect to RabbitMQ")
				time.Sleep(1 * time.Second)
				continue
			} else {
				defer conn.Close()
			}
			ch, _ := conn.Channel()
			defer ch.Close()
			body := bodyFrom(os.Args)
			q, err := ch.QueueDeclare(
				"task_queue", // name
				true,         // durable
				false,        // delete when unused
				false,        // exclusive
				false,        // no-wait
				nil,          // arguments
			)
			err = ch.Publish(
				"",     // exchange
				q.Name, // routing key
				false,  // mandatory
				false,
				amqp.Publishing{
					DeliveryMode: amqp.Persistent,
					ContentType:  "text/plain",
					Body:         []byte(body),
				})
			if err == nil {
				log.Printf(" [x] Sent %s", body)
			}
		}
		time.Sleep(1 * time.Second)
	}
}
コード例 #5
0
ファイル: changes.go プロジェクト: blockloop/unison
func WaitForFile(channel *amqp.Channel, filePath string, queueName string) {
	defer channel.Close()

	<-LockFile(filePath)
	defer UnlockFile(filePath)

	msgs, err := channel.Consume(
		queueName, // queue
		"",        // consumer
		true,      // auto-ack
		true,      // exclusive
		false,     // no-local
		false,     // no-wait
		nil,       // args
	)
	if err != nil {
		panic(fmt.Errorf("Failed to register a consumer: %s", err.Error()))
	}
	log.Println("waiting for file to be transfered")

	fullPath := filepath.Join(rootDir, filePath)
	file, err := os.OpenFile(fullPath, os.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC, 0600)

	if err != nil {
		log.Fatalf("couldn't open local file %s", fullPath, err)
	}
	defer file.Close()

	for {
		msg := <-msgs
		order, err := strconv.Atoi(msg.Headers["Order"].(string))
		if err != nil {
			log.Println(fmt.Errorf("no order was present in the header for file transfer"))
			break
		}
		chunk := &FileChunk{
			order,
			msg.Body,
		}
		if chunk.Order == -1 {
			break
		}

		log.Printf("writing chunk %d for file %s\n", chunk.Order, file.Name())
		file.Write(chunk.Chunk)
	}

	log.Println("finished receiving file", filePath)
}
コード例 #6
0
ファイル: amqp.go プロジェクト: gooops/machinery
// Closes the connection
func (amqpBroker *AMQPBroker) close(channel *amqp.Channel, conn *amqp.Connection) error {
	if channel != nil {
		if err := channel.Close(); err != nil {
			return fmt.Errorf("Channel Close: %s", err)
		}
	}

	if conn != nil {
		if err := conn.Close(); err != nil {
			return fmt.Errorf("Connection Close: %s", err)
		}
	}

	return nil
}
コード例 #7
0
ファイル: rabbitmq.go プロジェクト: yuvista/rabbitmq
// shutdownChannel is a general closer function for channels
func shutdownChannel(channel *amqp.Channel, tag string) error {
	// This waits for a server acknowledgment which means the sockets will have
	// flushed all outbound publishings prior to returning.  It's important to
	// block on Close to not lose any publishings.
	if err := channel.Cancel(tag, true); err != nil {
		if amqpError, isAmqpError := err.(*amqp.Error); isAmqpError && amqpError.Code != 504 {
			return fmt.Errorf("AMQP connection close error: %s", err)
		}
	}

	if err := channel.Close(); err != nil {
		return err
	}

	return nil
}
コード例 #8
0
ファイル: main.go プロジェクト: presbrey/mqslice
func (s *Server) consumer() {
	var (
		ch  *amqp.Channel
		err error
		q   amqp.Queue

		consumer <-chan amqp.Delivery
	)
	qArgs := amqp.Table{}
	if *ttl > 0 {
		qArgs["x-message-ttl"] = int64(*ttl)
	}
	if s.Ttl > 0 {
		qArgs["x-message-ttl"] = int64(s.Ttl)
	}
	for s.alive {
		if err != nil {
			ch = nil
			log.Println(err)
			err = nil
			time.Sleep(*backoff)
			continue
		}
		if ch == nil {
			ch, err = s.dial()
		}
		if ch != nil {
			q, err = ch.QueueDeclare(*queue, false, true, true, false, qArgs)
			if err != nil {
				continue
			}
			err = ch.QueueBind(q.Name, s.Source, s.Source, false, nil)
			if err != nil {
				continue
			}
			consumer, err = ch.Consume(q.Name, "mqslice", true, true, false, false, nil)
			if err != nil {
				continue
			}
			for elt := range consumer {
				s.ch <- elt.Body
			}
			ch.Close()
			ch = nil
		}
	}
}
コード例 #9
0
ファイル: fromAMQP.go プロジェクト: jprobinson/streamtools
// connects to an AMQP topic and emits each message into streamtools.
func (b *FromAMQP) Run() {
	var err error
	var conn *amqp.Connection
	var amqp_chan *amqp.Channel
	toOut := make(blocks.MsgChan)
	toError := make(chan error)

	host := "localhost"
	port := "5672"
	username := "******"
	password := "******"
	routingkey := "#"
	exchange := "amq.topic"
	exchange_type := "topic"

	for {
		select {
		case msg := <-toOut:
			b.out <- msg
		case err := <-toError:
			b.Error(err)
		case ruleI := <-b.inrule:
			rule := ruleI.(map[string]interface{})

			routingkey, err = util.ParseString(rule, "RoutingKey")
			if err != nil {
				b.Error(err)
				continue
			}
			exchange, err = util.ParseString(rule, "Exchange")
			if err != nil {
				b.Error(err)
				continue
			}
			exchange_type, err = util.ParseString(rule, "ExchangeType")
			if err != nil {
				b.Error(err)
				continue
			}
			host, err = util.ParseString(rule, "Host")
			if err != nil {
				b.Error(err)
				continue
			}
			port, err = util.ParseString(rule, "Port")
			if err != nil {
				b.Error(err)
				continue
			}
			username, err = util.ParseString(rule, "Username")
			if err != nil {
				b.Error(err)
				continue
			}
			password, err = util.ParseString(rule, "Password")
			if err != nil {
				b.Error(err)
				continue
			}

			conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
			if err != nil {
				b.Error(err)
				continue
			}

			amqp_chan, err = conn.Channel()
			if err != nil {
				b.Error(err)
				continue
			}

			err = amqp_chan.ExchangeDeclare(
				exchange,      // name
				exchange_type, // type
				true,          // durable
				false,         // auto-deleted
				false,         // internal
				false,         // noWait
				nil,           // arguments
			)
			if err != nil {
				b.Error(err)
				continue
			}

			queue, err := amqp_chan.QueueDeclare(
				"",    // name
				false, // durable
				true,  // delete when unused
				false, // exclusive
				false, // noWait
				nil,   // arguments
			)
			if err != nil {
				b.Error(err)
				continue
			}

			err = amqp_chan.QueueBind(
				queue.Name, // queue name
				routingkey, // routing key
				exchange,   // exchange
				false,
				nil,
			)

			if err != nil {
				b.Error(err)
				continue
			}

			deliveries, err := amqp_chan.Consume(
				queue.Name, // name
				"",         // consumerTag
				true,       // noAck
				false,      // exclusive
				false,      // noLocal
				false,      // noWait
				nil,        // arguments
			)
			if err != nil {
				b.Error(err)
				continue
			}

			h := readWriteAMQPHandler{toOut, toError}
			go h.handle(deliveries)
		case <-b.quit:
			if amqp_chan != nil {
				amqp_chan.Close()
			}
			if conn != nil {
				conn.Close()
			}
			return
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Host":         host,
				"Port":         port,
				"Username":     username,
				"Password":     password,
				"Exchange":     exchange,
				"ExchangeType": exchange_type,
				"RoutingKey":   routingkey,
			}
		}
	}
}
コード例 #10
0
ファイル: toAMQP.go プロジェクト: jprobinson/streamtools
// connects to an AMQP topic and emits each message into streamtools.
func (b *ToAMQP) Run() {
	var err error
	var conn *amqp.Connection
	var amqp_chan *amqp.Channel

	host := "localhost"
	port := "5672"
	username := "******"
	password := "******"
	routingkey := "streamtools"
	exchange := "amq.topic"
	exchange_type := "topic"

	for {
		select {
		case ruleI := <-b.inrule:

			routingkey, err = util.ParseString(ruleI, "RoutingKey")
			if err != nil {
				b.Error(err)
				continue
			}
			exchange, err = util.ParseString(ruleI, "Exchange")
			if err != nil {
				b.Error(err)
				continue
			}
			exchange_type, err = util.ParseString(ruleI, "ExchangeType")
			if err != nil {
				b.Error(err)
				continue
			}
			host, err = util.ParseString(ruleI, "Host")
			if err != nil {
				b.Error(err)
				continue
			}
			port, err = util.ParseString(ruleI, "Port")
			if err != nil {
				b.Error(err)
				continue
			}
			username, err = util.ParseString(ruleI, "Username")
			if err != nil {
				b.Error(err)
				continue
			}
			password, err = util.ParseString(ruleI, "Password")
			if err != nil {
				b.Error(err)
				continue
			}

			conn, err = amqp.Dial("amqp://" + username + ":" + password + "@" + host + ":" + port + "/")
			if err != nil {
				b.Error(err)
				continue
			}

			amqp_chan, err = conn.Channel()
			if err != nil {
				b.Error(err)
				continue
			}

			err = amqp_chan.ExchangeDeclare(
				exchange,      // name
				exchange_type, // type
				true,          // durable
				false,         // auto-deleted
				false,         // internal
				false,         // noWait
				nil,           // arguments
			)
			if err != nil {
				b.Error(err)
				continue
			}

		case msg := <-b.in:
			if conn == nil || amqp_chan == nil {
				continue
			}

			msgBytes, err := json.Marshal(msg)

			// Make the output JSON if msg wasn't JSON already
			if err != nil {
				json_msg := map[string]interface{}{
					"data": msg,
				}
				msgBytes, err = json.Marshal(json_msg)

				if err != nil {
					b.Error(err)
					continue
				}
			}

			if len(msgBytes) == 0 {
				b.Error("Zero byte length message")
				continue
			}

			err = amqp_chan.Publish(
				exchange,
				routingkey,
				false,
				false,
				amqp.Publishing{
					Headers:         amqp.Table{},
					ContentType:     "text/plain",
					ContentEncoding: "",
					Body:            msgBytes,
					DeliveryMode:    amqp.Transient,
					Priority:        0,
				},
			)
			if err != nil {
				b.Error(err)
				continue
			}
		case <-b.quit:
			if amqp_chan != nil {
				amqp_chan.Close()
			}
			if conn != nil {
				conn.Close()
			}
			return
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Host":         host,
				"Port":         port,
				"Username":     username,
				"Password":     password,
				"Exchange":     exchange,
				"ExchangeType": exchange_type,
				"RoutingKey":   routingkey,
			}
		}
	}
}
コード例 #11
0
func (c *Consumer) Consume() {
	c.InfLogger.Println("Registering consumer... ")
	msgs, err := c.Channel.Consume(c.Queue, "", false, false, false, false, nil)
	if err != nil {
		c.ErrLogger.Fatalf("Failed to register a consumer: %s", err)
	}
	c.InfLogger.Println("Succeeded registering consumer.")

	var sendCh *amqp.Channel

	if c.DeadLetter {
		var err error
		sendCh, err = c.Connection.Channel()
		if err != nil {
			c.ErrLogger.Println("Could not open channel to republish failed jobs %s", err)
		}
		defer sendCh.Close()
	}

	defer c.Connection.Close()
	defer c.Channel.Close()

	forever := make(chan bool)

	go func() {
		for d := range msgs {
			c.InfLogger.Println("reading deliveries")
			input := d.Body
			if c.Compression {
				var b bytes.Buffer
				w, err := zlib.NewWriterLevel(&b, zlib.BestCompression)
				if err != nil {
					c.ErrLogger.Println("Could not create zlib handler")

					d.Nack(true, true)
				}
				c.InfLogger.Println("Compressed message")
				w.Write(input)
				w.Close()

				input = b.Bytes()
			}

			if c.DeadLetter {
				var retryCount int
				if d.Headers == nil {
					d.Headers = make(map[string]interface{}, 0)
				}
				retry, ok := d.Headers["retry_count"]
				if !ok {
					retry = "0"
				}
				c.InfLogger.Println(fmt.Sprintf("retry %s", retry))

				retryCount, err = strconv.Atoi(retry.(string))
				if err != nil {
					c.ErrLogger.Fatal("could not parse retry header")
				}

				c.InfLogger.Println(fmt.Sprintf("retryCount : %d max retries: %d", retryCount, c.Retry))

				var cmd *exec.Cmd

				if c.WriteToPath != "" {

					var filepath string
					filepath += c.WriteToPath
					filepath += c.Queue
					filepath += "-"
					filepath += time.Now().Format("20060102150405")
					filepath += "-"
					filepath += strconv.FormatInt(rand.Int63(), 10) // Convert random int to string
					filepath += ".json"

					cmd = c.Factory.Create(filepath)

					jsonContent := jsonPrettyPrint(input) // Pretty print

					err := ioutil.WriteFile(filepath, []byte(jsonContent), 0644)
					if err != nil {
						c.ErrLogger.Println("Could not write input to file")
						d.Nack(true, true)
					}
				} else {
					cmd = c.Factory.Create(base64.StdEncoding.EncodeToString(input))
				}

				if c.Executer.Execute(cmd, d.Body[:]) {
					d.Ack(true)
				} else if retryCount >= c.Retry {
					d.Nack(true, false)
				} else {
					//republish message with new retry header
					retryCount++
					d.Headers["retry_count"] = strconv.Itoa(retryCount)
					republish := amqp.Publishing{
						ContentType:     d.ContentType,
						ContentEncoding: d.ContentEncoding,
						Timestamp:       time.Now(),
						Body:            d.Body,
						Headers:         d.Headers,
					}
					err = sendCh.Publish("", c.Queue, false, false, republish)
					if err != nil {
						c.ErrLogger.Println("error republish %s", err)
					}
					d.Ack(true)
				}
			} else {
				cmd := c.Factory.Create(base64.StdEncoding.EncodeToString(input))
				if c.Executer.Execute(cmd, d.Body[:]) {
					d.Ack(true)
				} else {
					d.Nack(true, false)
				}
			}

		}
	}()
	c.InfLogger.Println("Waiting for messages...")
	<-forever
}
コード例 #12
0
ファイル: mq.go プロジェクト: Cepave/fe
func Start() {
	mq := g.Config().Mq
	nodes := map[string]interface{}{}

	// Retry RetriedLimit times if there is some problem during connecting
	var ch *amqp.Channel
	var conn *amqp.Connection
	var err error
	for i := 0; i < RetriedLimit; i++ {
		if conn, ch, err = setup(mq.Queue); err != nil {
			time.Sleep(time.Second * SleepTimePeriod)
		}
	}
	if err != nil {
		log.Println(ExitStringPrefix + "retried too many times.")
		return
	}
	defer conn.Close()
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"mute", // name
		false,  // durable
		false,  // delete when usused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to declare a queue", err)
		return
	}

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to register a consumer", err)
		return
	}

	forever := make(chan bool)

	var pkt Message
	go func() {
		for d := range msgs {
			json.Unmarshal(d.Body, &pkt)
			log.Printf("Received a message: %v", pkt)
			params := map[string]string{
				"host": pkt.Hostname,
				"mute": "",
			}
			if pkt.Mute {
				params["mute"] = "1"
			} else {
				params["mute"] = "0"
			}
			payload := map[string]interface{}{
				"method": "host.update",
				"params": params,
			}
			log.Println("payload =", payload)

			s, err := json.Marshal(payload)
			if err != nil {
				log.Println("json.Marshal Error:", err.Error())
			}

			url := mq.Consumer
			log.Println("url =", url)
			reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s)))
			if err != nil {
				log.Println("reqPost Error:", err.Error())
			}
			reqPost.Header.Set("Content-Type", "application/json")

			client := &http.Client{}
			resp, err := client.Do(reqPost)
			if err != nil {
				log.Println("resp Error:", err.Error())
			} else {
				defer resp.Body.Close()
				body, _ := ioutil.ReadAll(resp.Body)
				err = json.Unmarshal(body, &nodes)
				if err != nil {
					log.Println("Unmarshal Error:", err.Error())
				}
				log.Println("nodes =", nodes)
			}
		}
	}()
	log.Println("mq.Start ok. Waiting for messages.")
	<-forever
}