コード例 #1
0
func (s *queueServer) doUpload(w http.ResponseWriter, file multipart.File, jid int64, filename string) error {

	conn, err := cfg.NewRabbitmqConn()
	if err != nil {
		glog.Errorln("Couldn't connect to rabbitmq", err)
		error500(w)
		return err
	}
	defer conn.Close()

	ch, err := conn.Channel()
	if err != nil {
		glog.Errorln("Couldn't open rabbitmq channel", err)
		error500(w)
		return err
	}
	defer ch.Close()

	f, err := createJobInputFile(jid)
	if err != nil {
		glog.Errorln("couldn't create file", err)
		error500(w)
		return err
	}
	defer f.Close()
	fpath := f.Name()

	h := sha1.New()
	limit := int64(s.SizeLimit)
	glog.Infoln("job file name ", filename)
	n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(file, limit))
	if n == limit {
		glog.Errorln("Error body too long", filename, err)
		http.Error(w, "Request body too long", http.StatusBadRequest)
		os.Remove(fpath)
		return errors.New("Error body too long")
	}
	if err != nil {
		glog.Errorln("Error writing request to file", fpath, err)
		error500(w)
		return err
	}
	err = f.Close()
	if err != nil {
		glog.Errorln("Error closing file", fpath, err)
		error500(w)
		return err
	}

	glog.Infoln("job file", s.Name, filename, "wrote bytes", n)
	hash := hex.EncodeToString(h.Sum(nil))
	glog.Infoln("Wrote job with hash", hash, "to file", fpath)
	// TODO check if hash is already queued and if it is abort and return the job id
	// TODO store hash state in db
	// TODO configurable ttr?
	q, err := ch.QueueDeclare(
		s.Name, // name
		true,   // durable
		false,  // delete when unused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	if err != nil {
		glog.Errorln("Couldn't open rabbitmq queue", s.Name, err)
		error500(w)
		return err
	}

	qe := QueueEntry{JobId: fmt.Sprint(jid), Filename: filename}
	qem, err := json.Marshal(qe)
	glog.Infoln("job file name ", filename)
	if err != nil {
		glog.Errorln("could not marshal queue entry to json", err)
		error500(w)
		return err
	}
	err = ch.Publish(
		"",     // exchange
		q.Name, // routing key
		false,  // mandatory
		false,
		amqp.Publishing{
			DeliveryMode: amqp.Persistent,
			ContentType:  "application/json",
			Body:         qem,
		})
	if err != nil {
		glog.Errorln("Error storing job for file", fpath, "with hash", hash, "to rabbitmq")
		return err
	}
	response := UploadRensponse{JobId: fmt.Sprint(jid)}
	json_response, err := json.Marshal(response)
	if err != nil {
		glog.Errorln("json response", err)
		error500(w)
		return err
	}
	glog.Infoln("json response", string(json_response))
	w.Write(json_response)
	w.Header().Set("Content-Type", "application/json")
	return nil
}
コード例 #2
0
func main() {
	cfg.Init()
	defer cfg.Finalize()

	engine = NewEngineContainerWrap(*engineType, *workerIndex)
	// TODO: pull image before reading from the queue
	conn, err := cfg.NewRabbitmqConn()
	if err != nil {
		glog.Errorln("Couldn't connect to rabbitmq", err)
		return
	}
	defer conn.Close()

	ch, err := conn.Channel()
	if err != nil {
		glog.Errorln("Couldn't open rabbitmq channel", err)
		return
	}
	defer ch.Close()

	q, err := ch.QueueDeclare(
		*engineType, // name
		true,        // durable
		false,       // delete when unused
		false,       // exclusive
		false,       // no-wait
		nil,         // arguments
	)
	if err != nil {
		glog.Errorln("Couldn't open rabbitmq queue", *engineType, err)
		return
	}

	// make sure we fetch one at a time
	err = ch.Qos(
		1,     // prefetch count
		0,     // prefetch size
		false, // global
	)
	if err != nil {
		glog.Errorln("Failed to set QoS to RmQ channel")
		return
	}

	consumer := fmt.Sprintf("pid-%d", os.Getpid())
	msgs, err := ch.Consume(
		q.Name,   // queue
		consumer, // consumer string
		false,    // auto-ack
		false,    // exclusive
		false,    // no-local
		false,    // no-wait
		nil,      // args
	)
	if err != nil {
		glog.Errorln("Failed to start RmQ consumption loop")
		return
	}

	for i := 0; i < *joblimit || *joblimit == 0; i++ {
		if d, gotMsg := <-msgs; gotMsg {
			work(d)
		} else {
			glog.Warningln("Message channel closed. Exiting.")
			break
		}
	}
	ch.Cancel(consumer, true)
}