func (s *queueServer) doUpload(w http.ResponseWriter, file multipart.File, jid int64, filename string) error {
conn, err := cfg.NewRabbitmqConn()
if err != nil {
glog.Errorln("Couldn't connect to rabbitmq", err)
error500(w)
return err
}
defer conn.Close()
ch, err := conn.Channel()
if err != nil {
glog.Errorln("Couldn't open rabbitmq channel", err)
error500(w)
return err
}
defer ch.Close()
f, err := createJobInputFile(jid)
if err != nil {
glog.Errorln("couldn't create file", err)
error500(w)
return err
}
defer f.Close()
fpath := f.Name()
h := sha1.New()
limit := int64(s.SizeLimit)
glog.Infoln("job file name ", filename)
n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(file, limit))
if n == limit {
glog.Errorln("Error body too long", filename, err)
http.Error(w, "Request body too long", http.StatusBadRequest)
os.Remove(fpath)
return errors.New("Error body too long")
}
if err != nil {
glog.Errorln("Error writing request to file", fpath, err)
error500(w)
return err
}
err = f.Close()
if err != nil {
glog.Errorln("Error closing file", fpath, err)
error500(w)
return err
}
glog.Infoln("job file", s.Name, filename, "wrote bytes", n)
hash := hex.EncodeToString(h.Sum(nil))
glog.Infoln("Wrote job with hash", hash, "to file", fpath)
// TODO check if hash is already queued and if it is abort and return the job id
// TODO store hash state in db
// TODO configurable ttr?
q, err := ch.QueueDeclare(
s.Name, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
glog.Errorln("Couldn't open rabbitmq queue", s.Name, err)
error500(w)
return err
}
qe := QueueEntry{JobId: fmt.Sprint(jid), Filename: filename}
qem, err := json.Marshal(qe)
glog.Infoln("job file name ", filename)
if err != nil {
glog.Errorln("could not marshal queue entry to json", err)
error500(w)
return err
}
err = ch.Publish(
"", // exchange
q.Name, // routing key
false, // mandatory
false,
amqp.Publishing{
DeliveryMode: amqp.Persistent,
ContentType: "application/json",
Body: qem,
})
if err != nil {
glog.Errorln("Error storing job for file", fpath, "with hash", hash, "to rabbitmq")
return err
}
response := UploadRensponse{JobId: fmt.Sprint(jid)}
json_response, err := json.Marshal(response)
if err != nil {
glog.Errorln("json response", err)
error500(w)
return err
}
glog.Infoln("json response", string(json_response))
w.Write(json_response)
w.Header().Set("Content-Type", "application/json")
return nil
}
func main() {
cfg.Init()
defer cfg.Finalize()
engine = NewEngineContainerWrap(*engineType, *workerIndex)
// TODO: pull image before reading from the queue
conn, err := cfg.NewRabbitmqConn()
if err != nil {
glog.Errorln("Couldn't connect to rabbitmq", err)
return
}
defer conn.Close()
ch, err := conn.Channel()
if err != nil {
glog.Errorln("Couldn't open rabbitmq channel", err)
return
}
defer ch.Close()
q, err := ch.QueueDeclare(
*engineType, // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
glog.Errorln("Couldn't open rabbitmq queue", *engineType, err)
return
}
// make sure we fetch one at a time
err = ch.Qos(
1, // prefetch count
0, // prefetch size
false, // global
)
if err != nil {
glog.Errorln("Failed to set QoS to RmQ channel")
return
}
consumer := fmt.Sprintf("pid-%d", os.Getpid())
msgs, err := ch.Consume(
q.Name, // queue
consumer, // consumer string
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
glog.Errorln("Failed to start RmQ consumption loop")
return
}
for i := 0; i < *joblimit || *joblimit == 0; i++ {
if d, gotMsg := <-msgs; gotMsg {
work(d)
} else {
glog.Warningln("Message channel closed. Exiting.")
break
}
}
ch.Cancel(consumer, true)
}
