Beispiel #1
0
func (r *Replyer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	base := filepath.Base(req.URL.Path)
	mylog.Debug("request for response ", base)
	session := r.SessionSeed.New()
	defer session.Close()
	session.SetMode(mgo.Eventual, true)
	db := session.DB(r.Cfg.Database)
	respColl := db.C(r.Cfg.ResponsesColl)
	rpl := &Reply{}
	mylog.Debug("searching response", base)
	e := respColl.Find(bson.M{"id": base}).One(&rpl)
	if e != nil {
		mylog.Debug("reply not found ", base)
		http.Error(w, e.Error(), http.StatusNotFound)
		return
	}
	mylog.Debug("json marshaling response", base)
	text, e := json.MarshalIndent(rpl, "", " ")
	if e != nil {
		mylog.Debug("error marshaling JSON ", base, e)
		http.Error(w, e.Error(), http.StatusInternalServerError)
		return
	}
	mylog.Debugf("returning JSON %q for %v", text, base)
	w.Header().Set("Content-Type", "application/json")
	w.Write(text)
}
Beispiel #2
0
//process recreates the request that should be sent to the target host
//it stores the response in the store of replies.
func (c *Consumer) process(petition *Petition) {
	var (
		req   *http.Request
		resp  *http.Response
		reply *Reply
		start = bson.Now()
	)

	db := c.SessionSeed.DB(c.Cfg.Database)
	petColl := db.C(c.Cfg.Instance + c.Cfg.PetitionsColl)
	replyColl := db.C(c.Cfg.ResponsesColl)
	errColl := db.C(c.Cfg.ErrorsColl)

	mylog.Debugf("processing petition %+v", petition)
	req, err := petition.Request()
	if err != nil {
		mylog.Alert(petition.ID, err)
		return
	}
	mylog.Debugf("restored request %+v", req)
	mylog.Debug("before making request", petition.ID)
	resp, err = c.doRequest(req, petition.ID)
	if err == nil {
		mylog.Debug("after making request", petition.ID)
		defer func() {
			mylog.Debug("closing response body", petition.ID)
			resp.Body.Close()
		}()
	}

	reply = newReply(resp, petition, err)
	reply.Created = start
	mylog.Debugf("created reply %+v", reply)
	if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 {
		e := errColl.Insert(reply)
		if e != nil {
			mylog.Alert("ERROR inserting erroneous reply", petition.ID, err)
			c.SessionSeed.Refresh()
		}
	}
	mylog.Debugf("before insert reply %+v", reply)
	err = replyColl.Insert(reply)
	mylog.Debugf("after insert reply %+v", reply)
	if err != nil {
		mylog.Alert("ERROR inserting reply", petition.ID, err)
		c.SessionSeed.Refresh()
	}
	mylog.Debugf("before remove petition %+v", petition)
	err = petColl.Remove(bson.M{"id": petition.ID})
	mylog.Debugf("after remove petition %+v", petition)
	if err != nil {
		mylog.Alert("ERROR removing petition", petition.ID, err)
		c.SessionSeed.Refresh()
	}

}
Beispiel #3
0
//Start starts n goroutines for taking Petitions from the GetFrom channel.
//It returns a channel for notifying when the consumer has ended (hopefully after a Stop() method invocation).
func (c *Consumer) Start(n int) <-chan bool {
	mylog.Debugf("starting consumer %+v", c)
	c.n = n
	finalDone := make(chan bool)
	c.endChan = make(chan struct{})
	c.wg.Add(c.n)
	for i := 0; i < c.n; i++ {
		go c.relay()
	}
	go func() {
		c.wg.Wait()
		mylog.Debug("consumer waiting for children")
		finalDone <- true
		mylog.Debug("all consumer's children finished")
	}()

	return finalDone
}
Beispiel #4
0
//Recover gets all the petitions stored and sends them to a channel for processing by a consumer.
//It returns when all of them are re-enqueued or when an error happens. It should be run before starting
//a listener (with the same PetitionStore) or new petitions could be enqueued twice. Listeners with a different PetitionStore
//should not be a problem. A Consumer can be started before with the same PetitionStore to avoid overflowing the queue.
func (r *Recoverer) Recover() error {
	mylog.Debug("begin recoverer")
	db := r.SessionSeed.DB(r.Cfg.Database)
	petColl := db.C(r.Cfg.Instance + r.Cfg.PetitionsColl)
	p := Petition{}
	iter := petColl.Find(nil).Iter()
	for iter.Next(&p) {
		paux := p
		mylog.Debugf("re-enqueue petition %+v", paux)
		r.SendTo <- &paux
	}
	//iter.Err()
	if err := iter.Close(); err != nil {
		mylog.Alertf("error closing cursor %+v", err)
		return err
	}
	mylog.Debug("end recoverer")
	return nil
}
Beispiel #5
0
//ServeHTTP implements HTTP handler interface
func (l *Listener) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	mylog.Debugf("received request %+v", r)
	w.Header().Set("Content-Type", "application/json")
	if l.Stopped() {
		mylog.Debug("warning client server is stopping")
		http.Error(w, `{"error":"Server is shutting down"}`, 503)
		return
	}
	relayedRequest, e := newPetition(r)
	if e != nil {
		mylog.Debug("petition with error", e)
		http.Error(w, fmt.Sprintf(`{"error": %q }`, e), 400)
		return
	}
	db := l.SessionSeed.DB(l.Cfg.Database)
	petColl := db.C(l.Cfg.Instance + l.Cfg.PetitionsColl)
	mylog.Debugf("petition created %+v", relayedRequest)
	e = petColl.Insert(relayedRequest)
	if e != nil {
		http.Error(w, fmt.Sprintf(`{"error": %q, "ref":%q }`, e, relayedRequest.ID), 500)
		mylog.Alert("ERROR inserting", relayedRequest.ID, e)
		l.SessionSeed.Refresh()
		return
	}
	select {
	case l.SendTo <- relayedRequest:
		mylog.Debug("enqueued petition", relayedRequest)
		fmt.Fprintf(w, "{\"id\":%q}\n", relayedRequest.ID)
	default:
		mylog.Alert("server is busy")
		http.Error(w, `{"error":"Server is busy"}`, 500)
		mylog.Debugf("before remove petition", relayedRequest.ID)
		err := petColl.Remove(bson.M{"id": relayedRequest.ID})
		mylog.Debugf("after remove petition", relayedRequest.ID)
		if err != nil {
			mylog.Alert("ERROR removing petition", relayedRequest.ID, e)
			l.SessionSeed.Refresh()
			return
		}
		return
	}
}
Beispiel #6
0
//newReply returns the Reply for the Petition made, the http.Response gotten and the possible error
func newReply(resp *http.Response, p *Petition, e error) *Reply {
	var reply = &Reply{ID: p.ID, Petition: p, TraceID: p.TraceID}
	if e != nil {
		reply.Error = e.Error()
		return reply
	}
	reply.StatusCode = resp.StatusCode
	reply.Proto = resp.Proto
	reply.Header = resp.Header
	reply.Trailer = resp.Trailer
	mylog.Debug("before reading response body", p.ID)
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		mylog.Debugf("error reading response body %v %+v", err, p)
		reply.Error = e.Error()
	} else {
		mylog.Debug("after reading response body", p.ID)
		reply.Body = body
	}
	reply.Done = bson.Now()
	mylog.Debugf("reply done %+v", reply)
	return reply
}
Beispiel #7
0
//doRequest makes and retries the request as many times as is set, increasing the time between retries, doubling the initial time
func (c *Consumer) doRequest(req *http.Request, petid string) (resp *http.Response, err error) {
	resp, err = c.Client.Do(req)
	if err == nil && resp.StatusCode != 503 { //Good, not error and non challenging response
		return resp, nil
	}
	mylog.Debug("error making request", petid, err)
	var retryTime = time.Duration(c.Cfg.RetryTime) * time.Millisecond
	var retries = c.Cfg.Retries
	for i := 0; i < retries; i++ {
		time.Sleep(retryTime)
		mylog.Debugf("retrying request %v retry #%v after %v error %v", petid, i+1, retryTime, err)
		resp, err = c.Client.Do(req)
		if err == nil && resp.StatusCode != 503 {
			break
		}
		retryTime *= 2
	}
	return resp, err
}
Beispiel #8
0
//newPetition creates a petition from an http.Request. It checks header fields and make necessary transformations.
//The body is read and saved as a slice of byte.
func newPetition(original *http.Request) (*Petition, error) {
	targetHost := original.Header.Get(RelayerHost)
	if targetHost == "" {
		return nil, fmt.Errorf("gridas: Missing mandatory header %s", RelayerHost)
	}
	original.Header.Del(RelayerHost)
	scheme := strings.ToLower(original.Header.Get(RelayerProtocol))
	switch scheme {
	case "http", "https":
	case "":
		scheme = "http"
	default:
		mylog.Debug("unsupported protocol", scheme)
		return nil, fmt.Errorf("gridas: unsupported protocol %s", scheme)

	}
	original.Header.Del(RelayerProtocol)
	traceID := original.Header.Get(RelayerTraceID)
	if traceID == "" {
		//Just in case an older version client using "Topic"
		traceID = original.Header.Get(RelayerTopic)
	}
	original.Header.Del(RelayerTraceID)
	original.Header.Del(RelayerTopic)

	//Delete older header fields, ignore them, do nothing yet
	original.Header.Del(RelayerProxy)
	original.Header.Del(RelayerRetry)

	{
		//Hack for clients of older version
		const HTTPS = "https://"
		const HTTPSLen = len(HTTPS)
		const HTTP = "http://"
		const HTTPLen = len(HTTP)
		if strings.HasPrefix(targetHost, HTTPS) {
			targetHost = targetHost[HTTPSLen:]
			scheme = "https"
		} else if strings.HasPrefix(targetHost, HTTP) {
			targetHost = targetHost[HTTPLen:]
			scheme = "http"
		}
	}
	//save body content
	body, err := ioutil.ReadAll(original.Body)
	if err != nil {
		mylog.Debugf("error reading body request %v %+v", err, original)
		return nil, err
	}
	id := uuid.New()
	relayedRequest := &Petition{
		ID:           id,
		Body:         body,
		Method:       original.Method,
		URL:          original.URL,
		Proto:        original.Proto, // "HTTP/1.0"
		Header:       original.Header,
		Trailer:      original.Trailer,
		RemoteAddr:   original.RemoteAddr,
		RequestURI:   original.RequestURI,
		TargetHost:   targetHost,
		TargetScheme: scheme,
		Created:      bson.Now(),
		TraceID:      traceID}
	return relayedRequest, nil
}
Beispiel #9
0
//Stop asks consumer to stop taking petitions. When the stop is complete,
//the fact will be notified through the channel returned by the Start() method.
func (c *Consumer) Stop() {
	mylog.Debug("closing consumer end channel")
	close(c.endChan)
}
Beispiel #10
0
func main() {
	var cfgFile = flag.String("config", "./gridas.yaml", "configuration file")
	flag.Parse()
	cfg, err := config.ReadConfig(*cfgFile)
	if err != nil {
		mylog.Alert(err)
		os.Exit(-1)
	}
	fmt.Printf("configuration %q %+v\n", *cfgFile, cfg)

	mylog.SetLevel(cfg.LogLevel)
	mylog.Alert("hello World!")
	reqChan := make(chan *gridas.Petition, cfg.QueueSize)
	session, err := mgo.Dial(cfg.Mongo)
	if err != nil {
		mylog.Alert(err)
		panic(err)
	}
	defer func() {
		session.Close()
		mylog.Debugf("mongo session closed %+v", session)
	}()

	mylog.Debugf("mongo session %+v", session)

	db := session.DB(cfg.Database)
	mylog.Debug("mongo database", db)
	listener := &gridas.Listener{SendTo: reqChan, Cfg: cfg, SessionSeed: session}
	mylog.Debugf("listener %+v", listener)
	consumer := &gridas.Consumer{GetFrom: reqChan, Cfg: cfg, SessionSeed: session}
	mylog.Debugf("consumer %+v", consumer)
	rplyr := &gridas.Replyer{Cfg: cfg, SessionSeed: session}
	mylog.Debugf("replyer %+v", rplyr)
	rcvr := &gridas.Recoverer{SendTo: reqChan, Cfg: cfg, SessionSeed: session}
	mylog.Debugf("recoverer %+v", rcvr)
	endConsumers := consumer.Start(cfg.Consumers)
	if err := rcvr.Recover(); err != nil {
		mylog.Alert(err)
		os.Exit(-1)
	}
	http.Handle("/", listener)
	http.Handle("/responses/", http.StripPrefix("/responses/", rplyr))
	go func() {
		mylog.Debug("starting HTTP server (listener)")
		err := http.ListenAndServe(":"+cfg.Port, nil)
		if err != nil {
			mylog.Alert(err)
			os.Exit(-1)
		}
	}()

	onEnd(func() {
		mylog.Info("shutting down gridas ...")
		listener.Stop()
		mylog.Debug("listener stopped")
		consumer.Stop()
		mylog.Debug("consumer stopped")
	})
	<-endConsumers
	mylog.Alert("bye World!")
}
Beispiel #11
0
//Stop asks listener to stop receiving petitions
func (l *Listener) Stop() {
	mylog.Debug("listener received stop")
	atomic.StoreUint64(&l.stopping, 1)
}