コード例 #1
0
ファイル: client.go プロジェクト: jamloop/rtbkit
func (c *Client) Get(ctx context.Context, url string) (r *http.Response, err error) {
	ctx = trace.Enter(ctx, "HTTP")

	if maximum := c.MaximumInFlight; maximum != 0 {
		defer atomic.AddInt64(&c.inflight, -1)

		// too many in-flight?
		if n := atomic.AddInt64(&c.inflight, 1); n >= maximum {
			trace.Leave(ctx, "Errors.TooManyInFlight")
			return
		}
	}

	r, err = defaults.Client(c.Client).Get(url)
	if err != nil {
		atomic.AddInt64(&c.count, -1)
		trace.Error(ctx, "Errors.Fail", err)
		return
	}

	if r.StatusCode != http.StatusOK && r.StatusCode != http.StatusNoContent {
		atomic.AddInt64(&c.count, -1)
		trace.Error(ctx, "Errors.Status", fmt.Errorf("%s", r.Status))
		return
	}

	trace.Leave(ctx, "Check")
	return
}
コード例 #2
0
ファイル: rtbkit.go プロジェクト: jamloop/rtbkit
func (b *Bidders) Start() (err error) {
	b.state.Store(make(map[string]*Agent))

	//log.Println("START")

	update := func() (err error) {
		b.mu.Lock()
		defer b.mu.Unlock()

		log.Println("checking configurations...")

		ctx := trace.Start(trace.SetHandler(context.Background(), b.Tracer), b.Name+".Tick", "")
		last := b.state.Load().(map[string]*Agent)

		agents, err := Import(b.Pattern)
		if err != nil {
			log.Println(err)
			err = nil
		}

		if len(agents) == 0 {
			log.Println("no files found: ", b.Pattern)
			trace.Leave(ctx, "Errors.NoFiles")
			return
		}

		next := make(map[string]*Agent)
		now := time.Now().UTC()

		for _, agent := range agents {
			id := fmt.Sprintf("%d", agent.ID)
			//log.Println(id, agent.Account[0])
			next[id] = agent
			if a := last[id]; a == nil {
				agent.Begin()
			} else {
				agent.state = a.state
				agent.Update(ctx, now)
			}
		}

		b.state.Store(next)
		trace.Leave(ctx, "Done")
		return
	}

	if err = update(); err != nil {
		log.Println(err)
	}

	b.tick = rtb.PeriodicFunc(rtb.Every(time.Minute), func() {
		err := update()
		if err != nil {
			log.Println(err)
		}
	})

	return
}
コード例 #3
0
ファイル: openrtb_test.go プロジェクト: jamloop/rtbkit
func TestOpenRTB(t *testing.T) {
	m := make(map[string]interface{})

	if err := json.Unmarshal(sample, &m); err != nil {
		t.Fatal(err)
	}

	br, err := json.Marshal(m)
	if err != nil {
		t.Fatal(err)
	}

	log.Println(string(br))

	b := &Bidders{
		Pattern: "../../../../configs/bidders/*.json",
	}

	c := trace.Start(trace.SetHandler(context.Background(), nil), "test", "")

	b.Start()
	e := &Exchange{Bidders: b}

	w := httptest.NewRecorder()
	e.ServeHTTP(c, w, &http.Request{Body: ioutil.NopCloser(bytes.NewReader(br))})

	trace.Leave(c, "done")

	log.Println(w.Code, w.Body.String())
}
コード例 #4
0
ファイル: openrtb.go プロジェクト: jamloop/rtbkit
func (e *Exchange) forensiqRiskScore(ctx context.Context, r rtb.Request) (value float64) {
	ctx = trace.Enter(ctx, "Forensiq")

	result, err := e.Client.NewRequest(r)
	if err != nil {
		trace.Error(ctx, "Error.NewRequest", err)
		return
	}

	p := result.(rtb.Processor)

	if err := p.Process(ctx, result); err != nil {
		trace.Error(ctx, "Error.Process", err)
		return
	}

	resp := result.Component("forensiq")
	if resp == nil {
		trace.Error(ctx, "Error.NoResponse", err)
		return
	}

	extractor, ok := resp.(rtb.Extractor)
	if !ok {
		trace.Error(ctx, "Error.NoExtractor", err)
		return
	}

	score := extractor.Extract("riskScore")
	if score == nil {
		trace.Leave(ctx, "Error.NoRiskScore")
		return
	}

	value, ok = score.(float64)
	if !ok {
		trace.Leave(ctx, "Error.NoRiskScoreValue")
		return
	}

	trace.Leave(ctx, "Requested")
	return
}
コード例 #5
0
ファイル: rtbkit.go プロジェクト: jamloop/rtbkit
func (a *Agent) Update(ctx context.Context, now time.Time) {
	ctx = trace.Enter(ctx, a.Account[0]+".Pace")
	a.post()

	p := a.state.Load().(*Pacing)

	dt := now.Sub(p.timestamp)
	n := atomic.LoadInt64(&p.requests)
	qps := float64(n) / dt.Seconds()

	m := 0.0
	s := 1.0

	bps := 0.0
	if a.Parameters != nil {
		pace, err := strconv.Atoi(strings.TrimSuffix(a.Parameters.Pace, "USD/1M"))
		if err != nil {
			pace = 0
		}

		price, err := strconv.Atoi(strings.TrimSuffix(a.Parameters.Price, "USD/1M"))
		if err != nil {
			price = 0
		}

		if price != 0 {
			bps = float64(pace) / float64(price) / 60.0 / 2.0
			if bps >= qps {
				s = 1.0
			} else {
				s = bps / qps
			}

			m = bps * dt.Seconds()
		}
	}

	ema := 0.8*p.sampling + 0.2*s

	trace.Set(ctx, "BPS", bps)
	trace.Set(ctx, "QPS", qps)
	//trace.Set(ctx, "SmoothQPS", qps)
	trace.Set(ctx, "Sampling", s)
	trace.Set(ctx, "SamplingEMA", ema)

	a.state.Store(&Pacing{
		bids:      int64(m),
		timestamp: now,
		qps:       qps,
		sampling:  ema,
	})

	trace.Leave(ctx, "Done")
}
コード例 #6
0
ファイル: server.go プロジェクト: jamloop/rtbkit
// ServeHTTP creates the root context and pass it to the HTTP request handler.
// The default /ready route is also handled.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if atomic.LoadInt64(&s.ready) == 0 {
		http.Error(w, "server is closed", http.StatusServiceUnavailable)
		return
	}

	// start trace
	key := r.Header.Get(trace.HeaderKey)
	ctx := trace.Start(trace.SetHandler(context.Background(), s.Tracer), s.name, key)

	// ready?
	if r.URL.Path == "/ready" && r.Method == "GET" {
		trace.Leave(ctx, "Ready")
		w.WriteHeader(http.StatusOK)
		return
	}

	if s.Handler != nil {
		s.Handler.ServeHTTP(ctx, w, r)
	}

	trace.Leave(ctx, "Served")
}
コード例 #7
0
ファイル: forensiq.go プロジェクト: jamloop/rtbkit
// Process sends a request to query the /check endpoint.
// The response is attached to the request.
func (c *Client) Process(ctx context.Context, r rtb.Request) (err error) {
	ctx = trace.Enter(ctx, "Forensiq")

	// ready?
	if !c.HTTP.Ready() {
		err = ErrUnavailable
		trace.Error(ctx, "Errors.Ready", err)
		return
	}

	var args url.Values

	// fast path in case the request is already prepared
	if p, ok := r.(*request); ok {
		args = p.values
	} else {
		args, err = c.prepare(r)
		if err != nil {
			trace.Error(ctx, "Errors.Prepare", err)
			return
		}
	}

	qs := args.Encode()

	// look into the memory cache or perform the query
	value := c.fromCache(qs)
	if value != nil {
		trace.Count(ctx, "CacheHit", 1)
	} else {
		value, err = c.query(ctx, qs)
		if err != nil {
			trace.Error(ctx, "Errors.Request", err)
			return
		}

		c.intoCache(qs, value)
	}

	// hold the result
	r.Attach(defaults.String(c.Target, "forensiq"), value)

	trace.Leave(ctx, "Check")
	return
}
コード例 #8
0
ファイル: jamloop.go プロジェクト: jamloop/rtbkit
func (e *NoExchange) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	ctx = trace.Enter(ctx, "Request")

	vast := func() {
		value := []byte(`<VAST xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="vast.xsd" version="2.0"/>`)
		w.Header().Set("Content-Type", "application/xml")
		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(value)))
		w.Write(value)
	}

	u := e.URL
	if list := strings.Split(u, ","); len(list) > 1 {
		n := atomic.AddInt64(&e.random, 1)
		u = list[n%int64(len(list))]
	}

	req, err := http.NewRequest("GET", u+r.URL.Path, nil)
	if err != nil {
		trace.Error(ctx, "Errors.Request", err)
		vast()
		return
	}

	if _, err := url.QueryUnescape(r.URL.RawQuery); err != nil {
		log.Println(r.URL.RawQuery)
		trace.Error(ctx, "Errors.BadQueryString", err)
		vast()
		return
	}

	values := r.URL.Query()

	for k, v := range values {
		p, ok := parameters[k]
		if !ok || len(v) != 1 || v[0] == "" {
			delete(values, k)
			continue
		}

		if _, err := p.Parse(v[0]); err != nil {
			delete(values, k)
			continue
		}
	}

	values.Set("id", NewUUID())

	req.URL.RawQuery = values.Encode()

	n := atomic.AddInt64(&e.inflight, 1)
	if n > 32 {
		atomic.AddInt64(&e.inflight, -1)
		trace.Leave(ctx, "Errors.TooManyInFlight")
		vast()
		return
	}

	result := make(chan func() (*http.Response, error), 1)
	go func() {
		c := e.Client
		if nil == c {
			c = client
		}

		resp, err := c.Do(req)
		result <- func() (*http.Response, error) {
			return resp, err
		}

		atomic.AddInt64(&e.inflight, -1)
	}()

	select {
	case <-time.After(50 * time.Millisecond):
		trace.Leave(ctx, "Errors.Timeout")
		vast()

		go func() {
			f := <-result

			resp, err := f()
			if err != nil {
				//trace.Error(ctx, "Errors.TimeoutFailed", err)
				return
			}

			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}()

		return
	case f := <-result:
		resp, err := f()
		if err != nil {
			trace.Error(ctx, "Errors.Failed", err)
			vast()
			return
		}

		h := w.Header()
		for k, v := range resp.Header {
			h[k] = v
		}

		if _, err := io.Copy(w, resp.Body); err != nil {
			trace.Error(ctx, "Errors.Copy", err)
			vast()
			return
		}

		resp.Body.Close()
	}

	trace.Leave(ctx, "Done")
	return
}
コード例 #9
0
ファイル: server.go プロジェクト: jamloop/rtbkit
// Start installs the server and starts serving requests until the server is closed.
func (s *Server) Start() (err error) {
	s.Server.Handler = s

	// open socket
	s.listen, err = net.Listen("tcp", defaults.String(s.Server.Addr, ":http"))
	if err != nil {
		return
	}

	s.name = defaults.String(s.Name, s.listen.Addr().String())

	// provide a default read timeout if missing
	if 0 == s.Server.ReadTimeout {
		s.Server.ReadTimeout = time.Minute
	}

	// track new/closed HTTP connections
	s.conns = make(map[net.Conn]http.ConnState)

	cs := s.Server.ConnState
	s.Server.ConnState = func(conn net.Conn, state http.ConnState) {
		s.mu.Lock()
		defer s.mu.Unlock()

		switch state {
		case http.StateNew:
			atomic.AddInt64(&s.count, +1)
			s.wg.Add(1)
			s.conns[conn] = state
		case http.StateClosed, http.StateHijacked:
			atomic.AddInt64(&s.count, -1)
			s.wg.Done()
			delete(s.conns, conn)
		case http.StateActive, http.StateIdle:
			s.conns[conn] = state
		}

		if nil != cs {
			cs(conn, state)
		}
	}

	// update metrics
	s.tick = Tick(time.Second, func() {
		ctx := trace.Start(trace.SetHandler(context.Background(), s.Tracer), s.name+".Tick", "")
		trace.Set(ctx, "Connections", atomic.LoadInt64(&s.count))
		trace.Set(ctx, "State", atomic.LoadInt64(&s.ready))
		trace.Leave(ctx, "Ticked")
	})

	// serve requests
	go func() {
		s.Server.Serve(s.listen)
	}()

	// set ready
	if !atomic.CompareAndSwapInt64(&s.ready, 0, 1) {
		panic("server is already ready to serve requests")
	}

	return
}
コード例 #10
0
ファイル: openrtb.go プロジェクト: jamloop/rtbkit
func (e *Exchange) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	ctx = trace.Enter(ctx, "OpenRTB")

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		trace.Error(ctx, "Errors.Read", err)
		w.WriteHeader(http.StatusNoContent)
		return
	}

	value := &jq.Value{}

	if err := value.Unmarshal(body); err != nil {
		trace.Error(ctx, "Errors.Unmarshal", err)
		w.WriteHeader(http.StatusNoContent)
		return
	}

	p := value.NewQuery()
	impid, err := p.String("imp", "@0", "id")
	if err != nil {
		trace.Error(ctx, "Errors.MissingImpressionID", err)
		w.WriteHeader(http.StatusNoContent)
		return
	}

	q := value.NewQuery()
	if err := q.FindObject("imp", "@0", "ext", "creative-ids"); err != nil {
		trace.Error(ctx, "Errors.MissingIDs", err)
		w.WriteHeader(http.StatusNoContent)
		return
	}

	allowed := make(map[string][]string)
	if q.Down() {
		for {
			if q.Down() {
				list := make([]string, 0, q.Count())
				for {
					list = append(list, q.Value())
					if q.Next() == false {
						break
					}
				}

				q.Up()
				allowed[q.Key()] = list
			}

			if q.Next() == false {
				break
			}
		}
	}

	if len(allowed) == 0 {
		trace.Leave(ctx, "NoAllowedBidders")
		w.WriteHeader(http.StatusNoContent)
		return
	}

	ids := make([]string, 0, len(allowed))
	for i := range allowed {
		ids = append(ids, i)
	}

	bidders := e.Bidders.Bidders(ids)

	bestPrice := ""
	bestPriority := ""
	best := -1
	for i := range bidders {
		if bidders[i] == nil {
			continue
		}

		price, priority := bidders[i].Bid(ctx)
		if price == "" {
			continue
		}

		if best != -1 {
			if len(priority) < len(bestPriority) {
				continue
			}

			if len(priority) == len(bestPriority) && priority < bestPriority {
				continue
			}

			if priority == bestPriority {
				if len(price) < len(bestPrice) {
					continue
				}

				if len(price) == len(bestPrice) && price < bestPrice {
					continue
				}
			}
		}

		bestPrice = price
		bestPriority = priority
		best = i
	}

	if best == -1 {
		trace.Leave(ctx, "NoAllowedBidders")
		w.WriteHeader(http.StatusNoContent)
		return
	}

	text := `{"seatbid":[{"bid":[{"impid":"%s","price":%f,"crid":"%s","ext":{"priority":%s,"external-id":%s}}]}]}`

	cpm := 0.0
	if money, err := strconv.Atoi(strings.TrimSuffix(bestPrice, "USD/1M")); err != nil {
		trace.Error(ctx, "Errors.Price", err)
		w.WriteHeader(http.StatusNoContent)
		return
	} else {
		cpm = float64(money) / 1000.0
	}

	cid := ids[best]

	augmenters := bidders[best].Augmenters
	if augmenters != nil {
		if f := augmenters.Forensiq; e.Client != nil && f != nil {
			r := &rtb.Components{}
			r.Attach("fields", value)

			score := e.forensiqRiskScore(ctx, r)
			trace.Record(ctx, "Score", score)

			if score > f.Configuration.RiskScore {
				trace.Leave(ctx, "ForensiqRiskScore")
				w.WriteHeader(http.StatusNoContent)
				return
			}
		}
	}

	crid := allowed[cid][0]

	b := string(body)
	if strings.Contains(b, cid) == false {
		log.Println(b)
		log.Println(allowed, ids, cid, crid, best)
	}

	w.Header().Set("Content-Type", "application/json")
	if _, err := fmt.Fprintf(w, text, impid, cpm, crid, bestPriority, cid); err != nil {
		trace.Error(ctx, "Errors.Response", err)
		return
	}

	//jsons.Put(value)

	trace.Leave(ctx, "Responded")
}