func (c *Client) Get(ctx context.Context, url string) (r *http.Response, err error) {
ctx = trace.Enter(ctx, "HTTP")
if maximum := c.MaximumInFlight; maximum != 0 {
defer atomic.AddInt64(&c.inflight, -1)
// too many in-flight?
if n := atomic.AddInt64(&c.inflight, 1); n >= maximum {
trace.Leave(ctx, "Errors.TooManyInFlight")
return
}
}
r, err = defaults.Client(c.Client).Get(url)
if err != nil {
atomic.AddInt64(&c.count, -1)
trace.Error(ctx, "Errors.Fail", err)
return
}
if r.StatusCode != http.StatusOK && r.StatusCode != http.StatusNoContent {
atomic.AddInt64(&c.count, -1)
trace.Error(ctx, "Errors.Status", fmt.Errorf("%s", r.Status))
return
}
trace.Leave(ctx, "Check")
return
}
func (b *Bidders) Start() (err error) {
b.state.Store(make(map[string]*Agent))
//log.Println("START")
update := func() (err error) {
b.mu.Lock()
defer b.mu.Unlock()
log.Println("checking configurations...")
ctx := trace.Start(trace.SetHandler(context.Background(), b.Tracer), b.Name+".Tick", "")
last := b.state.Load().(map[string]*Agent)
agents, err := Import(b.Pattern)
if err != nil {
log.Println(err)
err = nil
}
if len(agents) == 0 {
log.Println("no files found: ", b.Pattern)
trace.Leave(ctx, "Errors.NoFiles")
return
}
next := make(map[string]*Agent)
now := time.Now().UTC()
for _, agent := range agents {
id := fmt.Sprintf("%d", agent.ID)
//log.Println(id, agent.Account[0])
next[id] = agent
if a := last[id]; a == nil {
agent.Begin()
} else {
agent.state = a.state
agent.Update(ctx, now)
}
}
b.state.Store(next)
trace.Leave(ctx, "Done")
return
}
if err = update(); err != nil {
log.Println(err)
}
b.tick = rtb.PeriodicFunc(rtb.Every(time.Minute), func() {
err := update()
if err != nil {
log.Println(err)
}
})
return
}
func TestOpenRTB(t *testing.T) {
m := make(map[string]interface{})
if err := json.Unmarshal(sample, &m); err != nil {
t.Fatal(err)
}
br, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
log.Println(string(br))
b := &Bidders{
Pattern: "../../../../configs/bidders/*.json",
}
c := trace.Start(trace.SetHandler(context.Background(), nil), "test", "")
b.Start()
e := &Exchange{Bidders: b}
w := httptest.NewRecorder()
e.ServeHTTP(c, w, &http.Request{Body: ioutil.NopCloser(bytes.NewReader(br))})
trace.Leave(c, "done")
log.Println(w.Code, w.Body.String())
}
func (e *Exchange) forensiqRiskScore(ctx context.Context, r rtb.Request) (value float64) {
ctx = trace.Enter(ctx, "Forensiq")
result, err := e.Client.NewRequest(r)
if err != nil {
trace.Error(ctx, "Error.NewRequest", err)
return
}
p := result.(rtb.Processor)
if err := p.Process(ctx, result); err != nil {
trace.Error(ctx, "Error.Process", err)
return
}
resp := result.Component("forensiq")
if resp == nil {
trace.Error(ctx, "Error.NoResponse", err)
return
}
extractor, ok := resp.(rtb.Extractor)
if !ok {
trace.Error(ctx, "Error.NoExtractor", err)
return
}
score := extractor.Extract("riskScore")
if score == nil {
trace.Leave(ctx, "Error.NoRiskScore")
return
}
value, ok = score.(float64)
if !ok {
trace.Leave(ctx, "Error.NoRiskScoreValue")
return
}
trace.Leave(ctx, "Requested")
return
}
func (a *Agent) Update(ctx context.Context, now time.Time) {
ctx = trace.Enter(ctx, a.Account[0]+".Pace")
a.post()
p := a.state.Load().(*Pacing)
dt := now.Sub(p.timestamp)
n := atomic.LoadInt64(&p.requests)
qps := float64(n) / dt.Seconds()
m := 0.0
s := 1.0
bps := 0.0
if a.Parameters != nil {
pace, err := strconv.Atoi(strings.TrimSuffix(a.Parameters.Pace, "USD/1M"))
if err != nil {
pace = 0
}
price, err := strconv.Atoi(strings.TrimSuffix(a.Parameters.Price, "USD/1M"))
if err != nil {
price = 0
}
if price != 0 {
bps = float64(pace) / float64(price) / 60.0 / 2.0
if bps >= qps {
s = 1.0
} else {
s = bps / qps
}
m = bps * dt.Seconds()
}
}
ema := 0.8*p.sampling + 0.2*s
trace.Set(ctx, "BPS", bps)
trace.Set(ctx, "QPS", qps)
//trace.Set(ctx, "SmoothQPS", qps)
trace.Set(ctx, "Sampling", s)
trace.Set(ctx, "SamplingEMA", ema)
a.state.Store(&Pacing{
bids: int64(m),
timestamp: now,
qps: qps,
sampling: ema,
})
trace.Leave(ctx, "Done")
}
// ServeHTTP creates the root context and pass it to the HTTP request handler.
// The default /ready route is also handled.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if atomic.LoadInt64(&s.ready) == 0 {
http.Error(w, "server is closed", http.StatusServiceUnavailable)
return
}
// start trace
key := r.Header.Get(trace.HeaderKey)
ctx := trace.Start(trace.SetHandler(context.Background(), s.Tracer), s.name, key)
// ready?
if r.URL.Path == "/ready" && r.Method == "GET" {
trace.Leave(ctx, "Ready")
w.WriteHeader(http.StatusOK)
return
}
if s.Handler != nil {
s.Handler.ServeHTTP(ctx, w, r)
}
trace.Leave(ctx, "Served")
}
// Process sends a request to query the /check endpoint.
// The response is attached to the request.
func (c *Client) Process(ctx context.Context, r rtb.Request) (err error) {
ctx = trace.Enter(ctx, "Forensiq")
// ready?
if !c.HTTP.Ready() {
err = ErrUnavailable
trace.Error(ctx, "Errors.Ready", err)
return
}
var args url.Values
// fast path in case the request is already prepared
if p, ok := r.(*request); ok {
args = p.values
} else {
args, err = c.prepare(r)
if err != nil {
trace.Error(ctx, "Errors.Prepare", err)
return
}
}
qs := args.Encode()
// look into the memory cache or perform the query
value := c.fromCache(qs)
if value != nil {
trace.Count(ctx, "CacheHit", 1)
} else {
value, err = c.query(ctx, qs)
if err != nil {
trace.Error(ctx, "Errors.Request", err)
return
}
c.intoCache(qs, value)
}
// hold the result
r.Attach(defaults.String(c.Target, "forensiq"), value)
trace.Leave(ctx, "Check")
return
}
func (e *NoExchange) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
ctx = trace.Enter(ctx, "Request")
vast := func() {
value := []byte(`<VAST xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="vast.xsd" version="2.0"/>`)
w.Header().Set("Content-Type", "application/xml")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(value)))
w.Write(value)
}
u := e.URL
if list := strings.Split(u, ","); len(list) > 1 {
n := atomic.AddInt64(&e.random, 1)
u = list[n%int64(len(list))]
}
req, err := http.NewRequest("GET", u+r.URL.Path, nil)
if err != nil {
trace.Error(ctx, "Errors.Request", err)
vast()
return
}
if _, err := url.QueryUnescape(r.URL.RawQuery); err != nil {
log.Println(r.URL.RawQuery)
trace.Error(ctx, "Errors.BadQueryString", err)
vast()
return
}
values := r.URL.Query()
for k, v := range values {
p, ok := parameters[k]
if !ok || len(v) != 1 || v[0] == "" {
delete(values, k)
continue
}
if _, err := p.Parse(v[0]); err != nil {
delete(values, k)
continue
}
}
values.Set("id", NewUUID())
req.URL.RawQuery = values.Encode()
n := atomic.AddInt64(&e.inflight, 1)
if n > 32 {
atomic.AddInt64(&e.inflight, -1)
trace.Leave(ctx, "Errors.TooManyInFlight")
vast()
return
}
result := make(chan func() (*http.Response, error), 1)
go func() {
c := e.Client
if nil == c {
c = client
}
resp, err := c.Do(req)
result <- func() (*http.Response, error) {
return resp, err
}
atomic.AddInt64(&e.inflight, -1)
}()
select {
case <-time.After(50 * time.Millisecond):
trace.Leave(ctx, "Errors.Timeout")
vast()
go func() {
f := <-result
resp, err := f()
if err != nil {
//trace.Error(ctx, "Errors.TimeoutFailed", err)
return
}
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}()
return
case f := <-result:
resp, err := f()
if err != nil {
trace.Error(ctx, "Errors.Failed", err)
vast()
return
}
h := w.Header()
for k, v := range resp.Header {
h[k] = v
}
if _, err := io.Copy(w, resp.Body); err != nil {
trace.Error(ctx, "Errors.Copy", err)
vast()
return
}
resp.Body.Close()
}
trace.Leave(ctx, "Done")
return
}
// Start installs the server and starts serving requests until the server is closed.
func (s *Server) Start() (err error) {
s.Server.Handler = s
// open socket
s.listen, err = net.Listen("tcp", defaults.String(s.Server.Addr, ":http"))
if err != nil {
return
}
s.name = defaults.String(s.Name, s.listen.Addr().String())
// provide a default read timeout if missing
if 0 == s.Server.ReadTimeout {
s.Server.ReadTimeout = time.Minute
}
// track new/closed HTTP connections
s.conns = make(map[net.Conn]http.ConnState)
cs := s.Server.ConnState
s.Server.ConnState = func(conn net.Conn, state http.ConnState) {
s.mu.Lock()
defer s.mu.Unlock()
switch state {
case http.StateNew:
atomic.AddInt64(&s.count, +1)
s.wg.Add(1)
s.conns[conn] = state
case http.StateClosed, http.StateHijacked:
atomic.AddInt64(&s.count, -1)
s.wg.Done()
delete(s.conns, conn)
case http.StateActive, http.StateIdle:
s.conns[conn] = state
}
if nil != cs {
cs(conn, state)
}
}
// update metrics
s.tick = Tick(time.Second, func() {
ctx := trace.Start(trace.SetHandler(context.Background(), s.Tracer), s.name+".Tick", "")
trace.Set(ctx, "Connections", atomic.LoadInt64(&s.count))
trace.Set(ctx, "State", atomic.LoadInt64(&s.ready))
trace.Leave(ctx, "Ticked")
})
// serve requests
go func() {
s.Server.Serve(s.listen)
}()
// set ready
if !atomic.CompareAndSwapInt64(&s.ready, 0, 1) {
panic("server is already ready to serve requests")
}
return
}
func (e *Exchange) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
ctx = trace.Enter(ctx, "OpenRTB")
body, err := ioutil.ReadAll(r.Body)
if err != nil {
trace.Error(ctx, "Errors.Read", err)
w.WriteHeader(http.StatusNoContent)
return
}
value := &jq.Value{}
if err := value.Unmarshal(body); err != nil {
trace.Error(ctx, "Errors.Unmarshal", err)
w.WriteHeader(http.StatusNoContent)
return
}
p := value.NewQuery()
impid, err := p.String("imp", "@0", "id")
if err != nil {
trace.Error(ctx, "Errors.MissingImpressionID", err)
w.WriteHeader(http.StatusNoContent)
return
}
q := value.NewQuery()
if err := q.FindObject("imp", "@0", "ext", "creative-ids"); err != nil {
trace.Error(ctx, "Errors.MissingIDs", err)
w.WriteHeader(http.StatusNoContent)
return
}
allowed := make(map[string][]string)
if q.Down() {
for {
if q.Down() {
list := make([]string, 0, q.Count())
for {
list = append(list, q.Value())
if q.Next() == false {
break
}
}
q.Up()
allowed[q.Key()] = list
}
if q.Next() == false {
break
}
}
}
if len(allowed) == 0 {
trace.Leave(ctx, "NoAllowedBidders")
w.WriteHeader(http.StatusNoContent)
return
}
ids := make([]string, 0, len(allowed))
for i := range allowed {
ids = append(ids, i)
}
bidders := e.Bidders.Bidders(ids)
bestPrice := ""
bestPriority := ""
best := -1
for i := range bidders {
if bidders[i] == nil {
continue
}
price, priority := bidders[i].Bid(ctx)
if price == "" {
continue
}
if best != -1 {
if len(priority) < len(bestPriority) {
continue
}
if len(priority) == len(bestPriority) && priority < bestPriority {
continue
}
if priority == bestPriority {
if len(price) < len(bestPrice) {
continue
}
if len(price) == len(bestPrice) && price < bestPrice {
continue
}
}
}
bestPrice = price
bestPriority = priority
best = i
}
if best == -1 {
trace.Leave(ctx, "NoAllowedBidders")
w.WriteHeader(http.StatusNoContent)
return
}
text := `{"seatbid":[{"bid":[{"impid":"%s","price":%f,"crid":"%s","ext":{"priority":%s,"external-id":%s}}]}]}`
cpm := 0.0
if money, err := strconv.Atoi(strings.TrimSuffix(bestPrice, "USD/1M")); err != nil {
trace.Error(ctx, "Errors.Price", err)
w.WriteHeader(http.StatusNoContent)
return
} else {
cpm = float64(money) / 1000.0
}
cid := ids[best]
augmenters := bidders[best].Augmenters
if augmenters != nil {
if f := augmenters.Forensiq; e.Client != nil && f != nil {
r := &rtb.Components{}
r.Attach("fields", value)
score := e.forensiqRiskScore(ctx, r)
trace.Record(ctx, "Score", score)
if score > f.Configuration.RiskScore {
trace.Leave(ctx, "ForensiqRiskScore")
w.WriteHeader(http.StatusNoContent)
return
}
}
}
crid := allowed[cid][0]
b := string(body)
if strings.Contains(b, cid) == false {
log.Println(b)
log.Println(allowed, ids, cid, crid, best)
}
w.Header().Set("Content-Type", "application/json")
if _, err := fmt.Fprintf(w, text, impid, cpm, crid, bestPriority, cid); err != nil {
trace.Error(ctx, "Errors.Response", err)
return
}
//jsons.Put(value)
trace.Leave(ctx, "Responded")
}