func POST(w http.ResponseWriter, r *http.Request) {
//read the raw post data nad marhsal it s
bytes, _ := ioutil.ReadAll(r.Body)
var allFields AllFields
json.Unmarshal(bytes, &allFields)
//get the time sturct work right
if allFields.TempTime != "" {
t, err := time.Parse("2006-01-02 15:04:05", allFields.TempTime)
if err != nil {
structured.Warn("", "", "can't parse the time", 0, nil)
} else {
allFields.Created = t
}
}
//find the log types
logType := allFields.LogType
if logType == "impress" {
impression(allFields)
} else if logType == "click" {
click(allFields)
} else if logType == "install" {
install(allFields)
} else if logType == "event" {
event(allFields)
} else if logType == "open" {
open(allFields)
}
}
func click(allFields AllFields) {
//set up unqiue fields first
var uniFields Click /////
uniFields.Id = allFields.Id
uniFields.Created = allFields.Created
uniFields.DeviceIp = allFields.DeviceIp
uniFields.CountryCode = allFields.CountryCode
uniFields.RegionCode = allFields.RegionCode
uniFields.PostalCode = allFields.PostalCode
uniFields.Location = strconv.FormatFloat(allFields.Latitude, 'E', -1, 64) + "," + strconv.FormatFloat(allFields.Longitude, 'E', -1, 64)
uniFields.WurflBrandName = allFields.WurflBrandName
uniFields.WurflModelName = allFields.WurflModelName
uniFields.WurflDeviceOs = allFields.WurflDeviceOs
//search for realationship with install first ******click id aleays exist
termQuery := elastic.NewTermQuery("ClickInstallId", uniFields.Id)
searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()
//******************** find the only install
if searchResult.TotalHits() == 1 {
hits := searchResult.Hits.Hits
firstHit := hits[0]
parentId := firstHit.Id
//update the parent doc's clickInstallId -----the click id which realted to that install
var clickID UpdateClickId
clickID.ClickInstallId = uniFields.Id
client.Update().Index("alls").Type("Common").Id(parentId).Doc(clickID).Do()
//index the click doc as child doc to this parent id
client.Index().Index("alls").Type("Click").Parent(parentId).BodyJson(uniFields).Do()
} else if searchResult.TotalHits() == 0 {
//************find no install--- find the event and open first *****click id always exists
termQuery = elastic.NewTermQuery("StatClickId", uniFields.Id)
//update the events to have reengaement click struct
searchEvent, _ := client.Search().Index("alls").Type("Event").Query(&termQuery).Pretty(true).Do()
for _, hit := range searchEvent.Hits.Hits {
var update UpdateClick
update.Click = uniFields
docId := hit.Id
//get the parent id
var event Acticity
json.Unmarshal(*hit.Source, &event)
client.Update().Index("alls").Type("Event").Parent(event.ParentId).Id(docId).Doc(update).Do()
}
//update the opens to have this reengaement click
searchOpen, _ := client.Search().Index("alls").Type("Open").Query(&termQuery).Pretty(true).Do()
for _, hit := range searchOpen.Hits.Hits {
var update UpdateClick
update.Click = uniFields
docId := hit.Id
//get the parent id
var open Acticity
json.Unmarshal(*hit.Source, &open)
client.Update().Index("alls").Type("Open").Parent(open.ParentId).Id(docId).Doc(update).Do()
}
//************if there is no install and no matter whether it found a event/open, post the click
//post the parent first
var common Common
common.GoogleAid = allFields.GoogleAid
common.WindowsAid = allFields.WindowsAid
common.IosIfa = allFields.IosIfa
common.Language = allFields.Language
common.CurrencyCode = allFields.CurrencyCode
common.SiteId = allFields.SiteId
common.AdvertiserId = allFields.AdvertiserId
common.PackageName = allFields.PackageName
common.PublisherId = allFields.PublisherId
common.AdNetworkId = allFields.AdNetworkId
common.AgencyId = allFields.AgencyId
common.CampaignId = allFields.CampaignId
common.PublisherUserId = allFields.PublisherUserId
//the click id which future install may contributed to
common.ClickInstallId = uniFields.Id /////
//index the parent common field first
indexParent, _ := client.Index().Index("alls").Type("Common").BodyJson(common).Do()
parentId := indexParent.Id
//index the click as the children
client.Index().Index("alls").Type("Click").Parent(parentId).BodyJson(uniFields).Do() /////
} else {
//*************find multiple install related to this click
structured.Warn(uniFields.Id, "Click", "the click has multiple install contributed to it ", int(allFields.SiteId), nil) /////
}
}
func helper(allFields AllFields, eventType string) {
var event Acticity
event.Id = allFields.Id
event.Created = allFields.Created
event.DeviceIp = allFields.DeviceIp
event.StatImpressionId = allFields.StatImpressionId
event.StatClickId = allFields.StatClickId
event.StatInstallId = allFields.StatInstallId
event.CountryCode = allFields.CountryCode
event.RegionCode = allFields.RegionCode
event.PostalCode = allFields.PostalCode
event.Location = strconv.FormatFloat(allFields.Latitude, 'E', -1, 64) + "," + strconv.FormatFloat(allFields.Longitude, 'E', -1, 64)
event.WurflBrandName = allFields.WurflBrandName
event.WurflDeviceOs = allFields.WurflDeviceOs
event.WurflModelName = allFields.WurflModelName
//************find whether it related to the install first
noHit := true
if event.StatInstallId != "" {
termQuery := elastic.NewTermQuery("StatInstallId", event.StatInstallId)
searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()
if searchResult.TotalHits() == 1 {
//already find that only install related
noHit = false
hit := searchResult.Hits.Hits[0]
parentId := hit.Id
event.ParentId = parentId //event need to own a parent id and for later update
//unmarhsal the parent
var parent Common
json.Unmarshal(*hit.Source, &parent)
nohitClickImp := true
//*************************if event has stat click id and different with parent
if event.StatClickId != "" && parent.ClickInstallId != event.StatClickId {
termQuery = elastic.NewTermQuery("Id", event.StatClickId)
searchResult, _ = client.Search().Index("alls").Type("Click").Query(&termQuery).Pretty(true).Do()
if searchResult.TotalHits() > 0 {
if searchResult.TotalHits() == 1 {
nohitClickImp = false
//put the click struct inside the event sturct
var found Click
hits := searchResult.Hits.Hits[0]
json.Unmarshal(*hits.Source, &found)
event.Click = found
//put the event as a child
client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do()
} else {
structured.Warn(event.Id, eventType, "this event/open has multiple clicks related to it ", int(allFields.SiteId), nil)
}
}
} else if event.StatImpressionId != "" && parent.ImpressionInstallId != event.StatImpressionId {
//*************************if event has stat impression id and different with parent
termQuery = elastic.NewTermQuery("Id", event.StatImpressionId)
searchResult, _ = client.Search().Index("alls").Type("Impression").Query(&termQuery).Pretty(true).Do()
if searchResult.TotalHits() > 0 {
if searchResult.TotalHits() == 1 {
nohitClickImp = false
//put the impression struct inside the event sturct
var found Impression
hits := searchResult.Hits.Hits[0]
json.Unmarshal(*hits.Source, &found)
event.Impression = found
//put the event as a child
client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do()
} else {
structured.Warn(event.Id, eventType, "this event/open has multiple impression related to it ", int(allFields.SiteId), nil)
}
}
}
if nohitClickImp {
//if the event no related with any click or impresssion
client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do()
}
} else if searchResult.TotalHits() > 1 {
structured.Warn(event.Id, eventType, "has multiple event/open contributed to that", int(allFields.SiteId), nil)
}
//else is no hit with current install leave nohit to be true
}
//**********no install related to it, post all parent and children doc have ttl and then ignore it in ES
if noHit {
var common Common
common.GoogleAid = allFields.GoogleAid
common.WindowsAid = allFields.WindowsAid
common.IosIfa = allFields.IosIfa
common.Language = allFields.Language
common.CurrencyCode = allFields.CurrencyCode
common.SiteId = allFields.SiteId
common.AdvertiserId = allFields.AdvertiserId
common.PackageName = allFields.PackageName
common.PublisherId = allFields.PublisherId
common.AdNetworkId = allFields.AdNetworkId
common.AgencyId = allFields.AgencyId
common.CampaignId = allFields.CampaignId
common.PublisherUserId = allFields.PublisherUserId
indexParent, _ := client.Index().Index("alls").Type("Common").BodyJson(common).Do()
parentId := indexParent.Id
event.ParentId = parentId //event need to own a parent id and for later update
client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do() ///////////
}
}
//click/impression----X---alone
//assume all events happen after install
func install(allFields AllFields) {
//************build unique install struct first
var installUni Install
installUni.Id = allFields.Id
installUni.Created = allFields.Created
installUni.DeviceIp = allFields.DeviceIp
installUni.StatImpressionId = allFields.StatImpressionId
installUni.StatClickId = allFields.StatClickId
installUni.CountryCode = allFields.CountryCode
installUni.RegionCode = allFields.RegionCode
installUni.PostalCode = allFields.PostalCode
installUni.Location = strconv.FormatFloat(allFields.Latitude, 'E', -1, 64) + "," + strconv.FormatFloat(allFields.Longitude, 'E', -1, 64)
installUni.WurflBrandName = allFields.WurflBrandName
installUni.WurflModelName = allFields.WurflModelName
installUni.WurflDeviceOs = allFields.WurflDeviceOs
noHit := true
//*************************search for realtionship with click by using Common field clickinsatllid if provide statclickid
if installUni.StatClickId != "" {
termQuery := elastic.NewTermQuery("ClickInstallId", installUni.StatClickId)
searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()
if searchResult.TotalHits() != 0 {
if searchResult.TotalHits() != 1 {
structured.Warn(installUni.Id, "install", "the click is not only one when related to this install", int(allFields.SiteId), nil)
} else {
noHit = false
hit := searchResult.Hits.Hits[0]
parentId := hit.Id
//update the statinstallid in common field
var update UpdateInstallId
update.StatInstallId = installUni.Id
client.Update().Index("alls").Type("Common").Id(parentId).Doc(update).Do()
//post install data as child of this parent doc
client.Index().Index("alls").Type("Install").Parent(parentId).BodyJson(installUni).Do()
}
}
} else if installUni.StatImpressionId != "" {
//**********************search for relationship with impression bY using common field impressioninstallid if provide statimpression id
termQuery := elastic.NewTermQuery("ImpressionInstallId", installUni.StatImpressionId)
searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()
if searchResult.TotalHits() != 0 {
if searchResult.TotalHits() != 1 {
structured.Warn(installUni.Id, "installs", "this install has not onlt one impression related to it", int(allFields.SiteId), nil)
} else {
noHit = false
hit := searchResult.Hits.Hits[0]
parentId := hit.Id
//update the statInstallId in common field
var update UpdateInstallId
update.StatInstallId = installUni.Id
client.Update().Index("alls").Type("Common").Id(parentId).Doc(update).Do()
//post install data as child of this parent doc
client.Index().Index("alls").Type("Install").Parent(parentId).BodyJson(installUni).Do()
}
}
}
//***********post the install data alone when it find no realtionship with existing ***outside click or impression
// if find no realtionship with click/ impression, 1. this install may not have click/imrpession id 2. the click/impression id is not posted yet
if noHit {
//post the common field first
var common Common
common.GoogleAid = allFields.GoogleAid
common.WindowsAid = allFields.WindowsAid
common.IosIfa = allFields.IosIfa
common.Language = allFields.Language
common.CurrencyCode = allFields.CurrencyCode
common.SiteId = allFields.SiteId
common.AdvertiserId = allFields.AdvertiserId
common.PackageName = allFields.PackageName
common.PublisherId = allFields.PublisherId
common.AdNetworkId = allFields.AdNetworkId
common.AgencyId = allFields.AgencyId
common.CampaignId = allFields.CampaignId
common.PublisherUserId = allFields.PublisherUserId
//stat install id is the install's id
common.StatInstallId = installUni.Id
/********this is for avoiding the speical ---- event happen before install, still have click id even the click is in inner struct of opens/events
********the trade off is not storing the related click outside but it would only happen when the first event is logging before install and click and also click
********is logging after install****************/
common.ClickInstallId = installUni.StatClickId
common.ImpressionInstallId = installUni.StatImpressionId
//post the parent data frist and then post the install as a child
indexParent, _ := client.Index().Index("alls").Type("Common").BodyJson(common).Refresh(true).Do()
parentId := indexParent.Id
client.Index().Index("alls").Type("Install").Parent(parentId).BodyJson(installUni).Pretty(true).Do()
}
}