Пример #1
0
func POST(w http.ResponseWriter, r *http.Request) {
	//read the raw post data nad marhsal it s
	bytes, _ := ioutil.ReadAll(r.Body)
	var allFields AllFields
	json.Unmarshal(bytes, &allFields)

	//get the time sturct work right
	if allFields.TempTime != "" {
		t, err := time.Parse("2006-01-02 15:04:05", allFields.TempTime)
		if err != nil {
			structured.Warn("", "", "can't parse the time", 0, nil)
		} else {
			allFields.Created = t
		}
	}

	//find the log types
	logType := allFields.LogType

	if logType == "impress" {
		impression(allFields)
	} else if logType == "click" {
		click(allFields)
	} else if logType == "install" {
		install(allFields)
	} else if logType == "event" {
		event(allFields)
	} else if logType == "open" {
		open(allFields)
	}

}
Пример #2
0
func click(allFields AllFields) {
	//set up unqiue fields first
	var uniFields Click /////
	uniFields.Id = allFields.Id
	uniFields.Created = allFields.Created
	uniFields.DeviceIp = allFields.DeviceIp
	uniFields.CountryCode = allFields.CountryCode
	uniFields.RegionCode = allFields.RegionCode
	uniFields.PostalCode = allFields.PostalCode
	uniFields.Location = strconv.FormatFloat(allFields.Latitude, 'E', -1, 64) + "," + strconv.FormatFloat(allFields.Longitude, 'E', -1, 64)
	uniFields.WurflBrandName = allFields.WurflBrandName
	uniFields.WurflModelName = allFields.WurflModelName
	uniFields.WurflDeviceOs = allFields.WurflDeviceOs

	//search for realationship with install first ******click id aleays exist
	termQuery := elastic.NewTermQuery("ClickInstallId", uniFields.Id)
	searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()

	//******************** find the only install
	if searchResult.TotalHits() == 1 {
		hits := searchResult.Hits.Hits

		firstHit := hits[0]
		parentId := firstHit.Id

		//update the parent doc's clickInstallId -----the click id which realted to that install
		var clickID UpdateClickId
		clickID.ClickInstallId = uniFields.Id
		client.Update().Index("alls").Type("Common").Id(parentId).Doc(clickID).Do()

		//index the click doc as child doc to this parent id
		client.Index().Index("alls").Type("Click").Parent(parentId).BodyJson(uniFields).Do()

	} else if searchResult.TotalHits() == 0 {
		//************find no install--- find the event and open first *****click id always exists

		termQuery = elastic.NewTermQuery("StatClickId", uniFields.Id)

		//update the events to have reengaement  click struct
		searchEvent, _ := client.Search().Index("alls").Type("Event").Query(&termQuery).Pretty(true).Do()

		for _, hit := range searchEvent.Hits.Hits {

			var update UpdateClick
			update.Click = uniFields
			docId := hit.Id

			//get the parent id
			var event Acticity
			json.Unmarshal(*hit.Source, &event)

			client.Update().Index("alls").Type("Event").Parent(event.ParentId).Id(docId).Doc(update).Do()

		}

		//update the opens to have this reengaement click
		searchOpen, _ := client.Search().Index("alls").Type("Open").Query(&termQuery).Pretty(true).Do()
		for _, hit := range searchOpen.Hits.Hits {
			var update UpdateClick
			update.Click = uniFields
			docId := hit.Id

			//get the parent id
			var open Acticity
			json.Unmarshal(*hit.Source, &open)

			client.Update().Index("alls").Type("Open").Parent(open.ParentId).Id(docId).Doc(update).Do()
		}

		//************if there is no install and no matter whether it found a event/open, post the click

		//post the parent first
		var common Common
		common.GoogleAid = allFields.GoogleAid
		common.WindowsAid = allFields.WindowsAid
		common.IosIfa = allFields.IosIfa
		common.Language = allFields.Language
		common.CurrencyCode = allFields.CurrencyCode
		common.SiteId = allFields.SiteId
		common.AdvertiserId = allFields.AdvertiserId
		common.PackageName = allFields.PackageName
		common.PublisherId = allFields.PublisherId
		common.AdNetworkId = allFields.AdNetworkId
		common.AgencyId = allFields.AgencyId
		common.CampaignId = allFields.CampaignId
		common.PublisherUserId = allFields.PublisherUserId
		//the click id which future install  may contributed to
		common.ClickInstallId = uniFields.Id /////

		//index the parent common field first
		indexParent, _ := client.Index().Index("alls").Type("Common").BodyJson(common).Do()
		parentId := indexParent.Id
		//index the click as the children
		client.Index().Index("alls").Type("Click").Parent(parentId).BodyJson(uniFields).Do() /////

	} else {
		//*************find multiple install related to this click
		structured.Warn(uniFields.Id, "Click", "the click has multiple install contributed to it ", int(allFields.SiteId), nil) /////
	}

}
Пример #3
0
func helper(allFields AllFields, eventType string) {

	var event Acticity
	event.Id = allFields.Id
	event.Created = allFields.Created
	event.DeviceIp = allFields.DeviceIp
	event.StatImpressionId = allFields.StatImpressionId
	event.StatClickId = allFields.StatClickId
	event.StatInstallId = allFields.StatInstallId
	event.CountryCode = allFields.CountryCode
	event.RegionCode = allFields.RegionCode
	event.PostalCode = allFields.PostalCode
	event.Location = strconv.FormatFloat(allFields.Latitude, 'E', -1, 64) + "," + strconv.FormatFloat(allFields.Longitude, 'E', -1, 64)
	event.WurflBrandName = allFields.WurflBrandName
	event.WurflDeviceOs = allFields.WurflDeviceOs
	event.WurflModelName = allFields.WurflModelName

	//************find whether it related to the install first
	noHit := true

	if event.StatInstallId != "" {
		termQuery := elastic.NewTermQuery("StatInstallId", event.StatInstallId)
		searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()

		if searchResult.TotalHits() == 1 {
			//already find that only install related
			noHit = false
			hit := searchResult.Hits.Hits[0]
			parentId := hit.Id
			event.ParentId = parentId //event need to own a parent id and for later update

			//unmarhsal the parent
			var parent Common
			json.Unmarshal(*hit.Source, &parent)
			nohitClickImp := true

			//*************************if event has stat click id and different with parent
			if event.StatClickId != "" && parent.ClickInstallId != event.StatClickId {
				termQuery = elastic.NewTermQuery("Id", event.StatClickId)
				searchResult, _ = client.Search().Index("alls").Type("Click").Query(&termQuery).Pretty(true).Do()
				if searchResult.TotalHits() > 0 {
					if searchResult.TotalHits() == 1 {
						nohitClickImp = false
						//put the click struct inside the event sturct
						var found Click
						hits := searchResult.Hits.Hits[0]
						json.Unmarshal(*hits.Source, &found)
						event.Click = found

						//put the event as a child
						client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do()

					} else {
						structured.Warn(event.Id, eventType, "this event/open has multiple clicks related to it ", int(allFields.SiteId), nil)
					}
				}

			} else if event.StatImpressionId != "" && parent.ImpressionInstallId != event.StatImpressionId {
				//*************************if event has stat impression id and different with parent

				termQuery = elastic.NewTermQuery("Id", event.StatImpressionId)
				searchResult, _ = client.Search().Index("alls").Type("Impression").Query(&termQuery).Pretty(true).Do()
				if searchResult.TotalHits() > 0 {
					if searchResult.TotalHits() == 1 {
						nohitClickImp = false
						//put the impression struct inside the event sturct
						var found Impression
						hits := searchResult.Hits.Hits[0]
						json.Unmarshal(*hits.Source, &found)
						event.Impression = found

						//put the event as a child
						client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do()

					} else {
						structured.Warn(event.Id, eventType, "this event/open has multiple impression related to it ", int(allFields.SiteId), nil)
					}
				}

			}

			if nohitClickImp {
				//if the event no related with any click or impresssion
				client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do()
			}

		} else if searchResult.TotalHits() > 1 {
			structured.Warn(event.Id, eventType, "has multiple event/open contributed to that", int(allFields.SiteId), nil)
		}

		//else is no hit with current install leave nohit to be true

	}
	//**********no install related to it, post all parent and children doc have ttl and then ignore it in ES
	if noHit {
		var common Common
		common.GoogleAid = allFields.GoogleAid
		common.WindowsAid = allFields.WindowsAid
		common.IosIfa = allFields.IosIfa
		common.Language = allFields.Language
		common.CurrencyCode = allFields.CurrencyCode
		common.SiteId = allFields.SiteId
		common.AdvertiserId = allFields.AdvertiserId
		common.PackageName = allFields.PackageName
		common.PublisherId = allFields.PublisherId
		common.AdNetworkId = allFields.AdNetworkId
		common.AgencyId = allFields.AgencyId
		common.CampaignId = allFields.CampaignId
		common.PublisherUserId = allFields.PublisherUserId

		indexParent, _ := client.Index().Index("alls").Type("Common").BodyJson(common).Do()
		parentId := indexParent.Id
		event.ParentId = parentId //event need to own a parent id and for later update

		client.Index().Index("alls").Type(eventType).Parent(parentId).BodyJson(event).Do() ///////////
	}

}
Пример #4
0
//click/impression----X---alone
//assume all events happen after install
func install(allFields AllFields) {
	//************build unique install struct first
	var installUni Install
	installUni.Id = allFields.Id
	installUni.Created = allFields.Created
	installUni.DeviceIp = allFields.DeviceIp
	installUni.StatImpressionId = allFields.StatImpressionId
	installUni.StatClickId = allFields.StatClickId
	installUni.CountryCode = allFields.CountryCode
	installUni.RegionCode = allFields.RegionCode
	installUni.PostalCode = allFields.PostalCode
	installUni.Location = strconv.FormatFloat(allFields.Latitude, 'E', -1, 64) + "," + strconv.FormatFloat(allFields.Longitude, 'E', -1, 64)
	installUni.WurflBrandName = allFields.WurflBrandName
	installUni.WurflModelName = allFields.WurflModelName
	installUni.WurflDeviceOs = allFields.WurflDeviceOs

	noHit := true
	//*************************search for realtionship with click  by using Common field clickinsatllid if provide statclickid
	if installUni.StatClickId != "" {

		termQuery := elastic.NewTermQuery("ClickInstallId", installUni.StatClickId)
		searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()
		if searchResult.TotalHits() != 0 {

			if searchResult.TotalHits() != 1 {
				structured.Warn(installUni.Id, "install", "the click is not only one when related to this install", int(allFields.SiteId), nil)
			} else {
				noHit = false
				hit := searchResult.Hits.Hits[0]
				parentId := hit.Id

				//update the statinstallid in common field
				var update UpdateInstallId
				update.StatInstallId = installUni.Id
				client.Update().Index("alls").Type("Common").Id(parentId).Doc(update).Do()

				//post install data as child of this parent doc
				client.Index().Index("alls").Type("Install").Parent(parentId).BodyJson(installUni).Do()
			}
		}
	} else if installUni.StatImpressionId != "" {

		//**********************search for relationship with impression bY using common field impressioninstallid if provide statimpression id
		termQuery := elastic.NewTermQuery("ImpressionInstallId", installUni.StatImpressionId)
		searchResult, _ := client.Search().Index("alls").Type("Common").Query(&termQuery).Pretty(true).Do()

		if searchResult.TotalHits() != 0 {
			if searchResult.TotalHits() != 1 {
				structured.Warn(installUni.Id, "installs", "this install has not onlt one impression related to it", int(allFields.SiteId), nil)
			} else {
				noHit = false
				hit := searchResult.Hits.Hits[0]
				parentId := hit.Id

				//update the statInstallId in common field
				var update UpdateInstallId
				update.StatInstallId = installUni.Id
				client.Update().Index("alls").Type("Common").Id(parentId).Doc(update).Do()

				//post install data as child of this parent doc
				client.Index().Index("alls").Type("Install").Parent(parentId).BodyJson(installUni).Do()
			}
		}
	}

	//***********post the install data alone when it find no realtionship with existing ***outside click or impression
	// if find no realtionship with click/ impression, 1. this install may not have click/imrpession id 2. the click/impression id is not posted yet

	if noHit {

		//post the common field first
		var common Common
		common.GoogleAid = allFields.GoogleAid
		common.WindowsAid = allFields.WindowsAid
		common.IosIfa = allFields.IosIfa
		common.Language = allFields.Language
		common.CurrencyCode = allFields.CurrencyCode
		common.SiteId = allFields.SiteId
		common.AdvertiserId = allFields.AdvertiserId
		common.PackageName = allFields.PackageName
		common.PublisherId = allFields.PublisherId
		common.AdNetworkId = allFields.AdNetworkId
		common.AgencyId = allFields.AgencyId
		common.CampaignId = allFields.CampaignId
		common.PublisherUserId = allFields.PublisherUserId
		//stat install id is the install's id
		common.StatInstallId = installUni.Id

		/********this is for avoiding the speical ---- event happen before install, still have click id even the click is in inner struct of opens/events
		********the trade off is not storing the related click outside but it would only happen when the first event is logging before install and click and also click
		********is logging after install****************/
		common.ClickInstallId = installUni.StatClickId
		common.ImpressionInstallId = installUni.StatImpressionId

		//post the parent data frist and then post the install as a child
		indexParent, _ := client.Index().Index("alls").Type("Common").BodyJson(common).Refresh(true).Do()
		parentId := indexParent.Id
		client.Index().Index("alls").Type("Install").Parent(parentId).BodyJson(installUni).Pretty(true).Do()
	}

}