//the function which handle the post method //post the json data from broswer to the server and sql databases func POST(w http.ResponseWriter, r *http.Request) { //time when do request RequestStart := time.Now() //get the raw bytes of input data bytes, errs := ioutil.ReadAll(r.Body) if errs != nil { errString := fmt.Sprintf("buffer overflow %s", errs) response(w, errString, "", http.StatusBadRequest) return } //store the raw bytes to a temporary struct and log the Json invalid format var point Click errs = json.Unmarshal(bytes, &point) if errs != nil { errString := fmt.Sprintf("invalid Json format: %s", errs) response(w, errString, "", http.StatusBadRequest) structured.Error("", "", errString, 0, nil) return } //validate the input and log error input message if point.AdvertiserID == 0 || point.SiteID == 0 { errString := "your advertiserID or site ID may equals to 0" response(w, errString, "", http.StatusBadRequest) structured.Error("", "", errString, 0, nil) return } //generate a ramdom id for the post data and also get the ip address id := Id(point.AdvertiserID) ip := r.RemoteAddr //store the data from the struct to the sql databases and log the error or latency time QueryStart := time.Now() _, errs = db.Exec("INSERT INTO clicks(id, advertiser_id, site_id, ip, ios_ifa, google_aid, windows_aid, date_time) VALUES(?, ?, ?, ?, ?, ?, ?,?)", id, point.AdvertiserID, point.SiteID, ip, point.IosIfa, point.GoogleAid, point.WindowsAid, RequestStart) if errs != nil { fmt.Println(errs) //show to the sever protecter inside of users errString := "sorry, there is an error" response(w, errString, "", http.StatusInternalServerError) errString = fmt.Sprintf("database connection error : %s", errs) structured.Error("", "", errString, 0, nil) return } //sucess and log the request latency response(w, "", id, http.StatusOK) structured.Info(point.ID, point.Type, "Post successful!", point.SiteID, structured.ExtraFields{structured.RequestLatency: time.Since(RequestStart), structured.QueryLatency: time.Since(QueryStart)}) }
//function that handles the GET method //retrieve the json data form from database/server and output to the browser func GET(writer http.ResponseWriter, reader *http.Request) { // time the method var starttime = time.Now() ///get the id from the hashmap id := mux.Vars(reader)["id"] //select data from sql databases according to the id row := db.QueryRow("SELECT id, advertiser_id, site_id, ip, ios_ifa, google_aid, windows_aid, date_time FROM clicks WHERE id=?", id) //store the data from sql database in a temp struct var c Click err := row.Scan(&c.ID, &c.AdvertiserID, &c.SiteID, &c.IP, &c.IosIfa, &c.GoogleAid, &c.WindowsAid, &c.Date_time) //check for errors in scan (404 and 500) if err == sql.ErrNoRows { // fmt.Println(err) writer.WriteHeader(http.StatusNotFound) structured.Error(c.ID, c.Type, "Error 404: no rows found", c.SiteID, nil) return } else if err != nil { writer.WriteHeader(http.StatusInternalServerError) structured.Error(c.ID, c.Type, "Error 500: Internal server error", c.SiteID, nil) return } // log the event structured.Info(c.ID, c.Type, "GET successful", c.SiteID, structured.ExtraFields{structured.RequestLatency: time.Since(starttime)}) }
func init() { structured.AddHookToSyslog("tcp", "localhost:10514", syslog.LOG_EMERG, "log messages ") structured.AddHookToElasticsearch("localhost", "9200", "clients", "ids", "") var err error //set up the database db, err = sql.Open("mysql", "root@tcp(localhost:3306)/logs?parseTime=true") if err != nil { structured.Error("", "", "can't open databases", 0, nil) return } // seed the random generator to generate IDs rand.Seed(time.Now().UTC().UnixNano()) }
//the function which handle the post method //post the json data from broswer to the server and sql databases func POST(w http.ResponseWriter, r *http.Request) { url := "http://dp-joshp01-dev.sea1.office.priv:9200/testDatabase/testdata" var DocumentToInsert Document // Create a client client, err := elastic.NewClient() if err != nil { fmt.Println(err) } // Create an index _, err = client.CreateIndex("alldata").Do() if err != nil { fmt.Println(err) } //get a raw byte[] from input data bytes, errs := ioutil.ReadAll(r.Body) if errs != nil { errString := fmt.Sprintf("buffer overflow %s", errs) response(w, errString, "", http.StatusBadRequest) return } //store the raw byte[] to a temporary struct and varify the JSON is in the correct format var temp AllFieldsStr errs = json.Unmarshal(bytes, &temp) if errs != nil { errString := fmt.Sprintf("invalid Json format: %s", errs) response(w, errString, "", http.StatusBadRequest) structured.Error("", "", errString, 0, nil) return } // Search with a term query termQuery := elastic.NewTermQuery(temp.id) searchResult, err := client.Search(). Index("database"). // search in index "database" Query(&termQuery). // specify the query From(0).Size(10). // take documents 0-9 Pretty(true). // pretty print request and response JSON Do() // execute if err != nil { // Handle error panic(err) } // Here's how you iterate through results with full control over each step. if searchResult.Hits != nil { fmt.Printf("Found a total of %d matches \n", searchResult.Hits.TotalHits) // Iterate through results for _, hit := range searchResult.Hits.Hits { // hit.Index contains the name of the index // Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}). var matchedDocument Document err := json.Unmarshal(*hit.Source, &matchedDocument) if err != nil { fmt.Println("Deserialization failed") } switch temp.LogType { case "impression": DocumentToInsert = AddImpression(DocumentToInsert, temp) case "click": DocumentToInsert = AddClick(DocumentToInsert, temp) case "install": DocumentToInsert = AddInstall(DocumentToInsert, temp) case "open": DocumentToInsert = AddOpen(DocumentToInsert, temp) case "event": DocumentToInsert = AddEvent(DocumentToInsert, temp) default: panic("unrecognized escape character") } } } else { // No hits fmt.Print("Found no matches, inserting new data\n") switch temp.LogType { case "impression": DocumentToInsert = NewImpression(temp) case "click": DocumentToInsert = NewClick(temp) case "install": DocumentToInsert = NewInstall(temp) case "open": DocumentToInsert = NewOpen(temp) case "event": DocumentToInsert = NewEvent(temp) default: panic("unrecognized escape character") } } eventJSONstring, err := json.Marshal(DocumentToInsert) if err != nil { fmt.Println(err) return } // fmt.Println(string(eventJSONstring)) req, err := http.NewRequest("POST", url, bytes.NewBuffer(eventJSONstring)) if err != nil { fmt.Println(err) } client := &http.Client{} resp, err := client.Do(req) if err != nil { fmt.Println(err) } //temp, _ := ioutil.ReadAll(resp.Body) // fmt.Println(string(temp)) resp.Body.Close() }