func (self Time) MarshalJSON(args ...interface{}) ([]byte, error) {
if len(args) == 1 {
if s, ok := args[0].(string); ok && s == "bigquery" {
return json.Marshal(self.Time)
}
}
return json.Marshal(self.Time.Format(ISO8601DateTimeFormat))
}
func (self StatusMap) MarshalJSON() (b []byte, err error) {
tmpMap := map[string]interface{}{}
for status, num := range self {
tmpMap[fmt.Sprint(status)] = num
}
return json.Marshal(tmpMap)
}
/*
AddToIndex adds source to a search index.
Source must have a field `Id *datastore.key`.
*/
func AddToIndex(c ElasticConnector, index string, source interface{}) (err error) {
sourceVal := reflect.ValueOf(source)
if sourceVal.Kind() != reflect.Ptr {
err = errors.Errorf("%#v is not a pointer", source)
return
}
if sourceVal.Elem().Kind() != reflect.Struct {
err = errors.Errorf("%#v is not a pointer to a struct", source)
return
}
index = processIndexName(index)
value := reflect.ValueOf(source).Elem()
id := value.FieldByName("Id").Interface().(key.Key).Encode()
name := value.Type().Name()
json, err := json.Marshal(source)
if err != nil {
return
}
url := fmt.Sprintf("%s/%s/%s/%s",
c.GetElasticService(),
index,
name,
id)
updatedAtField := value.FieldByName("UpdatedAt")
if updatedAtField.IsValid() {
updatedAtUnixNano := updatedAtField.MethodByName("UnixNano")
if updatedAtUnixNano.IsValid() {
if unixNano, ok := updatedAtUnixNano.Call(nil)[0].Interface().(int64); ok && unixNano > 0 {
millis := unixNano / 1000000
url = fmt.Sprintf("%v?version_type=external_gte&version=%v", url, millis)
}
}
}
request, err := http.NewRequest("PUT", url, bytes.NewBuffer(json))
if err != nil {
return
}
if c.GetElasticUsername() != "" {
request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
}
response, err := c.Client().Do(request)
if err != nil {
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusOK && response.StatusCode != http.StatusConflict {
body, _ := ioutil.ReadAll(response.Body)
err = errors.Errorf("Bad status code from elasticsearch %v: %v, %v", url, response.Status, string(body))
return
}
return
}
func ExampleIndent() {
type Road struct {
Name string
Number int
}
roads := []Road{
{"Diamond Fork", 29},
{"Sheep Creek", 51},
}
b, err := json.Marshal(roads)
if err != nil {
log.Fatal(err)
}
var out bytes.Buffer
json.Indent(&out, b, "=", "\t")
out.WriteTo(os.Stdout)
// Output:
// [
// = {
// = "Name": "Diamond Fork",
// = "Number": 29
// = },
// = {
// = "Name": "Sheep Creek",
// = "Number": 51
// = }
// =]
}
func ExampleMarshal() {
type ColorGroup struct {
ID int
Name string
Colors []string
}
group := ColorGroup{
ID: 1,
Name: "Reds",
Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
}
b, err := json.Marshal(group)
if err != nil {
fmt.Println("error:", err)
}
os.Stdout.Write(b)
// Output:
// {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]}
}
func (self *SearchResponse) Copy(result interface{}) (err error) {
sources := make(Sources, len(self.Hits.Hits))
for index, hit := range self.Hits.Hits {
sources[index] = hit.Source
}
buf, err := json.Marshal(sources)
if err != nil {
return
}
resultValue := reflect.ValueOf(result).Elem()
for resultValue.Kind() == reflect.Ptr {
resultValue = resultValue.Elem()
}
if err = json.Unmarshal(buf, resultValue.FieldByName("Items").Addr().Interface()); err != nil {
return
}
resultValue.FieldByName("Total").Set(reflect.ValueOf(self.Hits.Total))
resultValue.FieldByName("Page").Set(reflect.ValueOf(self.Page))
resultValue.FieldByName("PerPage").Set(reflect.ValueOf(self.PerPage))
return
}
func UpdateDoc(c ElasticConnector, index string, id key.Key, groovyCode string, params map[string]interface{}) (err error) {
index = processIndexName(index)
url := fmt.Sprintf("%s/%s/%s/%s/_update?retry_on_conflict=%v",
c.GetElasticService(),
index,
id.Kind(),
id.Encode(),
UpdateConflictRetries)
json, err := json.Marshal(UpdateRequest{
Script: groovyCode,
Params: params,
})
if err != nil {
return
}
request, err := http.NewRequest("POST", url, bytes.NewBuffer(json))
if err != nil {
return
}
if c.GetElasticUsername() != "" {
request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
}
response, err := c.Client().Do(request)
if err != nil {
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(response.Body)
err = errors.Errorf("Bad status code from elasticsearch %v: %v, %v", url, response.Status, string(body))
return
}
return
}
func TestToAndFromJSONInsideWrapper(t *testing.T) {
for i := 0; i < 1000; i++ {
k := randomKey(5)
w := &testWrapper{
Id: k,
Name: "hehu",
}
enc, err := json.Marshal(w)
if err != nil {
t.Fatalf(err.Error())
}
var i interface{}
err = json.Unmarshal(enc, &i)
if err != nil {
t.Fatalf("Bad json: %#v: %v", string(enc), err.Error())
}
w2 := &testWrapper{}
if err := json.Unmarshal(enc, w2); err != nil {
t.Fatalf(err.Error())
}
if !reflect.DeepEqual(w, w2) {
t.Fatalf("%+v != %+v", w, w2)
}
w3 := &testWrapperString{}
if err := json.Unmarshal(enc, w3); err != nil {
t.Fatalf(err.Error())
}
k2, err := Decode(w3.Id)
if err != nil {
t.Fatalf(err.Error())
}
if !k.Equal(k2) {
t.Fatalf("%v != %v", k, k2)
}
}
}
func createIndexDef(c ElasticConnector, path string, def interface{}) (err error) {
url := c.GetElasticService() + path
b, err := json.Marshal(def)
if err != nil {
return
}
request, err := http.NewRequest("PUT", url, bytes.NewBuffer(b))
if err != nil {
return
}
if c.GetElasticUsername() != "" {
request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
}
response, err := c.Client().Do(request)
if err != nil {
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
err = errors.Errorf("Bad status trying to create index template in elasticsearch %v: %v, body: %v", url, response.Status, string(b))
return
}
return
}
func RemoveFromIndex(c ElasticConnector, index string, source interface{}) (err error) {
index = processIndexName(index)
value := reflect.ValueOf(source)
id := value.Elem().FieldByName("Id").Interface().(key.Key).Encode()
name := value.Elem().Type().Name()
url := fmt.Sprintf("%s/%s/%s/%s",
c.GetElasticService(),
index,
name,
id)
json, err := json.Marshal(source)
if err != nil {
return
}
request, err := http.NewRequest("DELETE", url, bytes.NewBuffer(json))
if err != nil {
return
}
if c.GetElasticUsername() != "" {
request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
}
response, err := c.Client().Do(request)
if err != nil {
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
err = errors.Errorf("Bad status code from elasticsearch %v: %v", url, response.Status)
return
}
return
}
/*
InsertTable data will assume that AssertTable for the type of the provided interface{} has been called beforehand, and
push this particular instance into BigQuery.
*/
func (self *BigQuery) InsertTableData(i interface{}) (err error) {
j := map[string]gbigquery.JsonValue{}
b, err := json.Marshal(i, "bigquery")
if err != nil {
return
}
if err = json.Unmarshal(b, &j); err != nil {
return
}
cropStrings(j)
if b, err = time.Now().MarshalJSON(); err != nil {
return
}
s := ""
if err = json.Unmarshal(b, &s); err != nil {
return
}
j["_inserted_at"] = s
request := &gbigquery.TableDataInsertAllRequest{
Rows: []*gbigquery.TableDataInsertAllRequestRows{
&gbigquery.TableDataInsertAllRequestRows{
Json: j,
},
},
}
typ := reflect.TypeOf(i)
for typ.Kind() == reflect.Ptr {
typ = typ.Elem()
}
for i := 0; i < typ.NumField(); i++ {
if typ.Field(i).Tag.Get("bigquery") == "-" {
name := typ.Field(i).Name
if jsonTag := typ.Field(i).Tag.Get("json"); jsonTag != "" {
if splitTag := strings.Split(jsonTag, ","); splitTag[0] != "" {
name = splitTag[0]
}
}
delete(j, name)
}
}
tabledataService := gbigquery.NewTabledataService(self.GetService())
tableDataList, err := tabledataService.InsertAll(self.GetProjectId(), self.GetDatasetId(), typ.Name(), request).Do()
if err != nil {
return
}
// Build insert errors error message
if len(tableDataList.InsertErrors) != 0 {
prettyJ := utils.Prettify(j)
errorStrings := []string{}
for _, errors := range tableDataList.InsertErrors {
for _, errorProto := range errors.Errors {
errorStrings = append(errorStrings, fmt.Sprintf("\nReason:%v,\nMessage:%v,\nLocation:%v", errorProto.Reason, errorProto.Message, errorProto.Location))
}
}
errorStrings = append(errorStrings, fmt.Sprintf("BigQuery: Error inserting json %v into table %v:", prettyJ, typ.Name()))
err = errors.Errorf(strings.Join(errorStrings, "\n"))
}
return
}
func Search(c ElasticSearchContext, query *SearchRequest, index, typ string) (result *SearchResponse, err error) {
if query.Size == 0 {
query.Size = 10
}
index = processIndexName(index)
url := c.GetElasticService()
if index == "" {
url += "/_all"
} else {
url += "/" + index
}
if typ != "" {
url += "/" + typ
}
url += "/_search"
b, err := json.Marshal(query)
if err != nil {
return
}
request, err := http.NewRequest("POST", url, bytes.NewBuffer(b))
if err != nil {
return
}
if c.GetElasticUsername() != "" {
request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
}
response, err := c.Client().Do(request)
if err != nil {
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
err = errors.Errorf("Bad status trying to search in elasticsearch %v: %v", url, response.Status)
return
}
result = &SearchResponse{}
bodyBytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return
}
if err = json.Unmarshal(bodyBytes, &result); err != nil {
var secondTry interface{}
if err = json.Unmarshal(bodyBytes, &secondTry); err != nil {
return
}
err = errors.Errorf("Unable to marshal %v into %#v", utils.Prettify(secondTry), result)
return
}
c.Debugf("Elasticsearch took %v, url:%s", result.Took, url)
result.Page = 1 + (query.From / query.Size)
result.PerPage = query.Size
return
}
func (self ByteString) MarshalJSON() ([]byte, error) {
return json.Marshal(string(self.Bytes))
}
func (self Base64String) MarshalJSON() (result []byte, err error) {
if _, err = base64.StdEncoding.DecodeString(string(self)); err != nil {
return
}
return json.Marshal(string(self))
}