コード例 #1
0
ファイル: utils.go プロジェクト: DaviWei/utils-1
func (self Time) MarshalJSON(args ...interface{}) ([]byte, error) {
	if len(args) == 1 {
		if s, ok := args[0].(string); ok && s == "bigquery" {
			return json.Marshal(self.Time)
		}
	}
	return json.Marshal(self.Time.Format(ISO8601DateTimeFormat))
}
コード例 #2
0
ファイル: gae.go プロジェクト: DaviWei/utils-1
func (self StatusMap) MarshalJSON() (b []byte, err error) {
	tmpMap := map[string]interface{}{}
	for status, num := range self {
		tmpMap[fmt.Sprint(status)] = num
	}
	return json.Marshal(tmpMap)
}
コード例 #3
0
ファイル: elasticsearch.go プロジェクト: DaviWei/utils-1
/*
AddToIndex adds source to a search index.
Source must have a field `Id *datastore.key`.
*/
func AddToIndex(c ElasticConnector, index string, source interface{}) (err error) {
	sourceVal := reflect.ValueOf(source)
	if sourceVal.Kind() != reflect.Ptr {
		err = errors.Errorf("%#v is not a pointer", source)
		return
	}
	if sourceVal.Elem().Kind() != reflect.Struct {
		err = errors.Errorf("%#v is not a pointer to a struct", source)
		return
	}
	index = processIndexName(index)

	value := reflect.ValueOf(source).Elem()
	id := value.FieldByName("Id").Interface().(key.Key).Encode()

	name := value.Type().Name()

	json, err := json.Marshal(source)
	if err != nil {
		return
	}

	url := fmt.Sprintf("%s/%s/%s/%s",
		c.GetElasticService(),
		index,
		name,
		id)

	updatedAtField := value.FieldByName("UpdatedAt")
	if updatedAtField.IsValid() {
		updatedAtUnixNano := updatedAtField.MethodByName("UnixNano")
		if updatedAtUnixNano.IsValid() {
			if unixNano, ok := updatedAtUnixNano.Call(nil)[0].Interface().(int64); ok && unixNano > 0 {
				millis := unixNano / 1000000
				url = fmt.Sprintf("%v?version_type=external_gte&version=%v", url, millis)
			}
		}
	}

	request, err := http.NewRequest("PUT", url, bytes.NewBuffer(json))
	if err != nil {
		return
	}

	if c.GetElasticUsername() != "" {
		request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
	}
	response, err := c.Client().Do(request)
	if err != nil {
		return
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusOK && response.StatusCode != http.StatusConflict {
		body, _ := ioutil.ReadAll(response.Body)
		err = errors.Errorf("Bad status code from elasticsearch %v: %v, %v", url, response.Status, string(body))
		return
	}
	return
}
コード例 #4
0
ファイル: example_test.go プロジェクト: DaviWei/utils-1
func ExampleIndent() {
	type Road struct {
		Name   string
		Number int
	}
	roads := []Road{
		{"Diamond Fork", 29},
		{"Sheep Creek", 51},
	}

	b, err := json.Marshal(roads)
	if err != nil {
		log.Fatal(err)
	}

	var out bytes.Buffer
	json.Indent(&out, b, "=", "\t")
	out.WriteTo(os.Stdout)
	// Output:
	// [
	// =	{
	// =		"Name": "Diamond Fork",
	// =		"Number": 29
	// =	},
	// =	{
	// =		"Name": "Sheep Creek",
	// =		"Number": 51
	// =	}
	// =]
}
コード例 #5
0
ファイル: example_test.go プロジェクト: DaviWei/utils-1
func ExampleMarshal() {
	type ColorGroup struct {
		ID     int
		Name   string
		Colors []string
	}
	group := ColorGroup{
		ID:     1,
		Name:   "Reds",
		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
	}
	b, err := json.Marshal(group)
	if err != nil {
		fmt.Println("error:", err)
	}
	os.Stdout.Write(b)
	// Output:
	// {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]}
}
コード例 #6
0
ファイル: elasticsearch.go プロジェクト: DaviWei/utils-1
func (self *SearchResponse) Copy(result interface{}) (err error) {
	sources := make(Sources, len(self.Hits.Hits))
	for index, hit := range self.Hits.Hits {
		sources[index] = hit.Source
	}
	buf, err := json.Marshal(sources)
	if err != nil {
		return
	}
	resultValue := reflect.ValueOf(result).Elem()
	for resultValue.Kind() == reflect.Ptr {
		resultValue = resultValue.Elem()
	}
	if err = json.Unmarshal(buf, resultValue.FieldByName("Items").Addr().Interface()); err != nil {
		return
	}
	resultValue.FieldByName("Total").Set(reflect.ValueOf(self.Hits.Total))
	resultValue.FieldByName("Page").Set(reflect.ValueOf(self.Page))
	resultValue.FieldByName("PerPage").Set(reflect.ValueOf(self.PerPage))

	return
}
コード例 #7
0
ファイル: elasticsearch.go プロジェクト: DaviWei/utils-1
func UpdateDoc(c ElasticConnector, index string, id key.Key, groovyCode string, params map[string]interface{}) (err error) {
	index = processIndexName(index)

	url := fmt.Sprintf("%s/%s/%s/%s/_update?retry_on_conflict=%v",
		c.GetElasticService(),
		index,
		id.Kind(),
		id.Encode(),
		UpdateConflictRetries)

	json, err := json.Marshal(UpdateRequest{
		Script: groovyCode,
		Params: params,
	})
	if err != nil {
		return
	}

	request, err := http.NewRequest("POST", url, bytes.NewBuffer(json))
	if err != nil {
		return
	}

	if c.GetElasticUsername() != "" {
		request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
	}
	response, err := c.Client().Do(request)
	if err != nil {
		return
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(response.Body)
		err = errors.Errorf("Bad status code from elasticsearch %v: %v, %v", url, response.Status, string(body))
		return
	}
	return
}
コード例 #8
0
ファイル: key_test.go プロジェクト: DaviWei/utils-1
func TestToAndFromJSONInsideWrapper(t *testing.T) {
	for i := 0; i < 1000; i++ {
		k := randomKey(5)
		w := &testWrapper{
			Id:   k,
			Name: "hehu",
		}
		enc, err := json.Marshal(w)
		if err != nil {
			t.Fatalf(err.Error())
		}
		var i interface{}
		err = json.Unmarshal(enc, &i)
		if err != nil {
			t.Fatalf("Bad json: %#v: %v", string(enc), err.Error())
		}
		w2 := &testWrapper{}
		if err := json.Unmarshal(enc, w2); err != nil {
			t.Fatalf(err.Error())
		}
		if !reflect.DeepEqual(w, w2) {
			t.Fatalf("%+v != %+v", w, w2)
		}
		w3 := &testWrapperString{}
		if err := json.Unmarshal(enc, w3); err != nil {
			t.Fatalf(err.Error())
		}
		k2, err := Decode(w3.Id)
		if err != nil {
			t.Fatalf(err.Error())
		}
		if !k.Equal(k2) {
			t.Fatalf("%v != %v", k, k2)
		}
	}

}
コード例 #9
0
ファイル: elasticsearch.go プロジェクト: DaviWei/utils-1
func createIndexDef(c ElasticConnector, path string, def interface{}) (err error) {
	url := c.GetElasticService() + path
	b, err := json.Marshal(def)
	if err != nil {
		return
	}
	request, err := http.NewRequest("PUT", url, bytes.NewBuffer(b))
	if err != nil {
		return
	}
	if c.GetElasticUsername() != "" {
		request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
	}
	response, err := c.Client().Do(request)
	if err != nil {
		return
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		err = errors.Errorf("Bad status trying to create index template in elasticsearch %v: %v, body: %v", url, response.Status, string(b))
		return
	}
	return
}
コード例 #10
0
ファイル: elasticsearch.go プロジェクト: DaviWei/utils-1
func RemoveFromIndex(c ElasticConnector, index string, source interface{}) (err error) {
	index = processIndexName(index)
	value := reflect.ValueOf(source)
	id := value.Elem().FieldByName("Id").Interface().(key.Key).Encode()

	name := value.Elem().Type().Name()
	url := fmt.Sprintf("%s/%s/%s/%s",
		c.GetElasticService(),
		index,
		name,
		id)

	json, err := json.Marshal(source)
	if err != nil {
		return
	}
	request, err := http.NewRequest("DELETE", url, bytes.NewBuffer(json))
	if err != nil {
		return
	}

	if c.GetElasticUsername() != "" {
		request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
	}
	response, err := c.Client().Do(request)
	if err != nil {
		return
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		err = errors.Errorf("Bad status code from elasticsearch %v: %v", url, response.Status)
		return
	}
	return
}
コード例 #11
0
ファイル: bigquery.go プロジェクト: DaviWei/utils-1
/*
InsertTable data will assume that AssertTable for the type of the provided interface{} has been called beforehand, and
push this particular instance into BigQuery.
*/
func (self *BigQuery) InsertTableData(i interface{}) (err error) {
	j := map[string]gbigquery.JsonValue{}

	b, err := json.Marshal(i, "bigquery")
	if err != nil {
		return
	}
	if err = json.Unmarshal(b, &j); err != nil {
		return
	}
	cropStrings(j)
	if b, err = time.Now().MarshalJSON(); err != nil {
		return
	}
	s := ""
	if err = json.Unmarshal(b, &s); err != nil {
		return
	}
	j["_inserted_at"] = s

	request := &gbigquery.TableDataInsertAllRequest{
		Rows: []*gbigquery.TableDataInsertAllRequestRows{
			&gbigquery.TableDataInsertAllRequestRows{
				Json: j,
			},
		},
	}

	typ := reflect.TypeOf(i)
	for typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}

	for i := 0; i < typ.NumField(); i++ {
		if typ.Field(i).Tag.Get("bigquery") == "-" {
			name := typ.Field(i).Name
			if jsonTag := typ.Field(i).Tag.Get("json"); jsonTag != "" {
				if splitTag := strings.Split(jsonTag, ","); splitTag[0] != "" {
					name = splitTag[0]
				}
			}
			delete(j, name)
		}
	}

	tabledataService := gbigquery.NewTabledataService(self.GetService())
	tableDataList, err := tabledataService.InsertAll(self.GetProjectId(), self.GetDatasetId(), typ.Name(), request).Do()
	if err != nil {
		return
	}

	// Build insert errors error message
	if len(tableDataList.InsertErrors) != 0 {
		prettyJ := utils.Prettify(j)
		errorStrings := []string{}
		for _, errors := range tableDataList.InsertErrors {
			for _, errorProto := range errors.Errors {
				errorStrings = append(errorStrings, fmt.Sprintf("\nReason:%v,\nMessage:%v,\nLocation:%v", errorProto.Reason, errorProto.Message, errorProto.Location))
			}
		}
		errorStrings = append(errorStrings, fmt.Sprintf("BigQuery: Error inserting json %v into table %v:", prettyJ, typ.Name()))
		err = errors.Errorf(strings.Join(errorStrings, "\n"))
	}

	return
}
コード例 #12
0
ファイル: elasticsearch.go プロジェクト: DaviWei/utils-1
func Search(c ElasticSearchContext, query *SearchRequest, index, typ string) (result *SearchResponse, err error) {
	if query.Size == 0 {
		query.Size = 10
	}
	index = processIndexName(index)

	url := c.GetElasticService()
	if index == "" {
		url += "/_all"
	} else {
		url += "/" + index
	}
	if typ != "" {
		url += "/" + typ
	}
	url += "/_search"

	b, err := json.Marshal(query)
	if err != nil {
		return
	}

	request, err := http.NewRequest("POST", url, bytes.NewBuffer(b))
	if err != nil {
		return
	}

	if c.GetElasticUsername() != "" {
		request.SetBasicAuth(c.GetElasticUsername(), c.GetElasticPassword())
	}

	response, err := c.Client().Do(request)
	if err != nil {
		return
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		err = errors.Errorf("Bad status trying to search in elasticsearch %v: %v", url, response.Status)
		return
	}

	result = &SearchResponse{}
	bodyBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return
	}
	if err = json.Unmarshal(bodyBytes, &result); err != nil {
		var secondTry interface{}
		if err = json.Unmarshal(bodyBytes, &secondTry); err != nil {
			return
		}
		err = errors.Errorf("Unable to marshal %v into %#v", utils.Prettify(secondTry), result)
		return
	}

	c.Debugf("Elasticsearch took %v, url:%s", result.Took, url)
	result.Page = 1 + (query.From / query.Size)
	result.PerPage = query.Size
	return
}
コード例 #13
0
ファイル: utils.go プロジェクト: DaviWei/utils-1
func (self ByteString) MarshalJSON() ([]byte, error) {
	return json.Marshal(string(self.Bytes))
}
コード例 #14
0
ファイル: utils.go プロジェクト: DaviWei/utils-1
func (self Base64String) MarshalJSON() (result []byte, err error) {
	if _, err = base64.StdEncoding.DecodeString(string(self)); err != nil {
		return
	}
	return json.Marshal(string(self))
}