func Assign(server string, count uint64, replication string, collection string, ttl string) (*AssignResult, error) {
values := make(url.Values)
values.Add("count", strconv.FormatUint(count, 10))
if replication != "" {
values.Add("replication", replication)
}
if collection != "" {
values.Add("collection", collection)
}
if ttl != "" {
values.Add("ttl", ttl)
}
jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
glog.V(2).Info("assign result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret AssignResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, fmt.Errorf("/dir/assign result JSON unmarshal error:%v, json:%s", err, string(jsonBlob))
}
if ret.Count <= 0 {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
func vacuumVolume_Commit(urlLocation string, vid storage.VolumeId) error {
values := make(url.Values)
values.Add("volume", vid.String())
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum/commit", values)
if err != nil {
return err
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err
}
if ret.Error != "" {
return errors.New(ret.Error)
}
return nil
}
func do_lookup(server string, vid string) (*LookupResult, error) {
values := make(url.Values)
values.Add("volumeId", vid)
jsonBlob, err := util.Post("http://"+server+"/dir/lookup", values)
if err != nil {
return nil, err
}
var ret LookupResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
func call(server string, request ApiRequest, ret interface{}) error {
b, err := json.Marshal(request)
if err != nil {
fmt.Println("error:", err)
return nil
}
values := make(url.Values)
values.Add("request", string(b))
jsonBlob, err := util.Post("http://"+server+"/__api__", values)
if err != nil {
return err
}
err = json.Unmarshal(jsonBlob, ret)
if err != nil {
return err
}
return nil
}
func GetVolumeSyncStatus(server string, vid string) (*SyncVolumeResponse, error) {
values := make(url.Values)
values.Add("volume", vid)
jsonBlob, err := util.Post("http://"+server+"/admin/sync/status", values)
glog.V(2).Info("sync volume result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret SyncVolumeResponse
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, fmt.Errorf("Volume %s get sync status error: %s", vid, ret.Error)
}
return &ret, nil
}
func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThreshold string) (error, bool) {
values := make(url.Values)
values.Add("volume", vid.String())
values.Add("garbageThreshold", garbageThreshold)
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum/check", values)
if err != nil {
glog.V(0).Infoln("parameters:", values)
return err, false
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err, false
}
if ret.Error != "" {
return errors.New(ret.Error), false
}
return nil, ret.Result
}
func AllocateVolume(dn *DataNode, vid storage.VolumeId, option *VolumeGrowOption) error {
values := make(url.Values)
values.Add("volume", vid.String())
values.Add("collection", option.Collection)
values.Add("replication", option.ReplicaPlacement.String())
values.Add("ttl", option.Ttl.String())
jsonBlob, err := util.Post("http://"+dn.Url()+"/admin/assign_volume", values)
if err != nil {
return err
}
var ret AllocateVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return fmt.Errorf("Invalid JSON result for %s: %s", "/admin/assign_volum", string(jsonBlob))
}
if ret.Error != "" {
return errors.New(ret.Error)
}
return nil
}
// LookupVolumeIds find volume locations by cache and actual lookup
func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, error) {
ret := make(map[string]LookupResult)
var unknown_vids []string
//check vid cache first
for _, vid := range vids {
locations, cache_err := vc.Get(vid)
if cache_err == nil {
ret[vid] = LookupResult{VolumeId: vid, Locations: locations}
} else {
unknown_vids = append(unknown_vids, vid)
}
}
//return success if all volume ids are known
if len(unknown_vids) == 0 {
return ret, nil
}
//only query unknown_vids
values := make(url.Values)
for _, vid := range unknown_vids {
values.Add("volumeId", vid)
}
jsonBlob, err := util.Post("http://"+server+"/vol/lookup", values)
if err != nil {
return nil, err
}
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, errors.New(err.Error() + " " + string(jsonBlob))
}
//set newly checked vids to cache
for _, vid := range unknown_vids {
locations := ret[vid].Locations
vc.Set(vid, locations, 10*time.Minute)
}
return ret, nil
}
func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
vid_to_fileIds := make(map[string][]string)
ret := &DeleteFilesResult{}
var vids []string
for _, fileId := range fileIds {
vid, _, err := ParseFileId(fileId)
if err != nil {
ret.Results = append(ret.Results, DeleteResult{Fid: vid, Error: err.Error()})
continue
}
if _, ok := vid_to_fileIds[vid]; !ok {
vid_to_fileIds[vid] = make([]string, 0)
vids = append(vids, vid)
}
vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
}
lookupResults, err := LookupVolumeIds(master, vids)
if err != nil {
return ret, err
}
server_to_fileIds := make(map[string][]string)
for vid, result := range lookupResults {
if result.Error != "" {
ret.Errors = append(ret.Errors, result.Error)
continue
}
for _, location := range result.Locations {
if _, ok := server_to_fileIds[location.Url]; !ok {
server_to_fileIds[location.Url] = make([]string, 0)
}
server_to_fileIds[location.Url] = append(
server_to_fileIds[location.Url], vid_to_fileIds[vid]...)
}
}
var wg sync.WaitGroup
for server, fidList := range server_to_fileIds {
wg.Add(1)
go func(server string, fidList []string) {
defer wg.Done()
values := make(url.Values)
for _, fid := range fidList {
values.Add("fid", fid)
}
jsonBlob, err := util.Post("http://"+server+"/delete", values)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
var result []DeleteResult
err = json.Unmarshal(jsonBlob, &result)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
ret.Results = append(ret.Results, result...)
}(server, fidList)
}
wg.Wait()
return ret, nil
}