func Assign(server string, count uint64, replication string, collection string, ttl string) (*AssignResult, error) {
values := make(url.Values)
values.Add("count", strconv.FormatUint(count, 10))
if replication != "" {
values.Add("replication", replication)
}
if collection != "" {
values.Add("collection", collection)
}
if ttl != "" {
values.Add("ttl", ttl)
}
jsonBlob, err := util.Post(server, "/dir/assign", values)
glog.V(2).Info("assign result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret AssignResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, fmt.Errorf("/dir/assign result JSON unmarshal error:%v, json:%s", err, string(jsonBlob))
}
if ret.Count <= 0 {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
func RegisterFile(filer string, path string, fileId string, secret security.Secret) error {
// TODO: jwt need to be used
_ = security.GenJwt(secret, fileId)
values := make(url.Values)
values.Add("path", path)
values.Add("fileId", fileId)
_, err := util.Post("http://"+filer+"/admin/register", values)
if err != nil {
return fmt.Errorf("Failed to register path:%s on filer:%s to file id:%s", path, filer, fileId)
}
return nil
}
func vacuumVolume_Commit(urlLocation string, vid storage.VolumeId) error {
values := make(url.Values)
values.Add("volume", vid.String())
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum/commit", values)
if err != nil {
return err
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err
}
if ret.Error != "" {
return errors.New(ret.Error)
}
return nil
}
func do_lookup(server string, vid string) (*LookupResult, error) {
values := make(url.Values)
values.Add("volumeId", vid)
jsonBlob, err := util.Post("http://"+server+"/dir/lookup", values)
if err != nil {
return nil, err
}
var ret LookupResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
func GetVolumeSyncStatus(server string, vid string) (*SyncVolumeResponse, error) {
values := make(url.Values)
values.Add("volume", vid)
jsonBlob, err := util.Post("http://"+server+"/admin/sync/status", values)
glog.V(2).Info("sync volume result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret SyncVolumeResponse
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, fmt.Errorf("Volume %s get sync status error: %s", vid, ret.Error)
}
return &ret, nil
}
func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThreshold string) (error, bool) {
values := make(url.Values)
values.Add("volume", vid.String())
values.Add("garbageThreshold", garbageThreshold)
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum/check", values)
if err != nil {
glog.V(0).Infoln("parameters:", values)
return err, false
}
var ret VacuumVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err, false
}
if ret.Error != "" {
return errors.New(ret.Error), false
}
return nil, ret.Result
}
func call(server string, request ApiRequest, ret interface{}) error {
b, err := json.Marshal(request)
if err != nil {
fmt.Println("error:", err)
return nil
}
values := make(url.Values)
values.Add("request", string(b))
jsonBlob, err := util.Post("http://"+server+"/__api__", values)
if err != nil {
return err
}
err = json.Unmarshal(jsonBlob, ret)
if err != nil {
return err
}
return nil
}
func AllocateVolume(dn *DataNode, vid storage.VolumeId, option *VolumeGrowOption) error {
values := make(url.Values)
values.Add("volume", vid.String())
values.Add("collection", option.Collection)
values.Add("ttl", option.Ttl.String())
jsonBlob, err := util.Post(dn.Url(), "/admin/assign_volume", values)
if err != nil {
return err
}
var ret AllocateVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return fmt.Errorf("Invalid JSON result for %s: %s", "/admin/assign_volum", string(jsonBlob))
}
if ret.Error != "" {
return errors.New(ret.Error)
}
return nil
}
func do_lookup(server, vid, collection string) (*LookupResult, error) {
values := make(url.Values)
values.Add("volumeId", vid)
if collection != "" {
values.Set("collection", collection)
}
jsonBlob, err := util.Post(server, "/dir/lookup", values)
if err != nil {
return nil, err
}
var ret LookupResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
// LookupVolumeIds find volume locations by cache and actual lookup
func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, error) {
ret := make(map[string]LookupResult)
var unknown_vids []string
//check vid cache first
for _, vid := range vids {
locations, cache_err := vc.Get(vid)
if cache_err == nil {
ret[vid] = LookupResult{VolumeId: vid, Locations: locations}
} else {
unknown_vids = append(unknown_vids, vid)
}
}
//return success if all volume ids are known
if len(unknown_vids) == 0 {
return ret, nil
}
//only query unknown_vids
values := make(url.Values)
for _, vid := range unknown_vids {
values.Add("volumeId", vid)
}
jsonBlob, err := util.Post(server, "/vol/lookup", values)
if err != nil {
return nil, err
}
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, errors.New(err.Error() + " " + string(jsonBlob))
}
//set newly checked vids to cache
for _, vid := range unknown_vids {
locations := ret[vid].Locations
vc.Set(vid, locations, 10*time.Minute)
}
return ret, nil
}
func Assign(server string, r *VolumeAssignRequest) (*AssignResult, error) {
values := make(url.Values)
values.Add("count", strconv.FormatUint(r.Count, 10))
if r.Replication != "" {
values.Add("replication", r.Replication)
}
if r.Collection != "" {
values.Add("collection", r.Collection)
}
if r.Ttl != "" {
values.Add("ttl", r.Ttl)
}
if r.DataCenter != "" {
values.Add("dataCenter", r.DataCenter)
}
if r.Rack != "" {
values.Add("rack", r.Rack)
}
if r.DataNode != "" {
values.Add("dataNode", r.DataNode)
}
jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
glog.V(2).Info("assign result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret AssignResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, fmt.Errorf("/dir/assign result JSON unmarshal error:%v, json:%s", err, string(jsonBlob))
}
if ret.Count <= 0 {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
func (ms *MasterServer) batchSetVolumeOption(settingKey, settingValue string, volumes, collections []string) (result map[string]interface{}) {
forms := url.Values{}
forms.Set("key", settingKey)
forms.Set("value", settingValue)
if len(volumes) == 0 && len(collections) == 0 {
forms.Set("all", "true")
} else {
forms["volume"] = volumes
forms["collection"] = collections
}
var wg sync.WaitGroup
ms.Topo.WalkDataNode(func(dn *topology.DataNode) (e error) {
wg.Add(1)
go func(server string, values url.Values) {
defer wg.Done()
jsonBlob, e := util.Post(server, "/admin/setting", values)
if e != nil {
result[server] = map[string]interface{}{
"error": e.Error() + " " + string(jsonBlob),
}
}
var ret interface{}
if e := json.Unmarshal(jsonBlob, ret); e == nil {
result[server] = ret
} else {
result[server] = map[string]interface{}{
"error": e.Error() + " " + string(jsonBlob),
}
}
}(dn.Url(), forms)
return nil
})
wg.Wait()
return
}
func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
vid_to_fileIds := make(map[string][]string)
ret := &DeleteFilesResult{}
var vids []string
for _, fileId := range fileIds {
vid, _, err := ParseFileId(fileId)
if err != nil {
ret.Results = append(ret.Results, DeleteResult{
Fid: vid,
Status: http.StatusBadRequest,
Error: err.Error()},
)
continue
}
if _, ok := vid_to_fileIds[vid]; !ok {
vid_to_fileIds[vid] = make([]string, 0)
vids = append(vids, vid)
}
vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
}
lookupResults, err := LookupVolumeIds(master, vids)
if err != nil {
return ret, err
}
server_to_fileIds := make(map[string][]string)
for vid, result := range lookupResults {
if result.Error != "" {
ret.Errors = append(ret.Errors, result.Error)
continue
}
for _, location := range result.Locations {
if _, ok := server_to_fileIds[location.Url]; !ok {
server_to_fileIds[location.Url] = make([]string, 0)
}
server_to_fileIds[location.Url] = append(
server_to_fileIds[location.Url], vid_to_fileIds[vid]...)
}
}
var wg sync.WaitGroup
for server, fidList := range server_to_fileIds {
wg.Add(1)
go func(server string, fidList []string) {
defer wg.Done()
values := make(url.Values)
for _, fid := range fidList {
values.Add("fid", fid)
}
jsonBlob, err := util.Post(server, "/delete", values)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
var result []DeleteResult
err = json.Unmarshal(jsonBlob, &result)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
ret.Results = append(ret.Results, result...)
}(server, fidList)
}
wg.Wait()
return ret, nil
}
