/*
# Basic Usage:
curl http://localhost:9333/dir/assign
{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080",
"publicUrl":"localhost:8080"}
# To assign with a specific replication type:
curl "http://localhost:9333/dir/assign?replication=001"
# To specify how many file ids to reserve
curl "http://localhost:9333/dir/assign?count=5"
# To assign a specific data center
curl "http://localhost:9333/dir/assign?dataCenter=dc1"
*/
func Assign(server string, count int, replication string, collection string, ttl string) (*AssignResult, error) {
values := make(url.Values)
values.Add("count", strconv.Itoa(count))
if replication != "" {
values.Add("replication", replication)
}
if collection != "" {
values.Add("collection", collection)
}
if ttl != "" {
values.Add("ttl", ttl)
}
jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
glog.V(2).Info("assign result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret AssignResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Count <= 0 {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
//按volume查找
//http://192.168.1.2:80/vol/lookup
func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, error) {
values := make(url.Values)
for _, vid := range vids {
values.Add("volumeId", vid)
}
jsonBlob, err := util.Post("http://"+server+"/vol/lookup", values)
if err != nil {
return nil, err
}
ret := make(map[string]LookupResult)
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, errors.New(err.Error() + " " + string(jsonBlob))
}
return ret, nil
}
//http://192.168.1.2:80/dir/lookup
func do_lookup(server string, vid string) (*LookupResult, error) {
values := make(url.Values)
values.Add("volumeId", vid)
jsonBlob, err := util.Post("http://"+server+"/dir/lookup", values)
if err != nil {
return nil, err
}
var ret LookupResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
func call(server string, request ApiRequest, ret interface{}) error {
b, err := json.Marshal(request)
if err != nil {
fmt.Println("error:", err)
return nil
}
values := make(url.Values)
values.Add("request", string(b))
jsonBlob, err := util.Post("http://"+server+"/__api__", values)
if err != nil {
return err
}
err = json.Unmarshal(jsonBlob, ret)
if err != nil {
return err
}
return nil
}
//http://192.168.1.1:8080/admin/assign_volume
func AllocateVolume(dn *DataNode, vid storage.VolumeId, option *VolumeGrowOption) error {
values := make(url.Values)
values.Add("volume", vid.String())
values.Add("collection", option.Collection)
values.Add("replication", option.ReplicaPlacement.String())
values.Add("ttl", option.Ttl.String())
jsonBlob, err := util.Post("http://"+dn.PublicUrl+"/admin/assign_volume", values)
if err != nil {
return err
}
var ret AllocateVolumeResult
if err := json.Unmarshal(jsonBlob, &ret); err != nil {
return err
}
if ret.Error != "" {
return errors.New(ret.Error)
}
return nil
}
func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
vid_to_fileIds := make(map[string][]string)
ret := &DeleteFilesResult{}
var vids []string
for _, fileId := range fileIds {
vid, _, err := ParseFileId(fileId)
if err != nil {
ret.Results = append(ret.Results, DeleteResult{Fid: vid, Error: err.Error()})
continue
}
if _, ok := vid_to_fileIds[vid]; !ok {
vid_to_fileIds[vid] = make([]string, 0)
vids = append(vids, vid)
}
vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
}
lookupResults, err := LookupVolumeIds(master, vids)
if err != nil {
return ret, err
}
server_to_fileIds := make(map[string][]string)
for vid, result := range lookupResults {
if result.Error != "" {
ret.Errors = append(ret.Errors, result.Error)
continue
}
for _, location := range result.Locations {
if _, ok := server_to_fileIds[location.Url]; !ok {
server_to_fileIds[location.Url] = make([]string, 0)
}
server_to_fileIds[location.Url] = append(
server_to_fileIds[location.Url], vid_to_fileIds[vid]...)
}
}
var wg sync.WaitGroup
for server, fidList := range server_to_fileIds {
wg.Add(1)
go func(server string, fidList []string) {
defer wg.Done()
values := make(url.Values)
for _, fid := range fidList {
values.Add("fid", fid)
}
jsonBlob, err := util.Post("http://"+server+"/delete", values)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
var result []DeleteResult
err = json.Unmarshal(jsonBlob, &result)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
ret.Results = append(ret.Results, result...)
}(server, fidList)
}
wg.Wait()
return ret, nil
}
/*
此函数的作用是从集群中删除一组文件它的执行步骤如下:
1.将文件按volume 分组
2.将volume归并出最终的volume集合
3.从master中获得volumes所在服务器的列表,因为一个server可能会有多个volumes
4.将文件按服务器分组
5.从服务器上将所持有的文件删除
*/
func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
vid_to_fileIds := make(map[string][]string)
ret := &DeleteFilesResult{}
var vids []string
/*
此循环做了两件事情:
1.将参数fileIds所指定的所有的file所存入的volume找出来放到vids中 即统计出哪些volume参与了操作
2.将fileIds按volume分组
*/
for _, fileId := range fileIds {
vid, _, err := ParseFileId(fileId)
if err != nil {
ret.Results = append(ret.Results, DeleteResult{Fid: vid, Error: err.Error()})
continue
}
//如果此vid尚未cache
if _, ok := vid_to_fileIds[vid]; !ok {
vid_to_fileIds[vid] = make([]string, 0)
vids = append(vids, vid) //cache新的vid
}
//某个volume下的所有field
vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
}
//从master获得所有的vid的urls
lookupResults, err := LookupVolumeIds(master, vids)
if err != nil {
return ret, err
}
server_to_fileIds := make(map[string][]string)
for vid, result := range lookupResults {
if result.Error != "" {
ret.Errors = append(ret.Errors, result.Error)
continue
}
//此循环的作用是按PublicUrl即server将所有的files分组
for _, location := range result.Locations {
if _, ok := server_to_fileIds[location.PublicUrl]; !ok {
server_to_fileIds[location.PublicUrl] = make([]string, 0)
}
server_to_fileIds[location.PublicUrl] = append(
server_to_fileIds[location.PublicUrl], vid_to_fileIds[vid]...)
}
}
var wg sync.WaitGroup
for server, fidList := range server_to_fileIds {
wg.Add(1)
go func(server string, fidList []string) {
defer wg.Done()
values := make(url.Values)
for _, fid := range fidList {
values.Add("fid", fid)
}
//从某个server删除此server上的所有files
jsonBlob, err := util.Post("http://"+server+"/delete", values)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
var result []DeleteResult
err = json.Unmarshal(jsonBlob, &result)
if err != nil {
ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
return
}
ret.Results = append(ret.Results, result...)
}(server, fidList)
}
wg.Wait()
return ret, nil
}