Exemple #1
0
func (s *Store) Join() error {
	stats := new([]*VolumeInfo)
	for k, v := range s.volumes {
		s := &VolumeInfo{Id: VolumeId(k), Size: uint64(v.Size()),
			RepType: v.ReplicaType, Version: v.Version(),
			FileCount:        v.nm.FileCount(),
			DeleteCount:      v.nm.DeletedCount(),
			DeletedByteCount: v.nm.DeletedSize(),
			ReadOnly:         v.readOnly}
		*stats = append(*stats, s)
	}
	bytes, _ := json.Marshal(stats)
	values := make(url.Values)
	if !s.connected {
		values.Add("init", "true")
	}
	values.Add("port", strconv.Itoa(s.Port))
	values.Add("ip", s.Ip)
	values.Add("publicUrl", s.PublicUrl)
	values.Add("volumes", string(bytes))
	values.Add("maxVolumeCount", strconv.Itoa(s.MaxVolumeCount))
	jsonBlob, err := util.Post("http://"+s.masterNode+"/dir/join", values)
	if err != nil {
		return err
	}
	var ret JoinResult
	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
		return err
	}
	s.volumeSizeLimit = ret.VolumeSizeLimit
	s.connected = true
	return nil
}
Exemple #2
0
func Assign(server string, count int, replication string, collection string, ttl string) (*AssignResult, error) {
	values := make(url.Values)
	values.Add("count", strconv.Itoa(count))
	if replication != "" {
		values.Add("replication", replication)
	}
	if collection != "" {
		values.Add("collection", collection)
	}
	if ttl != "" {
		values.Add("ttl", ttl)
	}
	jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
	glog.V(2).Info("assign result :", string(jsonBlob))
	if err != nil {
		return nil, err
	}
	var ret AssignResult
	err = json.Unmarshal(jsonBlob, &ret)
	if err != nil {
		return nil, err
	}
	if ret.Count <= 0 {
		return nil, errors.New(ret.Error)
	}
	return &ret, nil
}
Exemple #3
0
func vacuumVolume_Commit(urlLocation string, vid storage.VolumeId) error {
	values := make(url.Values)
	values.Add("volume", vid.String())
	jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_commit", values)
	if err != nil {
		return err
	}
	var ret VacuumVolumeResult
	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
		return err
	}
	if ret.Error != "" {
		return errors.New(ret.Error)
	}
	return nil
}
Exemple #4
0
func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, error) {
	values := make(url.Values)
	for _, vid := range vids {
		values.Add("volumeId", vid)
	}
	jsonBlob, err := util.Post("http://"+server+"/vol/lookup", values)
	if err != nil {
		return nil, err
	}
	ret := make(map[string]LookupResult)
	err = json.Unmarshal(jsonBlob, &ret)
	if err != nil {
		return nil, errors.New(err.Error() + " " + string(jsonBlob))
	}
	return ret, nil
}
//TODO: Add a caching for vid here
func Lookup(server string, vid storage.VolumeId) (*LookupResult, error) {
	values := make(url.Values)
	values.Add("volumeId", vid.String())
	jsonBlob, err := util.Post("http://"+server+"/dir/lookup", values)
	if err != nil {
		return nil, err
	}
	var ret LookupResult
	err = json.Unmarshal(jsonBlob, &ret)
	if err != nil {
		return nil, err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	return &ret, nil
}
func AllocateVolume(dn *topology.DataNode, vid storage.VolumeId, repType storage.ReplicationType) error {
	values := make(url.Values)
	values.Add("volume", vid.String())
	values.Add("replicationType", repType.String())
	jsonBlob, err := util.Post("http://"+dn.Url()+"/admin/assign_volume", values)
	if err != nil {
		return err
	}
	var ret AllocateVolumeResult
	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
		return err
	}
	if ret.Error != "" {
		return errors.New(ret.Error)
	}
	return nil
}
Exemple #7
0
func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThreshold string) (error, bool) {
	values := make(url.Values)
	values.Add("volume", vid.String())
	values.Add("garbageThreshold", garbageThreshold)
	jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_check", values)
	if err != nil {
		fmt.Println("parameters:", values)
		return err, false
	}
	var ret VacuumVolumeResult
	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
		return err, false
	}
	if ret.Error != "" {
		return errors.New(ret.Error), false
	}
	return nil, ret.Result
}
Exemple #8
0
func call(server string, request ApiRequest, ret interface{}) error {
	b, err := json.Marshal(request)
	if err != nil {
		fmt.Println("error:", err)
		return nil
	}
	values := make(url.Values)
	values.Add("request", string(b))
	jsonBlob, err := util.Post("http://"+server+"/__api__", values)
	if err != nil {
		return err
	}
	err = json.Unmarshal(jsonBlob, ret)
	if err != nil {
		return err
	}
	return nil
}
Exemple #9
0
func AllocateVolume(dn *DataNode, vid storage.VolumeId, collection string, rp *storage.ReplicaPlacement) error {
	values := make(url.Values)
	values.Add("volume", vid.String())
	values.Add("collection", collection)
	values.Add("replication", rp.String())
	jsonBlob, err := util.Post("http://"+dn.PublicUrl+"/admin/assign_volume", values)
	if err != nil {
		return err
	}
	var ret AllocateVolumeResult
	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
		return err
	}
	if ret.Error != "" {
		return errors.New(ret.Error)
	}
	return nil
}
Exemple #10
0
func assign(count int) (*AssignResult, error) {
	values := make(url.Values)
	values.Add("count", strconv.Itoa(count))
	values.Add("replication", *uploadReplication)
	jsonBlob, err := util.Post("http://"+*server+"/dir/assign", values)
	debug("assign result :", string(jsonBlob))
	if err != nil {
		return nil, err
	}
	var ret AssignResult
	err = json.Unmarshal(jsonBlob, &ret)
	if err != nil {
		return nil, err
	}
	if ret.Count <= 0 {
		return nil, errors.New(ret.Error)
	}
	return &ret, nil
}
Exemple #11
0
func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
	vid_to_fileIds := make(map[string][]string)
	ret := &DeleteFilesResult{}
	var vids []string
	for _, fileId := range fileIds {
		vid, _, err := ParseFileId(fileId)
		if err != nil {
			ret.Results = append(ret.Results, DeleteResult{Fid: vid, Error: err.Error()})
			continue
		}
		if _, ok := vid_to_fileIds[vid]; !ok {
			vid_to_fileIds[vid] = make([]string, 0)
			vids = append(vids, vid)
		}
		vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
	}

	lookupResults, err := LookupVolumeIds(master, vids)
	if err != nil {
		return ret, err
	}

	server_to_fileIds := make(map[string][]string)
	for vid, result := range lookupResults {
		if result.Error != "" {
			ret.Errors = append(ret.Errors, result.Error)
			continue
		}
		for _, location := range result.Locations {
			if _, ok := server_to_fileIds[location.PublicUrl]; !ok {
				server_to_fileIds[location.PublicUrl] = make([]string, 0)
			}
			server_to_fileIds[location.PublicUrl] = append(
				server_to_fileIds[location.PublicUrl], vid_to_fileIds[vid]...)
		}
	}

	var wg sync.WaitGroup
	for server, fidList := range server_to_fileIds {
		wg.Add(1)
		go func(server string, fidList []string) {
			defer wg.Done()
			values := make(url.Values)
			for _, fid := range fidList {
				values.Add("fid", fid)
			}
			jsonBlob, err := util.Post("http://"+server+"/delete", values)
			if err != nil {
				ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
				return
			}
			var result []DeleteResult
			err = json.Unmarshal(jsonBlob, &result)
			if err != nil {
				ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
				return
			}
			ret.Results = append(ret.Results, result...)
		}(server, fidList)
	}
	wg.Wait()

	return ret, nil
}