func NewFilePart(fullPathFilename string) (ret FilePart, err error) {
fh, openErr := os.Open(fullPathFilename)
if openErr != nil {
glog.V(0).Info("Failed to open file: ", fullPathFilename)
return ret, openErr
}
ret.Reader = fh
if fi, fiErr := fh.Stat(); fiErr != nil {
glog.V(0).Info("Failed to stat file:", fullPathFilename)
return ret, fiErr
} else {
ret.ModTime = fi.ModTime().UTC().Unix()
}
ext := strings.ToLower(path.Ext(fullPathFilename))
ret.IsGzipped = ext == ".gz"
if ret.IsGzipped {
ret.FileName = fullPathFilename[0 : len(fullPathFilename)-3]
}
ret.FileName = fullPathFilename
if ext != "" {
ret.MimeType = mime.TypeByExtension(ext)
}
return ret, nil
}
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
var ret []operation.DeleteResult
for _, fid := range r.Form["fid"] {
vid, id_cookie, err := operation.ParseFileId(fid)
if err != nil {
ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
continue
}
n := new(storage.Needle)
volumeId, _ := storage.NewVolumeId(vid)
n.ParsePath(id_cookie)
glog.V(4).Infoln("batch deleting", n)
cookie := n.Cookie
if _, err := vs.store.Read(volumeId, n); err != nil {
ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
continue
}
if n.Cookie != cookie {
ret = append(ret, operation.DeleteResult{Fid: fid, Error: "File Random Cookie does not match."})
glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
return
}
if size, err := vs.store.Delete(volumeId, n); err != nil {
ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
} else {
ret = append(ret, operation.DeleteResult{Fid: fid, Size: int(size)})
}
}
w.WriteHeader(http.StatusAccepted)
writeJsonQuiet(w, r, ret)
}
func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
nm := NewNeedleMap(file)
e := WalkIndexFile(file, func(key uint64, offset, size uint32) error {
if key > nm.MaximumFileKey {
nm.MaximumFileKey = key
}
nm.FileCounter++
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
if offset > 0 {
oldSize := nm.m.Set(Key(key), offset, size)
glog.V(3).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
if oldSize > 0 {
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
} else {
oldSize := nm.m.Delete(Key(key))
glog.V(3).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
return nil
})
glog.V(1).Infoln("max file key:", nm.MaximumFileKey)
return nm, e
}
// walks through the index file, calls fn function with each key, offset, size
// stops with the error returned by the fn function
func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error {
var readerOffset int64
bytes := make([]byte, 16*RowsToRead)
count, e := r.ReadAt(bytes, readerOffset)
glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
readerOffset += int64(count)
var (
key uint64
offset, size uint32
i int
)
for count > 0 && e == nil || e == io.EOF {
for i = 0; i+16 <= count; i += 16 {
key = util.BytesToUint64(bytes[i : i+8])
offset = util.BytesToUint32(bytes[i+8 : i+12])
size = util.BytesToUint32(bytes[i+12 : i+16])
if e = fn(key, offset, size); e != nil {
return e
}
}
if e == io.EOF {
return nil
}
count, e = r.ReadAt(bytes, readerOffset)
glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
readerOffset += int64(count)
}
return e
}
func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) {
go func() {
for {
if t.IsLeader() {
freshThreshHold := time.Now().Unix() - 3*t.pulse //3 times of sleep interval
t.CollectDeadNodeAndFullVolumes(freshThreshHold, t.volumeSizeLimit)
}
time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
}
}()
go func(garbageThreshold string) {
c := time.Tick(15 * time.Minute)
if t.IsLeader() {
for _ = range c {
t.Vacuum(garbageThreshold)
}
}
}(garbageThreshold)
go func() {
for {
select {
case v := <-t.chanFullVolumes:
t.SetVolumeCapacityFull(v)
case dn := <-t.chanRecoveredDataNodes:
t.RegisterRecoveredDataNode(dn)
glog.V(0).Infoln("DataNode", dn, "is back alive!")
case dn := <-t.chanDeadDataNodes:
t.UnRegisterDataNode(dn)
glog.V(0).Infoln("DataNode", dn, "is dead!")
}
}
}()
}
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error {
var e error
fileName := v.FileName()
if exists, canRead, canWrite, _ := checkFile(fileName + ".dat"); exists {
if !canRead {
return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
}
if canWrite {
v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
} else {
glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode")
v.dataFile, e = os.Open(fileName + ".dat")
v.readOnly = true
}
} else {
if createDatIfMissing {
v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
} else {
return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
}
}
if e != nil {
if !os.IsPermission(e) {
return fmt.Errorf("cannot load Volume Data %s.dat: %s", fileName, e.Error())
}
}
if v.ReplicaPlacement == nil {
e = v.readSuperBlock()
} else {
e = v.maybeWriteSuperBlock()
}
if e == nil && alsoLoadIndex {
if v.readOnly {
if v.ensureConvertIdxToCdb(fileName) {
v.nm, e = OpenCdbMap(fileName + ".cdb")
return e
}
}
var indexFile *os.File
if v.readOnly {
glog.V(1).Infoln("open to read file", fileName+".idx")
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil {
return fmt.Errorf("cannot read Volume Index %s.idx: %s", fileName, e.Error())
}
} else {
glog.V(1).Infoln("open to write file", fileName+".idx")
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil {
return fmt.Errorf("cannot write Volume Index %s.idx: %s", fileName, e.Error())
}
}
glog.V(0).Infoln("loading file", fileName+".idx", "readonly", v.readOnly)
if v.nm, e = LoadNeedleMap(indexFile); e != nil {
glog.V(0).Infoln("loading error:", e)
}
}
return e
}
// a workaround because http POST following redirection misses request body
func postFollowingOneRedirect(target string, contentType string, b *bytes.Buffer) error {
backupReader := bytes.NewReader(b.Bytes())
resp, err := http.Post(target, contentType, b)
if err != nil {
return err
}
defer resp.Body.Close()
reply, _ := ioutil.ReadAll(resp.Body)
statusCode := resp.StatusCode
if statusCode == http.StatusMovedPermanently {
var urlStr string
if urlStr = resp.Header.Get("Location"); urlStr == "" {
return errors.New(fmt.Sprintf("%d response missing Location header", resp.StatusCode))
}
glog.V(0).Infoln("Post redirected to ", urlStr)
resp2, err2 := http.Post(urlStr, contentType, backupReader)
if err2 != nil {
return err2
}
defer resp2.Body.Close()
reply, _ = ioutil.ReadAll(resp2.Body)
statusCode = resp2.StatusCode
}
glog.V(0).Infoln("Post returned status: ", statusCode, string(reply))
if statusCode != http.StatusOK {
return errors.New(string(reply))
}
return nil
}
func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
writeJsonError(w, r, err)
return
}
joinMessage := &operation.JoinMessage{}
if err = proto.Unmarshal(body, joinMessage); err != nil {
writeJsonError(w, r, err)
return
}
if *joinMessage.Ip == "" {
*joinMessage.Ip = r.RemoteAddr[0:strings.Index(r.RemoteAddr, ":")]
}
if glog.V(4) {
if jsonData, jsonError := json.Marshal(joinMessage); jsonError != nil {
glog.V(0).Infoln("json marshaling error: ", jsonError)
writeJsonError(w, r, jsonError)
return
} else {
glog.V(4).Infoln("Proto size", len(body), "json size", len(jsonData), string(jsonData))
}
}
ms.Topo.ProcessJoinMessage(joinMessage)
writeJsonQuiet(w, r, operation.JoinResult{VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024})
}
// Join joins an existing cluster.
func (s *RaftServer) Join(peers []string) error {
command := &raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: "http://" + s.httpAddr,
}
var err error
var b bytes.Buffer
json.NewEncoder(&b).Encode(command)
for _, m := range peers {
if m == s.httpAddr {
continue
}
target := fmt.Sprintf("http://%s/cluster/join", strings.TrimSpace(m))
glog.V(0).Infoln("Attempting to connect to:", target)
err = postFollowingOneRedirect(target, "application/json", &b)
if err != nil {
glog.V(0).Infoln("Post returned error: ", err.Error())
if _, ok := err.(*url.Error); ok {
// If we receive a network error try the next member
continue
}
} else {
return nil
}
}
return errors.New("Could not connect to any cluster peers")
}
func batchVacuumVolumeCompact(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
vl.removeFromWritable(vid)
ch := make(chan bool, locationlist.Length())
for index, dn := range locationlist.list {
go func(index int, url string, vid storage.VolumeId) {
glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
if e := vacuumVolume_Compact(url, vid); e != nil {
glog.V(0).Infoln(index, "Error when vacuuming", vid, "on", url, e)
ch <- false
} else {
glog.V(0).Infoln(index, "Complete vacuuming", vid, "on", url)
ch <- true
}
}(index, dn.Url(), vid)
}
isVacuumSuccess := true
for _ = range locationlist.list {
select {
case _ = <-ch:
case <-time.After(30 * time.Minute):
isVacuumSuccess = false
break
}
}
return isVacuumSuccess
}
func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
if e := r.ParseForm(); e != nil {
glog.V(0).Infoln("form parse error:", e)
writeJsonError(w, r, e)
return
}
vid, _, _, _, _ := parseURLPath(r.URL.Path)
volumeId, ve := storage.NewVolumeId(vid)
if ve != nil {
glog.V(0).Infoln("NewVolumeId error:", ve)
writeJsonError(w, r, ve)
return
}
needle, ne := storage.NewNeedle(r)
if ne != nil {
writeJsonError(w, r, ne)
return
}
ret := operation.UploadResult{}
size, errorStatus := topology.ReplicatedWrite(vs.masterNode, vs.store, volumeId, needle, r)
if errorStatus == "" {
w.WriteHeader(http.StatusCreated)
} else {
w.WriteHeader(http.StatusInternalServerError)
ret.Error = errorStatus
}
if needle.HasName() {
ret.Name = string(needle.Name)
}
ret.Size = size
writeJsonQuiet(w, r, ret)
}
func (v *Volume) ensureConvertIdxToCdb(fileName string) (cdbCanRead bool) {
var indexFile *os.File
var e error
_, cdbCanRead, cdbCanWrite, cdbModTime := checkFile(fileName + ".cdb")
_, idxCanRead, _, idxModeTime := checkFile(fileName + ".idx")
if cdbCanRead && cdbModTime.After(idxModeTime) {
return true
}
if !cdbCanWrite {
return false
}
if !idxCanRead {
glog.V(0).Infoln("Can not read file", fileName+".idx!")
return false
}
glog.V(2).Infoln("opening file", fileName+".idx")
if indexFile, e = os.Open(fileName + ".idx"); e != nil {
glog.V(0).Infoln("Failed to read file", fileName+".idx !")
return false
}
defer indexFile.Close()
glog.V(0).Infof("converting %s.idx to %s.cdb", fileName, fileName)
if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil {
glog.V(0).Infof("error converting %s.idx to %s.cdb: %s", fileName, fileName, e.Error())
return false
}
return true
}
func NewVolumeServer(r *http.ServeMux, ip string, port int, publicIp string, folders []string, maxCounts []int,
masterNode string, pulseSeconds int,
dataCenter string, rack string,
whiteList []string, fixJpgOrientation bool) *VolumeServer {
publicUrl := publicIp + ":" + strconv.Itoa(port)
vs := &VolumeServer{
masterNode: masterNode,
pulseSeconds: pulseSeconds,
dataCenter: dataCenter,
rack: rack,
whiteList: whiteList,
FixJpgOrientation: fixJpgOrientation,
}
vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts)
r.HandleFunc("/submit", secure(vs.whiteList, vs.submitFromVolumeServerHandler))
r.HandleFunc("/status", secure(vs.whiteList, vs.statusHandler))
r.HandleFunc("/admin/assign_volume", secure(vs.whiteList, vs.assignVolumeHandler))
r.HandleFunc("/admin/vacuum_volume_check", secure(vs.whiteList, vs.vacuumVolumeCheckHandler))
r.HandleFunc("/admin/vacuum_volume_compact", secure(vs.whiteList, vs.vacuumVolumeCompactHandler))
r.HandleFunc("/admin/vacuum_volume_commit", secure(vs.whiteList, vs.vacuumVolumeCommitHandler))
r.HandleFunc("/admin/freeze_volume", secure(vs.whiteList, vs.freezeVolumeHandler))
r.HandleFunc("/admin/delete_collection", secure(vs.whiteList, vs.deleteCollectionHandler))
r.HandleFunc("/stats/counter", secure(vs.whiteList, statsCounterHandler))
r.HandleFunc("/stats/memory", secure(vs.whiteList, statsMemoryHandler))
r.HandleFunc("/stats/disk", secure(vs.whiteList, vs.statsDiskHandler))
r.HandleFunc("/delete", secure(vs.whiteList, vs.batchDeleteHandler))
r.HandleFunc("/", vs.storeHandler)
go func() {
connected := true
vs.store.SetBootstrapMaster(vs.masterNode)
vs.store.SetDataCenter(vs.dataCenter)
vs.store.SetRack(vs.rack)
for {
master, err := vs.store.Join()
if err == nil {
if !connected {
connected = true
glog.V(0).Infoln("Volume Server Connected with master at", master)
}
} else {
glog.V(4).Infoln("Volume Server Failed to talk with master:", err.Error())
if connected {
connected = false
}
}
if connected {
time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*(1+rand.Float32())) * time.Millisecond)
} else {
time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*0.25) * time.Millisecond)
}
}
}()
return vs
}
func (v *Volume) Compact() error {
glog.V(3).Infof("Compacting ...")
v.accessLock.Lock()
defer v.accessLock.Unlock()
glog.V(3).Infof("Got Compaction lock...")
filePath := v.FileName()
glog.V(3).Infof("creating copies for volume %d ...", v.Id)
return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
}
func (s *RaftServer) redirectToLeader(w http.ResponseWriter, req *http.Request) {
if leader, e := s.topo.Leader(); e == nil {
//http.StatusMovedPermanently does not cause http POST following redirection
glog.V(0).Infoln("Redirecting to", http.StatusMovedPermanently, "http://"+leader+req.URL.Path)
http.Redirect(w, req, "http://"+leader+req.URL.Path, http.StatusMovedPermanently)
} else {
glog.V(0).Infoln("Error: Leader Unknown")
http.Error(w, "Leader unknown", http.StatusInternalServerError)
}
}
func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) {
form, fe := r.MultipartReader()
if fe != nil {
glog.V(0).Infoln("MultipartReader [ERROR]", fe)
e = fe
return
}
part, fe := form.NextPart()
if fe != nil {
glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
e = fe
return
}
fileName = part.FileName()
if fileName != "" {
fileName = path.Base(fileName)
}
data, e = ioutil.ReadAll(part)
if e != nil {
glog.V(0).Infoln("Reading Content [ERROR]", e)
return
}
dotIndex := strings.LastIndex(fileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(fileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := part.Header.Get("Content-Type")
if contentType != "" && mtype != contentType {
mimeType = contentType //only return mime type if not deductable
mtype = contentType
}
if part.Header.Get("Content-Encoding") == "gzip" {
isGzipped = true
} else if IsGzippable(ext, mtype) {
if data, e = GzipData(data); e != nil {
return
}
isGzipped = true
}
if ext == ".gz" {
isGzipped = true
}
if strings.HasSuffix(fileName, ".gz") {
fileName = fileName[:len(fileName)-3]
}
modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
ttl, _ = ReadTTL(r.FormValue("ttl"))
return
}
func (vs *VolumeServer) deleteCollectionHandler(w http.ResponseWriter, r *http.Request) {
if "benchmark" != r.FormValue("collection") {
glog.V(0).Infoln("deleting collection =", r.FormValue("collection"), "!!!")
return
}
err := vs.store.DeleteCollection(r.FormValue("collection"))
if err == nil {
writeJsonQuiet(w, r, map[string]string{"error": ""})
} else {
writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
}
glog.V(2).Infoln("deleting collection =", r.FormValue("collection"), ", error =", err)
}
func GzipData(input []byte) ([]byte, error) {
buf := new(bytes.Buffer)
w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)
if _, err := w.Write(input); err != nil {
glog.V(2).Infoln("error compressing data:", err)
return nil, err
}
if err := w.Close(); err != nil {
glog.V(2).Infoln("error closing compressed data:", err)
return nil, err
}
return buf.Bytes(), nil
}
func (v *Volume) load(alsoLoadIndex bool) error {
var e error
fileName := path.Join(v.dir, v.Id.String())
if exists, canRead, canWrite, _ := checkFile(fileName + ".dat"); exists && !canRead {
return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
} else if !exists || canWrite {
v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
} else if exists && canRead {
glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode")
v.dataFile, e = os.Open(fileName + ".dat")
v.readOnly = true
} else {
return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName)
}
if e != nil {
if !os.IsPermission(e) {
return fmt.Errorf("cannot load Volume Data %s.dat: %s", fileName, e.Error())
}
}
if v.ReplicaType == CopyNil {
e = v.readSuperBlock()
} else {
e = v.maybeWriteSuperBlock()
}
if e == nil && alsoLoadIndex {
if v.readOnly {
if v.ensureConvertIdxToCdb(fileName) {
v.nm, e = OpenCdbMap(fileName + ".cdb")
return e
}
}
var indexFile *os.File
if v.readOnly {
glog.V(1).Infoln("open to read file", fileName+".idx")
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil {
return fmt.Errorf("cannot read Volume Data %s.dat: %s", fileName, e.Error())
}
} else {
glog.V(1).Infoln("open to write file", fileName+".idx")
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil {
return fmt.Errorf("cannot write Volume Data %s.dat: %s", fileName, e.Error())
}
}
glog.V(0).Infoln("loading file", fileName+".idx", "readonly", v.readOnly)
if v.nm, e = LoadNeedleMap(indexFile); e != nil {
glog.V(0).Infoln("loading error:", e)
}
}
return e
}
func (vg *VolumeGrowth) grow(topo *Topology, vid storage.VolumeId, option *VolumeGrowOption, servers ...*DataNode) error {
for _, server := range servers {
if err := AllocateVolume(server, vid, option.Collection, option.ReplicaPlacement); err == nil {
vi := storage.VolumeInfo{Id: vid, Size: 0, Collection: option.Collection, ReplicaPlacement: option.ReplicaPlacement, Version: storage.CurrentVersion}
server.AddOrUpdateVolume(vi)
topo.RegisterVolumeLayout(vi, server)
glog.V(0).Infoln("Created Volume", vid, "on", server)
} else {
glog.V(0).Infoln("Failed to assign", vid, "to", servers, "error", err)
return fmt.Errorf("Failed to assign %s: %s", vid.String(), err.Error())
}
}
return nil
}
func (vg *VolumeGrowth) grow(topo *topology.Topology, vid storage.VolumeId, repType storage.ReplicationType, servers ...*topology.DataNode) error {
for _, server := range servers {
if err := operation.AllocateVolume(server, vid, repType); err == nil {
vi := storage.VolumeInfo{Id: vid, Size: 0, RepType: repType, Version: storage.CurrentVersion}
server.AddOrUpdateVolume(vi)
topo.RegisterVolumeLayout(&vi, server)
glog.V(0).Infoln("Created Volume", vid, "on", server)
} else {
glog.V(0).Infoln("Failed to assign", vid, "to", servers, "error", err)
return errors.New("Failed to assign " + vid.String() + ", " + err.Error())
}
}
return nil
}
func Post(url string, values url.Values) ([]byte, error) {
r, err := client.PostForm(url, values)
if err != nil {
glog.V(0).Infoln(err)
return nil, err
}
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
if err != nil {
glog.V(0).Infoln("read post result from", url, err)
return nil, err
}
return b, nil
}
func PostBytes(url string, body []byte) ([]byte, error) {
r, err := client.Post(url, "application/octet-stream", bytes.NewReader(body))
if err != nil {
glog.V(0).Infoln(err)
return nil, err
}
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
if err != nil {
glog.V(0).Infoln("read post result from", url, err)
return nil, err
}
return b, nil
}
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string) (*UploadResult, error) {
body_buf := bytes.NewBufferString("")
body_writer := multipart.NewWriter(body_buf)
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
if mtype == "" {
mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
}
if mtype != "" {
h.Set("Content-Type", mtype)
}
if isGzipped {
h.Set("Content-Encoding", "gzip")
}
file_writer, err := body_writer.CreatePart(h)
if err != nil {
glog.V(0).Infoln("error creating form file", err)
return nil, err
}
if err = fillBufferFunction(file_writer); err != nil {
glog.V(0).Infoln("error copying data", err)
return nil, err
}
content_type := body_writer.FormDataContentType()
if err = body_writer.Close(); err != nil {
glog.V(0).Infoln("error closing body", err)
return nil, err
}
resp, err := http.Post(uploadUrl, content_type, body_buf)
if err != nil {
glog.V(0).Infoln("failing to upload to", uploadUrl)
return nil, err
}
defer resp.Body.Close()
resp_body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
var ret UploadResult
err = json.Unmarshal(resp_body, &ret)
if err != nil {
glog.V(0).Infoln("failing to read upload resonse", uploadUrl, string(resp_body))
return nil, err
}
if ret.Error != "" {
return nil, errors.New(ret.Error)
}
return &ret, nil
}
// the first node must satisfy filterFirstNodeFn(), the rest nodes must have one free slot
func (n *NodeImpl) RandomlyPickNodes(numberOfNodes int, filterFirstNodeFn func(dn Node) error) (firstNode Node, restNodes []Node, err error) {
candidates := make([]Node, 0, len(n.children))
errs := make([]string, 0)
for _, node := range n.children {
if err := filterFirstNodeFn(node); err == nil {
candidates = append(candidates, node)
} else {
errs = append(errs, string(node.Id())+":"+err.Error())
}
}
if len(candidates) == 0 {
return nil, nil, errors.New("No matching data node found! \n" + strings.Join(errs, "\n"))
}
firstNode = candidates[rand.Intn(len(candidates))]
glog.V(2).Infoln(n.Id(), "picked main node:", firstNode.Id())
restNodes = make([]Node, numberOfNodes-1)
candidates = candidates[:0]
for _, node := range n.children {
if node.Id() == firstNode.Id() {
continue
}
if node.FreeSpace() <= 0 {
continue
}
glog.V(2).Infoln("select rest node candidate:", node.Id())
candidates = append(candidates, node)
}
glog.V(2).Infoln(n.Id(), "picking", numberOfNodes-1, "from rest", len(candidates), "node candidates")
ret := len(restNodes) == 0
for k, node := range candidates {
if k < len(restNodes) {
restNodes[k] = node
if k == len(restNodes)-1 {
ret = true
}
} else {
r := rand.Intn(k + 1)
if r < len(restNodes) {
restNodes[r] = node
}
}
}
if !ret {
glog.V(2).Infoln(n.Id(), "failed to pick", numberOfNodes-1, "from rest", len(candidates), "node candidates")
err = errors.New("Not enough data node found!")
}
return
}
func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
ms.Topo.RaftServer = raftServer.raftServer
ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
if ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
}
})
if ms.Topo.IsLeader() {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!")
} else {
if ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
}
}
}
func (v *Volume) freeze() error {
if v.readOnly {
return nil
}
nm, ok := v.nm.(*NeedleMap)
if !ok {
return nil
}
v.accessLock.Lock()
defer v.accessLock.Unlock()
bn, _ := nakeFilename(v.dataFile.Name())
cdbFn := bn + ".cdb"
glog.V(0).Infof("converting %s to %s", nm.indexFile.Name(), cdbFn)
err := DumpNeedleMapToCdb(cdbFn, nm)
if err != nil {
return err
}
if v.nm, err = OpenCdbMap(cdbFn); err != nil {
return err
}
nm.indexFile.Close()
os.Remove(nm.indexFile.Name())
v.readOnly = true
return nil
}
func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
isCommitSuccess := true
for _, dn := range locationlist.list {
glog.V(0).Infoln("Start Commiting vacuum", vid, "on", dn.Url())
if e := vacuumVolume_Commit(dn.Url(), vid); e != nil {
glog.V(0).Infoln("Error when committing vacuum", vid, "on", dn.Url(), e)
isCommitSuccess = false
} else {
glog.V(0).Infoln("Complete Commiting vacuum", vid, "on", dn.Url())
}
if isCommitSuccess {
vl.SetVolumeAvailable(dn, vid)
}
}
return isCommitSuccess
}
func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []FileEntry) {
glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
dirKey := genKey(dirId, "")
iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
limitCounter := 0
for iter.Next() {
key := iter.Key()
if !bytes.HasPrefix(key, dirKey) {
break
}
fileName := string(key[len(dirKey):])
if fileName == lastFileName {
continue
}
limitCounter++
if limit > 0 {
if limitCounter > limit {
break
}
}
files = append(files, FileEntry{Name: fileName, Id: FileId(string(iter.Value()))})
}
iter.Release()
return
}
func Assign(server string, count int, replication string, collection string, ttl string) (*AssignResult, error) {
values := make(url.Values)
values.Add("count", strconv.Itoa(count))
if replication != "" {
values.Add("replication", replication)
}
if collection != "" {
values.Add("collection", collection)
}
if ttl != "" {
values.Add("ttl", ttl)
}
jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
glog.V(2).Info("assign result :", string(jsonBlob))
if err != nil {
return nil, err
}
var ret AssignResult
err = json.Unmarshal(jsonBlob, &ret)
if err != nil {
return nil, err
}
if ret.Count <= 0 {
return nil, errors.New(ret.Error)
}
return &ret, nil
}