Esempio n. 1
0
func DeleteFile(master string, fileId string) error {
	fileUrl, err := LookupFileId(master, fileId)
	if err != nil {
		return err
	}
	return util.Delete(fileUrl)
}
Esempio n. 2
0
func ReplicatedDelete(masterNode string, store *storage.Store, volumeId storage.VolumeId, n *storage.Needle, r *http.Request) (ret uint32) {
	ret, err := store.Delete(volumeId, n)
	if err != nil {
		glog.V(0).Infoln("delete error:", err)
		return
	}

	needToReplicate := !store.HasVolume(volumeId)
	if !needToReplicate && ret > 0 {
		needToReplicate = store.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "replicate" {
			if !distributedOperation(masterNode, store, volumeId, func(location operation.Location) bool {
				return nil == util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate")
			}) {
				ret = 0
			}
		}
	}
	return
}
Esempio n. 3
0
func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) {
	ret, err := s.Write(volumeId, needle)
	needToReplicate := !s.HasVolume(volumeId)
	if err != nil {
		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
	} else if ret > 0 {
		needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
	} else {
		errorStatus = "Failed to write to local disk"
	}
	if !needToReplicate && ret > 0 {
		needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
	}
	if needToReplicate { //send to other replica locations
		if r.FormValue("type") != "replicate" {
			if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
				_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified, 10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime))
				return err == nil
			}) {
				ret = 0
				errorStatus = "Failed to write to replicas for volume " + volumeId.String()
			}
		}
	}
	if errorStatus != "" {
		if _, err = s.Delete(volumeId, needle); err != nil {
			errorStatus += "\nCannot delete " + strconv.FormatUint(needle.Id, 10) + " from " +
				volumeId.String() + ": " + err.Error()
		} else {
			distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
				return nil == util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate")
			})
		}
	}
	size = ret
	return
}
Esempio n. 4
0
func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) {
	delayedDeleteChan := make(chan *delayedFile, 100)
	var waitForDeletions sync.WaitGroup
	for i := 0; i < 7; i++ {
		go func() {
			waitForDeletions.Add(1)
			for df := range delayedDeleteChan {
				if df == nil {
					break
				}
				if df.enterTime.After(time.Now()) {
					time.Sleep(df.enterTime.Sub(time.Now()))
				}
				fp := df.fp
				serverLimitChan[fp.Server] <- true
				if e := util.Delete("http://" + fp.Server + "/" + fp.Fid); e == nil {
					s.completed++
				} else {
					s.failed++
				}
				<-serverLimitChan[fp.Server]
			}
			waitForDeletions.Done()
		}()
	}

	for {
		if id, ok := <-idChan; ok {
			start := time.Now()
			fileSize := int64(*b.fileSize + rand.Intn(64))
			fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
			if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil {
				fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection
				if _, ok := serverLimitChan[fp.Server]; !ok {
					serverLimitChan[fp.Server] = make(chan bool, 7)
				}
				serverLimitChan[fp.Server] <- true
				if _, err := fp.Upload(0, *b.server); err == nil {
					if rand.Intn(100) < *b.deletePercentage {
						s.total++
						delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
					} else {
						fileIdLineChan <- fp.Fid
					}
					s.completed++
					s.transferred += fileSize
				} else {
					s.failed++
				}
				writeStats.addSample(time.Now().Sub(start))
				<-serverLimitChan[fp.Server]
				if *cmdBenchmark.IsDebug {
					fmt.Printf("writing %d file %s\n", id, fp.Fid)
				}
			} else {
				s.failed++
				println("writing file error:", err.Error())
			}
		} else {
			break
		}
	}
	close(delayedDeleteChan)
	waitForDeletions.Wait()
	wait.Done()
}