func (this *FD) MergeNext() error {
if tmpErr := this.ReadInNumberZero(); tmpErr != nil {
return tmpErr
}
// Read one patch file , get ready for merge
this.updateChainLock.RLock()
var nextEmptyPatch = this.nextAvailablePosition
this.updateChainLock.RUnlock()
this.contentLock.Lock()
defer this.contentLock.Unlock()
if nextEmptyPatch == this.nextToBeMerge {
return NOTHING_TO_MERGE
}
var oldMerged = this.nextToBeMerge
var thePatch, meta, err = readInKvMapfile(this.io, this.GetPatchName(this.nextToBeMerge, -1))
// may happen due to the unsubmission of Submit() function
if thePatch == nil {
Secretary.Warn("distributedvc::FD.MergeNext()", "Fail to get a supposed-to-be patch for file "+this.filename)
if err == nil {
return MERGE_ERROR
} else {
return err
}
}
var theNext int
if tNext, ok := meta[INTRA_PATCH_METAKEY_NEXT_PATCH]; !ok {
Secretary.Warn("distributedvc::FD.MergeNext()", "Fail to get INTRA_PATCH_METAKEY_NEXT_PATCH for file "+this.filename)
theNext = this.nextToBeMerge + 1
} else {
if intTNext, err := strconv.Atoi(tNext); err != nil {
Secretary.Warn("distributedvc::FD.MergeNext()", "Fail to get INTRA_PATCH_METAKEY_NEXT_PATCH for file "+this.filename)
theNext = this.nextToBeMerge + 1
} else {
theNext = intTNext
}
}
tNew, err := this.numberZero.MergeWith(thePatch)
if err != nil {
Secretary.Warn("distributedvc::FD.MergeNext()", "Fail to merge patches for file "+this.filename)
return err
}
this.numberZero.MergeWith(filetype.FastMake(CONF_FLAG_PREFIX + NODE_SYNC_TIME_PREFIX + strconv.Itoa(NODE_NUMBER)))
this.numberZero = tNew
this.nextToBeMerge = theNext
this.modified = true
this.needsGossiped = true
Secretary.Log("distributedvc::FD.MergeNext()", "Successfully merged in patch #"+strconv.Itoa(oldMerged)+" for "+this.filename)
return nil
}
func _TestParallelCommit(t *testing.T) {
Insider.LogD("+++++ TestParallelCommit::start")
var wg sync.WaitGroup
var routine = func(number int) {
defer wg.Done()
var filename = strconv.Itoa(number)
{
var desParentMap = GetFD(filename, io)
if desParentMap == nil {
Insider.LogD("Fail to get foldermap fd for folder " + filename)
return
}
if err := desParentMap.Submit(filetype.FastMake("hasi")); err != nil {
Insider.LogD("Fail to submit foldermap patch for folder " + filename)
desParentMap.Release()
return
}
desParentMap.Release()
}
Insider.LogD("Succeed process for " + filename)
}
var numberOfThreads = 100
wg.Add(numberOfThreads)
for i := 0; i < numberOfThreads; i++ {
go routine(i)
}
wg.Wait()
Insider.LogD("----- TestParallelCommit::end")
for {
time.Sleep(time.Hour)
}
}
func TestPut(t *testing.T) {
// May returns 404 for unexist container
var io = NewSwiftio(DefaultConnector, "*****@*****.**")
fmt.Println(io.Put("1234", filetype.FastMake("12"), nil))
}
// @ Grasped Reader
// callback will get async invoked, int={
// 0: nothing posed;
// 1: post original gossip;
// 2: post temporarily nothing. A gossip will be posted when writing back
// }
// It will not writeback any change
func (this *FD) ASYNCMergeWithNodeX(context *gspdi.GossipEntry, callback func(int)) {
this.contentLock.RLock()
if this.needsGossiped {
go callback(2)
}
this.contentLock.RUnlock()
// Submit #0 patch if needed
this._LoadPointerMap_SyncUseOnly()
// read patch 0 from container. If just submit, the function will exit immediately
this.ReadInNumberZero()
this.contentLock.Lock()
if this.numberZero == nil {
Insider.Log("distributedvc::FD.ASYNCMergeWithNodeX", "Looks like a logical isle: this.numberZero==nil")
Secretary.Error("distributedvc::FD.ASYNCMergeWithNodeX", "Looks like a logical isle: this.numberZero==nil")
this.contentLock.Unlock()
go callback(1)
return
}
this.numberZero.CheckOut()
var keyStoreName = CONF_FLAG_PREFIX + NODE_SYNC_TIME_PREFIX + strconv.Itoa(context.NodeNumber)
var lastTime ClxTimestamp
if elem, ok := this.numberZero.Kvm[keyStoreName]; ok {
lastTime = elem.Timestamp
} else {
lastTime = 0
}
if lastTime >= context.UpdateTime {
this.contentLock.Unlock()
go callback(0)
return
}
go callback(1)
this.contentLock.Unlock()
var file, _, err = readInKvMapfile_NoWarning(this.io, this.GetPatchName(0, context.NodeNumber))
if file == nil {
if err == nil {
Secretary.Warn("distributedvc::FD.MergeWithNodeX", "Fail to get gossiped file: nonexist")
} else {
Secretary.Warn("distributedvc::FD.MergeWithNodeX", "Fail to get gossiped file: "+err.Error())
}
return
}
this.contentLock.Lock()
defer this.contentLock.Unlock()
if elem, ok := this.numberZero.Kvm[keyStoreName]; ok {
lastTime = elem.Timestamp
} else {
lastTime = 0
}
if lastTime >= context.UpdateTime {
return
}
if lastTime >= file.TGet() {
return
}
this.numberZero.MergeWith(file)
this.numberZero.MergeWith(filetype.FastMake(CONF_FLAG_PREFIX + NODE_SYNC_TIME_PREFIX + strconv.Itoa(NODE_NUMBER)))
this.modified = true
}
// Read and combine all the version from other nodes, providing the combined version.
// @ Get Reader Grasped
func (this *FD) __deprecated__Sync() error {
var nowTime = time.Now().Unix()
if this.lastSyncTime+SINGLE_FILE_SYNC_INTERVAL_MIN > nowTime {
// interval is too small, abort the sync.
return nil
}
// Submit #0 patch if needed
this._LoadPointerMap_SyncUseOnly()
// read patch 0 from container. If just submit, the function will exit immediately
this.ReadInNumberZero()
this.contentLock.Lock()
defer this.contentLock.Unlock()
if this.lastSyncTime+SINGLE_FILE_SYNC_INTERVAL_MIN > nowTime {
// interval is too small, abort the sync.
return nil
}
if this.numberZero == nil {
Insider.Log("distributedvc::FD.Sync()", "Looks like a logical isle: this.numberZero==nil")
Secretary.Error("distributedvc::FD.Sync()", "Looks like a logical isle: this.numberZero==nil")
return ex.LOGICAL_ERROR
}
// phase1: glean information from different nodes
// Attentez: the go routines will read numberZero.Kvm but will not write it. So not lock is required.
this.numberZero.CheckOut()
var updateChannel = make(chan int, NODE_NUMS_IN_ALL)
go (func() {
var wg = sync.WaitGroup{}
var gleanInfo = func(nodeNumber int) bool {
defer wg.Done()
if nodeNumber == NODE_NUMBER {
return false
}
var keyStoreName = CONF_FLAG_PREFIX + NODE_SYNC_TIME_PREFIX + strconv.Itoa(nodeNumber)
var lastTime ClxTimestamp
if elem, ok := this.numberZero.Kvm[keyStoreName]; ok {
lastTime = elem.Timestamp
} else {
lastTime = 0
}
if lastTime > 0 {
var meta, err = this.io.Getinfo(this.GetPatchName(0, nodeNumber))
if meta == nil || err != nil {
if err != nil {
Secretary.Warn("distributedvc::FD.Sync()", "Fail to get info from "+this.GetPatchName(0, nodeNumber)+": "+err.Error())
}
return false
}
var result, ok = meta[METAKEY_TIMESTAMP]
if !ok {
return false
}
var existRecentTS = String2ClxTimestamp(result)
if existRecentTS > lastTime {
updateChannel <- nodeNumber
return true
}
return false
}
updateChannel <- nodeNumber
return true
}
wg.Add(NODE_NUMS_IN_ALL)
for i := 0; i < NODE_NUMS_IN_ALL; i++ {
go gleanInfo(i)
}
wg.Wait()
close(updateChannel)
})()
// meanwhile: fetch corresponding patch as need
var thePatch = filetype.NewKvMap()
var changed = false
for i := range updateChannel {
var file, _, err = readInKvMapfile_NoWarning(this.io, this.GetPatchName(0, i))
if err != nil {
Secretary.Warn("distributedvc::FD.Sync()", "Fail to read supposed-to-be file "+this.GetPatchName(0, i))
continue
}
if file == nil {
continue
}
thePatch.MergeWith(file)
changed = true
}
// At this time, the channel must be closed and thus, all the reading routines of
// this.numberZero has terminated safely, on which any write is thread-safe.
if changed {
// merging to update modification timestamp
this.numberZero.MergeWith(filetype.FastMake(CONF_FLAG_PREFIX + NODE_SYNC_TIME_PREFIX + strconv.Itoa(NODE_NUMBER)))
this.numberZero.MergeWith(thePatch)
this.modified = true
this.needsGossiped = true
}
this.lastSyncTime = time.Now().Unix()
return nil
}