func ExportEChain(chainID string, db interfaces.DBOverlay) error {
fmt.Printf("ExportEChain %v\n", chainID)
id, err := primitives.NewShaHashFromStr(chainID)
if err != nil {
return err
}
eBlocks, err := db.FetchAllEBlocksByChain(id)
if err != nil {
return err
}
fmt.Printf("Fetched %v blocks\n", len(eBlocks))
sort.Sort(util.ByEBlockIDAccending(eBlocks))
for _, block := range eBlocks {
SaveBinary(block.(interfaces.DatabaseBatchable))
SaveJSON(block.(interfaces.DatabaseBatchable))
height := block.GetDatabaseHeight()
entryHashes := block.GetBody().GetEBEntries()
for _, hash := range entryHashes {
entry, err := db.FetchEntryByHash(hash)
if err != nil {
return err
}
err = ExportEntry(entry.(interfaces.DatabaseBatchable), height)
if err != nil {
return err
}
}
}
return nil
}
func ExportFctChain(db interfaces.DBOverlay) error {
fmt.Printf("ExportFctChain\n")
// get all aBlocks from db
fBlocks, err := db.FetchAllFBlocks()
if err != nil {
return err
}
sort.Sort(util.ByFBlockIDAccending(fBlocks))
for _, block := range fBlocks {
err = ExportBlock(block.(interfaces.DatabaseBatchable))
if err != nil {
return err
}
}
return nil
}
func (be *BlockExtractor) ExportAChain(db interfaces.DBOverlay) error {
fmt.Printf("ExportAChain\n")
// get all aBlocks from db
aBlocks, err := db.FetchAllABlocks()
if err != nil {
return err
}
sort.Sort(util.ByABlockIDAccending(aBlocks))
for _, block := range aBlocks {
err = be.ExportBlock(block.(interfaces.DatabaseBatchable))
if err != nil {
return err
}
}
return nil
}
func ExportAllEntryReceipts(dbo interfaces.DBOverlay) error {
entryIDs, err := dbo.FetchAllEntryIDs()
if err != nil {
return err
}
for i, entryID := range entryIDs {
err = ExportEntryReceipt(entryID.String(), dbo)
if err != nil {
if err.Error() != "dirBlockInfo not found" {
return err
} else {
fmt.Printf("dirBlockInfo not found for entry %v/%v - %v\n", i, len(entryIDs), entryID)
}
}
}
return nil
}
func ExportDirBlockInfo(db interfaces.DBOverlay) error {
fmt.Printf("ExportDirBlockInfo\n")
// get all aBlocks from db
dbi, err := db.FetchAllDirBlockInfos()
if err != nil {
return err
}
fmt.Printf("Fetched %v blocks\n", len(dbi))
sort.Sort(util.ByDirBlockInfoIDAccending(dbi))
for _, block := range dbi {
err = ExportBlock(block)
if err != nil {
return err
}
}
return nil
}
func (be *BlockExtractor) ExportDChain(db interfaces.DBOverlay) error {
fmt.Printf("ExportDChain\n")
// get all ecBlocks from db
dBlocks, err := db.FetchAllDBlocks()
if err != nil {
return err
}
sort.Sort(util.ByDBlockIDAccending(dBlocks))
for _, block := range dBlocks {
//Making sure Hash and KeyMR are set for the JSON export
block.GetFullHash()
block.GetKeyMR()
err = be.ExportBlock(block.(interfaces.DatabaseBatchable))
if err != nil {
return err
}
}
return nil
}
func FetchBlockSet(dbo interfaces.DBOverlay, dBlockHash interfaces.IHash) *BlockSet {
bs := new(BlockSet)
dBlock, err := dbo.FetchDBlock(dBlockHash)
if err != nil {
panic(err)
}
bs.DBlock = dBlock
if dBlock == nil {
return bs
}
entries := dBlock.GetDBEntries()
for _, entry := range entries {
switch entry.GetChainID().String() {
case "000000000000000000000000000000000000000000000000000000000000000a":
aBlock, err := dbo.FetchABlock(entry.GetKeyMR())
if err != nil {
panic(err)
}
bs.ABlock = aBlock
break
case "000000000000000000000000000000000000000000000000000000000000000c":
ecBlock, err := dbo.FetchECBlock(entry.GetKeyMR())
if err != nil {
panic(err)
}
bs.ECBlock = ecBlock
break
case "000000000000000000000000000000000000000000000000000000000000000f":
fBlock, err := dbo.FetchFBlock(entry.GetKeyMR())
if err != nil {
panic(err)
}
bs.FBlock = fBlock
break
default:
break
}
}
return bs
}
func CopyDB(dbase1, dbase2 interfaces.DBOverlay) {
processing := ""
defer func() {
if r := recover(); r != nil {
err := fmt.Errorf("Error processing: %v", processing)
panic(err)
}
}()
dBlocks, err := dbase1.FetchAllDBlocks()
if err != nil {
panic(err)
}
prevECHash := primitives.NewZeroHash()
for _, dBlock := range dBlocks {
dbase2.StartMultiBatch()
err := dbase2.ProcessDBlockMultiBatch(dBlock)
if err != nil {
panic(err)
}
for _, dbEntry := range dBlock.GetDBEntries() {
switch dbEntry.GetChainID().String() {
case "000000000000000000000000000000000000000000000000000000000000000a":
aBlock, err := dbase1.FetchABlock(dbEntry.GetKeyMR())
if err != nil {
panic(err)
}
err = dbase2.ProcessABlockMultiBatch(aBlock)
if err != nil {
panic(err)
}
break
case "000000000000000000000000000000000000000000000000000000000000000c":
ecBlock, err := dbase1.FetchECBlock(dbEntry.GetKeyMR())
if err != nil {
panic(err)
}
if ecBlock.GetHeader().GetPrevHeaderHash().IsSameAs(prevECHash) == false {
prev, err := dbase1.FetchECBlock(ecBlock.GetHeader().GetPrevHeaderHash())
if err != nil {
panic(err)
}
err = dbase2.ProcessECBlockMultiBatch(prev, true)
if err != nil {
panic(err)
}
}
err = dbase2.ProcessECBlockMultiBatch(ecBlock, true)
if err != nil {
panic(err)
}
prevECHash = dbEntry.GetKeyMR()
break
case "000000000000000000000000000000000000000000000000000000000000000f":
fBlock, err := dbase1.FetchFBlock(dbEntry.GetKeyMR())
if err != nil {
panic(err)
}
err = dbase2.ProcessFBlockMultiBatch(fBlock)
if err != nil {
panic(err)
}
break
default:
processing = fmt.Sprintf("%v - %v - %v", dBlock.GetKeyMR().String(), dbEntry.GetChainID().String(), dbEntry.GetKeyMR().String())
eBlock, err := dbase1.FetchEBlock(dbEntry.GetKeyMR())
if err != nil {
panic(err)
}
err = dbase2.ProcessEBlockMultiBatch(eBlock, true)
if err != nil {
panic(err)
}
for _, h := range eBlock.GetEntryHashes() {
entry, err := dbase1.FetchEntry(h)
if err != nil {
panic(err)
}
err = dbase2.InsertEntryMultiBatch(entry)
if err != nil {
panic(err)
}
}
break
}
}
if err := dbase2.ExecuteMultiBatch(); err != nil {
panic(err)
}
if dBlock.GetDatabaseHeight()%1000 == 0 {
fmt.Printf("Processed block #%v\n", dBlock.GetDatabaseHeight())
}
}
}
func CreateReceipt(dbo interfaces.DBOverlay, entryID interfaces.IHash) (*Receipt, error) {
receipt := new(Receipt)
receipt.Entry = new(JSON)
receipt.Entry.Key = entryID.String()
//EBlock
hash, err := dbo.FetchIncludedIn(entryID)
if err != nil {
return nil, err
}
if hash == nil {
return nil, fmt.Errorf("Block containing entry not found")
}
eBlock, err := dbo.FetchEBlock(hash)
if err != nil {
return nil, err
}
if eBlock == nil {
return nil, fmt.Errorf("EBlock not found")
}
hash = eBlock.DatabasePrimaryIndex()
receipt.EntryBlockKeyMR = hash.(*primitives.Hash)
entries := eBlock.GetEntryHashes()
//fmt.Printf("eBlock entries - %v\n\n", entries)
branch := primitives.BuildMerkleBranchForEntryHash(entries, entryID, true)
blockNode := new(primitives.MerkleNode)
left, err := eBlock.HeaderHash()
if err != nil {
return nil, err
}
blockNode.Left = left.(*primitives.Hash)
blockNode.Right = eBlock.BodyKeyMR().(*primitives.Hash)
blockNode.Top = hash.(*primitives.Hash)
//fmt.Printf("eBlock blockNode - %v\n\n", blockNode)
branch = append(branch, blockNode)
receipt.MerkleBranch = append(receipt.MerkleBranch, branch...)
//str, _ := eBlock.JSONString()
//fmt.Printf("eBlock - %v\n\n", str)
//DBlock
hash, err = dbo.FetchIncludedIn(hash)
if err != nil {
return nil, err
}
if hash == nil {
return nil, fmt.Errorf("Block containing EBlock not found")
}
dBlock, err := dbo.FetchDBlock(hash)
if err != nil {
return nil, err
}
if dBlock == nil {
return nil, fmt.Errorf("DBlock not found")
}
//str, _ = dBlock.JSONString()
//fmt.Printf("dBlock - %v\n\n", str)
entries = dBlock.GetEntryHashesForBranch()
//fmt.Printf("dBlock entries - %v\n\n", entries)
//merkleTree := primitives.BuildMerkleTreeStore(entries)
//fmt.Printf("dBlock merkleTree - %v\n\n", merkleTree)
branch = primitives.BuildMerkleBranchForEntryHash(entries, receipt.EntryBlockKeyMR, true)
blockNode = new(primitives.MerkleNode)
left, err = dBlock.HeaderHash()
if err != nil {
return nil, err
}
blockNode.Left = left.(*primitives.Hash)
blockNode.Right = dBlock.BodyKeyMR().(*primitives.Hash)
blockNode.Top = hash.(*primitives.Hash)
//fmt.Printf("dBlock blockNode - %v\n\n", blockNode)
branch = append(branch, blockNode)
receipt.MerkleBranch = append(receipt.MerkleBranch, branch...)
//DirBlockInfo
hash = dBlock.DatabasePrimaryIndex()
receipt.DirectoryBlockKeyMR = hash.(*primitives.Hash)
dirBlockInfo, err := dbo.FetchDirBlockInfoByKeyMR(hash)
if err != nil {
return nil, err
}
if dirBlockInfo != nil {
dbi := dirBlockInfo.(*dbInfo.DirBlockInfo)
receipt.BitcoinTransactionHash = dbi.BTCTxHash.(*primitives.Hash)
receipt.BitcoinBlockHash = dbi.BTCBlockHash.(*primitives.Hash)
}
return receipt, nil
}
func CheckDBlockEntries(dBlock interfaces.IDirectoryBlock, dbo interfaces.DBOverlay) {
entries := dBlock.GetDBEntries()
for {
missing := 0
for _, e := range entries {
HashMap[e.GetKeyMR().String()] = "OK"
switch e.GetChainID().String() {
case "000000000000000000000000000000000000000000000000000000000000000a":
aBlock, err := dbo.FetchABlock(e.GetKeyMR())
if err != nil {
panic(err)
}
if aBlock != nil {
break
}
fmt.Printf("Found missing aBlock in #%v\n", dBlock.GetDatabaseHeight())
missing++
aBlock, err = GetABlock(e.GetKeyMR().String())
if err != nil {
panic(err)
}
err = dbo.ProcessABlockBatchWithoutHead(aBlock)
if err != nil {
panic(err)
}
break
case "000000000000000000000000000000000000000000000000000000000000000f":
fBlock, err := dbo.FetchFBlock(e.GetKeyMR())
if err != nil {
panic(err)
}
if fBlock != nil {
break
}
fmt.Printf("Found missing fBlock in #%v\n", dBlock.GetDatabaseHeight())
missing++
fBlock, err = GetFBlock(e.GetKeyMR().String())
if err != nil {
panic(err)
}
err = dbo.ProcessFBlockBatchWithoutHead(fBlock)
if err != nil {
panic(err)
}
break
case "000000000000000000000000000000000000000000000000000000000000000c":
ecBlock, err := dbo.FetchECBlock(e.GetKeyMR())
if err != nil {
panic(err)
}
if ecBlock != nil {
break
}
fmt.Printf("Found missing ecBlock in #%v\n", dBlock.GetDatabaseHeight())
missing++
ecBlock, err = GetECBlock(e.GetKeyMR().String())
if err != nil {
panic(err)
}
err = dbo.ProcessECBlockBatchWithoutHead(ecBlock, true)
if err != nil {
panic(err)
}
break
default:
eBlock, err := dbo.FetchEBlock(e.GetKeyMR())
if err != nil {
if err.Error() != "EOF" {
panic(err)
}
}
if eBlock == nil {
fmt.Printf("Found missing eBlock in #%v\n", dBlock.GetDatabaseHeight())
missing++
eBlock, err = GetEBlock(e.GetKeyMR().String())
if err != nil {
panic(err)
}
err = dbo.ProcessEBlockBatchWithoutHead(eBlock, true)
if err != nil {
panic(err)
}
}
eBlockEntries := eBlock.GetEntryHashes()
for _, eHash := range eBlockEntries {
if eHash.IsMinuteMarker() == true {
continue
}
entry, err := dbo.FetchEntry(eHash)
if err != nil {
panic(err)
}
if entry == nil {
fmt.Printf("Found missing entry in #%v\n", dBlock.GetDatabaseHeight())
missing++
entry, err := GetEntry(eHash.String())
if err != nil {
fmt.Printf("Problem getting entry `%v` from block %v\n", eHash.String(), e.GetKeyMR().String())
panic(err)
}
err = dbo.InsertEntry(entry)
if err != nil {
panic(err)
}
}
}
break
}
}
if missing == 0 {
break
}
}
}
func CheckDatabaseForMissingEntries(dbo interfaces.DBOverlay) {
fmt.Printf("\t\tIterating over DBlocks\n")
prevD, err := dbo.FetchDBlockHead()
if err != nil {
panic(err)
}
HashMap = map[string]string{}
for {
CheckDBlockEntries(prevD, dbo)
HashMap[prevD.DatabasePrimaryIndex().String()] = "OK"
if prevD.GetHeader().GetPrevKeyMR().String() == "0000000000000000000000000000000000000000000000000000000000000000" {
break
}
dBlock, err := dbo.FetchDBlock(prevD.GetHeader().GetPrevKeyMR())
if err != nil {
panic(err)
}
if dBlock == nil {
fmt.Printf("Found a missing dblock - %v\n", prevD.GetHeader().GetPrevKeyMR().String())
dblock, err := GetDBlock(prevD.GetHeader().GetPrevKeyMR().String())
if err != nil {
panic(err)
}
err = dbo.ProcessDBlockBatchWithoutHead(dblock)
if err != nil {
panic(err)
}
} else {
//only iterate to the next block if it was properly fetched from the database
prevD = dBlock
}
}
fmt.Printf("\t\tIterating over ECBlocks\n")
prevEC, err := dbo.FetchECBlockHead()
if err != nil {
panic(err)
}
for {
HashMap[prevEC.DatabasePrimaryIndex().String()] = "OK"
if prevEC.GetHeader().GetPrevHeaderHash().String() == "0000000000000000000000000000000000000000000000000000000000000000" {
break
}
ecBlock, err := dbo.FetchECBlock(prevEC.GetHeader().GetPrevHeaderHash())
if err != nil {
panic(err)
}
if ecBlock == nil {
fmt.Printf("Found a missing ecblock - %v\n", prevEC.GetHeader().GetPrevHeaderHash().String())
ecblock, err := GetECBlock(prevEC.GetHeader().GetPrevHeaderHash().String())
if err != nil {
panic(err)
}
err = dbo.ProcessECBlockBatchWithoutHead(ecblock, true)
if err != nil {
panic(err)
}
} else {
//only iterate to the next block if it was properly fetched from the database
prevEC = ecBlock
}
}
fmt.Printf("\t\tIterating over FBlocks\n")
prevF, err := dbo.FetchFBlockHead()
if err != nil {
panic(err)
}
for {
HashMap[prevF.DatabasePrimaryIndex().String()] = "OK"
if prevF.GetPrevKeyMR().String() == "0000000000000000000000000000000000000000000000000000000000000000" {
break
}
fBlock, err := dbo.FetchFBlock(prevF.GetPrevKeyMR())
if err != nil {
panic(err)
}
if fBlock == nil {
fmt.Printf("Found a missing fblock - %v\n", prevF.GetPrevKeyMR().String())
fBlock, err := GetFBlock(prevF.GetPrevKeyMR().String())
if err != nil {
panic(err)
}
err = dbo.ProcessFBlockBatchWithoutHead(fBlock)
if err != nil {
panic(err)
}
} else {
//only iterate to the next block if it was properly fetched from the database
prevF = fBlock
}
}
fmt.Printf("\t\tIterating over ABlocks\n")
prevA, err := dbo.FetchABlockHead()
if err != nil {
panic(err)
}
for {
HashMap[prevA.DatabasePrimaryIndex().String()] = "OK"
if prevA.GetHeader().GetPrevBackRefHash().String() == "0000000000000000000000000000000000000000000000000000000000000000" {
break
}
aBlock, err := dbo.FetchABlock(prevA.GetHeader().GetPrevBackRefHash())
if err != nil {
panic(err)
}
if aBlock == nil {
fmt.Printf("Found a missing ablock - %v\n", prevA.GetHeader().GetPrevBackRefHash().String())
aBlock, err := GetABlock(prevA.GetHeader().GetPrevBackRefHash().String())
if err != nil {
panic(err)
}
err = dbo.ProcessABlockBatchWithoutHead(aBlock)
if err != nil {
panic(err)
}
} else {
//only iterate to the next block if it was properly fetched from the database
prevA = aBlock
}
}
fmt.Printf("\t\tFinding unused blocks\n")
hashes, err := dbo.FetchAllDBlockKeys()
if err != nil {
panic(err)
}
for _, h := range hashes {
if HashMap[h.String()] == "" {
fmt.Printf("Superfluous DBlock - %v\n", h)
dbo.Delete(databaseOverlay.DIRECTORYBLOCK, h.Bytes())
}
}
hashes, err = dbo.FetchAllABlockKeys()
if err != nil {
panic(err)
}
for _, h := range hashes {
if HashMap[h.String()] == "" {
fmt.Printf("Superfluous ABlock - %v\n", h)
dbo.Delete(databaseOverlay.ADMINBLOCK, h.Bytes())
}
}
hashes, err = dbo.FetchAllECBlockKeys()
if err != nil {
panic(err)
}
for _, h := range hashes {
if HashMap[h.String()] == "" {
fmt.Printf("Superfluous ECBlock - %v\n", h)
dbo.Delete(databaseOverlay.ENTRYCREDITBLOCK, h.Bytes())
}
}
hashes, err = dbo.FetchAllFBlockKeys()
if err != nil {
panic(err)
}
for _, h := range hashes {
if HashMap[h.String()] == "" {
fmt.Printf("Superfluous FBlock - %v\n", h)
dbo.Delete(databaseOverlay.FACTOIDBLOCK, h.Bytes())
}
}
}