func NewUpaxServer(ckPriv, skPriv *rsa.PrivateKey, cm *xcl.ClusterMember,
whichSHA int) (us *UpaxServer, err error) {
var (
count int
lfs string // path to local file system
f *os.File // file for debugging log
pathToLog string
logger *log.Logger
uDir u.UI
pathToU string
entries *xi.IDMap
ftLogFile *os.File
pathToFTLog string // conventionally lfs/U/L
)
if ckPriv == nil || ckPriv == nil {
err = NilRSAKey
} else if cm == nil {
err = NilClusterMember
}
if err == nil {
serverVersion, err = xu.ParseDecimalVersion(VERSION)
}
if err == nil {
// whatever created cm should have created the local file system
// and written the node configuration to
// LFS/.xlattice/cluster.member.config. Let's make sure that
// that exists before proceeding.
lfs = cm.GetLFS()
// This should be passed in opt.Logger
pathToLog = filepath.Join(lfs, "log")
f, err = os.OpenFile(pathToLog,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0640)
if err == nil {
logger = log.New(f, "", log.Ldate|log.Ltime)
}
pathToCfg := filepath.Join(
filepath.Join(lfs, ".xlattice"), "cluster.member.config")
var found bool
found, err = xf.PathExists(pathToCfg)
if err == nil && found == false {
err = ClusterConfigNotFound
}
}
if f != nil {
defer f.Close()
}
if err == nil {
// DEBUG
fmt.Printf("creating directory tree in %s\n", lfs)
// END
pathToU = filepath.Join(lfs, "U")
uDir, err = u.New(pathToU, u.DIR16x16, 0)
}
if err == nil {
entries, err = xi.NewNewIDMap() // with default depth
}
if err == nil {
var found bool
pathToFTLog = filepath.Join(pathToU, "L")
found, err = xf.PathExists(pathToFTLog)
if err == nil {
if found {
fmt.Printf("ftLog file exists\n")
count, err = loadEntries(pathToFTLog, entries, whichSHA)
if err == nil {
// reopen it 0600 for appending
ftLogFile, err = os.OpenFile(pathToFTLog,
os.O_WRONLY|os.O_APPEND, 0600)
}
} else {
// open it for appending
ftLogFile, err = os.OpenFile(pathToFTLog,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
}
}
}
if err == nil {
us = &UpaxServer{
DoneCh: make(chan bool, 2),
PathToDebugLog: pathToLog,
Logger: logger,
uDir: uDir,
entries: entries,
ftLogFile: ftLogFile,
pathToFTLog: pathToFTLog,
entryCount: count,
ckPriv: ckPriv,
skPriv: skPriv,
ClusterMember: *cm,
}
}
return
}
func (s *XLSuite) TestChunkListAssyDisassy(c *C) {
if VERBOSITY > 0 {
fmt.Println("TEST_CHUNK_LIST_ASSY_DISASSY")
}
rng := xr.MakeSimpleRNG()
// make a slice 3 to 7 chunks long, fill with random-ish data ---
chunkCount := 3 + rng.Intn(5) // so 3 to 7, inclusive
lastChunkLen := 1 + rng.Intn(MAX_DATA_BYTES-1)
dataLen := (chunkCount-1)*MAX_DATA_BYTES + lastChunkLen
data := make([]byte, dataLen)
rng.NextBytes(data)
// calculate datum, the SHA hash of the data --------------------
//d := sha3.NewKeccak256()
d := sha1.New()
d.Write(data)
hash := d.Sum(nil)
datum, err := xi.NewNodeID(hash)
c.Assert(err, IsNil)
// create tmp if it doesn't exist -------------------------------
found, err := xf.PathExists("tmp")
c.Assert(err, IsNil)
if !found {
err = os.MkdirAll("tmp", 0755)
c.Assert(err, IsNil)
}
// create scratch subdir with unique name -----------------------
var pathToU string
for {
dirName := rng.NextFileName(8)
pathToU = path.Join("tmp", dirName)
found, err = xf.PathExists(pathToU)
c.Assert(err, IsNil)
if !found {
break
}
}
// create a FLAT uDir at that point -----------------------------
myU, err := u.New(pathToU, u.DIR_FLAT, 0) // 0 means default perm
c.Assert(err, IsNil)
// write the test data into uDir --------------------------------
bytesWritten, key, err := myU.PutData(data, datum.Value())
c.Assert(err, IsNil)
c.Assert(bytes.Equal(datum.Value(), key), Equals, true)
c.Assert(bytesWritten, Equals, int64(dataLen))
skPriv, err := rsa.GenerateKey(rand.Reader, 1024) // cheap key
sk := &skPriv.PublicKey
c.Assert(err, IsNil)
c.Assert(skPriv, NotNil)
// Verify the file is present in uDir ---------------------------
// (yes this is a test of uDir logic but these are early days ---
// XXX uDir.Exist(arg) - arg should be []byte, no string
keyStr := hex.EncodeToString(key)
found, err = myU.HexKeyExists(keyStr)
c.Assert(err, IsNil)
c.Assert(found, Equals, true)
// use the data file to build a chunkList, writing the chunks ---
title := rng.NextFileName(8)
now := xu.Timestamp(time.Now().UnixNano())
// make a reader --------------------------------------
pathToData, err := myU.GetPathForHexKey(keyStr)
c.Assert(err, IsNil)
reader, err := os.Open(pathToData) // open for read only
c.Assert(err, IsNil)
defer reader.Close()
chunkList, err := NewChunkList(sk, title, now, reader, int64(dataLen), key, myU)
c.Assert(err, IsNil)
err = chunkList.Sign(skPriv)
c.Assert(err, IsNil)
digSig, err := chunkList.GetDigSig()
c.Assert(err, IsNil)
c.Assert(bytes.Equal(digSig, chunkList.digSig), Equals, true)
err = chunkList.Verify()
c.Assert(err, IsNil)
// REBUILD AND CHECK --------------------------------------------
// rebuild the complete file from the chunkList and files present
// in myU
u2, err := u.New(pathToU, u.DIR_FLAT, 0) // 0 means default perm
c.Assert(err, IsNil)
var data2 []byte // should become copy of original
count := chunkList.Size()
for i := uint(0); i < count; i++ {
chunkHash, err := chunkList.HashItem(i)
c.Assert(err, IsNil)
c.Assert(chunkHash, NotNil)
var raw []byte
// THIS IS A CHUNK, and so has a header, possibly some
// padding, and then its own hash :-). Need to verify
// and discard the last, then drop the padding.
// CORRECTION: hash should NOT have been written to disk
raw, err = u2.GetData(chunkHash)
c.Assert(err, IsNil)
chunk := &Chunk{packet: raw}
ndx := chunk.GetIndex()
c.Assert(ndx, Equals, uint32(i))
rawLen := uint32(len(raw))
dataLen := chunk.GetDataLen()
//fmt.Printf("chunk %2d: index is %8d\n", i, ndx)
//fmt.Printf(" len raw is %6d (%4x)\n", rawLen, rawLen)
//fmt.Printf(" dataLen is %6d (%4x)\n", dataLen, dataLen)
// if this isn't true, we get a panic
c.Assert(dataLen < rawLen, Equals, true)
payload := chunk.GetData()
data2 = append(data2, payload...)
}
// verify that the content key of the rebuilt file is identical to
// that of the original
//d2D := sha3.NewKeccak256()
d2D := sha1.New()
d2D.Write(data2)
hash2 := d2D.Sum(nil)
datum2, err := xi.NewNodeID(hash2)
c.Assert(err, IsNil)
// DEBUG
//fmt.Printf("datum: %x\ndatum2: %x\n", datum.Value(), datum2.Value())
//fmt.Printf("data: %x\ndata2: %x\n", data, data2)
// END
c.Assert(bytes.Equal(datum.Value(), datum2.Value()), Equals, true)
// presumably pure pedantry: verify that the file contents are
// also equal
c.Assert(bytes.Equal(data, data2), Equals, true)
}