func TestChunker(t *testing.T) {
	testFile := testutil.Setup(t)

	// Create 100MB file
	fo, err := os.Create(testFile)
	if err != nil {
		t.Error(err)
	}
	buf := make([]byte, 100000000)
	if _, err = fo.Write(buf); err != nil {
		t.Error(err)
	}
	if err = fo.Close(); err != nil {
		t.Error(err)
	}

	// Chunk file
	Chunk(testFile)

	// Check if original sha1 created
	isExist, err := util.CheckFileExists(testFile + ".sha1")
	if err != nil {
		t.Error(err)
	}
	if !isExist {
		t.Error("Original file's sha1 not created")
	}

	// Check if parts created
	for i := 0; i < 2; i++ {
		isExist, err := util.CheckFileExists(testFile + ".part" + strconv.Itoa(i))
		if err != nil {
			t.Error(err)
		}
		if !isExist {
			t.Error("Part", i, "not created")
		}
	}

	// Delete original file
	err = os.Remove(testFile)
	if err != nil {
		t.Error(err)
	}

	// Reconstruct file
	Reconstruct(testFile)

	// Check original sha1
	isMatch := hashutil.CompareChecksum(testFile)
	if !isMatch {
		t.Error("Reconstructed file not match original sha1")
	}

	testutil.Teardown(t)
}
Beispiel #2
0
func downloadIfBadFile(fileUrl string, fullPath string, fileName string) {
	var isFileExist bool
	var err error
	isFileExist, err = util.CheckFileExists(fullPath)
	if err != nil {
		log.Fatalln(err)
	} else if !isFileExist {
		downloadAndWrite(fileUrl, fullPath, fileName)
	}

	// Use fake sha1 if testing
	var sha1Url string
	if !*testDownloadTimeout {
		sha1Url = fileUrl + ".sha1"
	} else {
		sha1Url = fileUrl + ".sha1.fake"
	}

	isFileGood := false
	isFileGood = checkSum(fullPath, sha1Url)
	var numAttempt int

	for numAttempt = 0; numAttempt < 5 && !isFileGood; numAttempt++ {
		log.Println("File corrupted or outdated. Downloading new file")

		downloadAndWrite(fileUrl, fullPath, fileName)
		isFileGood = checkSum(fullPath, sha1Url)
	}

	if numAttempt == 5 {
		log.Fatalln(`FATAL: 5 failed attempts to download new`, fileName, `package`)
	} else {
		log.Println("File is good")
	}
}
Beispiel #3
0
func reconstruct(path string) {
	log.Println("Reconstruct file", path)

	// open output file
	fo, err := os.Create(path)
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()

	writeIndex := 0
	i := 0
	for {
		inFile := path + `.part` + strconv.Itoa(i)

		// Exit if no more part
		isFileExist, err := util.CheckFileExists(inFile)
		if err != nil {
			panic(err)
		}
		if !isFileExist {
			break
		}

		// open input file
		fi, err := os.Open(inFile)
		if err != nil {
			panic(err)
		}
		// close fi on exit and check for its returned error
		defer func() {
			if err := fi.Close(); err != nil {
				panic(err)
			}
		}()

		// make a buffer to keep chunks that are read
		buf := make([]byte, 50000000)
		// read a chunk
		byteRead, err := fi.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}

		// write a chunk
		if _, err := fo.WriteAt(buf[:byteRead], int64(writeIndex)); err != nil {
			panic(err)
		}

		writeIndex += byteRead
		i++
	}
}
Beispiel #4
0
func getOfflineFile(path string) string {
	var fileExists bool
	var err error
	if fileExists, err = util.CheckFileExists(path); err != nil {
		log.Fatalln(err)
	}

	if !fileExists {
		// Check if this is a big file and there exists its smaller parts
		var partFileExists bool
		if partFileExists, err = util.CheckFileExists(path + `.part0`); err != nil {
			log.Fatalln(err)
		}

		if !partFileExists {
			log.Fatalln(`File`, path, `does not exist`)
		}

		chunker.Reconstruct(path)
	}

	return path
}
Beispiel #5
0
func Setup(t *testing.T) string {
	testDir, _ := filepath.Abs("./test")
	testFile, _ := filepath.Abs(testDir + "/testfile")

	// Delete unwanted files if exists
	isExist, err := util.CheckFileExists(testDir)
	if err != nil {
		t.Error(err)
	}
	if isExist {
		err = os.RemoveAll(testDir)
		if err != nil {
			t.Error(err)
		}
	}
	err = os.Mkdir(testDir, 0755)
	if err != nil {
		t.Error(err)
	}

	return testFile
}