Esempio n. 1
0
func GetFsClient(config *Config) (fs.Client, error) {
	var client fs.Client
	var cltErr error
	switch config.IOConf.Fs {
	case "local":
		client = fs.NewLocalFSClient()
	case "hdfs":
		client, cltErr = fs.NewHdfsClient(
			config.IOConf.HdfsConf.NamenodeAddr,
			config.IOConf.HdfsConf.WebHdfsAddr,
			config.IOConf.HdfsConf.User,
		)
		if cltErr != nil {
			return nil, fmt.Errorf("Failed creating hdfs client %s", cltErr)
		}
	case "azure":
		client, cltErr = fs.NewAzureClient(
			config.IOConf.AzureConf.AccountName,
			config.IOConf.AzureConf.AccountKey,
			config.IOConf.AzureConf.BlogServiceBaseUrl,
			config.IOConf.AzureConf.ApiVersion,
			config.IOConf.AzureConf.UseHttps,
		)
		if cltErr != nil {
			return nil, fmt.Errorf("Failed creating azure client %s", cltErr)
		}
	default:
		return nil, fmt.Errorf("Unknow fs: %s", config.IOConf.Fs)
	}
	return client, nil
}
Esempio n. 2
0
func generateTestData(t *testing.T) {
	r0, c0, r1, c1 := getShards()
	fs := filesystem.NewLocalFSClient()
	wr0Err := bwmf.SaveMatrixShard(fs, r0, "../.tmp/row_shard.dat-000000")
	wr1Err := bwmf.SaveMatrixShard(fs, r1, "../.tmp/row_shard.dat-000001")
	wc0Err := bwmf.SaveMatrixShard(fs, c0, "../.tmp/column_shard.dat-000000")
	wc1Err := bwmf.SaveMatrixShard(fs, c1, "../.tmp/column_shard.dat-000001")
	if wr0Err != nil || wr1Err != nil || wc0Err != nil || wc1Err != nil {
		t.Errorf("Failed generating test data files: %s;%s;%s;%s.", wr0Err, wr1Err, wc0Err, wc1Err)
	}
}
Esempio n. 3
0
File: main.go Progetto: BaiGang/bwmf
func main() {
	etcdUrlList := flag.String("etcd_urls", "", "ETCD server lists, sep by a comma.")
	jobName := flag.String("job_name", "bwmf", "Job name in etcd path.")
	jobType := flag.String("job_type", "c", "Job type, either 'c' for controller or 't' for task.")
	numTasks := flag.Int("num_tasks", 1, "Num of tasks.")
	taskConfigFile := flag.String("task_config", "", "Path to task config json file.")

	flag.Parse()

	if *jobName == "" {
		log.Fatal("Job name is required.")
	}

	crd, oErr := filesystem.NewLocalFSClient().OpenReadCloser(*taskConfigFile)
	if oErr != nil {
		log.Fatalf("Failed opening task config file. %s", oErr)
	}
	confData, rdErr := ioutil.ReadAll(crd)
	if rdErr != nil {
		log.Fatalf("Failed reading task config. %s", rdErr)
	}
	log.Printf("conf data: %s", confData)

	if *etcdUrlList == "" {
		log.Fatal("Please specify the etcd server urls.")
	}
	etcdUrls := strings.Split(*etcdUrlList, ",")
	log.Println("etcd urls: ", etcdUrls)

	topoMaster := topo.NewFullTopologyOfMaster(uint64(*numTasks))
	topoNeighbors := topo.NewFullTopologyOfNeighbor(uint64(*numTasks))

	switch *jobType {
	case "t":
		bootstrap := framework.NewBootStrap(*jobName, etcdUrls, createListener(), nil)
		taskBuilder := &bwmf.BWMFTaskBuilder{
			NumOfTasks: uint64(*numTasks),
			ConfBytes:  confData,
		}
		bootstrap.SetTaskBuilder(taskBuilder)
		bootstrap.AddLinkage("Master", topoMaster)
		bootstrap.AddLinkage("Neighbors", topoNeighbors)
		log.Println("Starting task..")
		bootstrap.Start()
	case "c":
		controller := controller.New(*jobName, etcd.NewClient(etcdUrls), uint64(*numTasks), []string{"Master", "Neighbors"})
		controller.Start()
		log.Println("Controller started.")
		controller.WaitForJobDone()
		controller.Stop()
	default:
		log.Fatal("Please choose a type via '-jobtype': (c) controller, (t) task")
	}
}
Esempio n. 4
0
File: main.go Progetto: BaiGang/bwmf
func loadConfig(path string) (*bwmf.Config, error) {
	crd, err := fs.NewLocalFSClient().OpenReadCloser(path)
	if err != nil {
		return nil, err
	}
	confBytes, err := ioutil.ReadAll(crd)
	if err != nil {
		return nil, err
	}

	config := &bwmf.Config{}
	unmarshErr := json.Unmarshal(confBytes, config)
	if unmarshErr != nil {
		return nil, unmarshErr
	}
	return config, nil
}
Esempio n. 5
0
func TestShardIO(t *testing.T) {
	client := filesystem.NewLocalFSClient()
	path := "./.testShardIO.text.dat"
	m, n := uint32(2), uint32(3)

	oldShard := &pb.MatrixShard{
		IsSparse: false,
		M:        m,
		N:        n,
		Val:      []float32{0.70, 1.00, 0.90, 0.70, 0.80, 0.90},
	}

	client.Remove(path)
	sErr := SaveMatrixShard(client, oldShard, path)
	if sErr != nil {
		t.Errorf("Saving shard failed: %s", sErr)
	}

	newShard, lErr := LoadMatrixShard(client, path)
	if sErr != nil {
		t.Errorf("Loading shard failed: %s", lErr)
	}

	// shard should be:
	// -----------------------------
	// | 0.70 | 1.00 | 0.90 | 0.00 |
	// -----------------------------
	// | 0.00 | 0.70 | 0.80 | 0.90 |
	// -----------------------------
	fmt.Println("Original matrix shard is: ", oldShard)
	fmt.Println("Loaded matrix shard is: ", newShard)

	if len(newShard.Val) != len(oldShard.Val) {
		t.Errorf("num of values wrong. Expected %d, actual %d", len(oldShard.Val), len(newShard.Val))
	}

	for i := uint32(0); i < m; i++ {
		for j := uint32(0); j < n; j++ {
			if newShard.Val[i*n+j] != oldShard.Val[i*n+j] {
				t.Errorf("M[%d][%d] incorrect. Mismatched old %f, new %f", i, j, oldShard.Val[i*n+j], newShard.Val[i*n+j])
			}
		}
	}

	client.Remove(path)
}