func GetFsClient(config *Config) (fs.Client, error) { var client fs.Client var cltErr error switch config.IOConf.Fs { case "local": client = fs.NewLocalFSClient() case "hdfs": client, cltErr = fs.NewHdfsClient( config.IOConf.HdfsConf.NamenodeAddr, config.IOConf.HdfsConf.WebHdfsAddr, config.IOConf.HdfsConf.User, ) if cltErr != nil { return nil, fmt.Errorf("Failed creating hdfs client %s", cltErr) } case "azure": client, cltErr = fs.NewAzureClient( config.IOConf.AzureConf.AccountName, config.IOConf.AzureConf.AccountKey, config.IOConf.AzureConf.BlogServiceBaseUrl, config.IOConf.AzureConf.ApiVersion, config.IOConf.AzureConf.UseHttps, ) if cltErr != nil { return nil, fmt.Errorf("Failed creating azure client %s", cltErr) } default: return nil, fmt.Errorf("Unknow fs: %s", config.IOConf.Fs) } return client, nil }
func generateTestData(t *testing.T) { r0, c0, r1, c1 := getShards() fs := filesystem.NewLocalFSClient() wr0Err := bwmf.SaveMatrixShard(fs, r0, "../.tmp/row_shard.dat-000000") wr1Err := bwmf.SaveMatrixShard(fs, r1, "../.tmp/row_shard.dat-000001") wc0Err := bwmf.SaveMatrixShard(fs, c0, "../.tmp/column_shard.dat-000000") wc1Err := bwmf.SaveMatrixShard(fs, c1, "../.tmp/column_shard.dat-000001") if wr0Err != nil || wr1Err != nil || wc0Err != nil || wc1Err != nil { t.Errorf("Failed generating test data files: %s;%s;%s;%s.", wr0Err, wr1Err, wc0Err, wc1Err) } }
func main() { etcdUrlList := flag.String("etcd_urls", "", "ETCD server lists, sep by a comma.") jobName := flag.String("job_name", "bwmf", "Job name in etcd path.") jobType := flag.String("job_type", "c", "Job type, either 'c' for controller or 't' for task.") numTasks := flag.Int("num_tasks", 1, "Num of tasks.") taskConfigFile := flag.String("task_config", "", "Path to task config json file.") flag.Parse() if *jobName == "" { log.Fatal("Job name is required.") } crd, oErr := filesystem.NewLocalFSClient().OpenReadCloser(*taskConfigFile) if oErr != nil { log.Fatalf("Failed opening task config file. %s", oErr) } confData, rdErr := ioutil.ReadAll(crd) if rdErr != nil { log.Fatalf("Failed reading task config. %s", rdErr) } log.Printf("conf data: %s", confData) if *etcdUrlList == "" { log.Fatal("Please specify the etcd server urls.") } etcdUrls := strings.Split(*etcdUrlList, ",") log.Println("etcd urls: ", etcdUrls) topoMaster := topo.NewFullTopologyOfMaster(uint64(*numTasks)) topoNeighbors := topo.NewFullTopologyOfNeighbor(uint64(*numTasks)) switch *jobType { case "t": bootstrap := framework.NewBootStrap(*jobName, etcdUrls, createListener(), nil) taskBuilder := &bwmf.BWMFTaskBuilder{ NumOfTasks: uint64(*numTasks), ConfBytes: confData, } bootstrap.SetTaskBuilder(taskBuilder) bootstrap.AddLinkage("Master", topoMaster) bootstrap.AddLinkage("Neighbors", topoNeighbors) log.Println("Starting task..") bootstrap.Start() case "c": controller := controller.New(*jobName, etcd.NewClient(etcdUrls), uint64(*numTasks), []string{"Master", "Neighbors"}) controller.Start() log.Println("Controller started.") controller.WaitForJobDone() controller.Stop() default: log.Fatal("Please choose a type via '-jobtype': (c) controller, (t) task") } }
func loadConfig(path string) (*bwmf.Config, error) { crd, err := fs.NewLocalFSClient().OpenReadCloser(path) if err != nil { return nil, err } confBytes, err := ioutil.ReadAll(crd) if err != nil { return nil, err } config := &bwmf.Config{} unmarshErr := json.Unmarshal(confBytes, config) if unmarshErr != nil { return nil, unmarshErr } return config, nil }
func TestShardIO(t *testing.T) { client := filesystem.NewLocalFSClient() path := "./.testShardIO.text.dat" m, n := uint32(2), uint32(3) oldShard := &pb.MatrixShard{ IsSparse: false, M: m, N: n, Val: []float32{0.70, 1.00, 0.90, 0.70, 0.80, 0.90}, } client.Remove(path) sErr := SaveMatrixShard(client, oldShard, path) if sErr != nil { t.Errorf("Saving shard failed: %s", sErr) } newShard, lErr := LoadMatrixShard(client, path) if sErr != nil { t.Errorf("Loading shard failed: %s", lErr) } // shard should be: // ----------------------------- // | 0.70 | 1.00 | 0.90 | 0.00 | // ----------------------------- // | 0.00 | 0.70 | 0.80 | 0.90 | // ----------------------------- fmt.Println("Original matrix shard is: ", oldShard) fmt.Println("Loaded matrix shard is: ", newShard) if len(newShard.Val) != len(oldShard.Val) { t.Errorf("num of values wrong. Expected %d, actual %d", len(oldShard.Val), len(newShard.Val)) } for i := uint32(0); i < m; i++ { for j := uint32(0); j < n; j++ { if newShard.Val[i*n+j] != oldShard.Val[i*n+j] { t.Errorf("M[%d][%d] incorrect. Mismatched old %f, new %f", i, j, oldShard.Val[i*n+j], newShard.Val[i*n+j]) } } } client.Remove(path) }