func GetFsClient(config *Config) (fs.Client, error) {
var client fs.Client
var cltErr error
switch config.IOConf.Fs {
case "local":
client = fs.NewLocalFSClient()
case "hdfs":
client, cltErr = fs.NewHdfsClient(
config.IOConf.HdfsConf.NamenodeAddr,
config.IOConf.HdfsConf.WebHdfsAddr,
config.IOConf.HdfsConf.User,
)
if cltErr != nil {
return nil, fmt.Errorf("Failed creating hdfs client %s", cltErr)
}
case "azure":
client, cltErr = fs.NewAzureClient(
config.IOConf.AzureConf.AccountName,
config.IOConf.AzureConf.AccountKey,
config.IOConf.AzureConf.BlogServiceBaseUrl,
config.IOConf.AzureConf.ApiVersion,
config.IOConf.AzureConf.UseHttps,
)
if cltErr != nil {
return nil, fmt.Errorf("Failed creating azure client %s", cltErr)
}
default:
return nil, fmt.Errorf("Unknow fs: %s", config.IOConf.Fs)
}
return client, nil
}
func generateTestData(t *testing.T) {
r0, c0, r1, c1 := getShards()
fs := filesystem.NewLocalFSClient()
wr0Err := bwmf.SaveMatrixShard(fs, r0, "../.tmp/row_shard.dat-000000")
wr1Err := bwmf.SaveMatrixShard(fs, r1, "../.tmp/row_shard.dat-000001")
wc0Err := bwmf.SaveMatrixShard(fs, c0, "../.tmp/column_shard.dat-000000")
wc1Err := bwmf.SaveMatrixShard(fs, c1, "../.tmp/column_shard.dat-000001")
if wr0Err != nil || wr1Err != nil || wc0Err != nil || wc1Err != nil {
t.Errorf("Failed generating test data files: %s;%s;%s;%s.", wr0Err, wr1Err, wc0Err, wc1Err)
}
}
func main() {
etcdUrlList := flag.String("etcd_urls", "", "ETCD server lists, sep by a comma.")
jobName := flag.String("job_name", "bwmf", "Job name in etcd path.")
jobType := flag.String("job_type", "c", "Job type, either 'c' for controller or 't' for task.")
numTasks := flag.Int("num_tasks", 1, "Num of tasks.")
taskConfigFile := flag.String("task_config", "", "Path to task config json file.")
flag.Parse()
if *jobName == "" {
log.Fatal("Job name is required.")
}
crd, oErr := filesystem.NewLocalFSClient().OpenReadCloser(*taskConfigFile)
if oErr != nil {
log.Fatalf("Failed opening task config file. %s", oErr)
}
confData, rdErr := ioutil.ReadAll(crd)
if rdErr != nil {
log.Fatalf("Failed reading task config. %s", rdErr)
}
log.Printf("conf data: %s", confData)
if *etcdUrlList == "" {
log.Fatal("Please specify the etcd server urls.")
}
etcdUrls := strings.Split(*etcdUrlList, ",")
log.Println("etcd urls: ", etcdUrls)
topoMaster := topo.NewFullTopologyOfMaster(uint64(*numTasks))
topoNeighbors := topo.NewFullTopologyOfNeighbor(uint64(*numTasks))
switch *jobType {
case "t":
bootstrap := framework.NewBootStrap(*jobName, etcdUrls, createListener(), nil)
taskBuilder := &bwmf.BWMFTaskBuilder{
NumOfTasks: uint64(*numTasks),
ConfBytes: confData,
}
bootstrap.SetTaskBuilder(taskBuilder)
bootstrap.AddLinkage("Master", topoMaster)
bootstrap.AddLinkage("Neighbors", topoNeighbors)
log.Println("Starting task..")
bootstrap.Start()
case "c":
controller := controller.New(*jobName, etcd.NewClient(etcdUrls), uint64(*numTasks), []string{"Master", "Neighbors"})
controller.Start()
log.Println("Controller started.")
controller.WaitForJobDone()
controller.Stop()
default:
log.Fatal("Please choose a type via '-jobtype': (c) controller, (t) task")
}
}
func loadConfig(path string) (*bwmf.Config, error) {
crd, err := fs.NewLocalFSClient().OpenReadCloser(path)
if err != nil {
return nil, err
}
confBytes, err := ioutil.ReadAll(crd)
if err != nil {
return nil, err
}
config := &bwmf.Config{}
unmarshErr := json.Unmarshal(confBytes, config)
if unmarshErr != nil {
return nil, unmarshErr
}
return config, nil
}
func TestShardIO(t *testing.T) {
client := filesystem.NewLocalFSClient()
path := "./.testShardIO.text.dat"
m, n := uint32(2), uint32(3)
oldShard := &pb.MatrixShard{
IsSparse: false,
M: m,
N: n,
Val: []float32{0.70, 1.00, 0.90, 0.70, 0.80, 0.90},
}
client.Remove(path)
sErr := SaveMatrixShard(client, oldShard, path)
if sErr != nil {
t.Errorf("Saving shard failed: %s", sErr)
}
newShard, lErr := LoadMatrixShard(client, path)
if sErr != nil {
t.Errorf("Loading shard failed: %s", lErr)
}
// shard should be:
// -----------------------------
// | 0.70 | 1.00 | 0.90 | 0.00 |
// -----------------------------
// | 0.00 | 0.70 | 0.80 | 0.90 |
// -----------------------------
fmt.Println("Original matrix shard is: ", oldShard)
fmt.Println("Loaded matrix shard is: ", newShard)
if len(newShard.Val) != len(oldShard.Val) {
t.Errorf("num of values wrong. Expected %d, actual %d", len(oldShard.Val), len(newShard.Val))
}
for i := uint32(0); i < m; i++ {
for j := uint32(0); j < n; j++ {
if newShard.Val[i*n+j] != oldShard.Val[i*n+j] {
t.Errorf("M[%d][%d] incorrect. Mismatched old %f, new %f", i, j, oldShard.Val[i*n+j], newShard.Val[i*n+j])
}
}
}
client.Remove(path)
}