Ejemplo n.º 1
0
func main() {
	namenode := flag.String("namenode", "localhost:9000", "HDFS namenode address.")
	webhdfs := flag.String("webhdfs", "localhost:50070", "WebHDFS address.")
	user := flag.String("user", "", "HDFS username. Could be empty.")
	flag.Parse()
	fs.HookupHDFS(*namenode, *webhdfs, *user)

	dir := path.Join(fmt.Sprintf("/hdfs/tmp/test/github.com/wangkuiyi/file/%v", time.Now().UnixNano()))
	file := path.Join(dir, "hello.txt")
	content := "Hello World!\n"

	if e := fs.Mkdir(dir); e != nil {
		log.Panicf("Mkdir(%v) failed", dir)
	}

	if w, e := fs.Create(file); e != nil {
		log.Panicf("Create(%v) failed", file)
	} else {
		fmt.Fprintf(w, content)
		w.Close()
	}

	if _, e := fs.Stat(file); os.IsNotExist(e) {
		log.Panicf("Expecting file exists, but not")
	}

	if r, e := fs.Open(file); e == nil {
		b, _ := ioutil.ReadAll(r)
		fmt.Println(string(b))
		r.Close()
	}
}
Ejemplo n.º 2
0
func BenchmarkBuildVocabAndVSharder(t *testing.B) {
	f, e := fs.Open("testdata/internet-zh.num")
	if e != nil {
		t.Skip(e)
	}
	defer f.Close()

	_, _, e = BuildVocabAndVSharder(f, 10, true)
	if e != nil {
		t.Skip(e)
	}
}
Ejemplo n.º 3
0
func main() {
	freq := flag.String("tf", "", "Token frequency list. Each line consists of frequence and token.")
	shards := flag.Int("s", 10, "Hint of number of vshards. Might be less.")
	delUnbalanced := flag.Bool("d", true, "Delete singular and unbalanced vshards.")
	vocab := flag.String("vocab", "", "Gob encoded Vocab file")
	vshdr := flag.String("vshdr", "", "Gob encoded VSharder file.")
	flag.Parse()

	f, e := fs.Open(*freq)
	if e != nil {
		log.Fatal(e)
	}
	defer f.Close()

	v, e := fs.Create(*vocab)
	if e != nil {
		log.Fatal(e)
	}
	defer v.Close()

	s, e := fs.Create(*vshdr)
	if e != nil {
		log.Fatal(e)
	}
	defer s.Close()

	{
		vocab, vshdr, e := algo.BuildVocabAndVSharder(f, *shards, *delUnbalanced)
		if e != nil {
			log.Fatal(e)
		}

		fmt.Printf("Generated %v shards\n", vshdr.Num())

		if e := gob.NewEncoder(v).Encode(vocab); e != nil {
			log.Fatal(e)
		}

		if e := gob.NewEncoder(s).Encode(vshdr); e != nil {
			log.Fatal(e)
		}
	}
}