Beispiel #1
0
func TextFile(location string, lines chan string) {
	namenode, path, err := splitLocationToParts(location)
	if err != nil {
		return
	}

	client, err := hdfs.New(namenode)
	if err != nil {
		log.Fatalf("failed to create client to %s:%v\n", namenode, err)
	}

	file, err := client.Open(path)

	if err != nil {
		log.Fatalf("Can not open file %s: %v", location, err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines <- scanner.Text()
	}

	if err := scanner.Err(); err != nil {
		log.Printf("Scan file %s: %v", location, err)
	}
}
Beispiel #2
0
func getClient(connstr string) (client *dfs.Client, err error) {
	//log.Info("dpconn:", dpconn)
	//u, err := url.Parse("hdfs://" + dpconn)
	//if err != nil {
	//	return
	//}
	//userinfo := u.User
	//username := userinfo.Username()
	//password :=userinfo.Password()
	//host := u.Host
	client, err = dfs.New(connstr)
	if err != nil {
		return
	}

	info, err := client.Stat("/")
	if err != nil {
		return
	}
	hadoopUser := *info.Sys().(*hadoop_hdfs.HdfsFileStatusProto).Owner

	client, err = dfs.NewForUser(connstr, hadoopUser)
	if err != nil {
		return
	}

	return
}
Beispiel #3
0
func (jt *jobTracker) TestLogsDir() error {
	client, err := hdfs.New(*namenodeAddress)
	if err != nil {
		return err
	}

	_, err = client.ReadDir(*yarnLogDir)
	return err
}
Beispiel #4
0
func NewHDFS(namenode string) (Storage, error) {
	if client, err := hdfs.New(config.NameNode); err != nil {
		msg := fmt.Sprintf("connect to hdfs namenode failed: %s", config.NameNode)
		log.Fatal(msg)
		return nil, fmt.Errorf(msg)
	} else {
		return &HDFS{client: client}, nil
	}
}
Beispiel #5
0
func (jt *jobTracker) FetchLogs(id string) ([]string, error) {
	client, err := hdfs.New(*namenodeAddress)
	if err != nil {
		return nil, err
	}

	appID, jobID := hadoopIDs(id)
	user := jt.Jobs[jobID].Details.User
	dir := fmt.Sprintf("%s/%s/logs/%s/", *yarnLogDir, user, appID)

	files, err := client.ReadDir(dir)
	if err != nil {
		return nil, err
	}

	filenames := make(chan string, len(files))
	results := make(chan string, len(files))

	var wg sync.WaitGroup

	for x := 1; x <= 5; x++ {
		go func() {
			for name := range filenames {
				result, err := findStacktrace(client, name)
				if err != nil {
					log.Println("Error reading", name, err)
				} else if result != "" {
					results <- result
				}
				wg.Done()
			}
		}()
	}

	var logs []string

	wg.Add(len(files))
	for _, file := range files {
		filenames <- dir + file.Name()
	}
	close(filenames)

	wg.Wait()
	close(results)

	for result := range results {
		logs = append(logs, result)
	}

	return logs, nil
}
Beispiel #6
0
func setupHdfs(t *testing.T) *backend.HdfsBackend {
	nn := os.Getenv("HADOOP_NAMENODE")
	if nn == "" {
		t.Skip("Skipping hdfs tests because HADOOP_NAMENODE isn't set")
	}

	client, err := hdfs.New(nn)
	if err != nil {
		t.Fatal(err)
	}

	if _, err = client.Stat("/_test_sequins"); os.IsNotExist(err) {
		t.Skip("Skipping hdfs tests because there's no test data in /_test_sequins")
	}

	return backend.NewHdfsBackend(client, nn, "/_test_sequins")
}
Beispiel #7
0
func hdfsSetup(namenode string, path string, opts sequinsOptions) *sequins {
	client, err := hdfs.New(namenode)
	if err != nil {
		log.Fatal(fmt.Errorf("Error connecting to HDFS: %s", err))
	}

	backend := backend.NewHdfsBackend(client, namenode, path)
	if opts.LocalPath == "" {
		tmpDir, err := ioutil.TempDir("", "sequins-")
		if err != nil {
			log.Fatal(err)
		}

		opts.LocalPath = tmpDir
	}

	return newSequins(backend, opts)
}
Beispiel #8
0
func getClient(namenode string) (*hdfs.Client, error) {
	if cachedClient != nil {
		return cachedClient, nil
	}

	if namenode == "" {
		namenode = os.Getenv("HADOOP_NAMENODE")
	}

	if namenode == "" {
		return nil, errors.New("Couldn't find a namenode to connect to. You should specify hdfs://<namenode>:<port> in your paths, or set HADOOP_NAMENODE in your environment.")
	}

	c, err := hdfs.New(namenode)
	if err != nil {
		return nil, err
	}

	cachedClient = c
	return cachedClient, nil
}
Beispiel #9
0
// List generates a full list of file locations under the given
// location, which should have a prefix of hdfs://
func List(hdfsLocation string) (locations []string, err error) {

	namenode, path, err := splitLocationToParts(hdfsLocation)
	if err != nil {
		return
	}

	client, err := hdfs.New(namenode)
	if err != nil {
		return nil, fmt.Errorf("failed to create client to %s:%v\n", namenode, err)
	}

	fileInfos, err := client.ReadDir("/" + path)
	if err != nil {
		return nil, fmt.Errorf("failed to list files under /%s:%v\n", path, err)
	}

	for _, fi := range fileInfos {
		locations = append(locations, hdfsLocation+"/"+fi.Name())
	}

	return

}