Exemplo n.º 1
0
func (c *HdfsClient) OpenReadCloser(name string) (io.ReadCloser, error) {
	client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
	if err != nil {
		return nil, err
	}
	return client.Open(name)
}
Exemplo n.º 2
0
func (c *HdfsClient) Rename(oldpath, newpath string) error {
	client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
	if err != nil {
		return err
	}
	return client.Rename(oldpath, newpath)
}
Exemplo n.º 3
0
func getClient(connstr string) (client *dfs.Client, err error) {
	//log.Info("dpconn:", dpconn)
	//u, err := url.Parse("hdfs://" + dpconn)
	//if err != nil {
	//	return
	//}
	//userinfo := u.User
	//username := userinfo.Username()
	//password :=userinfo.Password()
	//host := u.Host
	client, err = dfs.New(connstr)
	if err != nil {
		return
	}

	info, err := client.Stat("/")
	if err != nil {
		return
	}
	hadoopUser := *info.Sys().(*hadoop_hdfs.HdfsFileStatusProto).Owner

	client, err = dfs.NewForUser(connstr, hadoopUser)
	if err != nil {
		return
	}

	return
}
Exemplo n.º 4
0
func (c *HdfsClient) Remove(name string) error {
	client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
	if err != nil {
		return err
	}
	return client.Remove(name)
}
Exemplo n.º 5
0
Arquivo: fs.go Projeto: cxwangyi/fs
func HookupHDFS(namenode, webapi, role string) error {
	err := ""

	if len(role) <= 0 {
		if u, e := user.Current(); e != nil {
			err += fmt.Sprintf("Unknown current user: %v\n", e)
		} else {
			role = u.Username
		}
	}

	log.Printf("Establish HDFS protobuf-based RPC connection as %s@%s", role, namenode)
	if fs, e := hdfs.NewForUser(namenode, role); e != nil {
		err += fmt.Sprintf("Cannot estabilish RPC connection to %s@%s: %v", role, namenode, e)
	} else {
		rpcfs = fs
	}

	log.Printf("Establish WebHDFS connection as %s@%s", role, webapi)
	if fs, e := gowfs.NewFileSystem(gowfs.Configuration{Addr: webapi, User: role}); e != nil {
		err += fmt.Sprintf("Cannot establish WebHDFS connection to %s@%s: %v", role, webapi, e)
	} else {
		webfs = fs
		if e := testConnection(); e != nil {
			err += fmt.Sprintf("Failed checking WebHDFS connection: %v", e)
		}
	}

	if len(err) > 0 {
		return fmt.Errorf(err)
	}
	return nil
}
Exemplo n.º 6
0
func (c *HdfsClient) Exists(name string) (bool, error) {
	client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
	if err != nil {
		return false, err
	}
	_, err = client.Stat(name)
	return existCommon(err)
}
Exemplo n.º 7
0
/*
createHDFSStorage creates a structure that writes all files to a Hadoop file system.
*/
func createHDFSStorage(address, user string) (*hdfsStorage, error) {
	// connect to URL
	client, err := hdfs.NewForUser(address, user)
	if err != nil {
		return nil, err
	}
	return &hdfsStorage{
		client: client}, nil
}
Exemplo n.º 8
0
func NewHdfsClient(namenodeAddr, webHdfsAddr, user string) (Client, error) {
	_, err := hdfs.NewForUser(namenodeAddr, user)
	if err != nil {
		return nil, err
	}
	return &HdfsClient{
		hdfsConfig: hdfsConfig{
			namenodeAddr: namenodeAddr,
			webHdfsAddr:  webHdfsAddr,
			user:         user,
		},
	}, nil
}
Exemplo n.º 9
0
func setupHdfsTest(t *testing.T) (Client, *hdfs.Client) {
	if namenodeAddr == "" || webhdfsAddr == "" || hdfsUser == "" {
		t.Skip("HDFS config not specified.")
	}
	client, err := NewHdfsClient(namenodeAddr, webhdfsAddr, hdfsUser)
	if err != nil {
		t.Fatalf("NewHdfsClient(%s, %s) failed: %v",
			namenodeAddr, webhdfsAddr, err)
	}
	hClient, err := hdfs.NewForUser(namenodeAddr, hdfsUser)
	if err != nil {
		t.Fatalf("hdfs.NewForUser(%s, %s) failed: %v",
			namenodeAddr, webhdfsAddr, err)
	}
	return client, hClient
}
Exemplo n.º 10
0
func (c *HdfsClient) glob(dir string, names []string) (m []string, err error) {

	client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
	if err != nil {
		return nil, err
	}
	name := names[0]
	var dirs []string
	if hasMeta(name) {
		fileInfos, err := client.ReadDir(dir)
		if err != nil {
			return nil, err
		}
		for _, fi := range fileInfos {
			matched, err := path.Match(name, fi.Name())
			if err != nil {
				return nil, err
			}
			if matched {
				dirs = append(dirs, path.Join(dir, fi.Name()))
			}
		}
	} else {
		dirs = append(dirs, path.Join(dir, name))
	}
	for _, pathname := range dirs {
		if len(names) == 1 {
			exist, err := c.Exists(pathname)
			if err != nil {
				return nil, err
			}
			if exist {
				m = append(m, pathname)
			}
		} else {
			return c.glob(pathname, names[1:len(names)])

		}
	}
	return
}
Exemplo n.º 11
0
func (c *HdfsClient) OpenWriteCloser(name string) (io.WriteCloser, error) {
	exist, err := c.Exists(name)
	if err != nil {
		return nil, err
	}
	if !exist {
		client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
		if err != nil {
			return nil, err
		}
		err = client.CreateEmptyFile(name)
		if err != nil {
			return nil, err
		}
	}
	return &HdfsFile{
		path:       name,
		logger:     log.New(os.Stdout, "", log.Lshortfile|log.LstdFlags),
		hdfsConfig: c.hdfsConfig,
	}, nil
}
Exemplo n.º 12
0
// New constructs a new driver
func New(params driverParameters) (storagedriver.StorageDriver, error) {

	// Setup the connection to hdfs
	client, err := hdfs.NewForUser(params.hdfsNameNode, params.hdfsUser)
	if err != nil {
		log.Fatal(err)
	}

	// Populate the driver
	d := &driver{
		hdfsRootDirectory: params.hdfsRootDirectory,
		hdfsNameNode:      params.hdfsNameNode,
		hdfsUser:          params.hdfsUser,
		directoryUmask:    params.directoryUmask,
		hdfsClient:        client,
	}

	// Return the StorageDriver
	return &base.Base{
		StorageDriver: d,
	}, nil
}
Exemplo n.º 13
0
func InitHdfsClient() {
	hdfsclient, err = hdfs.NewForUser(Config.HdfsAddress, Config.HdfsUsername)
	if err != nil {
		Log.Fatal(err)
	}
}