func (c *HdfsClient) OpenReadCloser(name string) (io.ReadCloser, error) {
client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
if err != nil {
return nil, err
}
return client.Open(name)
}
func (c *HdfsClient) Rename(oldpath, newpath string) error {
client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
if err != nil {
return err
}
return client.Rename(oldpath, newpath)
}
func getClient(connstr string) (client *dfs.Client, err error) {
//log.Info("dpconn:", dpconn)
//u, err := url.Parse("hdfs://" + dpconn)
//if err != nil {
// return
//}
//userinfo := u.User
//username := userinfo.Username()
//password :=userinfo.Password()
//host := u.Host
client, err = dfs.New(connstr)
if err != nil {
return
}
info, err := client.Stat("/")
if err != nil {
return
}
hadoopUser := *info.Sys().(*hadoop_hdfs.HdfsFileStatusProto).Owner
client, err = dfs.NewForUser(connstr, hadoopUser)
if err != nil {
return
}
return
}
func (c *HdfsClient) Remove(name string) error {
client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
if err != nil {
return err
}
return client.Remove(name)
}
func HookupHDFS(namenode, webapi, role string) error {
err := ""
if len(role) <= 0 {
if u, e := user.Current(); e != nil {
err += fmt.Sprintf("Unknown current user: %v\n", e)
} else {
role = u.Username
}
}
log.Printf("Establish HDFS protobuf-based RPC connection as %s@%s", role, namenode)
if fs, e := hdfs.NewForUser(namenode, role); e != nil {
err += fmt.Sprintf("Cannot estabilish RPC connection to %s@%s: %v", role, namenode, e)
} else {
rpcfs = fs
}
log.Printf("Establish WebHDFS connection as %s@%s", role, webapi)
if fs, e := gowfs.NewFileSystem(gowfs.Configuration{Addr: webapi, User: role}); e != nil {
err += fmt.Sprintf("Cannot establish WebHDFS connection to %s@%s: %v", role, webapi, e)
} else {
webfs = fs
if e := testConnection(); e != nil {
err += fmt.Sprintf("Failed checking WebHDFS connection: %v", e)
}
}
if len(err) > 0 {
return fmt.Errorf(err)
}
return nil
}
func (c *HdfsClient) Exists(name string) (bool, error) {
client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
if err != nil {
return false, err
}
_, err = client.Stat(name)
return existCommon(err)
}
/*
createHDFSStorage creates a structure that writes all files to a Hadoop file system.
*/
func createHDFSStorage(address, user string) (*hdfsStorage, error) {
// connect to URL
client, err := hdfs.NewForUser(address, user)
if err != nil {
return nil, err
}
return &hdfsStorage{
client: client}, nil
}
func NewHdfsClient(namenodeAddr, webHdfsAddr, user string) (Client, error) {
_, err := hdfs.NewForUser(namenodeAddr, user)
if err != nil {
return nil, err
}
return &HdfsClient{
hdfsConfig: hdfsConfig{
namenodeAddr: namenodeAddr,
webHdfsAddr: webHdfsAddr,
user: user,
},
}, nil
}
func setupHdfsTest(t *testing.T) (Client, *hdfs.Client) {
if namenodeAddr == "" || webhdfsAddr == "" || hdfsUser == "" {
t.Skip("HDFS config not specified.")
}
client, err := NewHdfsClient(namenodeAddr, webhdfsAddr, hdfsUser)
if err != nil {
t.Fatalf("NewHdfsClient(%s, %s) failed: %v",
namenodeAddr, webhdfsAddr, err)
}
hClient, err := hdfs.NewForUser(namenodeAddr, hdfsUser)
if err != nil {
t.Fatalf("hdfs.NewForUser(%s, %s) failed: %v",
namenodeAddr, webhdfsAddr, err)
}
return client, hClient
}
func (c *HdfsClient) glob(dir string, names []string) (m []string, err error) {
client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
if err != nil {
return nil, err
}
name := names[0]
var dirs []string
if hasMeta(name) {
fileInfos, err := client.ReadDir(dir)
if err != nil {
return nil, err
}
for _, fi := range fileInfos {
matched, err := path.Match(name, fi.Name())
if err != nil {
return nil, err
}
if matched {
dirs = append(dirs, path.Join(dir, fi.Name()))
}
}
} else {
dirs = append(dirs, path.Join(dir, name))
}
for _, pathname := range dirs {
if len(names) == 1 {
exist, err := c.Exists(pathname)
if err != nil {
return nil, err
}
if exist {
m = append(m, pathname)
}
} else {
return c.glob(pathname, names[1:len(names)])
}
}
return
}
func (c *HdfsClient) OpenWriteCloser(name string) (io.WriteCloser, error) {
exist, err := c.Exists(name)
if err != nil {
return nil, err
}
if !exist {
client, err := hdfs.NewForUser(c.hdfsConfig.namenodeAddr, c.hdfsConfig.user)
if err != nil {
return nil, err
}
err = client.CreateEmptyFile(name)
if err != nil {
return nil, err
}
}
return &HdfsFile{
path: name,
logger: log.New(os.Stdout, "", log.Lshortfile|log.LstdFlags),
hdfsConfig: c.hdfsConfig,
}, nil
}
// New constructs a new driver
func New(params driverParameters) (storagedriver.StorageDriver, error) {
// Setup the connection to hdfs
client, err := hdfs.NewForUser(params.hdfsNameNode, params.hdfsUser)
if err != nil {
log.Fatal(err)
}
// Populate the driver
d := &driver{
hdfsRootDirectory: params.hdfsRootDirectory,
hdfsNameNode: params.hdfsNameNode,
hdfsUser: params.hdfsUser,
directoryUmask: params.directoryUmask,
hdfsClient: client,
}
// Return the StorageDriver
return &base.Base{
StorageDriver: d,
}, nil
}
func InitHdfsClient() {
hdfsclient, err = hdfs.NewForUser(Config.HdfsAddress, Config.HdfsUsername)
if err != nil {
Log.Fatal(err)
}
}