func TextFile(location string, lines chan string) {
namenode, path, err := splitLocationToParts(location)
if err != nil {
return
}
client, err := hdfs.New(namenode)
if err != nil {
log.Fatalf("failed to create client to %s:%v\n", namenode, err)
}
file, err := client.Open(path)
if err != nil {
log.Fatalf("Can not open file %s: %v", location, err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines <- scanner.Text()
}
if err := scanner.Err(); err != nil {
log.Printf("Scan file %s: %v", location, err)
}
}
func getClient(connstr string) (client *dfs.Client, err error) {
//log.Info("dpconn:", dpconn)
//u, err := url.Parse("hdfs://" + dpconn)
//if err != nil {
// return
//}
//userinfo := u.User
//username := userinfo.Username()
//password :=userinfo.Password()
//host := u.Host
client, err = dfs.New(connstr)
if err != nil {
return
}
info, err := client.Stat("/")
if err != nil {
return
}
hadoopUser := *info.Sys().(*hadoop_hdfs.HdfsFileStatusProto).Owner
client, err = dfs.NewForUser(connstr, hadoopUser)
if err != nil {
return
}
return
}
func (jt *jobTracker) TestLogsDir() error {
client, err := hdfs.New(*namenodeAddress)
if err != nil {
return err
}
_, err = client.ReadDir(*yarnLogDir)
return err
}
func NewHDFS(namenode string) (Storage, error) {
if client, err := hdfs.New(config.NameNode); err != nil {
msg := fmt.Sprintf("connect to hdfs namenode failed: %s", config.NameNode)
log.Fatal(msg)
return nil, fmt.Errorf(msg)
} else {
return &HDFS{client: client}, nil
}
}
func (jt *jobTracker) FetchLogs(id string) ([]string, error) {
client, err := hdfs.New(*namenodeAddress)
if err != nil {
return nil, err
}
appID, jobID := hadoopIDs(id)
user := jt.Jobs[jobID].Details.User
dir := fmt.Sprintf("%s/%s/logs/%s/", *yarnLogDir, user, appID)
files, err := client.ReadDir(dir)
if err != nil {
return nil, err
}
filenames := make(chan string, len(files))
results := make(chan string, len(files))
var wg sync.WaitGroup
for x := 1; x <= 5; x++ {
go func() {
for name := range filenames {
result, err := findStacktrace(client, name)
if err != nil {
log.Println("Error reading", name, err)
} else if result != "" {
results <- result
}
wg.Done()
}
}()
}
var logs []string
wg.Add(len(files))
for _, file := range files {
filenames <- dir + file.Name()
}
close(filenames)
wg.Wait()
close(results)
for result := range results {
logs = append(logs, result)
}
return logs, nil
}
func setupHdfs(t *testing.T) *backend.HdfsBackend {
nn := os.Getenv("HADOOP_NAMENODE")
if nn == "" {
t.Skip("Skipping hdfs tests because HADOOP_NAMENODE isn't set")
}
client, err := hdfs.New(nn)
if err != nil {
t.Fatal(err)
}
if _, err = client.Stat("/_test_sequins"); os.IsNotExist(err) {
t.Skip("Skipping hdfs tests because there's no test data in /_test_sequins")
}
return backend.NewHdfsBackend(client, nn, "/_test_sequins")
}
func hdfsSetup(namenode string, path string, opts sequinsOptions) *sequins {
client, err := hdfs.New(namenode)
if err != nil {
log.Fatal(fmt.Errorf("Error connecting to HDFS: %s", err))
}
backend := backend.NewHdfsBackend(client, namenode, path)
if opts.LocalPath == "" {
tmpDir, err := ioutil.TempDir("", "sequins-")
if err != nil {
log.Fatal(err)
}
opts.LocalPath = tmpDir
}
return newSequins(backend, opts)
}
func getClient(namenode string) (*hdfs.Client, error) {
if cachedClient != nil {
return cachedClient, nil
}
if namenode == "" {
namenode = os.Getenv("HADOOP_NAMENODE")
}
if namenode == "" {
return nil, errors.New("Couldn't find a namenode to connect to. You should specify hdfs://<namenode>:<port> in your paths, or set HADOOP_NAMENODE in your environment.")
}
c, err := hdfs.New(namenode)
if err != nil {
return nil, err
}
cachedClient = c
return cachedClient, nil
}
// List generates a full list of file locations under the given
// location, which should have a prefix of hdfs://
func List(hdfsLocation string) (locations []string, err error) {
namenode, path, err := splitLocationToParts(hdfsLocation)
if err != nil {
return
}
client, err := hdfs.New(namenode)
if err != nil {
return nil, fmt.Errorf("failed to create client to %s:%v\n", namenode, err)
}
fileInfos, err := client.ReadDir("/" + path)
if err != nil {
return nil, fmt.Errorf("failed to list files under /%s:%v\n", path, err)
}
for _, fi := range fileInfos {
locations = append(locations, hdfsLocation+"/"+fi.Name())
}
return
}