// Get lists and retrieves s3 keys given a list of prefixes
// searchDepth can also be specified to increase speed of listing
func Get(prefixes []string, searchDepth int) {
if len(prefixes) == 0 {
fmt.Printf("No prefixes provided\n Usage: fasts3 get <prefix>")
return
}
getRequests := make(chan GetRequest, len(prefixes)*2+1)
var b *s3.Bucket = nil
go func() {
for _, prefix := range prefixes {
bucket, prefix := parseS3Uri(prefix)
if b == nil {
b = GetBucket(bucket)
}
keyExists, err := b.Exists(prefix)
if err != nil {
log.Fatalln(err)
}
if keyExists {
keyParts := strings.Split(prefix, "/")
ogPrefix := strings.Join(keyParts[0:len(keyParts)-1], "/") + "/"
getRequests <- GetRequest{Key: prefix, OriginalPrefix: ogPrefix}
} else {
for key := range s3wrapper.ListRecurse(b, prefix, searchDepth) {
getRequests <- GetRequest{Key: key.Key, OriginalPrefix: prefix}
}
}
}
close(getRequests)
}()
var wg sync.WaitGroup
msgs := make(chan string, 1000)
workingDirectory, err := os.Getwd()
if err != nil {
log.Fatalln(err)
}
for i := 1; i <= 10; i++ {
wg.Add(1)
go func() {
for rq := range getRequests {
dest := path.Join(workingDirectory, strings.Replace(rq.Key, rq.OriginalPrefix, "", 1))
msgs <- fmt.Sprintf("Getting %s -> %s\n", rq.Key, dest)
err := s3wrapper.GetToFile(b, rq.Key, dest)
if err != nil {
log.Fatalln(err)
}
}
wg.Done()
}()
}
go func() {
wg.Wait()
close(msgs)
}()
for msg := range msgs {
fmt.Print(msg)
}
}
// Stream takes a set of prefixes lists them and
// streams the contents by line
func Stream(prefixes []string, searchDepth int, keyRegex string, includeKeyName bool) {
if len(prefixes) == 0 {
fmt.Printf("No prefixes provided\n Usage: fasts3 get <prefix>")
return
}
keys := make(chan string, len(prefixes)*2+1)
var keyRegexFilter *regexp.Regexp
if keyRegex != "" {
keyRegexFilter = regexp.MustCompile(keyRegex)
} else {
keyRegexFilter = nil
}
var b *s3.Bucket = nil
go func() {
for _, prefix := range prefixes {
bucket, prefix := parseS3Uri(prefix)
if b == nil {
b = GetBucket(bucket)
}
keyExists, err := b.Exists(prefix)
if err != nil {
log.Fatalln(err)
}
if keyExists {
if keyRegexFilter != nil && !keyRegexFilter.MatchString(prefix) {
continue
}
keys <- prefix
} else {
for key := range s3wrapper.ListRecurse(b, prefix, searchDepth) {
if keyRegexFilter != nil && !keyRegexFilter.MatchString(key.Key) {
continue
}
keys <- key.Key
}
}
}
close(keys)
}()
var wg sync.WaitGroup
msgs := make(chan string, 1000)
for i := 1; i <= 10; i++ {
wg.Add(1)
go func() {
for key := range keys {
bts, err := s3wrapper.Get(b, key)
reader, err := getReaderByExt(bts, key)
if err != nil {
panic(err)
}
for {
line, _, err := reader.ReadLine()
if err != nil {
if err.Error() == "EOF" {
break
} else {
log.Fatalln(err)
}
}
msg := fmt.Sprintf("%s\n", string(line))
if includeKeyName {
msg = fmt.Sprintf("[%s] %s", key, msg)
}
msgs <- msg
}
}
wg.Done()
}()
}
go func() {
wg.Wait()
close(msgs)
}()
for msg := range msgs {
fmt.Print(msg)
}
}
// Del deletes a set of prefixes(s3 keys or partial keys
func Del(prefixes []string, searchDepth int, isRecursive bool) {
if len(*delPrefixes) == 0 {
fmt.Printf("No prefixes provided\n Usage: fasts3 del <prefix>")
return
}
keys := make(chan string, len(prefixes)*2+1)
var b *s3.Bucket = nil
go func() {
for _, delPrefix := range prefixes {
bucket, prefix := parseS3Uri(delPrefix)
if b == nil {
b = GetBucket(bucket)
}
keys <- prefix
if *delRecurse {
keyExists, err := b.Exists(prefix)
if err != nil {
log.Fatalln(err)
}
if keyExists {
keys <- prefix
} else if *delRecurse {
for key := range s3wrapper.ListRecurse(b, prefix, searchDepth) {
keys <- key.Key
}
} else {
fmt.Printf("trying to delete a prefix, please add --recursive or -r to proceed\n")
}
}
}
close(keys)
}()
var wg sync.WaitGroup
msgs := make(chan string, 1000)
for i := 1; i <= 10; i++ {
wg.Add(1)
go func() {
batch := make([]string, 0, 100)
for key := range keys {
batch = append(batch, key)
if len(batch) >= 100 {
err := s3wrapper.DeleteMulti(b, batch)
if err != nil {
log.Fatalln(err)
}
for _, k := range batch {
msgs <- fmt.Sprintf("File %s Deleted\n", k)
}
batch = batch[:0]
}
}
if len(batch) > 0 {
err := s3wrapper.DeleteMulti(b, batch)
if err != nil {
log.Fatalln(err)
}
for _, k := range batch {
msgs <- fmt.Sprintf("File %s Deleted\n", k)
}
}
wg.Done()
}()
}
go func() {
wg.Wait()
close(msgs)
}()
for msg := range msgs {
fmt.Print(msg)
}
}