// ExportZkns exports addresses from the VT serving graph to a legacy zkns server.
// Note these functions only work with a zktopo.
func (wr *Wrangler) ExportZkns(ctx context.Context, cell string) error {
zkTopo, ok := wr.ts.(*zktopo.Server)
if !ok {
return fmt.Errorf("ExportZkns only works with zktopo")
}
zconn := zkTopo.GetZConn()
vtNsPath := fmt.Sprintf("/zk/%v/vt/ns", cell)
zknsRootPath := fmt.Sprintf("/zk/%v/zkns/vt", cell)
children, err := zk.ChildrenRecursive(zconn, vtNsPath)
if err != nil {
return err
}
for _, child := range children {
addrPath := path.Join(vtNsPath, child)
zknsAddrPath := path.Join(zknsRootPath, child)
_, stat, err := zconn.Get(addrPath)
if err != nil {
return err
}
// Leaf nodes correspond to zkns vdns files in the old setup.
if stat.NumChildren() > 0 {
continue
}
if _, err = wr.exportVtnsToZkns(ctx, zconn, addrPath, zknsAddrPath); err != nil {
return err
}
}
return nil
}
// ExportZknsForKeyspace exports addresses from the VT serving graph to a legacy zkns server.
func (wr *Wrangler) ExportZknsForKeyspace(ctx context.Context, keyspace string) error {
zkTopo, ok := wr.ts.(*zktopo.Server)
if !ok {
return fmt.Errorf("ExportZknsForKeyspace only works with zktopo")
}
zconn := zkTopo.GetZConn()
shardNames, err := wr.ts.GetShardNames(ctx, keyspace)
if err != nil {
return err
}
// Scan the first shard to discover which cells need local serving data.
aliases, err := topo.FindAllTabletAliasesInShard(ctx, wr.ts, keyspace, shardNames[0])
if err != nil {
return err
}
cellMap := make(map[string]bool)
for _, alias := range aliases {
cellMap[alias.Cell] = true
}
for cell := range cellMap {
vtnsRootPath := fmt.Sprintf("/zk/%v/vt/ns/%v", cell, keyspace)
zknsRootPath := fmt.Sprintf("/zk/%v/zkns/vt/%v", cell, keyspace)
// Get the existing list of zkns children. If they don't get rewritten,
// delete them as stale entries.
zknsChildren, err := zk.ChildrenRecursive(zconn, zknsRootPath)
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
zknsChildren = make([]string, 0)
} else {
return err
}
}
staleZknsPaths := make(map[string]bool)
for _, child := range zknsChildren {
staleZknsPaths[path.Join(zknsRootPath, child)] = true
}
vtnsChildren, err := zk.ChildrenRecursive(zconn, vtnsRootPath)
if err != nil {
if zookeeper.IsError(err, zookeeper.ZNONODE) {
vtnsChildren = make([]string, 0)
} else {
return err
}
}
for _, child := range vtnsChildren {
vtnsAddrPath := path.Join(vtnsRootPath, child)
zknsAddrPath := path.Join(zknsRootPath, child)
_, stat, err := zconn.Get(vtnsAddrPath)
if err != nil {
return err
}
// Leaf nodes correspond to zkns vdns files in the old setup.
if stat.NumChildren() > 0 {
continue
}
zknsPathsWritten, err := wr.exportVtnsToZkns(ctx, zconn, vtnsAddrPath, zknsAddrPath)
if err != nil {
return err
}
log.V(6).Infof("zknsPathsWritten: %v", zknsPathsWritten)
for _, zkPath := range zknsPathsWritten {
delete(staleZknsPaths, zkPath)
}
}
log.V(6).Infof("staleZknsPaths: %v", staleZknsPaths)
prunePaths := make([]string, 0, len(staleZknsPaths))
for prunePath := range staleZknsPaths {
prunePaths = append(prunePaths, prunePath)
}
sort.Strings(prunePaths)
// Prune paths in reverse order so we remove children first
for i := len(prunePaths) - 1; i >= 0; i-- {
log.Infof("prune stale zkns path %v", prunePaths[i])
if err := zconn.Delete(prunePaths[i], -1); err != nil && !zookeeper.IsError(err, zookeeper.ZNOTEMPTY) {
return err
}
}
}
return nil
}
func cmdLs(subFlags *flag.FlagSet, args []string) error {
var (
longListing = subFlags.Bool("l", false, "long listing")
directoryListing = subFlags.Bool("d", false, "list directory instead of contents")
force = subFlags.Bool("f", false, "no warning on nonexistent node")
recursiveListing = subFlags.Bool("R", false, "recursive listing")
)
subFlags.Parse(args)
if subFlags.NArg() == 0 {
return fmt.Errorf("ls: no path specified")
}
// FIXME(szopa): shadowing?
resolved, err := zk.ResolveWildcards(zconn, subFlags.Args())
if err != nil {
return fmt.Errorf("ls: invalid wildcards: %v", err)
}
if len(resolved) == 0 {
// the wildcards didn't result in anything, we're
// done.
return nil
}
hasError := false
needsHeader := len(resolved) > 1 && !*directoryListing
for _, arg := range resolved {
zkPath := fixZkPath(arg)
var children []string
var err error
isDir := true
if *directoryListing {
children = []string{""}
isDir = false
} else if *recursiveListing {
children, err = zk.ChildrenRecursive(zconn, zkPath)
} else {
children, _, err = zconn.Children(zkPath)
// Assume this is a file node if it has no children.
if len(children) == 0 {
children = []string{""}
isDir = false
}
}
if err != nil {
hasError = true
if !*force || !zookeeper.IsError(err, zookeeper.ZNONODE) {
log.Warningf("ls: cannot access %v: %v", zkPath, err)
}
}
// Show the full path when it helps.
showFullPath := false
if *recursiveListing {
showFullPath = true
} else if *longListing && (*directoryListing || !isDir) {
showFullPath = true
}
if needsHeader {
fmt.Printf("%v:\n", zkPath)
}
if len(children) > 0 {
if *longListing && isDir {
fmt.Printf("total: %v\n", len(children))
}
sort.Strings(children)
stats := make([]zk.Stat, len(children))
wg := sync.WaitGroup{}
f := func(i int) {
localPath := path.Join(zkPath, children[i])
stat, err := zconn.Exists(localPath)
if err != nil {
if !*force || !zookeeper.IsError(err, zookeeper.ZNONODE) {
log.Warningf("ls: cannot access: %v: %v", localPath, err)
}
} else {
stats[i] = stat
}
wg.Done()
}
for i := range children {
wg.Add(1)
go f(i)
}
wg.Wait()
for i, child := range children {
localPath := path.Join(zkPath, child)
if stat := stats[i]; stat != nil {
fmtPath(stat, localPath, showFullPath, *longListing)
}
}
}
if needsHeader {
fmt.Println()
}
}
if hasError {
return fmt.Errorf("ls: some paths had errors")
}
return nil
}
// Store a zk tree in a zip archive. This won't be immediately useful to
// zip tools since even "directories" can contain data.
func cmdZip(subFlags *flag.FlagSet, args []string) error {
subFlags.Parse(args)
if subFlags.NArg() < 2 {
return fmt.Errorf("zip: need to specify source and destination paths")
}
dstPath := subFlags.Arg(subFlags.NArg() - 1)
paths := subFlags.Args()[:len(args)-1]
if !strings.HasSuffix(dstPath, ".zip") {
return fmt.Errorf("zip: need to specify destination .zip path: %v", dstPath)
}
zipFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("zip: error %v", err)
}
wg := sync.WaitGroup{}
items := make(chan *zkItem, 64)
for _, arg := range paths {
zkPath := fixZkPath(arg)
children, err := zk.ChildrenRecursive(zconn, zkPath)
if err != nil {
return fmt.Errorf("zip: error %v", err)
}
for _, child := range children {
toAdd := path.Join(zkPath, child)
wg.Add(1)
go func() {
data, stat, err := zconn.Get(toAdd)
items <- &zkItem{toAdd, data, stat, err}
wg.Done()
}()
}
}
go func() {
wg.Wait()
close(items)
}()
zipWriter := zip.NewWriter(zipFile)
for item := range items {
path, data, stat, err := item.path, item.data, item.stat, item.err
if err != nil {
return fmt.Errorf("zip: get failed: %v", err)
}
// Skip ephemerals - not sure why you would archive them.
if stat.EphemeralOwner() > 0 {
continue
}
fi := &zip.FileHeader{Name: path, Method: zip.Deflate}
fi.SetModTime(stat.MTime())
f, err := zipWriter.CreateHeader(fi)
if err != nil {
return fmt.Errorf("zip: create failed: %v", err)
}
_, err = f.Write([]byte(data))
if err != nil {
return fmt.Errorf("zip: create failed: %v", err)
}
}
err = zipWriter.Close()
if err != nil {
return fmt.Errorf("zip: close failed: %v", err)
}
zipFile.Close()
return nil
}
func cmdLs(args []string) {
if len(args) == 0 {
log.Fatal("ls: no path specified")
}
args, err := zk.ResolveWildcards(zconn, args)
if err != nil {
log.Fatalf("ls: invalid wildcards: %v", err)
}
if len(args) == 0 {
// the wildcards didn't result in anything, we're done
return
}
hasError := false
needsHeader := len(args) > 1 && !*directoryListing
for _, arg := range args {
zkPath := fixZkPath(arg)
var children []string
var err error
isDir := true
if *directoryListing {
children = []string{""}
isDir = false
} else if *recursiveListing {
children, err = zk.ChildrenRecursive(zconn, zkPath)
} else {
children, _, err = zconn.Children(zkPath)
// Assume this is a file node if it has no children.
if len(children) == 0 {
children = []string{""}
isDir = false
}
}
if err != nil {
hasError = true
if !*force || !zookeeper.IsError(err, zookeeper.ZNONODE) {
log.Printf("ls: cannot access %v: %v", zkPath, err)
}
}
// Show the full path when it helps.
showFullPath := false
if *recursiveListing {
showFullPath = true
} else if *longListing && (*directoryListing || !isDir) {
showFullPath = true
}
if needsHeader {
fmt.Printf("%v:\n", zkPath)
}
if len(children) > 0 {
if *longListing && isDir {
fmt.Printf("total: %v\n", len(children))
}
sort.Strings(children)
stats := make([]zk.Stat, len(children))
wg := sync.WaitGroup{}
f := func(i int) {
localPath := path.Join(zkPath, children[i])
stat, err := zconn.Exists(localPath)
if err != nil {
if !*force || !zookeeper.IsError(err, zookeeper.ZNONODE) {
log.Printf("ls: cannot access: %v: %v", localPath, err)
}
} else {
stats[i] = stat
}
wg.Done()
}
for i := range children {
wg.Add(1)
go f(i)
}
wg.Wait()
for i, child := range children {
localPath := path.Join(zkPath, child)
if stat := stats[i]; stat != nil {
fmtPath(stat, localPath, showFullPath)
}
}
}
if needsHeader {
fmt.Println()
}
}
if hasError {
os.Exit(1)
}
}