// Scrap will update the tablet type to 'Scrap', and remove it from
// the serving graph.
//
// 'force' means we are not on the tablet being scrapped, so it is
// probably dead. So if 'force' is true, we will also remove pending
// remote actions. And if 'force' is false, we also run an optional
// hook.
func Scrap(ts topo.Server, tabletAlias topo.TabletAlias, force bool) error {
tablet, err := ts.GetTablet(tabletAlias)
if err != nil {
return err
}
// If you are already scrap, skip updating replication data. It won't
// be there anyway.
wasAssigned := tablet.IsAssigned()
tablet.Type = topo.TYPE_SCRAP
tablet.Parent = topo.TabletAlias{}
// Update the tablet first, since that is canonical.
err = topo.UpdateTablet(ts, tablet)
if err != nil {
return err
}
// Remove any pending actions. Presumably forcing a scrap
// means you don't want the agent doing anything and the
// machine requires manual attention.
if force {
err := ts.PurgeTabletActions(tabletAlias, actionnode.ActionNodeCanBePurged)
if err != nil {
log.Warningf("purge actions failed: %v", err)
}
}
if wasAssigned {
err = topo.DeleteTabletReplicationData(ts, tablet.Tablet)
if err != nil {
if err == topo.ErrNoNode {
log.V(6).Infof("no ShardReplication object for cell %v", tablet.Alias.Cell)
err = nil
}
if err != nil {
log.Warningf("remove replication data for %v failed: %v", tablet.Alias, err)
}
}
}
// run a hook for final cleanup, only in non-force mode.
// (force mode executes on the vtctl side, not on the vttablet side)
if !force {
hk := hook.NewSimpleHook("postflight_scrap")
ConfigureTabletHook(hk, tablet.Alias)
if hookErr := hk.ExecuteOptional(); hookErr != nil {
// we don't want to return an error, the server
// is already in bad shape probably.
log.Warningf("Scrap: postflight_scrap failed: %v", hookErr)
}
}
return nil
}
// Make this external, since in needs to be forced from time to time.
func Scrap(ts topo.Server, tabletAlias topo.TabletAlias, force bool) error {
tablet, err := ts.GetTablet(tabletAlias)
if err != nil {
return err
}
// If you are already scrap, skip deleting the path. It won't
// be correct since the Parent will be cleared already.
wasAssigned := tablet.IsAssigned()
replicationPath := ""
if wasAssigned {
replicationPath = tablet.ReplicationPath()
}
tablet.Type = topo.TYPE_SCRAP
tablet.Parent = topo.TabletAlias{}
// Update the tablet first, since that is canonical.
err = topo.UpdateTablet(ts, tablet)
if err != nil {
return err
}
// Remove any pending actions. Presumably forcing a scrap means you don't
// want the agent doing anything and the machine requires manual attention.
if force {
err := ts.PurgeTabletActions(tabletAlias, ActionNodeCanBePurged)
if err != nil {
log.Warningf("purge actions failed: %v", err)
}
}
if wasAssigned {
err = ts.DeleteReplicationPath(tablet.Keyspace, tablet.Shard, replicationPath)
if err != nil {
switch err {
case topo.ErrNoNode:
log.V(6).Infof("no replication path: %v", replicationPath)
err = nil
case topo.ErrNotEmpty:
// If you are forcing the scrapping of a master, you can't update the
// replication graph yet, since other nodes are still under the impression
// they are slaved to this tablet.
// If the node was not empty, we can't do anything about it - the replication
// graph needs to be fixed by reparenting. If the action was forced, assume
// the user knows best and squelch the error.
if tablet.Parent.Uid == topo.NO_TABLET && force {
err = nil
}
}
if err != nil {
log.Warningf("remove replication path failed: %v %v", replicationPath, err)
}
}
}
// run a hook for final cleanup, only in non-force mode.
// (force mode executes on the vtctl side, not on the vttablet side)
if !force {
hk := hook.NewSimpleHook("postflight_scrap")
configureTabletHook(hk, tablet.Alias())
if hookErr := hk.ExecuteOptional(); hookErr != nil {
// we don't want to return an error, the server
// is already in bad shape probably.
log.Warningf("Scrap: postflight_scrap failed: %v", hookErr)
}
}
return nil
}
func CheckActions(t *testing.T, ts topo.Server) {
cell := getLocalCell(t, ts)
tablet := &topo.Tablet{
Alias: topo.TabletAlias{Cell: cell, Uid: 1},
Hostname: "localhost",
IPAddr: "10.11.12.13",
Portmap: map[string]int{
"vt": 3333,
"mysql": 3334,
},
Parent: topo.TabletAlias{},
Keyspace: "test_keyspace",
Type: topo.TYPE_MASTER,
State: topo.STATE_READ_WRITE,
KeyRange: newKeyRange("-10"),
}
if err := ts.CreateTablet(tablet); err != nil {
t.Fatalf("CreateTablet: %v", err)
}
tabletAlias := topo.TabletAlias{Cell: cell, Uid: 1}
actionPath, err := ts.WriteTabletAction(tabletAlias, "contents1")
if err != nil {
t.Fatalf("WriteTabletAction: %v", err)
}
interrupted := make(chan struct{}, 1)
if _, err := ts.WaitForTabletAction(actionPath, time.Second/100, interrupted); err != topo.ErrTimeout {
t.Errorf("WaitForTabletAction returned %v", err)
}
go func() {
time.Sleep(time.Second / 10)
close(interrupted)
}()
if _, err := ts.WaitForTabletAction(actionPath, time.Second*5, interrupted); err != topo.ErrInterrupted {
t.Errorf("WaitForTabletAction returned %v", err)
}
// wait for the result in one thread
wg1 := sync.WaitGroup{}
wg1.Add(1)
go func() {
defer wg1.Done()
interrupted := make(chan struct{}, 1)
var err error
for i := 0; i <= 30; i++ {
var result string
result, err = ts.WaitForTabletAction(actionPath, time.Second, interrupted)
if err == topo.ErrTimeout {
// we waited for one second, didn't see the
// result, try again up to 30 seconds.
t.Logf("WaitForTabletAction timed out at try %v/30", i)
continue
}
if err != nil {
t.Errorf("WaitForTabletAction returned: %v", err)
}
if result != "contents3" {
t.Errorf("WaitForTabletAction returned bad result: %v", result)
}
return
}
t.Errorf("WaitForTabletAction timed out: %v", err)
}()
// process the action in another thread
done := make(chan struct{}, 1)
wg2 := sync.WaitGroup{}
wg2.Add(1)
go ts.ActionEventLoop(tabletAlias, func(ap, data string) error {
// the actionPath sent back to the action processor
// is the exact one we have in normal cases,
// but for the tee, we add extra information.
if ap != actionPath && !strings.HasSuffix(ap, actionPath) {
t.Errorf("Bad action path: %v", ap)
}
if data != "contents1" {
t.Errorf("Bad data: %v", data)
}
ta, contents, version, err := ts.ReadTabletActionPath(ap)
if err != nil {
t.Errorf("Error from ReadTabletActionPath: %v", err)
}
if contents != data {
t.Errorf("Bad contents: %v", contents)
}
if ta != tabletAlias {
t.Errorf("Bad tablet alias: %v", ta)
}
if err := ts.UpdateTabletAction(ap, "contents2", version); err != nil {
t.Errorf("UpdateTabletAction failed: %v", err)
}
if err := ts.StoreTabletActionResponse(ap, "contents3"); err != nil {
t.Errorf("StoreTabletActionResponse failed: %v", err)
}
if err := ts.UnblockTabletAction(ap); err != nil {
t.Errorf("UnblockTabletAction failed: %v", err)
}
wg2.Done()
return nil
}, done)
// first wait for the processing to be done, then close the
// action loop, then wait for the response to be received.
wg2.Wait()
close(done)
wg1.Wait()
// start an action, and try to purge all actions, to test
// PurgeTabletActions
actionPath, err = ts.WriteTabletAction(tabletAlias, "contents2")
if err != nil {
t.Fatalf("WriteTabletAction(contents2): %v", err)
}
if err := ts.PurgeTabletActions(tabletAlias, func(data string) bool {
return true
}); err != nil {
t.Fatalf("PurgeTabletActions(contents2) failed: %v", err)
}
if _, _, _, err := ts.ReadTabletActionPath(actionPath); err != topo.ErrNoNode {
t.Fatalf("ReadTabletActionPath(contents2) should have failed with ErrNoNode: %v", err)
}
}