// writeDescriptor takes a Table or Database descriptor and writes it
// if needed, incrementing the descriptor counter.
func (p *planner) writeDescriptor(plainKey descriptorKey, descriptor descriptorProto, ifNotExists bool) error {
key := plainKey.Key()
// Check whether key exists.
gr, err := p.txn.Get(key)
if err != nil {
return err
}
if gr.Exists() {
if ifNotExists {
// Noop.
return nil
}
// Key exists, but we don't want it to: error out.
return fmt.Errorf("%s %q already exists", descriptor.TypeName(), plainKey.Name())
}
// Increment unique descriptor counter.
if ir, err := p.txn.Inc(keys.DescIDGenerator, 1); err == nil {
descriptor.SetID(structured.ID(ir.ValueInt() - 1))
} else {
return err
}
// TODO(pmattis): The error currently returned below is likely going to be
// difficult to interpret.
// TODO(pmattis): Need to handle if-not-exists here as well.
descKey := structured.MakeDescMetadataKey(descriptor.GetID())
b := client.Batch{}
b.CPut(key, descKey, nil)
b.CPut(descKey, descriptor, nil)
return p.txn.Run(&b)
}
func decodeIndexKey(desc *structured.TableDescriptor,
index structured.IndexDescriptor, vals map[string]parser.Datum, key []byte) ([]byte, error) {
if !bytes.HasPrefix(key, keys.TableDataPrefix) {
return nil, fmt.Errorf("%s: invalid key prefix: %q", desc.Name, key)
}
key = bytes.TrimPrefix(key, keys.TableDataPrefix)
var tableID uint64
key, tableID = encoding.DecodeUvarint(key)
if structured.ID(tableID) != desc.ID {
return nil, fmt.Errorf("%s: unexpected table ID: %d != %d", desc.Name, desc.ID, tableID)
}
var indexID uint64
key, indexID = encoding.DecodeUvarint(key)
if structured.ID(indexID) != index.ID {
return nil, fmt.Errorf("%s: unexpected index ID: %d != %d", desc.Name, index.ID, indexID)
}
for _, id := range index.ColumnIDs {
col, err := desc.FindColumnByID(id)
if err != nil {
return nil, err
}
switch col.Type.Kind {
case structured.ColumnType_BIT, structured.ColumnType_INT:
var i int64
key, i = encoding.DecodeVarint(key)
vals[col.Name] = parser.DInt(i)
case structured.ColumnType_FLOAT:
var f float64
key, f = encoding.DecodeNumericFloat(key)
vals[col.Name] = parser.DFloat(f)
case structured.ColumnType_CHAR, structured.ColumnType_TEXT,
structured.ColumnType_BLOB:
var r []byte
key, r = encoding.DecodeBytes(key, nil)
vals[col.Name] = parser.DString(r)
default:
return nil, util.Errorf("TODO(pmattis): decoded index key: %s", col.Type.Kind)
}
}
return key, nil
}
// decodeIndexKey decodes the values that are a part of the specified index
// key. Vals is a slice returned from makeIndexKeyVals. The remaining bytes in
// the index key are returned which will either be an encoded column ID for the
// primary key index, the primary key suffix for non-unique secondary indexes
// or unique secondary indexes containing NULL or empty.
func decodeIndexKey(desc *structured.TableDescriptor,
index structured.IndexDescriptor, vals []parser.Datum, key []byte) ([]byte, error) {
if !bytes.HasPrefix(key, keys.TableDataPrefix) {
return nil, fmt.Errorf("%s: invalid key prefix: %q", desc.Name, key)
}
key = bytes.TrimPrefix(key, keys.TableDataPrefix)
var tableID uint64
key, tableID = encoding.DecodeUvarint(key)
if structured.ID(tableID) != desc.ID {
return nil, fmt.Errorf("%s: unexpected table ID: %d != %d", desc.Name, desc.ID, tableID)
}
var indexID uint64
key, indexID = encoding.DecodeUvarint(key)
if structured.IndexID(indexID) != index.ID {
return nil, fmt.Errorf("%s: unexpected index ID: %d != %d", desc.Name, index.ID, indexID)
}
for j := range vals {
switch vals[j].(type) {
case parser.DInt:
var i int64
key, i = encoding.DecodeVarint(key)
vals[j] = parser.DInt(i)
case parser.DFloat:
var f float64
key, f = encoding.DecodeNumericFloat(key)
vals[j] = parser.DFloat(f)
case parser.DString:
var r []byte
key, r = encoding.DecodeBytes(key, nil)
vals[j] = parser.DString(r)
default:
return nil, util.Errorf("TODO(pmattis): decoded index key: %s", vals[j].Type())
}
}
return key, nil
}
func (n *scanNode) Next() bool {
if n.err != nil {
return false
}
if n.kvs == nil {
// Initialize our key/values.
if n.desc == nil {
// No table to read from, pretend there is a single empty row.
n.kvs = []client.KeyValue{}
n.primaryKey = []byte{}
} else {
// Retrieve all of the keys that start with our index key prefix.
startKey := proto.Key(structured.MakeIndexKeyPrefix(n.desc.ID, n.desc.PrimaryIndex.ID))
endKey := startKey.PrefixEnd()
// TODO(pmattis): Currently we retrieve all of the key/value pairs for
// the table. We could enhance this code so that it retrieves the
// key/value pairs in chunks.
n.kvs, n.err = n.db.Scan(startKey, endKey, 0)
if n.err != nil {
return false
}
}
}
// All of the columns for a particular row will be grouped together. We loop
// over the key/value pairs and decode the key to extract the columns encoded
// within the key and the column ID. We use the column ID to lookup the
// column and decode the value. All of these values go into a map keyed by
// column name. When the index key changes we output a row containing the
// current values.
for {
var kv client.KeyValue
if n.kvIndex < len(n.kvs) {
kv = n.kvs[n.kvIndex]
}
if n.primaryKey != nil &&
(n.kvIndex == len(n.kvs) || !bytes.HasPrefix(kv.Key, n.primaryKey)) {
// The current key belongs to a new row. Output the current row.
n.primaryKey = nil
var output bool
output, n.err = n.filterRow()
if n.err != nil {
return false
}
if output {
if n.err = n.renderRow(); n.err != nil {
return false
}
return true
}
}
if n.kvIndex == len(n.kvs) {
return false
}
if n.primaryKey == nil {
// This is the first key for the row, reset our vals map.
n.vals = valMap{}
}
var remaining []byte
remaining, n.err = decodeIndexKey(n.desc, n.desc.PrimaryIndex, n.vals, kv.Key)
if n.err != nil {
return false
}
n.primaryKey = []byte(kv.Key[:len(kv.Key)-len(remaining)])
// TODO(pmattis): We should avoid looking up the column name by column ID
// on every key. One possibility is that we could rewrite col-name
// references in expressions to refer to <table-id, col-id> tuples.
_, colID := encoding.DecodeUvarint(remaining)
var col *structured.ColumnDescriptor
col, n.err = n.desc.FindColumnByID(structured.ID(colID))
if n.err != nil {
return false
}
n.vals[col.Name] = unmarshalValue(*col, kv)
if log.V(2) {
log.Infof("Scan %q -> %v", kv.Key, n.vals[col.Name])
}
n.kvIndex++
}
}
func TestDatabaseDescriptor(t *testing.T) {
defer leaktest.AfterTest(t)
s, sqlDB, kvDB := setup(t)
defer cleanup(s, sqlDB)
// The first `MaxReservedDescID` (plus 0) are set aside.
expectedCounter := int64(structured.MaxReservedDescID + 1)
// Test values before creating the database.
// descriptor ID counter.
if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else if actual := ir.ValueInt(); actual != expectedCounter {
t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual)
}
// Database name.
nameKey := structured.MakeNameMetadataKey(structured.RootNamespaceID, "test")
if gr, err := kvDB.Get(nameKey); err != nil {
t.Fatal(err)
} else if gr.Exists() {
t.Fatal("expected non-existing key")
}
// Write some junk that is going to interfere with table creation.
dbDescKey := structured.MakeDescMetadataKey(structured.ID(expectedCounter))
if err := kvDB.CPut(dbDescKey, "foo", nil); err != nil {
t.Fatal(err)
}
// Table creation should fail, and nothing should have been written.
if _, err := sqlDB.Exec(`CREATE DATABASE test`); !testutils.IsError(err, "unexpected value") {
t.Fatalf("unexpected error %s", err)
}
if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else if actual := ir.ValueInt(); actual != expectedCounter {
t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual)
}
if kvs, err := kvDB.Scan(keys.NameMetadataPrefix, keys.NameMetadataPrefix.PrefixEnd(), 0); err != nil {
t.Fatal(err)
} else {
if a, e := len(kvs), 0; a != e {
t.Fatalf("expected %d keys to have been written, found %d keys", e, a)
}
}
// Remove the junk; allow table creation to proceed.
if err := kvDB.Del(dbDescKey); err != nil {
t.Fatal(err)
}
if _, err := sqlDB.Exec(`CREATE DATABASE test`); err != nil {
t.Fatal(err)
}
expectedCounter++
// Check keys again.
// descriptor ID counter.
if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else if actual := ir.ValueInt(); actual != expectedCounter {
t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual)
}
// Database name.
if gr, err := kvDB.Get(nameKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
// database descriptor.
if gr, err := kvDB.Get(dbDescKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
// Now try to create it again. We should fail, but not increment the counter.
if _, err := sqlDB.Exec(`CREATE DATABASE test`); err == nil {
t.Fatal("failure expected")
}
// Check keys again.
// descriptor ID counter.
if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else if actual := ir.ValueInt(); actual != expectedCounter {
t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, actual)
}
// Database name.
if gr, err := kvDB.Get(nameKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
// database descriptor.
if gr, err := kvDB.Get(dbDescKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
}
func TestDatabaseDescriptor(t *testing.T) {
defer leaktest.AfterTest(t)
s, sqlDB, kvDB := setup(t)
defer cleanup(s, sqlDB)
// The first `MaxReservedDescID` (plus 0) are set aside.
expectedCounter := int64(structured.MaxReservedDescID + 1)
// Test values before creating the database.
// descriptor ID counter.
if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else if ir.ValueInt() != expectedCounter {
t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, ir.ValueInt())
}
// Database name.
nameKey := structured.MakeNameMetadataKey(structured.RootNamespaceID, "test")
if gr, err := kvDB.Get(nameKey); err != nil {
t.Fatal(err)
} else if gr.Exists() {
t.Fatal("expected non-existing key")
}
if _, err := sqlDB.Exec(`CREATE DATABASE test`); err != nil {
t.Fatal(err)
}
expectedCounter++
// Check keys again.
// descriptor ID counter.
if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else if ir.ValueInt() != expectedCounter {
t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, ir.ValueInt())
}
// Database name.
if gr, err := kvDB.Get(nameKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
// database descriptor.
descKey := structured.MakeDescMetadataKey(structured.ID(expectedCounter - 1))
if gr, err := kvDB.Get(descKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
// Now try to create it again. We should fail, but not increment the counter.
if _, err := sqlDB.Exec(`CREATE DATABASE test`); err == nil {
t.Fatal("failure expected")
}
// Check keys again.
// descriptor ID counter.
if ir, err := kvDB.Get(keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else if ir.ValueInt() != expectedCounter {
t.Fatalf("expected descriptor ID == %d, got %d", expectedCounter, ir.ValueInt())
}
// Database name.
if gr, err := kvDB.Get(nameKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
// database descriptor.
if gr, err := kvDB.Get(descKey); err != nil {
t.Fatal(err)
} else if !gr.Exists() {
t.Fatal("key is missing")
}
}
func TestDropTable(t *testing.T) {
defer leaktest.AfterTest(t)
s, sqlDB, kvDB := setup(t)
defer cleanup(s, sqlDB)
if _, err := sqlDB.Exec("CREATE DATABASE t"); err != nil {
t.Fatal(err)
}
if _, err := sqlDB.Exec("CREATE TABLE t.kv (k CHAR PRIMARY KEY, v CHAR)"); err != nil {
t.Fatal(err)
}
if _, err := sqlDB.Exec(`INSERT INTO t.kv VALUES ('c', 'e'), ('a', 'c'),('b', 'd')`); err != nil {
t.Fatal(err)
}
// The first `MaxReservedDescID` (plus 0) are set aside.
expectedDBCounter := uint32(structured.MaxReservedDescID + 1)
nameKey := structured.MakeNameMetadataKey(structured.ID(expectedDBCounter), "kv")
gr, err := kvDB.Get(nameKey)
if err != nil {
t.Fatal(err)
}
if !gr.Exists() {
t.Fatalf("TableDescriptor %q does not exist", nameKey)
}
descKey := gr.ValueBytes()
desc := structured.TableDescriptor{}
if err := kvDB.GetProto(descKey, &desc); err != nil {
t.Fatal(err)
}
var tablePrefix []byte
tablePrefix = append(tablePrefix, keys.TableDataPrefix...)
tablePrefix = encoding.EncodeUvarint(tablePrefix, uint64(desc.ID))
tableStartKey := proto.Key(tablePrefix)
tableEndKey := tableStartKey.PrefixEnd()
if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil {
t.Fatal(err)
} else if l := 6; len(kvs) != l {
t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs))
}
if _, err := sqlDB.Exec("DROP TABLE t.kv"); err != nil {
t.Fatal(err)
}
if kvs, err := kvDB.Scan(tableStartKey, tableEndKey, 0); err != nil {
t.Fatal(err)
} else if l := 0; len(kvs) != l {
t.Fatalf("expected %d key value pairs, but got %d", l, len(kvs))
}
if gr, err := kvDB.Get(descKey); err != nil {
t.Fatal(err)
} else if gr.Exists() {
t.Fatalf("table descriptor still exists after the table is dropped")
}
if gr, err := kvDB.Get(nameKey); err != nil {
t.Fatal(err)
} else if gr.Exists() {
t.Fatalf("table namekey still exists after the table is dropped")
}
}