func TestEscapeFieldNames(t *testing.T) {
assert := assert.New(t)
ds := datas.NewDatabase(chunks.NewMemoryStore())
dataString := "1,2\n"
r := NewCSVReader(bytes.NewBufferString(dataString), ',')
headers := []string{"A A", "B"}
kinds := KindSlice{types.NumberKind, types.NumberKind}
l, _ := ReadToList(r, "test", headers, kinds, ds)
assert.Equal(uint64(1), l.Len())
assert.Equal(types.Number(1), l.Get(0).(types.Struct).Get(types.EscapeStructField("A A")))
r = NewCSVReader(bytes.NewBufferString(dataString), ',')
m := ReadToMap(r, headers, 1, kinds, ds)
assert.Equal(uint64(1), l.Len())
assert.Equal(types.Number(1), m.Get(types.Number(2)).(types.Struct).Get(types.EscapeStructField("A A")))
}
// NomsValueFromDecodedJSON takes a generic Go interface{} and recursively
// tries to resolve the types within so that it can build up and return
// a Noms Value with the same structure.
//
// Currently, the only types supported are the Go versions of legal JSON types:
// Primitives:
// - float64
// - bool
// - string
// - nil
//
// Composites:
// - []interface{}
// - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
switch o := o.(type) {
case string:
return types.String(o)
case bool:
return types.Bool(o)
case float64:
return types.Number(o)
case nil:
return nil
case []interface{}:
items := make([]types.Value, 0, len(o))
for _, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
items = append(items, nv)
}
}
return types.NewList(items...)
case map[string]interface{}:
var v types.Value
if useStruct {
fields := make(map[string]types.Value, len(o))
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
k := types.EscapeStructField(k)
fields[k] = nv
}
}
v = types.NewStruct("", fields)
} else {
kv := make([]types.Value, 0, len(o)*2)
for k, v := range o {
nv := NomsValueFromDecodedJSON(v, useStruct)
if nv != nil {
kv = append(kv, types.String(k), nv)
}
}
v = types.NewMap(kv...)
}
return v
default:
d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())
}
return nil
}
func ReadToMap(r *csv.Reader, headers_raw []string, pkIdx int, kinds KindSlice, vrw types.ValueReadWriter) (m types.Map) {
headers := make([]string, 0, len(headers_raw)-1)
for i, h := range headers_raw {
if i != pkIdx {
headers = append(headers, types.EscapeStructField(h))
}
}
var pkKind types.NomsKind
if len(kinds) == 0 {
pkKind = types.StringKind
} else {
pkKind = kinds[pkIdx]
kinds = append(kinds[:pkIdx], kinds[pkIdx+1:]...)
}
t := MakeStructTypeFromHeaders(headers, "", kinds)
kindMap := make(map[string]types.NomsKind, len(headers))
t.Desc.(types.StructDesc).IterFields(func(name string, t *types.Type) {
kindMap[name] = t.Kind()
})
m = types.NewMap()
fields := map[string]types.Value{}
var pk types.Value
for {
row, err := r.Read()
if err == io.EOF {
break
} else if err != nil {
panic(err)
}
fieldIndex := 0
for x, v := range row {
if x == pkIdx {
pk = StringToType(v, pkKind)
} else if fieldIndex < len(headers) {
name := headers[fieldIndex]
fields[name] = StringToType(v, kindMap[name])
fieldIndex++
}
}
m = m.Set(pk, types.NewStructWithType(t, fields))
}
return
}
// MakeStructTypeFromHeaders creates a struct type from the headers using |kinds| as the type of each field. If |kinds| is empty, default to strings.
func MakeStructTypeFromHeaders(headers []string, structName string, kinds KindSlice) (typ *types.Type, fieldOrder []int, kindMap []types.NomsKind) {
useStringType := len(kinds) == 0
d.Chk.True(useStringType || len(headers) == len(kinds))
fieldMap := make(types.TypeMap, len(headers))
origOrder := make(map[string]int, len(headers))
fieldNames := make(sort.StringSlice, len(headers))
for i, key := range headers {
fn := types.EscapeStructField(key)
origOrder[fn] = i
kind := types.StringKind
if !useStringType {
kind = kinds[i]
}
_, ok := fieldMap[fn]
d.PanicIfTrue(ok, `Duplicate field name "%s"`, key)
fieldMap[fn] = types.MakePrimitiveType(kind)
fieldNames[i] = fn
}
sort.Sort(fieldNames)
kindMap = make([]types.NomsKind, len(fieldMap))
fieldOrder = make([]int, len(fieldMap))
fieldTypes := make([]*types.Type, len(fieldMap))
for i, fn := range fieldNames {
typ := fieldMap[fn]
fieldTypes[i] = typ
kindMap[i] = typ.Kind()
fieldOrder[origOrder[fn]] = i
}
typ = types.MakeStructType(structName, fieldNames, fieldTypes)
return
}
// Read takes a CSV reader and reads it into a typed List of structs. Each row gets read into a struct named structName, described by headers. If the original data contained headers it is expected that the input reader has already read those and are pointing at the first data row.
// If kinds is non-empty, it will be used to type the fields in the generated structs; otherwise, they will be left as string-fields.
// In addition to the list, Read returns the typeRef for the structs in the list, and last the typeDef of the structs.
func ReadToList(r *csv.Reader, structName string, headers_raw []string, kinds KindSlice, vrw types.ValueReadWriter) (l types.List, t *types.Type) {
headers := make([]string, len(headers_raw))
for i, h := range headers_raw {
headers[i] = types.EscapeStructField(h)
}
t = MakeStructTypeFromHeaders(headers, structName, kinds)
valueChan := make(chan types.Value, 128) // TODO: Make this a function param?
listChan := types.NewStreamingList(vrw, valueChan)
kindMap := make(map[string]types.NomsKind, len(headers))
t.Desc.(types.StructDesc).IterFields(func(name string, t *types.Type) {
kindMap[name] = t.Kind()
})
for {
row, err := r.Read()
if err == io.EOF {
close(valueChan)
break
} else if err != nil {
panic(err)
}
fields := make(map[string]types.Value)
for i, v := range row {
if i < len(headers) {
name := headers[i]
fields[name] = StringToType(v, kindMap[name])
}
}
valueChan <- types.NewStructWithType(t, fields)
}
return <-listChan, t
}