func TestTransposeToColumns(t *testing.T) {
reader, e := dsv.NewReader("testdata/config_skip.dsv", nil)
if nil != e {
t.Fatal(e)
}
reader.SetMaxRows(-1)
_, e = dsv.Read(reader)
if e != io.EOF {
t.Fatal(e)
}
ds := reader.GetDataset().(tabula.DatasetInterface)
ds.TransposeToColumns()
exp := fmt.Sprint(expSkipColumnsAll)
columns := ds.GetDataAsColumns()
got := fmt.Sprint(*columns)
assert(t, exp, got, true)
e = reader.Close()
if e != nil {
t.Fatal(e)
}
}
func TestReaderToColumns(t *testing.T) {
reader := &dsv.Reader{}
e := dsv.ConfigParse(reader, []byte(jsonSample[4]))
if nil != e {
t.Fatal(e)
}
e = reader.Init("", nil)
if nil != e {
t.Fatal(e)
}
reader.SetDatasetMode(dsv.DatasetModeCOLUMNS)
var n, i int
for {
n, e = dsv.Read(reader)
if n > 0 {
ds := reader.GetDataset().(tabula.DatasetInterface)
ds.TransposeToRows()
r := fmt.Sprint(ds.GetData())
assert(t, expectation[i], r, true)
i++
} else if e == io.EOF {
// EOF
break
}
}
}
func TestMainOutput(t *testing.T) {
rw, e := dsv.New(fMainTestDsv, nil)
if e != nil {
t.Fatal(e)
}
for {
n, e := dsv.Read(&rw.Reader)
if e == io.EOF {
_, e = rw.Write(&rw.Reader)
break
}
if n > 0 {
_, e = rw.Write(&rw.Reader)
if e != nil {
t.Fatal(e)
}
}
}
e = rw.Writer.Flush()
if e != nil {
t.Fatal(e)
}
assertFile(t, rw.GetInput(), rw.GetOutput(), true)
}
/*
doReadWrite test reading and writing the DSV data.
*/
func doReadWrite(t *testing.T, dsvReader *dsv.Reader, dsvWriter *dsv.Writer,
expectation []string, check bool) {
var got string
i := 0
for {
n, e := dsv.Read(dsvReader)
if e == io.EOF {
_, e = dsvWriter.Write(dsvReader)
if e != nil {
t.Fatal(e)
}
break
}
if e != nil {
continue
}
if n > 0 {
if check {
ds := dsvReader.GetDataset().(tabula.DatasetInterface)
data := ds.GetData()
switch data.(type) {
case *tabula.Rows:
rows := data.(*tabula.Rows)
got = fmt.Sprint(*rows)
case *tabula.Columns:
cols := data.(*tabula.Columns)
got = fmt.Sprint(*cols)
case *tabula.Matrix:
matrix := data.(*tabula.Matrix)
got = fmt.Sprint(*matrix)
default:
fmt.Println("data type unknown")
}
assert(t, expectation[i], got, true)
i++
}
_, e = dsvWriter.Write(dsvReader)
if e != nil {
t.Fatal(e)
}
}
}
e := dsvWriter.Flush()
if e != nil {
t.Fatal(e)
}
}
func TestSplitRowsByValue(t *testing.T) {
reader, e := dsv.NewReader("testdata/config.dsv", nil)
if nil != e {
t.Fatal(e)
}
reader.SetMaxRows(256)
_, e = dsv.Read(reader)
if e != nil && e != io.EOF {
t.Fatal(e)
}
ds := reader.GetDataset().(tabula.DatasetInterface)
splitL, splitR, e := tabula.SplitRowsByValue(ds, 0, 6)
if e != nil {
t.Fatal(e)
}
// test left split
exp := ""
for x := 0; x < 4; x++ {
exp += expectation[x]
}
got := fmt.Sprint(splitL.GetDataAsRows())
assert(t, exp, got, true)
// test right split
exp = ""
for x := 4; x < len(expectation); x++ {
exp += expectation[x]
}
got = fmt.Sprint(splitR.GetDataAsRows())
assert(t, exp, got, true)
e = reader.Close()
if e != nil {
t.Fatal(e)
}
}
/*
Generate start computing the feature values which has been defined in
input file `finput`.
If `featureName` is not empty, it will be added to list of feature that will
be computed.
*/
func Generate(featureName, finput string) {
var ftrValues *tabula.Dataset
var e error
var n int
reader, writer := InitReadWriter(finput)
if featureName != "" {
ftrMd := dsv.Metadata{
Name: featureName,
}
writer.AddMetadata(ftrMd)
}
for {
n, e = dsv.Read(reader)
if n <= 0 {
break
}
ftrValues = computeFeatures(reader, writer)
_, ewrite := writer.WriteColumns(ftrValues.Columns, nil)
if ewrite != nil {
panic(e)
}
if e == io.EOF {
break
}
}
e = reader.Close()
if e != nil {
panic(e)
}
e = writer.Close()
if e != nil {
panic(e)
}
}
func TestSortColumnsByIndex(t *testing.T) {
reader, e := dsv.NewReader("testdata/config_skip.dsv", nil)
if nil != e {
t.Fatal(e)
}
reader.SetMaxRows(-1)
_, e = dsv.Read(reader)
if e != io.EOF {
t.Fatal(e)
}
// reverse the data
var idxReverse []int
var expReverse []string
for x := len(expSkip) - 1; x >= 0; x-- {
idxReverse = append(idxReverse, x)
expReverse = append(expReverse, expSkip[x])
}
ds := reader.GetDataset().(tabula.DatasetInterface)
tabula.SortColumnsByIndex(ds, idxReverse)
exp := strings.Join(expReverse, "")
got := fmt.Sprint(ds.GetDataAsRows())
assert(t, exp, got, true)
exp = "[" + strings.Join(expSkipColumnsAllRev, " ") + "]"
columns := ds.GetDataAsColumns()
got = fmt.Sprint(*columns)
assert(t, exp, got, true)
e = reader.Close()
if e != nil {
t.Fatal(e)
}
}
/*
doRead test reading the DSV data.
*/
func doRead(t *testing.T, dsvReader *dsv.Reader, exp []string) {
i := 0
n := 0
e := error(nil)
for {
n, e = dsv.Read(dsvReader)
if n > 0 {
r := fmt.Sprint(dsvReader.
GetDataset().(tabula.DatasetInterface).
GetDataAsRows())
assert(t, exp[i], r, true)
i++
} else if e == io.EOF {
// EOF
break
}
}
}
func TestMergeRows(t *testing.T) {
reader1, e := dsv.NewReader("testdata/config.dsv", nil)
if nil != e {
t.Fatal(e)
}
reader2, e := dsv.NewReader("testdata/config_skip.dsv", nil)
if nil != e {
t.Fatal(e)
}
reader1.SetMaxRows(-1)
reader2.SetMaxRows(-1)
_, e = dsv.Read(reader1)
if e != io.EOF {
t.Fatal(e)
}
_, e = dsv.Read(reader2)
if e != io.EOF {
t.Fatal(e)
}
e = reader1.Close()
if e != nil {
t.Fatal(e)
}
e = reader2.Close()
if e != nil {
t.Fatal(e)
}
reader1.MergeRows(reader2)
// write merged reader
writer, e := dsv.NewWriter("")
if e != nil {
t.Fatal(e)
}
outfile := "testdata/output_merge_rows.dat"
e = writer.OpenOutput(outfile)
if e != nil {
t.Fatal(e)
}
sep := "\t"
ds1 := reader1.GetDataset().(tabula.DatasetInterface)
_, e = writer.WriteRawDataset(ds1, &sep)
if e != nil {
t.Fatal(e)
}
e = writer.Close()
if e != nil {
t.Fatal(e)
}
expfile := "testdata/expected_merge_rows.dat"
assertFile(t, outfile, expfile, true)
}