Ejemplo n.º 1
0
func TestLNSmote(t *testing.T) {
	// Read sample dataset.
	dataset := tabula.Claset{}
	_, e := dsv.SimpleRead(fcfg, &dataset)
	if nil != e {
		t.Fatal(e)
	}

	fmt.Println("[lnsmote_test] Total samples:", dataset.GetNRow())

	// Write original samples.
	writer, e := dsv.NewWriter("")

	if nil != e {
		t.Fatal(e)
	}

	e = writer.OpenOutput("phoneme_lnsmote.csv")
	if e != nil {
		t.Fatal(e)
	}

	sep := dsv.DefSeparator
	_, e = writer.WriteRawRows(dataset.GetRows(), &sep)
	if e != nil {
		t.Fatal(e)
	}

	// Initialize LN-SMOTE.
	lnsmoteRun := lnsmote.New(100, 5, 5, "1", "lnsmote.outliers")

	e = lnsmoteRun.Resampling(&dataset)

	fmt.Println("[lnsmote_test] # synthetic:", lnsmoteRun.Synthetics.Len())

	sep = dsv.DefSeparator
	_, e = writer.WriteRawRows(lnsmoteRun.Synthetics.GetRows(), &sep)
	if e != nil {
		t.Fatal(e)
	}

	e = writer.Close()
	if e != nil {
		t.Fatal(e)
	}
}
Ejemplo n.º 2
0
/*
InitReadWriter initialize reader and writer.
*/
func InitReadWriter(finput string) (reader *wvcgen.Reader,
	writer *dsv.Writer) {
	reader, e := wvcgen.NewReader(finput)

	if e != nil {
		panic(e)
	}

	writer, e = dsv.NewWriter(finput)

	if e != nil {
		panic(e)
	}

	initReader(reader)

	return reader, writer
}
Ejemplo n.º 3
0
// writeOutliers will save the `outliers` to file specified by
// `OutliersFile`.
func (in *Runtime) writeOutliers() (e error) {
	writer, e := dsv.NewWriter("")
	if nil != e {
		return
	}

	e = writer.OpenOutput(in.OutliersFile)
	if e != nil {
		return
	}

	sep := dsv.DefSeparator
	_, e = writer.WriteRawRows(&in.outliers, &sep)
	if e != nil {
		return
	}

	return writer.Close()
}
Ejemplo n.º 4
0
//
// WriteSynthetics will write synthetic samples in resampling module `ri` into
// `file`.
//
func WriteSynthetics(ri Interface, file string) (e error) {
	writer, e := dsv.NewWriter("")
	if nil != e {
		return
	}

	e = writer.OpenOutput(file)
	if e != nil {
		return
	}

	sep := dsv.DefSeparator
	_, e = writer.WriteRawDataset(ri.GetSynthetics(), &sep)
	if e != nil {
		return
	}

	return writer.Close()
}
Ejemplo n.º 5
0
// runMerge will append original dataset to synthetic file.
func runMerge(smote *smote.Runtime, dataset *tabula.Claset) (e error) {
	writer, e := dsv.NewWriter("")
	if e != nil {
		return
	}

	e = writer.ReopenOutput(smote.SyntheticFile)
	if e != nil {
		return
	}

	sep := dsv.DefSeparator
	n, e := writer.WriteRawDataset(dataset, &sep)
	if e != nil {
		return
	}

	if DEBUG >= 1 {
		fmt.Println("[smote] # appended:", n)
	}

	return writer.Close()
}
Ejemplo n.º 6
0
func TestWriteRawColumns(t *testing.T) {
	dataset := tabula.NewDataset(tabula.DatasetModeColumns, nil, nil)

	populateWithColumns(t, dataset)

	writer, e := dsv.NewWriter("")
	if e != nil {
		t.Fatal(e)
	}

	outfile := "testdata/writerawcolumns.out"
	expfile := "testdata/writeraw.exp"

	e = writer.OpenOutput(outfile)

	_, e = writer.WriteRawDataset(dataset, nil)

	if e != nil {
		t.Fatal(e)
	}

	assertFile(t, outfile, expfile, true)
}
Ejemplo n.º 7
0
func TestMergeRows(t *testing.T) {
	reader1, e := dsv.NewReader("testdata/config.dsv", nil)
	if nil != e {
		t.Fatal(e)
	}

	reader2, e := dsv.NewReader("testdata/config_skip.dsv", nil)
	if nil != e {
		t.Fatal(e)
	}

	reader1.SetMaxRows(-1)
	reader2.SetMaxRows(-1)

	_, e = dsv.Read(reader1)
	if e != io.EOF {
		t.Fatal(e)
	}

	_, e = dsv.Read(reader2)
	if e != io.EOF {
		t.Fatal(e)
	}

	e = reader1.Close()
	if e != nil {
		t.Fatal(e)
	}

	e = reader2.Close()
	if e != nil {
		t.Fatal(e)
	}

	reader1.MergeRows(reader2)

	// write merged reader
	writer, e := dsv.NewWriter("")
	if e != nil {
		t.Fatal(e)
	}

	outfile := "testdata/output_merge_rows.dat"

	e = writer.OpenOutput(outfile)

	if e != nil {
		t.Fatal(e)
	}

	sep := "\t"
	ds1 := reader1.GetDataset().(tabula.DatasetInterface)
	_, e = writer.WriteRawDataset(ds1, &sep)

	if e != nil {
		t.Fatal(e)
	}

	e = writer.Close()
	if e != nil {
		t.Fatal(e)
	}

	expfile := "testdata/expected_merge_rows.dat"
	assertFile(t, outfile, expfile, true)
}