func TestCompareSimilarData(t *testing.T) {
a := tinystat.Summarize([]float64{1, 2, 3, 4})
b := tinystat.Summarize([]float64{1, 2, 3, 4})
d := tinystat.Compare(a, b, 80)
if v, want := d.Delta, 0.0; v != want {
t.Errorf("Delta was %v, but expected %v", v, want)
}
if v, want := d.Error, 1.314311166777796; v != want {
t.Errorf("Error was %v, but expected %v", v, want)
}
if v, want := d.RelDelta, 0.0; v != want {
t.Errorf("RelDelta was %v, but expected %v", v, want)
}
if v, want := d.RelError, 0.5257244667111184; v != want {
t.Errorf("RelError was %v, but expected %v", v, want)
}
if v, want := d.StdDev, 1.2909944487358056; v != want {
t.Errorf("StdDev was %v, but expected %v", v, want)
}
if v, want := d.Significant(), false; v != want {
t.Errorf("Significance was %v, but expected %v", v, want)
}
}
func TestCompareDifferentData(t *testing.T) {
a := tinystat.Summarize([]float64{1, 2, 3, 4})
b := tinystat.Summarize([]float64{10, 20, 30, 40})
d := tinystat.Compare(a, b, 80)
if v, want := d.Delta, 22.5; v != want {
t.Errorf("Delta was %v, but expected %v", v, want)
}
if v, want := d.Error, 9.33993571056027; v != want {
t.Errorf("Error was %v, but expected %v", v, want)
}
if v, want := d.RelDelta, 9.0; v != want {
t.Errorf("RelDelta was %v, but expected %v", v, want)
}
if v, want := d.RelError, 3.7359742842241084; v != want {
t.Errorf("RelError was %v, but expected %v", v, want)
}
if v, want := d.StdDev, 9.17423929634859; v != want {
t.Errorf("StdDev was %v, but expected %v", v, want)
}
if v, want := d.Significant(), true; v != want {
t.Errorf("Significance was %v, but expected %v", v, want)
}
}
func main() {
usage := `tinystat helps you conduct experiments.
Usage:
tinystat [options] <control> [<experiment>...]
Arguments:
<control> Input file containing control measurements.
<experiment> Input file containing experimental measurements.
Options:
-h --help Display this usage information.
-c --confidence=<pct> Specify confidence level for analysis. [default: 95]
-C --column=<num> The column to analyze. [default: 0]
-d --delimiter=(t|s|c) Tab, space, or comma delimited data. [default: t]
-n --no-chart Don't display the box chart.
-W --width=<chars> Width of box chart, in characters. [default: 74]
-H --height=<chars> Height of box chart, in characters. [default: 20]
`
args, err := docopt.Parse(usage, nil, true, "", true)
if err != nil {
panic(err)
}
confidence, err := strconv.ParseFloat(args["--confidence"].(string), 64)
if err != nil {
panic(err)
}
column, err := strconv.Atoi(args["--column"].(string))
if err != nil {
panic(err)
}
width, err := strconv.Atoi(args["--width"].(string))
if err != nil {
panic(err)
}
height, err := strconv.Atoi(args["--height"].(string))
if err != nil {
panic(err)
}
var delimiter rune
switch d := args["--delimiter"]; d {
case "t":
delimiter = '\t'
case "c":
delimiter = ','
case "s":
delimiter = ' '
default:
panic(fmt.Errorf("bad delimiter: %#v", d))
}
controlFilename := args["<control>"].(string)
experimentFilenames := args["<experiment>"].([]string)
// read the data
controlData, err := readFile(controlFilename, column, delimiter)
if err != nil {
panic(err)
}
experimentData := make(map[string][]float64)
for _, filename := range experimentFilenames {
expData, err := readFile(filename, column, delimiter)
if err != nil {
panic(err)
}
experimentData[filename] = expData
}
// chart the data
if args["--no-chart"] != true {
c := chart.BoxChart{}
c.XRange.Fixed(-1, float64(len(experimentFilenames))+1, 1)
c.XRange.Category = make([]string, len(experimentFilenames)+1)
c.XRange.Category[0] = path.Base(controlFilename)
for i, file := range experimentFilenames {
c.XRange.Category[i+1] = path.Base(file)
}
c.AddSet(0, controlData, true)
for i, filename := range experimentFilenames {
c.AddSet(float64(i+1), experimentData[filename], true)
}
txt := txtg.New(width, height)
c.Plot(txt)
fmt.Println(txt)
}
// compare the data
if len(experimentFilenames) > 0 {
control := tinystat.Summarize(controlData)
table := new(tabwriter.Writer)
table.Init(os.Stdout, 2, 0, 2, ' ', 0)
fmt.Fprintln(table, "Experiment\tResults\t")
for _, filename := range experimentFilenames {
experimental := tinystat.Summarize(experimentData[filename])
d := tinystat.Compare(control, experimental, confidence)
if d.Significant() {
fmt.Fprintf(table,
"%s\tDifference at %v%% confidence!\t\n",
filename, confidence)
fmt.Fprintf(table,
"\t %v +/- %v\t\n",
d.Delta, d.Error)
fmt.Fprintf(table,
"\t %v%% +/- %v%%\t\n",
d.RelDelta*100, d.RelError*100)
fmt.Fprintf(table,
"\t (Student's t, pooled s = %v)\t\n",
d.StdDev)
} else {
fmt.Fprintf(table,
"%s\tNo difference proven at %v%% confidence.\t\n",
filename, confidence)
}
}
_ = table.Flush()
}
}