func (n *InnerNormal) SetScale(data *mat64.Dense) error {
rows, dim := data.Dims()
if rows < 2 {
return errors.New("scale: less than two inputs")
}
means := make([]float64, dim)
stds := make([]float64, dim)
for i := 0; i < dim; i++ {
// Filter out the extremes
r := data.Col(nil, i)
if len(r) != rows {
panic("bad lengths")
}
sort.Float64s(r)
lowerIdx := int(math.Floor(float64(rows) * n.LowerQuantile))
upperIdx := int(math.Ceil(float64(rows) * n.UpperQuantile))
trimmed := r[lowerIdx:upperIdx]
mean, std := stat.MeanStdDev(trimmed, nil)
//std := stat.StdDev(trimmed, mean, nil)
means[i] = mean
stds[i] = std
}
n.Mu = means
n.Sigma = stds
fmt.Println(n.Mu, n.Sigma)
n.Dim = dim
n.Scaled = true
return nil
}
func randmatstat(t int) (float64, float64) {
n := 5
v := make([]float64, t)
w := make([]float64, t)
ad := make([]float64, n*n)
bd := make([]float64, n*n)
cd := make([]float64, n*n)
dd := make([]float64, n*n)
P := mat64.NewDense(n, 4*n, nil)
Q := mat64.NewDense(2*n, 2*n, nil)
pTmp := mat64.NewDense(4*n, 4*n, nil)
qTmp := mat64.NewDense(2*n, 2*n, nil)
for i := 0; i < t; i++ {
for i := range ad {
ad[i] = rnd.NormFloat64()
bd[i] = rnd.NormFloat64()
cd[i] = rnd.NormFloat64()
dd[i] = rnd.NormFloat64()
}
a := mat64.NewDense(n, n, ad)
b := mat64.NewDense(n, n, bd)
c := mat64.NewDense(n, n, cd)
d := mat64.NewDense(n, n, dd)
P.Copy(a)
P.View(0, n, n, n).(*mat64.Dense).Copy(b)
P.View(0, 2*n, n, n).(*mat64.Dense).Copy(c)
P.View(0, 3*n, n, n).(*mat64.Dense).Copy(d)
Q.Copy(a)
Q.View(0, n, n, n).(*mat64.Dense).Copy(b)
Q.View(n, 0, n, n).(*mat64.Dense).Copy(c)
Q.View(n, n, n, n).(*mat64.Dense).Copy(d)
pTmp.Mul(P.T(), P)
pTmp.Pow(pTmp, 4)
qTmp.Mul(Q.T(), Q)
qTmp.Pow(qTmp, 4)
v[i] = mat64.Trace(pTmp)
w[i] = mat64.Trace(qTmp)
}
mv, stdv := stat.MeanStdDev(v, nil)
mw, stdw := stat.MeanStdDev(v, nil)
return stdv / mv, stdw / mw
}
func main() {
log.SetPrefix("benchsum: ")
log.SetFlags(0)
flag.Parse()
if flag.NArg() == 0 {
b, err := ioutil.ReadAll(os.Stdin)
if err != nil {
log.Fatal(err)
}
read(b)
} else {
for _, p := range flag.Args() {
b, err := ioutil.ReadFile(p)
if err != nil {
log.Fatal(err)
}
read(b)
}
}
w := tabwriter.NewWriter(os.Stdout, 8, 8, 1, ' ', 0)
fmt.Fprint(w, "name\tmean\tstdev\tstat ...\n")
var allVals []benchValsPair
for name, statmap := range benchVals {
var stats []statPair
for st, results := range statmap {
stats = append(stats, statPair{stat: st, results: results})
}
sort.Sort(byStat(stats))
allVals = append(allVals, benchValsPair{name: name, stats: stats})
}
sort.Sort(byName(allVals))
for _, p := range allVals {
name := p.name
for _, p2 := range p.stats {
st := p2.stat
results := p2.results
μ, σ := stat.MeanStdDev(results.vals, results.niter)
if *noPercent {
fmt.Fprintf(w, "%s\t%.2e\t±%.2e\t%s\n", name, μ, σ, st)
} else {
σPercent := 100 * σ / μ
fmt.Fprintf(w, "%s\t%.2e\t±%.0f%%\t%s\n", name, μ, σPercent, st)
}
}
}
w.Flush()
}
func main() {
suppressWarnings := flag.Bool("suppress", false, "suppress warnings in the input data")
doPlot := flag.Bool("plot", false, "plot a histogram and the standard normal distribution")
flag.Parse()
c := csv.NewReader(bufio.NewReader(os.Stdin))
headers, err := c.Read()
if err != nil {
log.Fatalf("Could not read header: %v", err)
}
stats := make([]*columnStatistic, 0, len(headers))
for _, name := range headers {
stats = append(stats, &columnStatistic{
name: name,
})
}
line := 0
for {
line++
columns, err := c.Read()
if err != nil {
if err == io.EOF {
// Print statistics
w := tabwriter.NewWriter(os.Stdout, 5, 8, 1, '\t', tabwriter.AlignRight)
fmt.Fprint(w, "\n\tmin\tmax\tmean\tstddev\n")
for _, s := range stats {
mean, stddev := stat.MeanStdDev(s.obsv, nil)
fmt.Fprintf(w, "%s\t%f\t%f\t%f\t%f\n", s.name, s.min, s.max, mean, stddev)
}
w.Flush()
if *doPlot {
for _, s := range stats {
mean, stddev := stat.MeanStdDev(s.obsv, nil)
p, err := plot.New()
if err != nil {
panic(err)
}
p.X.Min = s.min - 1.5*stddev
p.X.Max = s.max + 1.5*stddev
p.Title.Text = fmt.Sprintf("Histogram for %s", s.name)
h, err := plotter.NewHist(plotter.Values(s.obsv), 16)
if err != nil {
panic(err)
}
h.Normalize(1)
p.Add(h)
norm := plotter.NewFunction(func(x float64) float64 {
return 1.0 / (stddev * math.Sqrt(2*math.Pi)) * math.Exp(-((x-mean)*(x-mean))/(2*stddev*stddev))
})
norm.Samples = int(p.X.Max-p.X.Min) + 100
norm.Color = color.RGBA{R: 255, A: 255}
norm.Width = vg.Points(2)
p.Add(norm)
// Save the plot to a PNG file.
if err := p.Save(4*vg.Inch, 4*vg.Inch, fmt.Sprintf("histogram-%s.png", s.name)); err != nil {
log.Fatal(err)
}
}
}
return
}
if !*suppressWarnings {
fmt.Fprintf(os.Stderr, "Could not parse line (line %d, '%s'): %v\n", line, columns, err)
}
continue
}
for i, data := range columns {
s := stats[i]
if s.typ == unknown {
// Determine the column's data type
_, err := strconv.ParseFloat(data, 64)
switch {
case err == nil:
// Is numeric
s.typ = numeric
case err != nil:
s.typ = text
}
}
var f float64
switch s.typ {
case numeric:
f, err = strconv.ParseFloat(data, 64)
if err != nil {
if !*suppressWarnings {
fmt.Fprintf(os.Stderr, "Could not parse numeric value (line %d, '%s'): %v\n", line, data, err)
}
continue
}
if math.IsNaN(f) {
if !*suppressWarnings {
fmt.Fprintf(os.Stderr, "Could not parse numeric value (line %d, '%s'): is not a number\n", line, data)
}
continue
}
if math.IsInf(f, 0) {
if !*suppressWarnings {
fmt.Fprintf(os.Stderr, "Could not parse numeric value (line %d, '%s'): infinity\n", line, data)
}
continue
}
case text:
// we take the text length as the according metric
f = float64(len(data))
default:
panic("unknown data type")
}
s.obsv = append(s.obsv, f)
if f < s.min {
s.min = f
}
if f > s.max {
s.max = f
}
}
}
}
func RunMaster(conf Config) {
// cria o emissor que envia as probabilidades para toda a rede na porta A
sender, _ := zmq.NewSocket(zmq.PUSH)
defer sender.Close()
sender.Bind("tcp://*:" + conf.Dist.PortA)
// cria o receptor que recebe os individuos de toda a rede na porta B
receiver, _ := zmq.NewSocket(zmq.PULL)
defer receiver.Close()
receiver.Bind("tcp://*:" + conf.Dist.PortB)
// gerar (ou ler -> TODO) as probabilidades iniciais
r, _ := rules.Create(conf.CA.InitStates, conf.CA.TransStates, conf.CA.HasJoker, conf.CA.R)
p := NewProbs(r.Prm)
// a população do DistEDA será de apenas vencedores dos torneios locais,
// realizados nos slaves. Portanto, população é igual população/n_torneio
var pop []Individual
pop = make([]Individual, conf.EDA.Population/conf.EDA.Tournament)
popFitness := make([]float64, conf.EDA.Population/conf.EDA.Tournament)
popQ3 := make([]float64, conf.EDA.Population/conf.EDA.Tournament)
// cria um arquivo de log onde serão salvas as estatisticas por geração, como
// média e variância do Q3
fstat, err := os.Create("log")
if err != nil {
panic(err)
}
defer fstat.Close()
// os slaves demoram um pouco para inicializar pois precisam acessar o DB e
// carregar os dados. O master precisa esperar os slaves estarem prontos. Por
// enquanto, o sinal de inicio é dado manualmente (TODO -> pensar numa forma
// automática)
fmt.Print("Press Enter when the workers are ready: ")
var line string
fmt.Scanln(&line)
fmt.Println("Sending tasks to workers...")
// Inicio do processamento
fmt.Println("RUNNING MASTER")
// para cada geracao
for g := 0; g < conf.EDA.Generations; g++ {
fmt.Println("GERACAO", g)
// atualizar as probabilidades de acordo com a populacao dessa geracao
if g != 0 {
// TODO refazer a funcao para trabalhar com toda a populacao. Acho que
// esta feito. Confirmar!!!
p.AdjustProbs(pop)
}
// Criar as probabilidades para serem enviadas no formato JSON.
// Será enviado o ID (PID) = hash da probabilidade, o número da geração e as
// probabilidades
tmp, _ := json.Marshal(p.probs)
pid := adler32.Checksum(tmp)
prob := &Probabilities{PID: pid, Generation: g, Data: p.probs}
b, _ := json.Marshal(prob)
// Para cada individuo que precisará retornar deve ser emitida uma
// probabilidade. Uma goroutine fica emitindo probabilidades que vão sendo
// capturados pelos slaves que após o torneio, devolvem o vencedor
go func(b *[]byte) {
for i := 0; i < len(pop); i++ {
sender.Send(string(*b), 0)
}
}(&b)
// Capta os individuos vencedores gerados pelos slaves
for i := 0; i < len(pop); {
m, err := receiver.Recv(0)
if err == nil {
json.Unmarshal([]byte(m), &pop[i])
// Checa pelo ID da probabilidade se o individuo vencedor que chegou foi
// gerado pela última probabilidade que foi emitida
if prob.PID == pop[i].PID {
// fmt.Printf("Individuo id: %d rid: %d g: %d, score: %f\n", g*len(pop)+i, pop[i].PID, pop[i].Generation, pop[i].Fitness)
popFitness[i] = pop[i].Fitness
popQ3[i] = pop[i].Q3
i++
} else {
fmt.Println(prob.PID, pop[i].PID)
}
} else {
fmt.Println(err)
}
}
// IMPORTANTE
// TODO criar um mecanismo para contornar falhas nos nós
// imprimir e as estatisticas// salva as probabilidades a cada geração
err := ioutil.WriteFile(conf.EDA.OutputProbs+"_g"+strconv.Itoa(g), []byte(p.String()), 0644)
if err != nil {
fmt.Println("Erro gravar as probabilidades")
fmt.Println(p)
}
// imprimir e as estatisticas
meanFit, stdFit := stat.MeanStdDev(popFitness, nil)
meanQ3, stdQ3 := stat.MeanStdDev(popQ3, nil)
fstat.WriteString(fmt.Sprintf("G: %d, Mean: %.5f, StdDev: %.5f, Mean Q3: %.5f, StdDev Q3: %.5f, \n", g, meanFit, stdFit, meanQ3, stdQ3))
fmt.Printf("G: %d, Mean: %.5f, StdDev: %.5f, Mean Q3: %.5f, StdDev Q3: %.5f, \n", g, meanFit, stdFit, meanQ3, stdQ3)
}
// salvar a melhor regra
}