Example #1
0
File: scale.go Project: reggo/reggo
func (n *InnerNormal) SetScale(data *mat64.Dense) error {
	rows, dim := data.Dims()
	if rows < 2 {
		return errors.New("scale: less than two inputs")
	}
	means := make([]float64, dim)
	stds := make([]float64, dim)
	for i := 0; i < dim; i++ {
		// Filter out the extremes
		r := data.Col(nil, i)
		if len(r) != rows {
			panic("bad lengths")
		}
		sort.Float64s(r)

		lowerIdx := int(math.Floor(float64(rows) * n.LowerQuantile))
		upperIdx := int(math.Ceil(float64(rows) * n.UpperQuantile))

		trimmed := r[lowerIdx:upperIdx]

		mean, std := stat.MeanStdDev(trimmed, nil)
		//std := stat.StdDev(trimmed, mean, nil)
		means[i] = mean
		stds[i] = std
	}
	n.Mu = means
	n.Sigma = stds
	fmt.Println(n.Mu, n.Sigma)
	n.Dim = dim
	n.Scaled = true
	return nil
}
Example #2
0
func randmatstat(t int) (float64, float64) {
	n := 5
	v := make([]float64, t)
	w := make([]float64, t)
	ad := make([]float64, n*n)
	bd := make([]float64, n*n)
	cd := make([]float64, n*n)
	dd := make([]float64, n*n)
	P := mat64.NewDense(n, 4*n, nil)
	Q := mat64.NewDense(2*n, 2*n, nil)
	pTmp := mat64.NewDense(4*n, 4*n, nil)
	qTmp := mat64.NewDense(2*n, 2*n, nil)
	for i := 0; i < t; i++ {
		for i := range ad {
			ad[i] = rnd.NormFloat64()
			bd[i] = rnd.NormFloat64()
			cd[i] = rnd.NormFloat64()
			dd[i] = rnd.NormFloat64()
		}
		a := mat64.NewDense(n, n, ad)
		b := mat64.NewDense(n, n, bd)
		c := mat64.NewDense(n, n, cd)
		d := mat64.NewDense(n, n, dd)
		P.Copy(a)
		P.View(0, n, n, n).(*mat64.Dense).Copy(b)
		P.View(0, 2*n, n, n).(*mat64.Dense).Copy(c)
		P.View(0, 3*n, n, n).(*mat64.Dense).Copy(d)

		Q.Copy(a)
		Q.View(0, n, n, n).(*mat64.Dense).Copy(b)
		Q.View(n, 0, n, n).(*mat64.Dense).Copy(c)
		Q.View(n, n, n, n).(*mat64.Dense).Copy(d)

		pTmp.Mul(P.T(), P)
		pTmp.Pow(pTmp, 4)

		qTmp.Mul(Q.T(), Q)
		qTmp.Pow(qTmp, 4)

		v[i] = mat64.Trace(pTmp)
		w[i] = mat64.Trace(qTmp)
	}
	mv, stdv := stat.MeanStdDev(v, nil)
	mw, stdw := stat.MeanStdDev(v, nil)
	return stdv / mv, stdw / mw
}
Example #3
0
File: main.go Project: uluyol/tools
func main() {
	log.SetPrefix("benchsum: ")
	log.SetFlags(0)
	flag.Parse()
	if flag.NArg() == 0 {
		b, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatal(err)
		}
		read(b)
	} else {
		for _, p := range flag.Args() {
			b, err := ioutil.ReadFile(p)
			if err != nil {
				log.Fatal(err)
			}
			read(b)
		}
	}

	w := tabwriter.NewWriter(os.Stdout, 8, 8, 1, ' ', 0)
	fmt.Fprint(w, "name\tmean\tstdev\tstat ...\n")
	var allVals []benchValsPair
	for name, statmap := range benchVals {
		var stats []statPair
		for st, results := range statmap {
			stats = append(stats, statPair{stat: st, results: results})
		}
		sort.Sort(byStat(stats))
		allVals = append(allVals, benchValsPair{name: name, stats: stats})
	}
	sort.Sort(byName(allVals))
	for _, p := range allVals {
		name := p.name
		for _, p2 := range p.stats {
			st := p2.stat
			results := p2.results
			μ, σ := stat.MeanStdDev(results.vals, results.niter)
			if *noPercent {
				fmt.Fprintf(w, "%s\t%.2e\t±%.2e\t%s\n", name, μ, σ, st)
			} else {
				σPercent := 100 * σ / μ
				fmt.Fprintf(w, "%s\t%.2e\t±%.0f%%\t%s\n", name, μ, σPercent, st)
			}
		}
	}
	w.Flush()
}
Example #4
0
func main() {
	suppressWarnings := flag.Bool("suppress", false, "suppress warnings in the input data")
	doPlot := flag.Bool("plot", false, "plot a histogram and the standard normal distribution")
	flag.Parse()

	c := csv.NewReader(bufio.NewReader(os.Stdin))

	headers, err := c.Read()
	if err != nil {
		log.Fatalf("Could not read header: %v", err)
	}

	stats := make([]*columnStatistic, 0, len(headers))
	for _, name := range headers {
		stats = append(stats, &columnStatistic{
			name: name,
		})
	}

	line := 0
	for {
		line++
		columns, err := c.Read()
		if err != nil {
			if err == io.EOF {
				// Print statistics
				w := tabwriter.NewWriter(os.Stdout, 5, 8, 1, '\t', tabwriter.AlignRight)
				fmt.Fprint(w, "\n\tmin\tmax\tmean\tstddev\n")

				for _, s := range stats {
					mean, stddev := stat.MeanStdDev(s.obsv, nil)
					fmt.Fprintf(w, "%s\t%f\t%f\t%f\t%f\n", s.name, s.min, s.max, mean, stddev)
				}

				w.Flush()

				if *doPlot {
					for _, s := range stats {
						mean, stddev := stat.MeanStdDev(s.obsv, nil)

						p, err := plot.New()
						if err != nil {
							panic(err)
						}
						p.X.Min = s.min - 1.5*stddev
						p.X.Max = s.max + 1.5*stddev
						p.Title.Text = fmt.Sprintf("Histogram for %s", s.name)

						h, err := plotter.NewHist(plotter.Values(s.obsv), 16)
						if err != nil {
							panic(err)
						}
						h.Normalize(1)
						p.Add(h)

						norm := plotter.NewFunction(func(x float64) float64 {
							return 1.0 / (stddev * math.Sqrt(2*math.Pi)) * math.Exp(-((x-mean)*(x-mean))/(2*stddev*stddev))
						})
						norm.Samples = int(p.X.Max-p.X.Min) + 100
						norm.Color = color.RGBA{R: 255, A: 255}
						norm.Width = vg.Points(2)
						p.Add(norm)

						// Save the plot to a PNG file.
						if err := p.Save(4*vg.Inch, 4*vg.Inch, fmt.Sprintf("histogram-%s.png", s.name)); err != nil {
							log.Fatal(err)
						}
					}
				}

				return
			}
			if !*suppressWarnings {
				fmt.Fprintf(os.Stderr, "Could not parse line (line %d, '%s'): %v\n", line, columns, err)
			}
			continue
		}

		for i, data := range columns {
			s := stats[i]

			if s.typ == unknown {
				// Determine the column's data type
				_, err := strconv.ParseFloat(data, 64)
				switch {
				case err == nil:
					// Is numeric
					s.typ = numeric
				case err != nil:
					s.typ = text
				}
			}

			var f float64
			switch s.typ {
			case numeric:
				f, err = strconv.ParseFloat(data, 64)
				if err != nil {
					if !*suppressWarnings {
						fmt.Fprintf(os.Stderr, "Could not parse numeric value (line %d, '%s'): %v\n", line, data, err)
					}
					continue
				}
				if math.IsNaN(f) {
					if !*suppressWarnings {
						fmt.Fprintf(os.Stderr, "Could not parse numeric value (line %d, '%s'): is not a number\n", line, data)
					}
					continue
				}
				if math.IsInf(f, 0) {
					if !*suppressWarnings {
						fmt.Fprintf(os.Stderr, "Could not parse numeric value (line %d, '%s'): infinity\n", line, data)
					}
					continue
				}
			case text:
				// we take the text length as the according metric
				f = float64(len(data))
			default:
				panic("unknown data type")
			}

			s.obsv = append(s.obsv, f)
			if f < s.min {
				s.min = f
			}
			if f > s.max {
				s.max = f
			}
		}
	}
}
Example #5
0
func RunMaster(conf Config) {

	// cria o emissor que envia as probabilidades para toda a rede na porta A
	sender, _ := zmq.NewSocket(zmq.PUSH)
	defer sender.Close()
	sender.Bind("tcp://*:" + conf.Dist.PortA)

	// cria o receptor que recebe os individuos de toda a rede na porta B
	receiver, _ := zmq.NewSocket(zmq.PULL)
	defer receiver.Close()
	receiver.Bind("tcp://*:" + conf.Dist.PortB)

	// gerar (ou ler -> TODO) as probabilidades iniciais
	r, _ := rules.Create(conf.CA.InitStates, conf.CA.TransStates, conf.CA.HasJoker, conf.CA.R)
	p := NewProbs(r.Prm)

	// a população do DistEDA será de apenas vencedores dos torneios locais,
	// realizados nos slaves. Portanto, população é igual população/n_torneio
	var pop []Individual
	pop = make([]Individual, conf.EDA.Population/conf.EDA.Tournament)

	popFitness := make([]float64, conf.EDA.Population/conf.EDA.Tournament)
	popQ3 := make([]float64, conf.EDA.Population/conf.EDA.Tournament)

	// cria um arquivo de log onde serão salvas as estatisticas por geração, como
	// média e variância do Q3
	fstat, err := os.Create("log")
	if err != nil {
		panic(err)
	}
	defer fstat.Close()

	// os slaves demoram um pouco para inicializar pois precisam acessar o DB e
	// carregar os dados. O master precisa esperar os slaves estarem prontos. Por
	// enquanto, o sinal de inicio é dado manualmente (TODO -> pensar numa forma
	// automática)
	fmt.Print("Press Enter when the workers are ready: ")
	var line string
	fmt.Scanln(&line)
	fmt.Println("Sending tasks to workers...")

	// Inicio do processamento
	fmt.Println("RUNNING MASTER")
	// para cada geracao
	for g := 0; g < conf.EDA.Generations; g++ {
		fmt.Println("GERACAO", g)
		// atualizar as probabilidades de acordo com a populacao dessa geracao
		if g != 0 {
			// TODO refazer a funcao para trabalhar com toda a populacao. Acho que
			// esta feito. Confirmar!!!
			p.AdjustProbs(pop)
		}

		// Criar as probabilidades para serem enviadas no formato JSON.
		// Será enviado o ID (PID) = hash da probabilidade, o número da geração e as
		// probabilidades
		tmp, _ := json.Marshal(p.probs)
		pid := adler32.Checksum(tmp)
		prob := &Probabilities{PID: pid, Generation: g, Data: p.probs}
		b, _ := json.Marshal(prob)

		// Para cada individuo que precisará retornar deve ser emitida uma
		// probabilidade. Uma goroutine fica emitindo probabilidades que vão sendo
		// capturados pelos slaves que após o torneio, devolvem o vencedor
		go func(b *[]byte) {
			for i := 0; i < len(pop); i++ {
				sender.Send(string(*b), 0)
			}
		}(&b)

		// Capta os individuos vencedores gerados pelos slaves
		for i := 0; i < len(pop); {
			m, err := receiver.Recv(0)
			if err == nil {
				json.Unmarshal([]byte(m), &pop[i])
				// Checa pelo ID da probabilidade se o individuo vencedor que chegou foi
				// gerado pela última probabilidade que foi emitida
				if prob.PID == pop[i].PID {
					// fmt.Printf("Individuo id: %d rid: %d g: %d, score: %f\n", g*len(pop)+i, pop[i].PID, pop[i].Generation, pop[i].Fitness)
					popFitness[i] = pop[i].Fitness
					popQ3[i] = pop[i].Q3
					i++
				} else {
					fmt.Println(prob.PID, pop[i].PID)
				}

			} else {
				fmt.Println(err)
			}
		}

		// IMPORTANTE
		// TODO criar um mecanismo para contornar falhas nos nós

		// imprimir e as estatisticas// salva as probabilidades a cada geração
		err := ioutil.WriteFile(conf.EDA.OutputProbs+"_g"+strconv.Itoa(g), []byte(p.String()), 0644)
		if err != nil {
			fmt.Println("Erro gravar as probabilidades")
			fmt.Println(p)
		}

		//  imprimir e as estatisticas
		meanFit, stdFit := stat.MeanStdDev(popFitness, nil)
		meanQ3, stdQ3 := stat.MeanStdDev(popQ3, nil)
		fstat.WriteString(fmt.Sprintf("G: %d, Mean: %.5f, StdDev: %.5f, Mean Q3: %.5f, StdDev Q3: %.5f, \n", g, meanFit, stdFit, meanQ3, stdQ3))
		fmt.Printf("G: %d, Mean: %.5f, StdDev: %.5f, Mean Q3: %.5f, StdDev Q3: %.5f, \n", g, meanFit, stdFit, meanQ3, stdQ3)

	}
	// salvar a melhor regra

}