Ejemplo n.º 1
0
// Returns min and range
func FindNormalizedVectors(in <-chan *matrix.Matrix) (*matrix.Matrix, *matrix.Matrix) {
	p := <-in
	max := make([]float64, numFeatures)
	min := make([]float64, numFeatures)

	copy(max, p.Vals)
	copy(min, p.Vals)

	for p = range in {
		for i, v := range p.Vals {
			if max[i] < v {
				max[i] = v
			}

			if min[i] > v {
				min[i] = v
			}
		}
	}

	maxVec := matrix.FromSlice(max, numFeatures, 1)
	minVec := matrix.FromSlice(min, numFeatures, 1)

	r, _ := maxVec.Sub(minVec)
	return minVec, r
}
Ejemplo n.º 2
0
func ProcessRecords(input <-chan *matrix.Matrix, n *NN, thresholdMat *matrix.Matrix) <-chan *matrix.Matrix {
	out := make(chan *matrix.Matrix)

	evaluateChannelIn := make(chan *matrix.Matrix)

	evaluateChannelOut := EvaluateNN(n, evaluateChannelIn)

	go func() {
		for m := range input {
			evaluateChannelIn <- m
			evOut := <-evaluateChannelOut

			diff, _ := m.Sub(evOut)
			predictionErr := diff.Power(2.0)

			wellnessSlice := make([]float64, numFeatures)

			for i, _ := range wellnessSlice {
				if predictionErr.Vals[i] <= thresholdMat.Vals[i] {
					wellnessSlice[i] = 1.0
				} else {
					wellnessSlice[i] = 0.0
				}
			}

			out <- matrix.FromSlice(wellnessSlice, numFeatures, 1)
		}
		close(out)
	}()

	return out
}
Ejemplo n.º 3
0
func UnrollParams(vals []*matrix.Matrix) *matrix.Matrix {
	unrolled := make([]float64, 0)

	for _, v := range vals {
		unrolled = append(unrolled, v.Values()...)
	}

	return matrix.FromSlice(unrolled, 1, len(unrolled))
}
Ejemplo n.º 4
0
func ReshapeParams(val *matrix.Matrix, layerSizes []int) []*matrix.Matrix {
	vals := val.Values()
	thetas := make(nn.Parameters, 0, len(layerSizes)-1)

	offset := 0
	for i := 0; i < len(layerSizes)-1; i++ {
		thetas = append(thetas, matrix.FromSlice(vals[offset:offset+(layerSizes[i+1]*layerSizes[i]+1)], layerSizes[i+1], layerSizes[i]+1))
		offset += layerSizes[i+1]*layerSizes[i] + 1
	}

	return thetas
}
Ejemplo n.º 5
0
func MakeFeatureVector(container *PageContainer) *matrix.Matrix {
	features := []float64{
		container.AvgSentenceLen,
		container.LinkDensity,
		float64(container.NumLinks),
		float64(container.NumExternalLinks),
		float64(container.NumHeadings),
		float64(container.NumRefs),
		float64(container.NumCategories),
		float64(container.NumSentences),
		float64(container.NumWords),

		//float64(len(container.Unigrams)),
		//float64(len(container.Bigrams)),
		//float64(len(container.Trigrams)),
	}

	return matrix.FromSlice(features, len(features), 1)
}
Ejemplo n.º 6
0
// Returns min, max of the deviation of the predictionError
func FindPredictionErrDeviation(input <-chan *PageContainer, n *NN, min, r *matrix.Matrix) (*matrix.Matrix, *matrix.Matrix) {
	evaluateChannelIn := make(chan *PageContainer)
	vectorizerChannelIn := make(chan *PageContainer)

	evaluateChannelOut := EvaluateNN(n, Normalizer(Vectorizer(evaluateChannelIn), min, r))
	vectorizerChannelOut := Normalizer(Vectorizer(vectorizerChannelIn), min, r)

	/*devMax := make([]float64, numFeatures)
	for i, _ := range devMax {
		devMax[i] = 0.0
	}

	for p := range input {
		evaluateChannelIn <- p
		vectorizerChannelIn <- p
		out := <-evaluateChannelOut
		in := <-vectorizerChannelOut

		diff, _ := in.Sub(out)
		predictionErr := diff.Power(2.0)

		for i, v := range predictionErr.Vals {
			if devMax[i] < v {
				devMax[i] = v
			}
		}

	}

	close(evaluateChannelIn)
	close(vectorizerChannelIn)

	return matrix.FromSlice(devMax, numFeatures, 1)*/

	vals := make([][]float64, numFeatures)
	for i, _ := range vals {
		vals[i] = make([]float64, 0, 1000)
	}

	for p := range input {
		evaluateChannelIn <- p
		vectorizerChannelIn <- p
		out := <-evaluateChannelOut
		in := <-vectorizerChannelOut

		diff, _ := in.Sub(out)
		predictionErr := diff.Power(2.0)

		for i, v := range predictionErr.Vals {
			vals[i] = append(vals[i], v)
		}

	}

	devs := make([]float64, numFeatures)
	means := make([]float64, numFeatures)

	for i, _ := range vals {
		devs[i], means[i] = calcStdDev(vals[i])
	}

	return matrix.FromSlice(devs, numFeatures, 1), matrix.FromSlice(means, numFeatures, 1)
}