// Returns min and range
func FindNormalizedVectors(in <-chan *matrix.Matrix) (*matrix.Matrix, *matrix.Matrix) {
p := <-in
max := make([]float64, numFeatures)
min := make([]float64, numFeatures)
copy(max, p.Vals)
copy(min, p.Vals)
for p = range in {
for i, v := range p.Vals {
if max[i] < v {
max[i] = v
}
if min[i] > v {
min[i] = v
}
}
}
maxVec := matrix.FromSlice(max, numFeatures, 1)
minVec := matrix.FromSlice(min, numFeatures, 1)
r, _ := maxVec.Sub(minVec)
return minVec, r
}
func ProcessRecords(input <-chan *matrix.Matrix, n *NN, thresholdMat *matrix.Matrix) <-chan *matrix.Matrix {
out := make(chan *matrix.Matrix)
evaluateChannelIn := make(chan *matrix.Matrix)
evaluateChannelOut := EvaluateNN(n, evaluateChannelIn)
go func() {
for m := range input {
evaluateChannelIn <- m
evOut := <-evaluateChannelOut
diff, _ := m.Sub(evOut)
predictionErr := diff.Power(2.0)
wellnessSlice := make([]float64, numFeatures)
for i, _ := range wellnessSlice {
if predictionErr.Vals[i] <= thresholdMat.Vals[i] {
wellnessSlice[i] = 1.0
} else {
wellnessSlice[i] = 0.0
}
}
out <- matrix.FromSlice(wellnessSlice, numFeatures, 1)
}
close(out)
}()
return out
}
func UnrollParams(vals []*matrix.Matrix) *matrix.Matrix {
unrolled := make([]float64, 0)
for _, v := range vals {
unrolled = append(unrolled, v.Values()...)
}
return matrix.FromSlice(unrolled, 1, len(unrolled))
}
func ReshapeParams(val *matrix.Matrix, layerSizes []int) []*matrix.Matrix {
vals := val.Values()
thetas := make(nn.Parameters, 0, len(layerSizes)-1)
offset := 0
for i := 0; i < len(layerSizes)-1; i++ {
thetas = append(thetas, matrix.FromSlice(vals[offset:offset+(layerSizes[i+1]*layerSizes[i]+1)], layerSizes[i+1], layerSizes[i]+1))
offset += layerSizes[i+1]*layerSizes[i] + 1
}
return thetas
}
func MakeFeatureVector(container *PageContainer) *matrix.Matrix {
features := []float64{
container.AvgSentenceLen,
container.LinkDensity,
float64(container.NumLinks),
float64(container.NumExternalLinks),
float64(container.NumHeadings),
float64(container.NumRefs),
float64(container.NumCategories),
float64(container.NumSentences),
float64(container.NumWords),
//float64(len(container.Unigrams)),
//float64(len(container.Bigrams)),
//float64(len(container.Trigrams)),
}
return matrix.FromSlice(features, len(features), 1)
}
// Returns min, max of the deviation of the predictionError
func FindPredictionErrDeviation(input <-chan *PageContainer, n *NN, min, r *matrix.Matrix) (*matrix.Matrix, *matrix.Matrix) {
evaluateChannelIn := make(chan *PageContainer)
vectorizerChannelIn := make(chan *PageContainer)
evaluateChannelOut := EvaluateNN(n, Normalizer(Vectorizer(evaluateChannelIn), min, r))
vectorizerChannelOut := Normalizer(Vectorizer(vectorizerChannelIn), min, r)
/*devMax := make([]float64, numFeatures)
for i, _ := range devMax {
devMax[i] = 0.0
}
for p := range input {
evaluateChannelIn <- p
vectorizerChannelIn <- p
out := <-evaluateChannelOut
in := <-vectorizerChannelOut
diff, _ := in.Sub(out)
predictionErr := diff.Power(2.0)
for i, v := range predictionErr.Vals {
if devMax[i] < v {
devMax[i] = v
}
}
}
close(evaluateChannelIn)
close(vectorizerChannelIn)
return matrix.FromSlice(devMax, numFeatures, 1)*/
vals := make([][]float64, numFeatures)
for i, _ := range vals {
vals[i] = make([]float64, 0, 1000)
}
for p := range input {
evaluateChannelIn <- p
vectorizerChannelIn <- p
out := <-evaluateChannelOut
in := <-vectorizerChannelOut
diff, _ := in.Sub(out)
predictionErr := diff.Power(2.0)
for i, v := range predictionErr.Vals {
vals[i] = append(vals[i], v)
}
}
devs := make([]float64, numFeatures)
means := make([]float64, numFeatures)
for i, _ := range vals {
devs[i], means[i] = calcStdDev(vals[i])
}
return matrix.FromSlice(devs, numFeatures, 1), matrix.FromSlice(means, numFeatures, 1)
}