func (wm *writtenMemory) backwardWErase() {
n := wm.N
m := len(wm.TopVal) / n
mgrad := make([]float64, n*m)
mGradG := blas64.General{Rows: n, Cols: m, Stride: m, Data: mgrad}
hEraseGrad := blas64.Vector{Inc: 1, Data: make([]float64, m)}
for i, weights := range wm.Ws {
erase := wm.erase[i]
add := wm.add[i]
eraseV := blas64.Vector{Inc: 1, Data: erase}
addV := blas64.Vector{Inc: 1, Data: add}
weightsVal := blas64.Vector{Inc: 1, Data: weights.TopVal}
for j := range mgrad {
mgrad[j] = 0
}
blas64.Ger(1, weightsVal, eraseV, mGradG)
wm.div1MWE(mgrad)
floats.Mul(mgrad, wm.TopGrad)
weightsV := blas64.Vector{Inc: 1, Data: weights.TopGrad}
blas64.Gemv(blas.NoTrans, -1, mGradG, eraseV, 1, weightsV)
blas64.Gemv(blas.NoTrans, 1, blas64.General{Rows: n, Cols: m, Stride: m, Data: wm.TopGrad}, addV, 1, weightsV)
hErase := wm.Heads[i].EraseGrad()
for j := range hEraseGrad.Data {
hEraseGrad.Data[j] = 0
}
blas64.Gemv(blas.Trans, -1, mGradG, weightsVal, 1, hEraseGrad)
for j, e := range erase {
hErase[j] += hEraseGrad.Data[j] * e * (1 - e)
}
}
}
func fitnessRMSE(ind, targ *imgut.Image) float64 {
// Images to vector
dataInd := imgut.ToSlice(ind)
dataTarg := imgut.ToSlice(targ)
// (root mean square) error
floats.Sub(dataInd, dataTarg)
// (root mean) square error
floats.Mul(dataInd, dataInd)
// (root) mean square error
totErr := floats.Sum(dataInd)
return math.Sqrt(totErr / float64(len(dataInd)))
}
func MakeFitLinScale(targetImage *imgut.Image) func(*imgut.Image) float64 {
// Pre-compute image to slice of floats
dataTarg := imgut.ToSlice(targetImage)
// Pre-compute average
avgt := floats.Sum(dataTarg) / float64(len(dataTarg))
return func(indImage *imgut.Image) float64 {
// Images to vector
dataInd := imgut.ToSlice(indImage)
// Compute average pixels
avgy := floats.Sum(dataInd) / float64(len(dataInd))
// Difference y - avgy
y_avgy := make([]float64, len(dataInd))
copy(y_avgy, dataInd)
floats.AddConst(-avgy, y_avgy)
// Difference t - avgt
t_avgt := make([]float64, len(dataTarg))
copy(t_avgt, dataTarg)
floats.AddConst(-avgt, t_avgt)
// Multuplication (t - avgt)(y - avgy)
floats.Mul(t_avgt, y_avgy)
// Summation
numerator := floats.Sum(t_avgt)
// Square (y - avgy)^2
floats.Mul(y_avgy, y_avgy)
denomin := floats.Sum(y_avgy)
// Compute b-value
b := numerator / denomin
// Compute a-value
a := avgt - b*avgy
// Compute now the scaled RMSE, using y' = a + b*y
floats.Scale(b, dataInd) // b*y
floats.AddConst(a, dataInd) // a + b*y
floats.Sub(dataInd, dataTarg) // (a + b * y - t)
floats.Mul(dataInd, dataInd) // (a + b * y - t)^2
total := floats.Sum(dataInd) // Sum(...)
return math.Sqrt(total / float64(len(dataInd)))
}
}
func MakeFitMSE(targetImage *imgut.Image) func(*imgut.Image) float64 {
dataTarg := imgut.ToSliceChans(targetImage, "R")
return func(indImage *imgut.Image) float64 {
// Get data
dataImg := imgut.ToSliceChans(indImage, "R")
// Difference (X - Y)
floats.Sub(dataImg, dataTarg)
// Squared (X - Y)^2
floats.Mul(dataImg, dataImg)
// Summation
return floats.Sum(dataImg) / float64(len(dataImg))
}
}
// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data, using a two-pass
// algorithm.
//
// The weights must have length equal to the number of rows in
// input data matrix x. If cov is nil, then a new matrix with appropriate size will
// be constructed. If cov is not nil, it should have the same number of columns as the
// input data matrix x, and it will be used as the destination for the covariance
// data. Weights must not be negative.
func CovarianceMatrix(cov *mat64.SymDense, x mat64.Matrix, weights []float64) *mat64.SymDense {
// This is the matrix version of the two-pass algorithm. It doesn't use the
// additional floating point error correction that the Covariance function uses
// to reduce the impact of rounding during centering.
r, c := x.Dims()
if cov == nil {
cov = mat64.NewSymDense(c, nil)
} else if n := cov.Symmetric(); n != c {
panic(matrix.ErrShape)
}
var xt mat64.Dense
xt.Clone(x.T())
// Subtract the mean of each of the columns.
for i := 0; i < c; i++ {
v := xt.RawRowView(i)
// This will panic with ErrShape if len(weights) != len(v), so
// we don't have to check the size later.
mean := Mean(v, weights)
floats.AddConst(-mean, v)
}
if weights == nil {
// Calculate the normalization factor
// scaled by the sample size.
cov.SymOuterK(1/(float64(r)-1), &xt)
return cov
}
// Multiply by the sqrt of the weights, so that multiplication is symmetric.
sqrtwts := make([]float64, r)
for i, w := range weights {
if w < 0 {
panic("stat: negative covariance matrix weights")
}
sqrtwts[i] = math.Sqrt(w)
}
// Weight the rows.
for i := 0; i < c; i++ {
v := xt.RawRowView(i)
floats.Mul(v, sqrtwts)
}
// Calculate the normalization factor
// scaled by the weighted sample size.
cov.SymOuterK(1/(floats.Sum(weights)-1), &xt)
return cov
}
func newWrittenMemory(ws []*refocus, heads []*Head, mtm1 *writtenMemory) *writtenMemory {
n := mtm1.N
m := len(mtm1.TopVal) / n
wm := writtenMemory{
Ws: ws,
Heads: heads,
Mtm1: mtm1,
N: mtm1.N,
TopVal: make([]float64, len(mtm1.TopVal)),
TopGrad: make([]float64, len(mtm1.TopVal)),
erase: makeTensor2(len(heads), m),
add: makeTensor2(len(heads), m),
erasures: make([]float64, len(mtm1.TopVal)),
}
for i, h := range wm.Heads {
erase := wm.erase[i]
add := wm.add[i]
addVec := h.AddVal()
for j, e := range h.EraseVal() {
erase[j] = Sigmoid(e)
add[j] = Sigmoid(addVec[j])
}
}
copy(wm.erasures, mtm1.TopVal)
we := make([]float64, n*m)
weG := blas64.General{Rows: n, Cols: m, Stride: m, Data: we}
for k, ws := range wm.Ws {
weights := blas64.Vector{Inc: 1, Data: ws.TopVal}
erase := blas64.Vector{Inc: 1, Data: wm.erase[k]}
for i := range we {
we[i] = 1
}
blas64.Ger(-1, weights, erase, weG)
floats.Mul(wm.erasures, we)
}
copy(wm.TopVal, wm.erasures)
topG := blas64.General{Rows: n, Cols: m, Stride: m, Data: wm.TopVal}
for k, ws := range wm.Ws {
weights := blas64.Vector{Inc: 1, Data: ws.TopVal}
add := blas64.Vector{Inc: 1, Data: wm.add[k]}
blas64.Ger(1, weights, add, topG)
}
return &wm
}
// CovarianceMatrix calculates a covariance matrix (also known as a
// variance-covariance matrix) from a matrix of data, using a two-pass
// algorithm. The matrix returned will be symmetric and square.
//
// The weights wts should have the length equal to the number of rows in
// input data matrix x. If c is nil, then a new matrix with appropriate size will
// be constructed. If c is not nil, it should be a square matrix with the same
// number of columns as the input data matrix x, and it will be used as the receiver
// for the covariance data. Weights cannot be negative.
func CovarianceMatrix(cov *mat64.Dense, x mat64.Matrix, wts []float64) *mat64.Dense {
// This is the matrix version of the two-pass algorithm. It doesn't use the
// additional floating point error correction that the Covariance function uses
// to reduce the impact of rounding during centering.
// TODO(jonlawlor): indicate that the resulting matrix is symmetric, and change
// the returned type from a *mat.Dense to a *mat.Symmetric.
r, c := x.Dims()
if cov == nil {
cov = mat64.NewDense(c, c, nil)
} else if covr, covc := cov.Dims(); covr != covc || covc != c {
panic(mat64.ErrShape)
}
var xt mat64.Dense
xt.TCopy(x)
// Subtract the mean of each of the columns.
for i := 0; i < c; i++ {
v := xt.RawRowView(i)
// This will panic with ErrShape if len(wts) != len(v), so
// we don't have to check the size later.
mean := Mean(v, wts)
floats.AddConst(-mean, v)
}
var n float64
if wts == nil {
n = float64(r)
cov.MulTrans(&xt, false, &xt, true)
// Scale by the sample size.
cov.Scale(1/(n-1), cov)
return cov
}
// Multiply by the sqrt of the weights, so that multiplication is symmetric.
sqrtwts := make([]float64, r)
for i, w := range wts {
if w < 0 {
panic("stat: negative covariance matrix weights")
}
sqrtwts[i] = math.Sqrt(w)
}
// Weight the rows.
for i := 0; i < c; i++ {
v := xt.RawRowView(i)
floats.Mul(v, sqrtwts)
}
// Calculate the normalization factor.
n = floats.Sum(wts)
cov.MulTrans(&xt, false, &xt, true)
// Scale by the sample size.
cov.Scale(1/(n-1), cov)
return cov
}