Example #1
0
func (l *lstmGate) Batch(in autofunc.Result, n int) autofunc.Result {
	if l.Peephole == nil {
		return l.Activation.Apply(l.Dense.Batch(in, n))
	}
	return autofunc.Pool(in, func(in autofunc.Result) autofunc.Result {
		vecSize := len(in.Output()) / n
		var weightedInputs []autofunc.Result
		var peepholed []autofunc.Result
		for i := 0; i < n; i++ {
			start := vecSize * i
			weightedEnd := start + vecSize - len(l.Peephole.Vector)
			weightedInputs = append(weightedInputs, autofunc.Slice(in, start, weightedEnd))
			peepholeMe := autofunc.Slice(in, weightedEnd, (i+1)*vecSize)
			peepholed = append(peepholed, autofunc.Mul(l.Peephole, peepholeMe))
		}
		weighted := l.Dense.Batch(autofunc.Concat(weightedInputs...), n)
		return l.Activation.Apply(autofunc.Add(autofunc.Concat(peepholed...), weighted))
	})
}
Example #2
0
func (s *LogSoftmaxLayer) Apply(in autofunc.Result) autofunc.Result {
	return autofunc.Pool(in, func(in autofunc.Result) autofunc.Result {
		// Compute the log of the sum of the exponents by
		// factoring out the largest exponent so that all
		// the exponentials fit nicely inside floats.
		maxIdx := maxVecIdx(in.Output())
		maxValue := autofunc.Slice(in, maxIdx, maxIdx+1)
		exponents := autofunc.AddFirst(in, autofunc.Scale(maxValue, -1))
		expSum := autofunc.SumAll(autofunc.Exp{}.Apply(exponents))
		expLog := autofunc.Log{}.Apply(expSum)
		denomLog := autofunc.Add(expLog, maxValue)
		return autofunc.AddFirst(in, autofunc.Scale(denomLog, -1))
	})
}