Exemple #1
0
func (_ SigmoidCECost) Cost(x linalg.Vector, a autofunc.Result) autofunc.Result {
	logsig := autofunc.LogSigmoid{}
	log := logsig.Apply(a)
	invLog := logsig.Apply(autofunc.Scale(a, -1))

	xVar := &autofunc.Variable{x}
	oneMinusX := autofunc.AddScaler(autofunc.Scale(xVar, -1), 1)

	sums := autofunc.Add(autofunc.Mul(xVar, log), autofunc.Mul(oneMinusX, invLog))
	return autofunc.Scale(autofunc.SumAll(sums), -1)
}
Exemple #2
0
func (_ CrossEntropyCost) Cost(x linalg.Vector, a autofunc.Result) autofunc.Result {
	return autofunc.Pool(a, func(a autofunc.Result) autofunc.Result {
		xVar := &autofunc.Variable{x}
		logA := autofunc.Log{}.Apply(a)
		oneMinusA := autofunc.AddScaler(autofunc.Scale(a, -1), 1)
		oneMinusX := autofunc.AddScaler(autofunc.Scale(xVar, -1), 1)
		log1A := autofunc.Log{}.Apply(oneMinusA)

		errorVec := autofunc.Add(autofunc.Mul(xVar, logA),
			autofunc.Mul(oneMinusX, log1A))
		return autofunc.Scale(autofunc.SumAll(errorVec), -1)
	})
}
Exemple #3
0
func (s *LogSoftmaxLayer) Apply(in autofunc.Result) autofunc.Result {
	return autofunc.Pool(in, func(in autofunc.Result) autofunc.Result {
		// Compute the log of the sum of the exponents by
		// factoring out the largest exponent so that all
		// the exponentials fit nicely inside floats.
		maxIdx := maxVecIdx(in.Output())
		maxValue := autofunc.Slice(in, maxIdx, maxIdx+1)
		exponents := autofunc.AddFirst(in, autofunc.Scale(maxValue, -1))
		expSum := autofunc.SumAll(autofunc.Exp{}.Apply(exponents))
		expLog := autofunc.Log{}.Apply(expSum)
		denomLog := autofunc.Add(expLog, maxValue)
		return autofunc.AddFirst(in, autofunc.Scale(denomLog, -1))
	})
}
Exemple #4
0
func (d *DropoutLayer) Apply(in autofunc.Result) autofunc.Result {
	if d.Training {
		return autofunc.Mul(in, d.dropoutMask(len(in.Output())))
	} else {
		return autofunc.Scale(in, d.KeepProbability)
	}
}
Exemple #5
0
func (r *RegularizingCost) Cost(a linalg.Vector, x autofunc.Result) autofunc.Result {
	regFunc := autofunc.SquaredNorm{}
	cost := r.CostFunc.Cost(a, x)
	for _, variable := range r.Variables {
		norm := regFunc.Apply(variable)
		cost = autofunc.Add(cost, autofunc.Scale(norm, r.Penalty))
	}
	return cost
}
Exemple #6
0
// ApplyBlock applies the block to an input.
func (g *GRU) ApplyBlock(s []State, in []autofunc.Result) BlockResult {
	stateVars, stateRes := PoolVecStates(s)
	var gateInputs []autofunc.Result
	for i, x := range stateRes {
		gateInputs = append(gateInputs, in[i], x)
	}
	n := len(in)

	gateInput := autofunc.Concat(gateInputs...)
	stateIn := autofunc.Concat(stateRes...)

	resetMask := g.resetGate.Batch(gateInput, n)
	updateMask := g.updateGate.Batch(gateInput, n)

	maskedByReset := autofunc.Mul(resetMask, stateIn)
	inputValue := autofunc.PoolSplit(n, maskedByReset,
		func(newStates []autofunc.Result) autofunc.Result {
			var newGateInputs []autofunc.Result
			for i, input := range in {
				newGateInputs = append(newGateInputs, input, newStates[i])
			}
			newIn := autofunc.Concat(newGateInputs...)
			return g.inputValue.Batch(newIn, n)
		})

	newState := autofunc.Pool(updateMask, func(umask autofunc.Result) autofunc.Result {
		updateComplement := autofunc.AddScaler(autofunc.Scale(umask, -1), 1)
		return autofunc.Add(autofunc.Mul(umask, stateIn),
			autofunc.Mul(updateComplement, inputValue))
	})

	return &gruResult{
		InStates: stateVars,
		Output:   newState,
	}
}
Exemple #7
0
func (r *RescaleLayer) Apply(in autofunc.Result) autofunc.Result {
	return autofunc.Scale(autofunc.AddScaler(in, r.Bias), r.Scale)
}
Exemple #8
0
func (_ HyperbolicTangent) Apply(r autofunc.Result) autofunc.Result {
	stretched := autofunc.Scale(autofunc.Sigmoid{}.Apply(autofunc.Scale(r, 2)), 2)
	return autofunc.AddScaler(stretched, -1)
}
Exemple #9
0
func (_ DotCost) Cost(x linalg.Vector, a autofunc.Result) autofunc.Result {
	xVar := &autofunc.Variable{x}
	return autofunc.Scale(autofunc.SumAll(autofunc.Mul(xVar, a)), -1)
}