open up possibilities for different density models

Author: fumin

addressing_test.go

@@ -81,7 +81,7 @@ func doAddressing(heads []*Head, memory [][]Unit) (weights [][]float64, reads []
 		wc := make([]float64, len(memory))
 		var sum float64 = 0
 		for j := 0; j < len(wc); j++ {
-			wc[j] = math.Exp(beta * cosineSimilarity(unitVals(h.K()), unitVals(memory[j])))
+			wc[j] = math.Exp(beta * cosineSimilarity(UnitVals(h.K()), UnitVals(memory[j])))
 			sum += wc[j]
 		}
 		for j := 0; j < len(wc); j++ {

cntl1.go

@@ -93,7 +93,7 @@ func (old *controller1) Forward(reads []*memRead, x []float64) Controller {
 			v += wyh1ij.Val * c.H1[j].Val
 		}
 		v += c.Wyh1[i][len(c.H1)].Val
-		c.y[i].Val = Sigmoid(v)
+		c.y[i].Val = v
 	}
 	memoryM := len(reads[0].Top)
 	for i, wuh1i := range c.Wuh1 {

cntl1_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 )
 
-func TestController1(t *testing.T) {
+func TestLogisticModel(t *testing.T) {
 	times := 10
 	x := MakeTensor2(times, 4)
 	for i := 0; i < len(x); i++ {
@@ -26,12 +26,40 @@ func TestController1(t *testing.T) {
 	numHeads := 2
 	c := NewEmptyController1(len(x[0]), len(y[0]), h1Size, numHeads, n, m)
 	c.Weights(func(u *Unit) { u.Val = 2 * rand.Float64() })
-	ForwardBackward(c, x, y)
 
-	l := loss(c, Controller1Forward, x, y)
-	checkGradients(t, c, Controller1Forward, x, y, l)
+	model := &LogisticModel{Y: y}
+	ForwardBackward(c, x, model)
+	checkGradients(t, c, Controller1Forward, x, model)
 }
 
+func TestMultinomialModel(t *testing.T) {
+	times := 10
+	x := MakeTensor2(times, 4)
+	for i := 0; i < len(x); i++ {
+		for j := 0; j < len(x[i]); j++ {
+			x[i][j] = rand.Float64()
+		}
+	}
+	outputSize := 4
+	y := make([]int, times)
+	for i := range y {
+		y[i] = rand.Intn(outputSize)
+	}
+	n := 3
+	m := 2
+	h1Size := 3
+	numHeads := 2
+	c := NewEmptyController1(len(x[0]), outputSize, h1Size, numHeads, n, m)
+	c.Weights(func(u *Unit) { u.Val = 2 * rand.Float64() })
+
+	model := &MultinomialModel{Y: y}
+	ForwardBackward(c, x, model)
+	checkGradients(t, c, Controller1Forward, x, model)
+}
+
+// A ControllerForward is a ground truth implementation of the forward pass of a controller.
+type ControllerForward func(c Controller, reads [][]float64, x []float64) (prediction []float64, heads []*Head)
+
 func Controller1Forward(c1 Controller, reads [][]float64, x []float64) ([]float64, []*Head) {
 	c := c1.(*controller1)
 	h1Size := len(c.Wh1r)
@@ -57,7 +85,7 @@ func Controller1Forward(c1 Controller, reads [][]float64, x []float64) ([]float6
 			v += c.Wyh1[i][j].Val * h1[j]
 		}
 		v += c.Wyh1[i][maxJ].Val
-		prediction[i] = Sigmoid(v)
+		prediction[i] = v
 	}
 	numHeads := len(c.Wh1r[0])
 	m := len(c.Wh1r[0][0])
@@ -75,7 +103,7 @@ func Controller1Forward(c1 Controller, reads [][]float64, x []float64) ([]float6
 	return prediction, heads
 }
 
-func loss(c Controller, forward func(Controller, [][]float64, []float64) ([]float64, []*Head), in, out [][]float64) float64 {
+func loss(c Controller, forward ControllerForward, in [][]float64, model DensityModel) float64 {
 	// Initialize memory as in the function ForwardBackward
 	mem := c.Mtm1BiasV().Top
 	wtm1Bs := c.Wtm1BiasV()
@@ -102,10 +130,11 @@ func loss(c Controller, forward func(Controller, [][]float64, []float64) ([]floa
 		}
 	}
 
-	prediction := make([][]float64, len(out))
+	prediction := make([][]float64, len(in))
 	var heads []*Head
 	for t := 0; t < len(in); t++ {
 		prediction[t], heads = forward(c, reads, in[t])
+		prediction[t] = computeDensity(t, prediction[t], model)
 		for i := 0; i < len(heads); i++ {
 			heads[i].Wtm1 = wtm1s[i]
 		}
@@ -115,24 +144,27 @@ func loss(c Controller, forward func(Controller, [][]float64, []float64) ([]floa
 		mem = transformMemFloat64(memFloat64)
 	}
 
-	var llh float64 = 0 // log likelihood
-	for t := 0; t < len(out); t++ {
-		for i := 0; i < len(out[t]); i++ {
-			p := prediction[t][i]
-			y := out[t][i]
-			llh += y*math.Log(p) + (1-y)*math.Log(1-p)
-		}
+	return model.Loss(prediction)
+}
+
+func computeDensity(timestep int, pred []float64, model DensityModel) []float64 {
+	units := make([]Unit, len(pred))
+	for j := range units {
+		units[j].Val = pred[j]
 	}
-	return -llh
+	model.Model(timestep, units)
+	return UnitVals(units)
 }
 
-func checkGradients(t *testing.T, c Controller, forward func(Controller, [][]float64, []float64) ([]float64, []*Head), in, out [][]float64, lx float64) {
+func checkGradients(t *testing.T, c Controller, forward ControllerForward, in [][]float64, model DensityModel) {
+	lx := loss(c, forward, in, model)
+
 	c.WeightsVerbose(func(tag string, w *Unit) {
 		x := w.Val
 		h := machineEpsilonSqrt * math.Max(math.Abs(x), 1)
 		xph := x + h
 		w.Val = xph
-		lxph := loss(c, forward, in, out)
+		lxph := loss(c, forward, in, model)
 		w.Val = x
 		grad := (lxph - lx) / (xph - x)
 

copytask/test/main.go

@@ -45,8 +45,9 @@ func main() {
 	runs := make([]Run, 0, len(seqLens))
 	for _, seql := range seqLens {
 		x, y := copytask.GenSeq(seql, vectorSize)
-		machines := ntm.ForwardBackward(c, x, y)
-		l := ntm.Loss(y, machines)
+		model := &ntm.LogisticModel{Y: y}
+		machines := ntm.ForwardBackward(c, x, model)
+		l := model.Loss(ntm.Predictions(machines))
 		bps := l / float64(len(y)*len(y[0]))
 		log.Printf("sequence length: %d, loss: %f", seql, bps)
 

copytask/train/main.go

@@ -75,9 +75,10 @@ func main() {
 	log.Printf("numweights: %d", c.NumWeights())
 	for i := 1; ; i++ {
 		x, y := copytask.GenSeq(rand.Intn(20)+1, vectorSize)
-		//machines := sgd.Train(x, y, 1e-4, 0.9)
-		machines := rmsp.Train(x, y, 0.95, 0.5, 1e-3, 1e-3)
-		l := ntm.Loss(y, machines)
+		model := &ntm.LogisticModel{Y: y}
+		//machines := sgd.Train(x, model, 1e-4, 0.9)
+		machines := rmsp.Train(x, model, 0.95, 0.5, 1e-3, 1e-3)
+		l := model.Loss(ntm.Predictions(machines))
 		if i%1000 == 0 {
 			bpc := l / float64(len(y)*len(y[0]))
 			losses = append(losses, bpc)

math.go

@@ -15,6 +15,13 @@ func Sigmoid(x float64) float64 {
 	return 1.0 / (1 + math.Exp(-x))
 }
 
+func delta(a, b int) float64 {
+	if a == b {
+		return 1
+	}
+	return 0
+}
+
 func cosineSimilarity(u, v []float64) float64 {
 	var sum float64 = 0
 	var usum float64 = 0

ngram/test/main.go

@@ -49,8 +49,9 @@ func main() {
 		sampletimes := 100
 		for j := 0; j < sampletimes; j++ {
 			x, y = ngram.GenSeq(prob)
-			machines = ntm.ForwardBackward(c, x, y)
-			l += ntm.Loss(y, machines)
+			model := &ntm.LogisticModel{Y: y}
+			machines = ntm.ForwardBackward(c, x, model)
+			l += model.Loss(ntm.Predictions(machines))
 			if (j+1)%10 == 0 {
 				log.Printf("%d %d %f", i, j+1, l/float64(j+1))
 			}

ngram/train/main.go

@@ -71,16 +71,17 @@ func main() {
 	log.Printf("seed: %d, numweights: %d, numHeads: %d", seed, c.NumWeights(), c.NumHeads())
 	for i := 1; ; i++ {
 		x, y := ngram.GenSeq(ngram.GenProb())
-		machines := rmsp.Train(x, y, 0.95, 0.5, 1e-3, 1e-3)
+		machines := rmsp.Train(x, &ntm.LogisticModel{Y: y}, 0.95, 0.5, 1e-3, 1e-3)
 
 		if i%10000 == 0 {
 			prob := ngram.GenProb()
 			var l float64 = 0
 			samn := 1000
 			for j := 0; j < samn; j++ {
 				x, y = ngram.GenSeq(prob)
-				machines = ntm.ForwardBackward(c, x, y)
-				l += ntm.Loss(y, machines)
+				model := &ntm.LogisticModel{Y: y}
+				machines = ntm.ForwardBackward(c, x, model)
+				l += model.Loss(ntm.Predictions(machines))
 			}
 			l = l / float64(samn)
 			losses = append(losses, l)