[neurons] leakyRelu

TODO.md

@@ -1,10 +1,8 @@
 # system-a
 
-[neurons] `leakyRelu`
-[neurons] `recuBlock`
-
-[neurons] `recu` -- rectifying correlational unit
 [neurons] `correlate`
+[neurons] `recu`
+[neurons] `recuBlock`
 
 [models] `morse/`
 [models] `tenserZip` -- `[d, n] -> [n, d]` -- like transposing a matrix

src/system-a/neurons/relu/denseBlock.ts

@@ -1,12 +1,12 @@
 import { Block } from "../../block/index.js"
-import { relu } from "../../neurons/relu/index.js"
 import type { Tensor } from "../../tensor/Tensor.js"
 import { randomTensor } from "../../tensor/randomTensor.js"
 import type { Shape } from "../../tensor/shape.js"
 import { zeroTensor } from "../../tensor/zeroTensor.js"
+import { leakyRelu } from "./leakyRelu.js"
 
 export function denseBlock(inputSize: number, layerWidth: number): Block {
-  return Block(relu, [[layerWidth, inputSize], [layerWidth]])
+  return Block(leakyRelu, [[layerWidth, inputSize], [layerWidth]])
 }
 
 export function denseBlockInitParameters(shapes: Array<Shape>): Array<Tensor> {

src/system-a/neurons/relu/index.ts

@@ -1,2 +1,3 @@
 export * from "./denseBlock.js"
+export * from "./leakyRelu.js"
 export * from "./relu.js"

src/system-a/neurons/relu/leakyRelu.ts

@@ -0,0 +1,13 @@
+import type { Scalar, Tensor } from "../../tensor/index.js"
+import { extend1, lt, mul } from "../../toys/index.js"
+import { linear } from "./linear.js"
+
+export function leakyRectifyScalar(s: Scalar): Scalar {
+  return lt(s, 0) ? (mul(-0.01, s) as Scalar) : s
+}
+
+export const leakyRectify = extend1(leakyRectifyScalar, 0)
+
+export function leakyRelu(t: Tensor): (weight: Tensor, bias: Tensor) => Tensor {
+  return (weight, bias) => leakyRectify(linear(t)(weight, bias))
+}

src/system-a/neurons/relu/relu.test.ts

@@ -1,55 +1,55 @@
-// import assert from "node:assert"
-// import { test } from "node:test"
-// import { assertTensorAlmostEqual } from "../../tensor/assertions.js"
-// import { rectify, relu } from "./relu.js"
+import assert from "node:assert"
+import { test } from "node:test"
+import { assertTensorAlmostEqual } from "../../tensor/assertions.js"
+import { rectify, relu } from "./relu.js"
 
-// test("rectify", () => {
-//   assert.deepStrictEqual(
-//     rectify([
-//       [-1, -2, -3],
-//       [1, 2, 3],
-//     ]),
-//     [
-//       [0, 0, 0],
-//       [1, 2, 3],
-//     ],
-//   )
-// })
+test("rectify", () => {
+  assert.deepStrictEqual(
+    rectify([
+      [-1, -2, -3],
+      [1, 2, 3],
+    ]),
+    [
+      [0, 0, 0],
+      [1, 2, 3],
+    ],
+  )
+})
 
-// test("relu", () => {
-//   assertTensorAlmostEqual(
-//     relu([2, 1, 3])(
-//       [
-//         [7.1, 4.3, -6.4],
-//         [7.1, 4.3, -6.4],
-//       ],
-//       [0.6, 1],
-//     ),
-//     [0, 0.3],
-//     0.01,
-//   )
+test("relu", () => {
+  assertTensorAlmostEqual(
+    relu([2, 1, 3])(
+      [
+        [7.1, 4.3, -6.4],
+        [7.1, 4.3, -6.4],
+      ],
+      [0.6, 1],
+    ),
+    [0, 0.3],
+    0.01,
+  )
 
-//   assertTensorAlmostEqual(
-//     relu([2, 1, 3])(
-//       [
-//         [7.1, 4.3, -6.4],
-//         [7.1, 4.3, -6.4],
-//       ],
-//       [0.6, 0.6],
-//     ),
-//     [0, 0],
-//     0,
-//   )
+  assertTensorAlmostEqual(
+    relu([2, 1, 3])(
+      [
+        [7.1, 4.3, -6.4],
+        [7.1, 4.3, -6.4],
+      ],
+      [0.6, 0.6],
+    ),
+    [0, 0],
+    0,
+  )
 
-//   assertTensorAlmostEqual(
-//     relu([2, 1, 3])(
-//       [
-//         [7.1, 4.3, -6.4],
-//         [7.1, 4.3, -6.4],
-//       ],
-//       0.6,
-//     ),
-//     [0, 0],
-//     0,
-//   )
-// })
+  assertTensorAlmostEqual(
+    relu([2, 1, 3])(
+      [
+        [7.1, 4.3, -6.4],
+        [7.1, 4.3, -6.4],
+      ],
+      0.6,
+    ),
+    [0, 0],
+    0,
+  )
+})

src/system-a/neurons/relu/relu.ts

@@ -1,9 +1,9 @@
 import type { Scalar, Tensor } from "../../tensor/index.js"
-import { extend1, lt, mul } from "../../toys/index.js"
+import { extend1, lt } from "../../toys/index.js"
 import { linear } from "./linear.js"
 
 export function rectifyScalar(s: Scalar): Scalar {
-  return lt(s, 0) ? (mul(-0.01, s) as Scalar) : s
+  return lt(s, 0) ? 0 : s
 }
 
 export const rectify = extend1(rectifyScalar, 0)