diff --git a/src/NeuralNet/Layers/AvgPool1D.php b/src/NeuralNet/Layers/AvgPool1D.php new file mode 100644 index 000000000..982141b6a --- /dev/null +++ b/src/NeuralNet/Layers/AvgPool1D.php @@ -0,0 +1,302 @@ + 0 ? $stride : $poolSize; + + $outputLength = (int) floor(($inputLength - $poolSize) / $stride) + 1; + + if ($outputLength < 1) { + throw new InvalidArgumentException('Output length must be' + . " greater than 0, $outputLength given. Check pool size and stride values."); + } + + $this->poolSize = $poolSize; + $this->inputLength = $inputLength; + $this->stride = $stride; + $this->outputLength = $outputLength; + } + + /** + * Return the width of the layer (same as input channels). + * + * @internal + * + * @throws RuntimeException + * @return positive-int + */ + public function width() : int + { + if ($this->inputChannels === null) { + throw new RuntimeException('Layer has not been initialized.'); + } + + return $this->inputChannels; + } + + /** + * Return the output length after pooling. + * + * @internal + * + * @return positive-int + */ + public function outputLength() : int + { + return $this->outputLength; + } + + /** + * Initialize the layer with the fan in from the previous layer and return + * the fan out for this layer. + * + * @internal + * + * @param positive-int $fanIn + * @return positive-int + */ + public function initialize(int $fanIn) : int + { + $this->inputChannels = $fanIn; + + return $fanIn * $this->outputLength; + } + + /** + * Compute a forward pass through the layer. + * + * @internal + * + * @param Matrix $input + * @throws RuntimeException + * @return Matrix + */ + public function forward(Matrix $input) : Matrix + { + if ($this->inputChannels === null) { + throw new RuntimeException('Layer has not been initialized.'); + } + + $inputChannels = $input->m(); + + if ($inputChannels !== $this->inputChannels) { + throw new RuntimeException('Input channels mismatch:' + . " expected {$this->inputChannels}, got {$inputChannels}."); + } + + $batchSize = (int) ($input->n() / $this->inputLength); + + $inputArray = $input->asArray(); + $output = []; + + $scale = 1.0 / $this->poolSize; + + foreach ($inputArray as $channel) { + $outputRow = []; + + for ($b = 0; $b < $batchSize; ++$b) { + $sampleOffset = $b * $this->inputLength; + + for ($t = 0; $t < $this->outputLength; ++$t) { + $startPos = $t * $this->stride; + $sum = 0.0; + + for ($p = 0; $p < $this->poolSize; ++$p) { + $pos = $startPos + $p; + $sum += $channel[$sampleOffset + $pos]; + } + + $outputRow[] = $sum * $scale; + } + } + + $output[] = $outputRow; + } + + return Matrix::quick($output); + } + + /** + * Compute an inference pass through the layer. + * + * @internal + * + * @param Matrix $input + * @throws RuntimeException + * @return Matrix + */ + public function infer(Matrix $input) : Matrix + { + return $this->forward($input); + } + + /** + * Calculate the gradient and update the parameters of the layer. + * + * @internal + * + * @param Deferred $prevGradient + * @param Optimizer $optimizer + * @throws RuntimeException + * @return Deferred + */ + public function back(Deferred $prevGradient, Optimizer $optimizer) : Deferred + { + if ($this->inputChannels === null) { + throw new RuntimeException('Layer has not been initialized.'); + } + + $inputChannels = $this->inputChannels; + $inputLength = $this->inputLength; + $poolSize = $this->poolSize; + $stride = $this->stride; + $outputLength = $this->outputLength; + + return new Deferred( + [$this, 'gradient'], + [$prevGradient, $inputChannels, $inputLength, $poolSize, $stride, $outputLength] + ); + } + + /** + * Calculate the gradient for the previous layer. + * + * @internal + * + * @param Deferred $prevGradient + * @param int $inputChannels + * @param int $inputLength + * @param int $poolSize + * @param int $stride + * @param int $outputLength + * @return Matrix + */ + public function gradient( + Deferred $prevGradient, + int $inputChannels, + int $inputLength, + int $poolSize, + int $stride, + int $outputLength + ) : Matrix { + $dOut = $prevGradient(); + $dOutArray = $dOut->asArray(); + + $batchSize = (int) (count($dOutArray[0]) / $outputLength); + + // Initialize gradient with zeros + $dInput = array_fill(0, $inputChannels, array_fill(0, $inputLength * $batchSize, 0.0)); + + $scale = 1.0 / $poolSize; + + // Distribute gradients evenly across pool window positions + foreach ($dOutArray as $c => $dOutRow) { + for ($b = 0; $b < $batchSize; ++$b) { + $sampleOffset = $b * $inputLength; + $outputOffset = $b * $outputLength; + + for ($t = 0; $t < $outputLength; ++$t) { + $startPos = $t * $stride; + $grad = $dOutRow[$outputOffset + $t] * $scale; + + for ($p = 0; $p < $poolSize; ++$p) { + $pos = $sampleOffset + $startPos + $p; + $dInput[$c][$pos] += $grad; + } + } + } + } + + return Matrix::quick($dInput); + } + + /** + * Return the string representation of the object. + * + * @internal + * + * @return string + */ + public function __toString() : string + { + return "AvgPool1D (pool size: {$this->poolSize}, input length: {$this->inputLength}," + . " stride: {$this->stride})"; + } +} diff --git a/src/NeuralNet/Layers/Conv1D.php b/src/NeuralNet/Layers/Conv1D.php new file mode 100644 index 000000000..e11a4d504 --- /dev/null +++ b/src/NeuralNet/Layers/Conv1D.php @@ -0,0 +1,774 @@ + + */ + protected int $padding; + + /** + * The amount of L2 regularization applied to the weights. + * + * @var float + */ + protected float $l2Penalty; + + /** + * Should the layer include a bias parameter? + * + * @var bool + */ + protected bool $bias; + + /** + * The kernel weight initializer. + * + * @var Initializer + */ + protected Initializer $kernelInitializer; + + /** + * The bias initializer. + * + * @var Initializer + */ + protected Initializer $biasInitializer; + + /** + * The kernel weights. + * + * @var Parameter|null + */ + protected ?Parameter $weights = null; + + /** + * The biases. + * + * @var Parameter|null + */ + protected ?Parameter $biases = null; + + /** + * The computed output length. + * + * @var positive-int + */ + protected int $outputLength; + + /** + * The memorized inputs to the layer. + * + * @var Matrix|null + */ + protected ?Matrix $input = null; + + /** + * The padded input (if padding was applied). + * + * @var Matrix|null + */ + protected ?Matrix $paddedInput = null; + + /** + * @param int $filters Number of output filters (output channels) + * @param int $kernelSize Size of the 1D convolution kernel + * @param int $inputLength Length of the input sequence + * @param int $inputChannels Number of input channels (default 1) + * @param int $stride Convolution stride (default 1) + * @param int $padding Zero-padding on both sides (default 0) + * @param float $l2Penalty L2 regularization (default 0.0) + * @param bool $bias Include bias parameter (default true) + * @param Initializer|null $kernelInitializer Weight initializer + * @param Initializer|null $biasInitializer Bias initializer + * @throws InvalidArgumentException + */ + public function __construct( + int $filters, + int $kernelSize, + int $inputLength, + int $inputChannels = 1, + int $stride = 1, + int $padding = 0, + float $l2Penalty = 0.0, + bool $bias = true, + ?Initializer $kernelInitializer = null, + ?Initializer $biasInitializer = null + ) { + if ($filters < 1) { + throw new InvalidArgumentException('Number of filters' + . " must be greater than 0, $filters given."); + } + + if ($kernelSize < 1) { + throw new InvalidArgumentException('Kernel size must be' + . " greater than 0, $kernelSize given."); + } + + if ($inputLength < 1) { + throw new InvalidArgumentException('Input length must be' + . " greater than 0, $inputLength given."); + } + + if ($inputChannels < 1) { + throw new InvalidArgumentException('Number of input channels' + . " must be greater than 0, $inputChannels given."); + } + + if ($stride < 1) { + throw new InvalidArgumentException('Stride must be' + . " greater than 0, $stride given."); + } + + if ($padding < 0) { + throw new InvalidArgumentException('Padding cannot be' + . " negative, $padding given."); + } + + if ($l2Penalty < 0.0) { + throw new InvalidArgumentException('L2 Penalty must be' + . " greater than or equal to 0, $l2Penalty given."); + } + + $outputLength = (int) floor(($inputLength + 2 * $padding - $kernelSize) / $stride) + 1; + + if ($outputLength < 1) { + throw new InvalidArgumentException('Output length must be' + . " greater than 0, $outputLength given. Check kernel size, stride, and padding values."); + } + + $this->filters = $filters; + $this->kernelSize = $kernelSize; + $this->inputLength = $inputLength; + $this->inputChannels = $inputChannels; + $this->stride = $stride; + $this->padding = $padding; + $this->l2Penalty = $l2Penalty; + $this->bias = $bias; + $this->kernelInitializer = $kernelInitializer ?? new He(); + $this->biasInitializer = $biasInitializer ?? new Constant(0.0); + $this->outputLength = $outputLength; + } + + /** + * Return the width of the layer (number of filters). + * + * @internal + * + * @return positive-int + */ + public function width() : int + { + return $this->filters; + } + + /** + * Return the output length after convolution. + * + * @internal + * + * @return positive-int + */ + public function outputLength() : int + { + return $this->outputLength; + } + + /** + * Return the kernel weight matrix. + * + * @internal + * + * @throws RuntimeException + * @return Matrix + */ + public function weights() : Matrix + { + if (!$this->weights) { + throw new RuntimeException('Layer is not initialized'); + } + + return $this->weights->param(); + } + + /** + * Initialize the layer with the fan in from the previous layer and return + * the fan out for this layer. + * + * @internal + * + * @param positive-int $fanIn (The fan in is not used; fan-in is calculated from inputChannels * kernelSize) + * @return positive-int + */ + public function initialize(int $fanIn) : int + { + $fanOut = $this->filters * $this->outputLength; + + // Initialize kernel weights: shape (filters, inputChannels * kernelSize) + $kernelFanIn = $this->inputChannels * $this->kernelSize; + $weights = $this->kernelInitializer->initialize($kernelFanIn, $this->filters); + + $this->weights = new Parameter($weights); + + if ($this->bias) { + $biases = $this->biasInitializer->initialize(1, $this->filters)->columnAsVector(0); + + $this->biases = new Parameter($biases); + } + + return $fanOut; + } + + /** + * Compute a forward pass through the layer. + * + * @internal + * + * @param Matrix $input + * @throws RuntimeException + * @return Matrix + */ + public function forward(Matrix $input) : Matrix + { + if (!$this->weights) { + throw new RuntimeException('Layer is not initialized'); + } + + // Input shape: (inputChannels, inputLength * batchSize) + $inputChannels = $input->m(); + + if ($inputChannels !== $this->inputChannels) { + throw new RuntimeException('Input channels mismatch:' + . " expected {$this->inputChannels}, got {$inputChannels}."); + } + + // Store the original input for backprop + $this->input = $input; + + // Apply padding if needed + $paddedInput = $this->padInput($input); + $this->paddedInput = $paddedInput; + + $paddedLength = $this->inputLength + 2 * $this->padding; + $batchSize = (int) ($input->n() / $this->inputLength); + + // Get kernel weights: shape (filters, inputChannels * kernelSize) + $kernel = $this->weights->param(); + + // Compute output: shape (filters, outputLength * batchSize) + $output = $this->computeConvolution($paddedInput, $kernel, $paddedLength, $batchSize); + + // Add bias if enabled + if ($this->biases) { + $output = $this->addBias($output, $batchSize); + } + + return $output; + } + + /** + * Compute an inference pass through the layer. + * + * @internal + * + * @param Matrix $input + * @throws RuntimeException + * @return Matrix + */ + public function infer(Matrix $input) : Matrix + { + if (!$this->weights) { + throw new RuntimeException('Layer is not initialized'); + } + + $inputChannels = $input->m(); + + if ($inputChannels !== $this->inputChannels) { + throw new RuntimeException('Input channels mismatch:' + . " expected {$this->inputChannels}, got {$inputChannels}."); + } + + $paddedInput = $this->padInput($input); + $paddedLength = $this->inputLength + 2 * $this->padding; + $batchSize = (int) ($input->n() / $this->inputLength); + + $kernel = $this->weights->param(); + + $output = $this->computeConvolution($paddedInput, $kernel, $paddedLength, $batchSize); + + if ($this->biases) { + $output = $this->addBias($output, $batchSize); + } + + return $output; + } + + /** + * Calculate the gradient and update the parameters of the layer. + * + * @internal + * + * @param Deferred $prevGradient + * @param Optimizer $optimizer + * @throws RuntimeException + * @return Deferred + */ + public function back(Deferred $prevGradient, Optimizer $optimizer) : Deferred + { + if (!$this->weights) { + throw new RuntimeException('Layer has not been initialized.'); + } + + if (!$this->input || !$this->paddedInput) { + throw new RuntimeException('Must perform forward pass' + . ' before backpropagating.'); + } + + $dOut = $prevGradient(); + + $batchSize = (int) ($this->input->n() / $this->inputLength); + $paddedLength = $this->inputLength + 2 * $this->padding; + + $kernel = $this->weights->param(); + + // Compute kernel gradient + $dKernel = $this->computeKernelGradient($this->paddedInput, $dOut, $paddedLength, $batchSize); + + // Apply L2 penalty if needed + if ($this->l2Penalty) { + $dKernel = $dKernel->add($kernel->multiply($this->l2Penalty)); + } + + $this->weights->update($dKernel, $optimizer); + + // Update biases if enabled + if ($this->biases) { + // Sum gradients across all output positions + $dB = $dOut->sum(); + + $this->biases->update($dB, $optimizer); + } + + $paddedInput = $this->paddedInput; + $input = $this->input; + + $this->input = null; + $this->paddedInput = null; + + return new Deferred([$this, 'gradient'], [$kernel, $dOut, $paddedInput, $input, $batchSize, $paddedLength]); + } + + /** + * Calculate the gradient for the previous layer. + * + * @internal + * + * @param Matrix $kernel + * @param Matrix $dOut + * @param Matrix $paddedInput + * @param Matrix $input + * @param int $batchSize + * @param int $paddedLength + * @return Matrix + */ + public function gradient( + Matrix $kernel, + Matrix $dOut, + Matrix $paddedInput, + Matrix $input, + int $batchSize, + int $paddedLength + ) : Matrix { + // Compute input gradient using full convolution with flipped kernel + $dPaddedInput = $this->computeInputGradient($kernel, $dOut, $paddedLength, $batchSize); + + // Remove padding from gradient if needed + if ($this->padding > 0) { + return $this->unpadGradient($dPaddedInput, $input); + } + + return $dPaddedInput; + } + + /** + * Return the parameters of the layer. + * + * @internal + * + * @throws RuntimeException + * @return Generator + */ + public function parameters() : Generator + { + if (!$this->weights) { + throw new RuntimeException('Layer has not been initialized.'); + } + + yield 'weights' => $this->weights; + + if ($this->biases) { + yield 'biases' => $this->biases; + } + } + + /** + * Restore the parameters in the layer from an associative array. + * + * @internal + * + * @param Parameter[] $parameters + */ + public function restore(array $parameters) : void + { + $this->weights = $parameters['weights']; + $this->biases = $parameters['biases'] ?? null; + } + + /** + * Pad the input matrix with zeros on both sides of the sequence. + * + * @param Matrix $input + * @return Matrix + */ + protected function padInput(Matrix $input) : Matrix + { + if ($this->padding === 0) { + return $input; + } + + $batchSize = (int) ($input->n() / $this->inputLength); + $paddedLength = $this->inputLength + 2 * $this->padding; + + $padded = []; + + foreach ($input->asArray() as $channel => $row) { + $paddedRow = []; + + for ($b = 0; $b < $batchSize; ++$b) { + // Add left padding (zeros) + for ($p = 0; $p < $this->padding; ++$p) { + $paddedRow[] = 0.0; + } + + // Add original sequence + for ($t = 0; $t < $this->inputLength; ++$t) { + $paddedRow[] = $row[$b * $this->inputLength + $t]; + } + + // Add right padding (zeros) + for ($p = 0; $p < $this->padding; ++$p) { + $paddedRow[] = 0.0; + } + } + + $padded[] = $paddedRow; + } + + return Matrix::quick($padded); + } + + /** + * Compute the convolution operation. + * + * @param Matrix $paddedInput Padded input matrix + * @param Matrix $kernel Kernel weights + * @param int $paddedLength Length of padded sequence + * @param int $batchSize Number of samples in batch + * @return Matrix + */ + protected function computeConvolution( + Matrix $paddedInput, + Matrix $kernel, + int $paddedLength, + int $batchSize + ) : Matrix { + $output = []; + + $kernelArray = $kernel->asArray(); + $inputArray = $paddedInput->asArray(); + + for ($f = 0; $f < $this->filters; ++$f) { + $outputRow = []; + + for ($b = 0; $b < $batchSize; ++$b) { + $sampleOffset = $b * $paddedLength; + + for ($t = 0; $t < $this->outputLength; ++$t) { + $startPos = $t * $this->stride; + $sum = 0.0; + + // Sum contributions across all input channels + for ($c = 0; $c < $this->inputChannels; ++$c) { + $kernelOffset = $c * $this->kernelSize; + + for ($k = 0; $k < $this->kernelSize; ++$k) { + $inputPos = $startPos + $k; + $sum += $inputArray[$c][$sampleOffset + $inputPos] + * $kernelArray[$f][$kernelOffset + $k]; + } + } + + $outputRow[] = $sum; + } + } + + $output[] = $outputRow; + } + + return Matrix::quick($output); + } + + /** + * Add bias to the output. + * + * @param Matrix $output + * @param int $batchSize + * @return Matrix + */ + protected function addBias(Matrix $output, int $batchSize) : Matrix + { + $biases = $this->biases; + + if (!$biases) { + return $output; + } + + $biasArray = $biases->param()->asArray(); + $outputArray = $output->asArray(); + + $biased = []; + + foreach ($outputArray as $f => $row) { + $bias = $biasArray[$f]; + $biasedRow = []; + + foreach ($row as $value) { + $biasedRow[] = $value + $bias; + } + + $biased[] = $biasedRow; + } + + return Matrix::quick($biased); + } + + /** + * Compute the kernel gradient. + * + * @param Matrix $paddedInput + * @param Matrix $dOut + * @param int $paddedLength + * @param int $batchSize + * @return Matrix + */ + protected function computeKernelGradient( + Matrix $paddedInput, + Matrix $dOut, + int $paddedLength, + int $batchSize + ) : Matrix { + $dKernel = []; + + $inputArray = $paddedInput->asArray(); + $dOutArray = $dOut->asArray(); + + for ($f = 0; $f < $this->filters; ++$f) { + $kernelRow = []; + + for ($c = 0; $c < $this->inputChannels; ++$c) { + for ($k = 0; $k < $this->kernelSize; ++$k) { + $grad = 0.0; + + for ($b = 0; $b < $batchSize; ++$b) { + $sampleOffset = $b * $paddedLength; + $outputOffset = $b * $this->outputLength; + + for ($t = 0; $t < $this->outputLength; ++$t) { + $inputPos = $t * $this->stride + $k; + $grad += $inputArray[$c][$sampleOffset + $inputPos] + * $dOutArray[$f][$outputOffset + $t]; + } + } + + $kernelRow[] = $grad; + } + } + + $dKernel[] = $kernelRow; + } + + return Matrix::quick($dKernel); + } + + /** + * Compute the input gradient using transposed convolution. + * + * @param Matrix $kernel + * @param Matrix $dOut + * @param int $paddedLength + * @param int $batchSize + * @return Matrix + */ + protected function computeInputGradient( + Matrix $kernel, + Matrix $dOut, + int $paddedLength, + int $batchSize + ) : Matrix { + $dInput = []; + + $kernelArray = $kernel->asArray(); + $dOutArray = $dOut->asArray(); + + // Flip kernel for transposed convolution + $flippedKernel = []; + + foreach ($kernelArray as $f => $row) { + $flippedKernel[$f] = array_reverse($row); + } + + for ($c = 0; $c < $this->inputChannels; ++$c) { + $inputRow = []; + + for ($b = 0; $b < $batchSize; ++$b) { + $sampleOffset = $b * $paddedLength; + $outputOffset = $b * $this->outputLength; + + for ($t = 0; $t < $paddedLength; ++$t) { + $grad = 0.0; + + for ($f = 0; $f < $this->filters; ++$f) { + for ($k = 0; $k < $this->kernelSize; ++$k) { + // Calculate which output position contributes to this input position + $outT = ($t - $this->kernelSize + 1 + $k) / $this->stride; + + if ($outT >= 0 && $outT < $this->outputLength && $outT == (int) $outT) { + $outT = (int) $outT; + $kernelOffset = $c * $this->kernelSize + $k; + $grad += $flippedKernel[$f][$kernelOffset] + * $dOutArray[$f][$outputOffset + $outT]; + } + } + } + + $inputRow[] = $grad; + } + } + + $dInput[] = $inputRow; + } + + return Matrix::quick($dInput); + } + + /** + * Remove padding from the gradient. + * + * @param Matrix $dPaddedInput + * @param Matrix $originalInput + * @return Matrix + */ + protected function unpadGradient(Matrix $dPaddedInput, Matrix $originalInput) : Matrix + { + $dInput = []; + $paddedArray = $dPaddedInput->asArray(); + + foreach ($paddedArray as $c => $row) { + $unpaddedRow = []; + $batchSize = (int) ($originalInput->n() / $this->inputLength); + $paddedLength = $this->inputLength + 2 * $this->padding; + + for ($b = 0; $b < $batchSize; ++$b) { + $paddedOffset = $b * $paddedLength + $this->padding; + + for ($t = 0; $t < $this->inputLength; ++$t) { + $unpaddedRow[] = $row[$paddedOffset + $t]; + } + } + + $dInput[] = $unpaddedRow; + } + + return Matrix::quick($dInput); + } + + /** + * Return the string representation of the object. + * + * @internal + * + * @return string + */ + public function __toString() : string + { + return "Conv1D (filters: {$this->filters}, kernel size: {$this->kernelSize}," + . " input length: {$this->inputLength}, input channels: {$this->inputChannels}," + . " stride: {$this->stride}, padding: {$this->padding}," + . " l2 penalty: {$this->l2Penalty}, bias: " . Params::toString($this->bias) . ',' + . " kernel initializer: {$this->kernelInitializer}," + . " bias initializer: {$this->biasInitializer})"; + } +} diff --git a/src/NeuralNet/Layers/MaxPool1D.php b/src/NeuralNet/Layers/MaxPool1D.php new file mode 100644 index 000000000..ef77dfdb4 --- /dev/null +++ b/src/NeuralNet/Layers/MaxPool1D.php @@ -0,0 +1,344 @@ +>|null + */ + protected ?array $maxIndices = null; + + /** + * @param int $poolSize Size of the pooling window + * @param int $inputLength Length of the input sequence + * @param int $stride Stride of the pooling operation (default: same as poolSize) + * @throws InvalidArgumentException + */ + public function __construct(int $poolSize, int $inputLength, int $stride = 0) + { + if ($poolSize < 1) { + throw new InvalidArgumentException('Pool size must be' + . " greater than 0, $poolSize given."); + } + + if ($inputLength < 1) { + throw new InvalidArgumentException('Input length must be' + . " greater than 0, $inputLength given."); + } + + if ($stride < 0) { + throw new InvalidArgumentException('Stride cannot be' + . " negative, $stride given."); + } + + $stride = $stride > 0 ? $stride : $poolSize; + + $outputLength = (int) floor(($inputLength - $poolSize) / $stride) + 1; + + if ($outputLength < 1) { + throw new InvalidArgumentException('Output length must be' + . " greater than 0, $outputLength given. Check pool size and stride values."); + } + + $this->poolSize = $poolSize; + $this->inputLength = $inputLength; + $this->stride = $stride; + $this->outputLength = $outputLength; + } + + /** + * Return the width of the layer (same as input channels). + * + * @internal + * + * @throws RuntimeException + * @return positive-int + */ + public function width() : int + { + if ($this->inputChannels === null) { + throw new RuntimeException('Layer has not been initialized.'); + } + + return $this->inputChannels; + } + + /** + * Return the output length after pooling. + * + * @internal + * + * @return positive-int + */ + public function outputLength() : int + { + return $this->outputLength; + } + + /** + * Initialize the layer with the fan in from the previous layer and return + * the fan out for this layer. + * + * @internal + * + * @param positive-int $fanIn + * @return positive-int + */ + public function initialize(int $fanIn) : int + { + $this->inputChannels = $fanIn; + + return $fanIn * $this->outputLength; + } + + /** + * Compute a forward pass through the layer. + * + * @internal + * + * @param Matrix $input + * @throws RuntimeException + * @return Matrix + */ + public function forward(Matrix $input) : Matrix + { + if ($this->inputChannels === null) { + throw new RuntimeException('Layer has not been initialized.'); + } + + $inputChannels = $input->m(); + + if ($inputChannels !== $this->inputChannels) { + throw new RuntimeException('Input channels mismatch:' + . " expected {$this->inputChannels}, got {$inputChannels}."); + } + + $batchSize = (int) ($input->n() / $this->inputLength); + + $inputArray = $input->asArray(); + $output = []; + $maxIndices = []; + + foreach ($inputArray as $c => $channel) { + $outputRow = []; + $indicesRow = []; + + for ($b = 0; $b < $batchSize; ++$b) { + $sampleOffset = $b * $this->inputLength; + + for ($t = 0; $t < $this->outputLength; ++$t) { + $startPos = $t * $this->stride; + $maxVal = -INF; + $maxIdx = $startPos; + + for ($p = 0; $p < $this->poolSize; ++$p) { + $pos = $startPos + $p; + $val = $channel[$sampleOffset + $pos]; + + if ($val > $maxVal) { + $maxVal = $val; + $maxIdx = $sampleOffset + $pos; + } + } + + $outputRow[] = $maxVal; + $indicesRow[] = $maxIdx; + } + } + + $output[] = $outputRow; + $maxIndices[] = $indicesRow; + } + + $this->maxIndices = $maxIndices; + + return Matrix::quick($output); + } + + /** + * Compute an inference pass through the layer. + * + * @internal + * + * @param Matrix $input + * @throws RuntimeException + * @return Matrix + */ + public function infer(Matrix $input) : Matrix + { + if ($this->inputChannels === null) { + throw new RuntimeException('Layer has not been initialized.'); + } + + $inputChannels = $input->m(); + + if ($inputChannels !== $this->inputChannels) { + throw new RuntimeException('Input channels mismatch:' + . " expected {$this->inputChannels}, got {$inputChannels}."); + } + + $batchSize = (int) ($input->n() / $this->inputLength); + + $inputArray = $input->asArray(); + $output = []; + + foreach ($inputArray as $channel) { + $outputRow = []; + + for ($b = 0; $b < $batchSize; ++$b) { + $sampleOffset = $b * $this->inputLength; + + for ($t = 0; $t < $this->outputLength; ++$t) { + $startPos = $t * $this->stride; + $maxVal = -INF; + + for ($p = 0; $p < $this->poolSize; ++$p) { + $pos = $startPos + $p; + $val = $channel[$sampleOffset + $pos]; + + if ($val > $maxVal) { + $maxVal = $val; + } + } + + $outputRow[] = $maxVal; + } + } + + $output[] = $outputRow; + } + + return Matrix::quick($output); + } + + /** + * Calculate the gradient and update the parameters of the layer. + * + * @internal + * + * @param Deferred $prevGradient + * @param Optimizer $optimizer + * @throws RuntimeException + * @return Deferred + */ + public function back(Deferred $prevGradient, Optimizer $optimizer) : Deferred + { + if ($this->maxIndices === null) { + throw new RuntimeException('Must perform forward pass before backpropagating.'); + } + + $maxIndices = $this->maxIndices; + $inputChannels = $this->inputChannels; + $inputLength = $this->inputLength; + + $this->maxIndices = null; + + return new Deferred([$this, 'gradient'], [$prevGradient, $maxIndices, $inputChannels, $inputLength]); + } + + /** + * Calculate the gradient for the previous layer. + * + * @internal + * + * @param Deferred $prevGradient + * @param array> $maxIndices + * @param int $inputChannels + * @param int $inputLength + * @return Matrix + */ + public function gradient( + Deferred $prevGradient, + array $maxIndices, + int $inputChannels, + int $inputLength + ) : Matrix { + $dOut = $prevGradient(); + $dOutArray = $dOut->asArray(); + + $batchSize = (int) (count($dOutArray[0]) / $this->outputLength); + + // Initialize gradient with zeros + $dInput = array_fill(0, $inputChannels, array_fill(0, $inputLength * $batchSize, 0.0)); + + // Distribute gradients to max positions + foreach ($maxIndices as $c => $indices) { + $outputPos = 0; + + foreach ($indices as $idx) { + $dInput[$c][$idx] += $dOutArray[$c][$outputPos]; + ++$outputPos; + } + } + + return Matrix::quick($dInput); + } + + /** + * Return the string representation of the object. + * + * @internal + * + * @return string + */ + public function __toString() : string + { + return "MaxPool1D (pool size: {$this->poolSize}, input length: {$this->inputLength}," + . " stride: {$this->stride})"; + } +} diff --git a/tests/NeuralNet/Layers/AvgPool1DTest.php b/tests/NeuralNet/Layers/AvgPool1DTest.php new file mode 100644 index 000000000..9a02a4fe2 --- /dev/null +++ b/tests/NeuralNet/Layers/AvgPool1DTest.php @@ -0,0 +1,291 @@ +inputChannels; ++$c) { + $row = []; + + for ($b = 0; $b < $this->batchSize; ++$b) { + for ($t = 0; $t < $this->inputLength; ++$t) { + $row[] = $t + $b * $this->inputLength + $c * 0.1; + } + } + + $inputData[] = $row; + } + + $this->input = Matrix::quick($inputData); + + $this->optimizer = new Stochastic(0.001); + + $this->layer = new AvgPool1D( + 3, + $this->inputLength, + 1 + ); + + srand(self::RANDOM_SEED); + } + + /** + * @test + */ + public function build() : void + { + $this->assertInstanceOf(AvgPool1D::class, $this->layer); + $this->assertInstanceOf(Layer::class, $this->layer); + $this->assertInstanceOf(Hidden::class, $this->layer); + } + + /** + * @test + */ + public function initializeForwardBackInfer() : void + { + $fanOut = $this->layer->initialize($this->inputChannels); + + // Width should be number of input channels + $this->assertEquals($this->inputChannels, $this->layer->width()); + + // Output length: floor((10 - 3) / 1) + 1 = 8 + $this->assertEquals(8, $this->layer->outputLength()); + + // Fan out should be inputChannels * outputLength = 2 * 8 = 16 + $this->assertEquals(16, $fanOut); + + // Forward pass + $forward = $this->layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + + // Output shape: (inputChannels, outputLength * batchSize) = (2, 8 * 3) = (2, 24) + $this->assertEquals($this->inputChannels, $forward->m()); + $this->assertEquals(8 * $this->batchSize, $forward->n()); + + // Backward pass + $outputLength = 8; + $gradData = []; + + for ($c = 0; $c < $this->inputChannels; ++$c) { + $row = []; + + for ($b = 0; $b < $this->batchSize; ++$b) { + for ($t = 0; $t < $outputLength; ++$t) { + $row[] = 0.01 * ($t + $c); + } + } + + $gradData[] = $row; + } + + $this->prevGrad = new Deferred(function () use ($gradData) { + return Matrix::quick($gradData); + }); + + $gradient = $this->layer->back($this->prevGrad, $this->optimizer)->compute(); + + $this->assertInstanceOf(Matrix::class, $gradient); + + // Gradient shape should match input shape + $this->assertEquals($this->inputChannels, $gradient->m()); + $this->assertEquals($this->inputLength * $this->batchSize, $gradient->n()); + + // Inference pass + $infer = $this->layer->infer($this->input); + + $this->assertInstanceOf(Matrix::class, $infer); + $this->assertEquals($this->inputChannels, $infer->m()); + $this->assertEquals(8 * $this->batchSize, $infer->n()); + } + + /** + * @test + */ + public function withStride() : void + { + // With stride=2, output length = floor((10 - 3) / 2) + 1 = 4 + $layer = new AvgPool1D( + 3, + 10, + 2 + ); + + srand(self::RANDOM_SEED); + + $layer->initialize($this->inputChannels); + + $this->assertEquals(4, $layer->outputLength()); + + $forward = $layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + + // Output shape: (2, 4 * 3) = (2, 12) + $this->assertEquals($this->inputChannels, $forward->m()); + $this->assertEquals(4 * $this->batchSize, $forward->n()); + } + + /** + * @test + */ + public function defaultStride() : void + { + // With default stride (= poolSize), output length = floor((10 - 3) / 3) + 1 = 3 + $layer = new AvgPool1D( + 3, + 10 + ); + + srand(self::RANDOM_SEED); + + $layer->initialize($this->inputChannels); + + $this->assertEquals(3, $layer->outputLength()); + + $forward = $layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + $this->assertEquals($this->inputChannels, $forward->m()); + $this->assertEquals(3 * $this->batchSize, $forward->n()); + } + + /** + * @test + */ + public function invalidPoolSize() : void + { + $this->expectException(\Rubix\ML\Exceptions\InvalidArgumentException::class); + + new AvgPool1D( + 15, + 10 + ); + } + + /** + * @test + */ + public function negativeStride() : void + { + $this->expectException(\Rubix\ML\Exceptions\InvalidArgumentException::class); + + new AvgPool1D( + 3, + 10, + -1 + ); + } + + /** + * @test + */ + public function gradientNumericalCheck() : void + { + $layer = new AvgPool1D(2, 4, 1); + $layer->initialize(1); + + $inputData = [[1.0, 2.0, 3.0, 4.0]]; + $input = Matrix::quick($inputData); + $layer->forward($input); + + $gradData = [[1.0, 1.0, 1.0]]; // output length = floor((4-2)/1) + 1 = 3 + $prevGrad = new Deferred(function () use ($gradData) { + return Matrix::quick($gradData); + }); + + $gradient = $layer->back($prevGrad, $this->optimizer)->compute(); + $analyticalGrad = $gradient->asArray()[0]; + + $epsilon = 1e-5; + $numericalGrad = []; + + for ($i = 0; $i < 4; ++$i) { + $plus = $inputData; + $minus = $inputData; + $plus[0][$i] += $epsilon; + $minus[0][$i] -= $epsilon; + + $layerPlus = new AvgPool1D(2, 4, 1); + $layerPlus->initialize(1); + $outPlus = $layerPlus->forward(Matrix::quick($plus)); + + $layerMinus = new AvgPool1D(2, 4, 1); + $layerMinus->initialize(1); + $outMinus = $layerMinus->forward(Matrix::quick($minus)); + + $sumPlus = array_sum($outPlus->asArray()[0]); + $sumMinus = array_sum($outMinus->asArray()[0]); + + $numericalGrad[$i] = ($sumPlus - $sumMinus) / (2 * $epsilon); + } + + for ($i = 0; $i < 4; ++$i) { + $this->assertEqualsWithDelta( + $numericalGrad[$i], + $analyticalGrad[$i], + 1e-4, + "Gradient mismatch at index {$i}: numerical={$numericalGrad[$i]}, analytical={$analyticalGrad[$i]}" + ); + } + + $this->assertEqualsWithDelta(0.5, $analyticalGrad[0], 1e-10); + $this->assertEqualsWithDelta(1.0, $analyticalGrad[1], 1e-10); + $this->assertEqualsWithDelta(1.0, $analyticalGrad[2], 1e-10); + $this->assertEqualsWithDelta(0.5, $analyticalGrad[3], 1e-10); + } +} diff --git a/tests/NeuralNet/Layers/Conv1DTest.php b/tests/NeuralNet/Layers/Conv1DTest.php new file mode 100644 index 000000000..d6af87a5d --- /dev/null +++ b/tests/NeuralNet/Layers/Conv1DTest.php @@ -0,0 +1,599 @@ +inputChannels; ++$c) { + $row = []; + + for ($b = 0; $b < $batchSize; ++$b) { + for ($t = 0; $t < $this->inputLength; ++$t) { + $row[] = $t + $b * $this->inputLength + $c * 0.1; + } + } + $inputData[] = $row; + } + + $this->input = Matrix::quick($inputData); + + // Create gradient for backprop: shape (filters, outputLength * batchSize) + $filters = 2; + $kernelSize = 3; + $outputLength = $this->inputLength - $kernelSize + 1; // 8 + + $gradData = []; + + for ($f = 0; $f < $filters; ++$f) { + $row = []; + + for ($b = 0; $b < $batchSize; ++$b) { + for ($t = 0; $t < $outputLength; ++$t) { + $row[] = 0.1 * ($t + $f); + } + } + $gradData[] = $row; + } + + $this->prevGrad = new Deferred(function () use ($gradData) { + return Matrix::quick($gradData); + }); + + $this->optimizer = new Stochastic(0.001); + + $this->layer = new Conv1D( + $filters, + $kernelSize, + $this->inputLength, + $this->inputChannels, + 1, // stride + 0, // padding + 0.0, // l2 penalty + true, // bias + new He(), + new Constant(0.0) + ); + + srand(self::RANDOM_SEED); + } + + /** + * @test + */ + public function build() : void + { + $this->assertInstanceOf(Conv1D::class, $this->layer); + $this->assertInstanceOf(Layer::class, $this->layer); + $this->assertInstanceOf(Hidden::class, $this->layer); + $this->assertInstanceOf(Parametric::class, $this->layer); + } + + /** + * @test + */ + public function initializeForwardBackInfer() : void + { + $fanOut = $this->layer->initialize($this->inputLength); + + // Width should be number of filters + $this->assertEquals(2, $this->layer->width()); + + // Output length should be inputLength - kernelSize + 1 = 10 - 3 + 1 = 8 + $this->assertEquals(8, $this->layer->outputLength()); + + // Fan out should be filters * outputLength = 2 * 8 = 16 + $this->assertEquals(16, $fanOut); + + // Forward pass + $forward = $this->layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + + // Output shape should be (filters, outputLength * batchSize) = (2, 8 * 3) = (2, 24) + $this->assertEquals(2, $forward->m()); + $this->assertEquals(24, $forward->n()); + + // Backward pass + $gradient = $this->layer->back($this->prevGrad, $this->optimizer)->compute(); + + $this->assertInstanceOf(Matrix::class, $gradient); + + // Gradient shape should match input shape + $this->assertEquals($this->inputChannels, $gradient->m()); + $this->assertEquals($this->input->n(), $gradient->n()); + + // Inference pass - should produce same shape as forward + $infer = $this->layer->infer($this->input); + + $this->assertInstanceOf(Matrix::class, $infer); + $this->assertEquals(2, $infer->m()); + $this->assertEquals(24, $infer->n()); + } + + /** + * @test + */ + public function withPadding() : void + { + // With padding=1, output length should be same as input length (for kernelSize=3) + // outputLength = floor((inputLength + 2*padding - kernelSize) / stride) + 1 + // = floor((10 + 2*1 - 3) / 1) + 1 = 10 + + $layer = new Conv1D( + 2, // filters + 3, // kernelSize + 10, // inputLength + 1, // inputChannels + 1, // stride + 1, // padding + 0.0, + true, + new He(), + new Constant(0.0) + ); + + srand(self::RANDOM_SEED); + + $layer->initialize(10); + + $this->assertEquals(10, $layer->outputLength()); + + $forward = $layer->forward($this->input); + + // Output shape: (2, 10 * 3) = (2, 30) + $this->assertEquals(2, $forward->m()); + $this->assertEquals(30, $forward->n()); + } + + /** + * @test + */ + public function withStride() : void + { + // With stride=2, output length should be half (roughly) + // outputLength = floor((inputLength - kernelSize) / stride) + 1 + // = floor((10 - 3) / 2) + 1 = 4 + + $layer = new Conv1D( + 2, // filters + 3, // kernelSize + 10, // inputLength + 1, // inputChannels + 2, // stride + 0, // padding + 0.0, + true, + new He(), + new Constant(0.0) + ); + + srand(self::RANDOM_SEED); + + $layer->initialize(10); + + $this->assertEquals(4, $layer->outputLength()); + + $forward = $layer->forward($this->input); + + // Output shape: (2, 4 * 3) = (2, 12) + $this->assertEquals(2, $forward->m()); + $this->assertEquals(12, $forward->n()); + } + + /** + * @test + */ + public function multiChannel() : void + { + // Test with 3 input channels + $inputChannels = 3; + $batchSize = 2; + + $inputData = []; + + for ($c = 0; $c < $inputChannels; ++$c) { + $row = []; + + for ($b = 0; $b < $batchSize; ++$b) { + for ($t = 0; $t < $this->inputLength; ++$t) { + $row[] = $t + $b * $this->inputLength + $c * 0.1; + } + } + $inputData[] = $row; + } + + $input = Matrix::quick($inputData); + + $layer = new Conv1D( + 4, // filters + 3, // kernelSize + 10, // inputLength + $inputChannels, // inputChannels + 1, // stride + 0, // padding + 0.0, + true, + new He(), + new Constant(0.0) + ); + + srand(self::RANDOM_SEED); + + $layer->initialize(10); + + $forward = $layer->forward($input); + + // Output shape: (4, 8 * 2) = (4, 16) + $this->assertEquals(4, $forward->m()); + $this->assertEquals(16, $forward->n()); + + // Backward pass + $gradData = []; + + for ($f = 0; $f < 4; ++$f) { + $row = []; + + for ($b = 0; $b < $batchSize; ++$b) { + for ($t = 0; $t < 8; ++$t) { + $row[] = 0.1 * ($t + $f); + } + } + $gradData[] = $row; + } + + $prevGrad = new Deferred(function () use ($gradData) { + return Matrix::quick($gradData); + }); + + $optimizer = new Stochastic(0.001); + + $gradient = $layer->back($prevGrad, $optimizer)->compute(); + + // Gradient shape should match input: (3, 10 * 2) = (3, 20) + $this->assertEquals($inputChannels, $gradient->m()); + $this->assertEquals($input->n(), $gradient->n()); + } + + /** + * @test + */ + public function invalidParameters() : void + { + // Test kernel size larger than input length with no padding + $this->expectException(\Rubix\ML\Exceptions\InvalidArgumentException::class); + + new Conv1D( + 2, // filters + 15, // kernelSize > inputLength + 10, // inputLength + 1, // inputChannels + 1, // stride + 0 // padding + ); + } + + /** + * @test + */ + public function negativeStride() : void + { + $this->expectException(\Rubix\ML\Exceptions\InvalidArgumentException::class); + + new Conv1D( + 2, // filters + 3, // kernelSize + 10, // inputLength + 1, // inputChannels + 0, // stride (invalid) + 0 // padding + ); + } + + /** + * @test + */ + public function negativePadding() : void + { + $this->expectException(\Rubix\ML\Exceptions\InvalidArgumentException::class); + + new Conv1D( + 2, // filters + 3, // kernelSize + 10, // inputLength + 1, // inputChannels + 1, // stride + -1 // padding (invalid) + ); + } + + /** + * @test + */ + public function l2Penalty() : void + { + $layer = new Conv1D( + 2, // filters + 3, // kernelSize + 10, // inputLength + 1, // inputChannels + 1, // stride + 0, // padding + 0.01, // l2 penalty + true, + new He(), + new Constant(0.0) + ); + + srand(self::RANDOM_SEED); + + $layer->initialize(10); + + $forward = $layer->forward($this->input); + + $gradData = []; + + for ($f = 0; $f < 2; ++$f) { + $row = []; + + for ($b = 0; $b < 3; ++$b) { + for ($t = 0; $t < 8; ++$t) { + $row[] = 0.1; + } + } + $gradData[] = $row; + } + + $prevGrad = new Deferred(function () use ($gradData) { + return Matrix::quick($gradData); + }); + + $optimizer = new Stochastic(0.001); + + $gradient = $layer->back($prevGrad, $optimizer)->compute(); + + $this->assertInstanceOf(Matrix::class, $gradient); + } + + /** + * @test + */ + public function noBias() : void + { + $layer = new Conv1D( + 2, // filters + 3, // kernelSize + 10, // inputLength + 1, // inputChannels + 1, // stride + 0, // padding + 0.0, + false, // no bias + new He(), + new Constant(0.0) + ); + + srand(self::RANDOM_SEED); + + $layer->initialize(10); + + $forward = $layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + + // Check that there's no bias parameter + $params = iterator_to_array($layer->parameters()); + $this->assertArrayHasKey('weights', $params); + $this->assertArrayNotHasKey('biases', $params); + } + + /** + * @test + */ + public function forwardPassCalculation() : void + { + $layer = new Conv1D( + 1, + 3, + 5, + 1, + 1, + 0, + 0.0, + false, + new Constant(0.1), + new Constant(0.0) + ); + + $layer->initialize(5); + + $weights = $layer->weights(); + $this->assertEqualsWithDelta(0.1, $weights[0][0], 1e-10); + $this->assertEqualsWithDelta(0.1, $weights[0][1], 1e-10); + $this->assertEqualsWithDelta(0.1, $weights[0][2], 1e-10); + + $input = Matrix::quick([[1.0, 2.0, 3.0, 4.0, 5.0]]); + + $output = $layer->forward($input); + $outputArray = $output->asArray()[0]; + + $this->assertEqualsWithDelta(0.6, $outputArray[0], 1e-10); + $this->assertEqualsWithDelta(0.9, $outputArray[1], 1e-10); + $this->assertEqualsWithDelta(1.2, $outputArray[2], 1e-10); + } + + /** + * @test + */ + public function backwardPassCalculation() : void + { + $layer = new Conv1D( + 1, + 3, + 5, + 1, + 1, + 0, + 0.0, + false, + new Constant(0.1), + new Constant(0.0) + ); + + $layer->initialize(5); + $weights = $layer->weights(); + $this->assertEqualsWithDelta(0.1, $weights[0][0], 1e-10); + $this->assertEqualsWithDelta(0.1, $weights[0][1], 1e-10); + $this->assertEqualsWithDelta(0.1, $weights[0][2], 1e-10); + + $input = Matrix::quick([[1.0, 2.0, 3.0, 4.0, 5.0]]); + + $layer->forward($input); + $dOut = Matrix::quick([[0.5, 0.3, 0.2]]); + + $prevGrad = new Deferred(function () use ($dOut) { + return $dOut; + }); + + $optimizer = new Stochastic(0.001); + $dInput = $layer->back($prevGrad, $optimizer)->compute(); + $dInputArray = $dInput->asArray()[0]; + + $this->assertEquals(5, count($dInputArray)); + $this->assertEqualsWithDelta(0.05, $dInputArray[0], 1e-10); + $this->assertEqualsWithDelta(0.08, $dInputArray[1], 1e-10); + $this->assertEqualsWithDelta(0.10, $dInputArray[2], 1e-10); + $this->assertEqualsWithDelta(0.05, $dInputArray[3], 1e-10); + $this->assertEqualsWithDelta(0.02, $dInputArray[4], 1e-10); + } + + /** + * @test + */ + public function gradientNumericalCheck() : void + { + $kernelSize = 3; + $inputLength = 5; + $filters = 1; + $inputChannels = 1; + + $layer = new Conv1D( + $filters, + $kernelSize, + $inputLength, + $inputChannels, + 1, + 0, + 0.0, + false, + new Constant(0.0), + new Constant(0.0) + ); + + $layer->initialize($inputLength); + $inputData = [[1.0, 2.0, 3.0, 4.0, 5.0]]; + $input = Matrix::quick($inputData); + + $layer->forward($input); + + $dOutData = [[1.0, 1.0, 1.0]]; + $dOut = Matrix::quick($dOutData); + + $prevGrad = new Deferred(function () use ($dOut) { + return $dOut; + }); + + $optimizer = new Stochastic(0.001); + + $dInput = $layer->back($prevGrad, $optimizer)->compute(); + $analyticalGrad = $dInput->asArray()[0]; + + $epsilon = 1e-5; + $numericalGrad = []; + + for ($i = 0; $i < $inputLength; ++$i) { + $plus = $inputData; + $minus = $inputData; + $plus[0][$i] += $epsilon; + $minus[0][$i] -= $epsilon; + + $layerPlus = new Conv1D($filters, $kernelSize, $inputLength, $inputChannels, 1, 0, 0.0, false, new Constant(0.0), new Constant(0.0)); + $layerPlus->initialize($inputLength); + $outPlus = $layerPlus->forward(Matrix::quick($plus)); + + $layerMinus = new Conv1D($filters, $kernelSize, $inputLength, $inputChannels, 1, 0, 0.0, false, new Constant(0.0), new Constant(0.0)); + $layerMinus->initialize($inputLength); + $outMinus = $layerMinus->forward(Matrix::quick($minus)); + + $sumPlus = array_sum($outPlus->asArray()[0]); + $sumMinus = array_sum($outMinus->asArray()[0]); + + $numericalGrad[$i] = ($sumPlus - $sumMinus) / (2 * $epsilon); + } + + for ($i = 0; $i < $inputLength; ++$i) { + $this->assertEqualsWithDelta( + $numericalGrad[$i], + $analyticalGrad[$i], + 1e-4, + "Input gradient mismatch at index {$i}: numerical={$numericalGrad[$i]}, analytical={$analyticalGrad[$i]}" + ); + } + } +} diff --git a/tests/NeuralNet/Layers/MaxPool1DTest.php b/tests/NeuralNet/Layers/MaxPool1DTest.php new file mode 100644 index 000000000..c3644bc71 --- /dev/null +++ b/tests/NeuralNet/Layers/MaxPool1DTest.php @@ -0,0 +1,297 @@ +inputChannels; ++$c) { + $row = []; + + for ($b = 0; $b < $this->batchSize; ++$b) { + for ($t = 0; $t < $this->inputLength; ++$t) { + $row[] = $t + $b * $this->inputLength + $c * 0.1; + } + } + + $inputData[] = $row; + } + + $this->input = Matrix::quick($inputData); + + $this->optimizer = new Stochastic(0.001); + + $this->layer = new MaxPool1D( + 3, + $this->inputLength, + 1 + ); + + srand(self::RANDOM_SEED); + } + + /** + * @test + */ + public function build() : void + { + $this->assertInstanceOf(MaxPool1D::class, $this->layer); + $this->assertInstanceOf(Layer::class, $this->layer); + $this->assertInstanceOf(Hidden::class, $this->layer); + } + + /** + * @test + */ + public function initializeForwardBackInfer() : void + { + $fanOut = $this->layer->initialize($this->inputChannels); + + // Width should be number of input channels + $this->assertEquals($this->inputChannels, $this->layer->width()); + + // Output length: floor((10 - 3) / 1) + 1 = 8 + $this->assertEquals(8, $this->layer->outputLength()); + + // Fan out should be inputChannels * outputLength = 2 * 8 = 16 + $this->assertEquals(16, $fanOut); + + // Forward pass + $forward = $this->layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + + // Output shape: (inputChannels, outputLength * batchSize) = (2, 8 * 3) = (2, 24) + $this->assertEquals($this->inputChannels, $forward->m()); + $this->assertEquals(8 * $this->batchSize, $forward->n()); + + // Backward pass + $outputLength = 8; + $gradData = []; + + for ($c = 0; $c < $this->inputChannels; ++$c) { + $row = []; + + for ($b = 0; $b < $this->batchSize; ++$b) { + for ($t = 0; $t < $outputLength; ++$t) { + $row[] = 0.01 * ($t + $c); + } + } + + $gradData[] = $row; + } + + $this->prevGrad = new Deferred(function () use ($gradData) { + return Matrix::quick($gradData); + }); + + $gradient = $this->layer->back($this->prevGrad, $this->optimizer)->compute(); + + $this->assertInstanceOf(Matrix::class, $gradient); + + // Gradient shape should match input shape + $this->assertEquals($this->inputChannels, $gradient->m()); + $this->assertEquals($this->inputLength * $this->batchSize, $gradient->n()); + + // Inference pass + $infer = $this->layer->infer($this->input); + + $this->assertInstanceOf(Matrix::class, $infer); + $this->assertEquals($this->inputChannels, $infer->m()); + $this->assertEquals(8 * $this->batchSize, $infer->n()); + } + + /** + * @test + */ + public function withStride() : void + { + // With stride=2, output length = floor((10 - 3) / 2) + 1 = 4 + $layer = new MaxPool1D( + 3, + 10, + 2 + ); + + srand(self::RANDOM_SEED); + + $layer->initialize($this->inputChannels); + + $this->assertEquals(4, $layer->outputLength()); + + $forward = $layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + + // Output shape: (2, 4 * 3) = (2, 12) + $this->assertEquals($this->inputChannels, $forward->m()); + $this->assertEquals(4 * $this->batchSize, $forward->n()); + } + + /** + * @test + */ + public function defaultStride() : void + { + // With default stride (= poolSize), output length = floor((10 - 3) / 3) + 1 = 3 + $layer = new MaxPool1D( + 3, + 10 + ); + + srand(self::RANDOM_SEED); + + $layer->initialize($this->inputChannels); + + $this->assertEquals(3, $layer->outputLength()); + + $forward = $layer->forward($this->input); + + $this->assertInstanceOf(Matrix::class, $forward); + $this->assertEquals($this->inputChannels, $forward->m()); + $this->assertEquals(3 * $this->batchSize, $forward->n()); + } + + /** + * @test + */ + public function invalidPoolSize() : void + { + $this->expectException(\Rubix\ML\Exceptions\InvalidArgumentException::class); + + new MaxPool1D( + 15, + 10 + ); + } + + /** + * @test + */ + public function negativeStride() : void + { + $this->expectException(\Rubix\ML\Exceptions\InvalidArgumentException::class); + + new MaxPool1D( + 3, + 10, + -1 + ); + } + + /** + * @test + */ + public function gradientNumericalCheck() : void + { + $layer = new MaxPool1D(2, 4, 1); + $layer->initialize(1); + + $inputData = [[1.0, 3.0, 2.0, 4.0]]; + $input = Matrix::quick($inputData); + $output = $layer->forward($input); + + $outputArray = $output->asArray()[0]; + $this->assertEquals(3.0, $outputArray[0]); // max(1, 3) = 3 + $this->assertEquals(3.0, $outputArray[1]); // max(3, 2) = 3 + $this->assertEquals(4.0, $outputArray[2]); // max(2, 4) = 4 + + $gradData = [[1.0, 1.0, 1.0]]; + $prevGrad = new Deferred(function () use ($gradData) { + return Matrix::quick($gradData); + }); + + $gradient = $layer->back($prevGrad, $this->optimizer)->compute(); + $analyticalGrad = $gradient->asArray()[0]; + + // Expected gradients: [0, 2, 0, 1] (sum of gradients at max positions) + $this->assertEquals(0.0, $analyticalGrad[0]); + $this->assertEquals(2.0, $analyticalGrad[1]); + $this->assertEquals(0.0, $analyticalGrad[2]); + $this->assertEquals(1.0, $analyticalGrad[3]); + + $epsilon = 1e-5; + $numericalGrad = []; + + for ($i = 0; $i < 4; ++$i) { + $plus = $inputData; + $minus = $inputData; + $plus[0][$i] += $epsilon; + $minus[0][$i] -= $epsilon; + + $layerPlus = new MaxPool1D(2, 4, 1); + $layerPlus->initialize(1); + $outPlus = $layerPlus->forward(Matrix::quick($plus)); + + $layerMinus = new MaxPool1D(2, 4, 1); + $layerMinus->initialize(1); + $outMinus = $layerMinus->forward(Matrix::quick($minus)); + + $sumPlus = array_sum($outPlus->asArray()[0]); + $sumMinus = array_sum($outMinus->asArray()[0]); + + $numericalGrad[$i] = ($sumPlus - $sumMinus) / (2 * $epsilon); + } + + for ($i = 0; $i < 4; ++$i) { + $this->assertEqualsWithDelta( + $numericalGrad[$i], + $analyticalGrad[$i], + 1e-4, + "Gradient mismatch at index {$i}: numerical={$numericalGrad[$i]}, analytical={$analyticalGrad[$i]}" + ); + } + } +}