From e6b6ebe0f6ebb7dabdf2da84c6571d64232338cf Mon Sep 17 00:00:00 2001
From: tahashieenavaz <tahashieenavaz@gmail.com>
Date: Wed, 5 Aug 2026 17:38:19 +0200
Subject: [PATCH 1/4] Adds GELU activation function

---
 neat/activations.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/neat/activations.py b/neat/activations.py
index df2b959f..98e60755 100644
--- a/neat/activations.py
+++ b/neat/activations.py
@@ -95,6 +95,8 @@ def square_activation(z):
 def cube_activation(z):
     return z ** 3
 
+def gelu_activation(z):
+    return 0.5 * z * (1 + math.erf(z / math.sqrt(2)))
 
 class InvalidActivationFunction(TypeError):
     pass

From de28d3db036d28107be8f1f5cf991f96631ba057 Mon Sep 17 00:00:00 2001
From: tahashieenavaz <tahashieenavaz@gmail.com>
Date: Wed, 5 Aug 2026 17:38:52 +0200
Subject: [PATCH 2/4] Add GELU approximation activation function

---
 neat/activations.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/neat/activations.py b/neat/activations.py
index 98e60755..21573cb0 100644
--- a/neat/activations.py
+++ b/neat/activations.py
@@ -95,9 +95,19 @@ def square_activation(z):
 def cube_activation(z):
     return z ** 3
 
+
 def gelu_activation(z):
     return 0.5 * z * (1 + math.erf(z / math.sqrt(2)))
 
+
+def gelu_approximation_activation(z):
+    return 0.5 * z * (
+        1 + math.tanh(
+            math.sqrt(2 / math.pi) * (z + 0.044715 * z**3)
+        )
+    )
+
+
 class InvalidActivationFunction(TypeError):
     pass
 

From e0b11e0ac34df454683e43612d90055a5d26aee0 Mon Sep 17 00:00:00 2001
From: tahashieenavaz <tahashieenavaz@gmail.com>
Date: Wed, 5 Aug 2026 17:49:28 +0200
Subject: [PATCH 3/4] Add unit tests for GELU activation function

---
 tests/test_activation.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/test_activation.py b/tests/test_activation.py
index 5e62acc8..39e5e65f 100644
--- a/tests/test_activation.py
+++ b/tests/test_activation.py
@@ -105,6 +105,21 @@ def test_cube():
     assert activations.cube_activation(0.5) == 0.125
     assert activations.cube_activation(1.0) == 1.0
 
+def test_gelu():
+    assert_almost_equal(activations.gelu_activation(-1.0), -0.15865525393145707)
+    assert_almost_equal(activations.gelu_activation(-0.5), -0.15426876936299347)
+    assert activations.gelu_activation(0.0) == 0.0
+    assert_almost_equal(activations.gelu_activation(0.5), 0.3457312306370065)
+    assert_almost_equal(activations.gelu_activation(1.0), 0.8413447460685429)
+
+
+def test_gelu_approximation():
+    assert_almost_equal(activations.gelu_approximation_activation(-1.0), -0.1588080093917233)
+    assert_almost_equal(activations.gelu_approximation_activation(-0.5), -0.15428599017485606)
+    assert activations.gelu_approximation_activation(0.0) == 0.0
+    assert_almost_equal(activations.gelu_approximation_activation(0.5), 0.34571400982514394)
+    assert_almost_equal(activations.gelu_approximation_activation(1.0), 0.8411919906082768)
+
 
 def plus_activation(x):
     """ Not useful - just a check. """
@@ -145,6 +160,8 @@ def test_function_set():
     assert s.get('hat') is not None
     assert s.get('square') is not None
     assert s.get('cube') is not None
+    assert s.get('gelu') is not None
+    assert s.get('gelu_approximation') is not None
 
     assert s.is_valid('sigmoid')
     assert s.is_valid('tanh')
@@ -163,6 +180,8 @@ def test_function_set():
     assert s.is_valid('hat')
     assert s.is_valid('square')
     assert s.is_valid('cube')
+    assert s.is_valid('gelu')
+    assert s.is_valid('gelu_approximation') 
 
     assert not s.is_valid('foo')
 
@@ -213,3 +232,5 @@ def test_bad_add2():
     test_hat()
     test_square()
     test_cube()
+    test_gelu()
+    test_gelu_approximation()

From 3b6e8142a1991e7c666f0cb4b348a51969e6767c Mon Sep 17 00:00:00 2001
From: tahashieenavaz <tahashieenavaz@gmail.com>
Date: Wed, 5 Aug 2026 17:49:57 +0200
Subject: [PATCH 4/4] Add GELU and GELU approximation activation functions to
 ActivationFunctionSet

---
 neat/activations.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/neat/activations.py b/neat/activations.py
index 21573cb0..936defeb 100644
--- a/neat/activations.py
+++ b/neat/activations.py
@@ -163,6 +163,8 @@ def __init__(self):
         self.add('hat', hat_activation)
         self.add('square', square_activation)
         self.add('cube', cube_activation)
+        self.add('gelu', gelu_activation)
+        self.add('gelu_approximation', gelu_approximation_activation)
 
     def add(self, name, function):
         validate_activation(function)