Add approximate parameter to GELU activation function

src/Native/LibTorchSharp/THSTensor.cpp

@@ -576,14 +576,14 @@ Tensor THSTensor_gather(
     CATCH_TENSOR(torch::gather(*tensor, dim, *index));
 }
 
-Tensor THSTensor_gelu(const Tensor tensor)
+Tensor THSTensor_gelu(const Tensor tensor, const char* approximate)
 {
-    CATCH_TENSOR(torch::gelu(*tensor));
+    CATCH_TENSOR(torch::gelu(*tensor, approximate));
 }
 
-Tensor THSTensor_gelu_(const Tensor tensor)
+Tensor THSTensor_gelu_(const Tensor tensor, const char* approximate)
 {
-    CATCH_TENSOR(torch::gelu_(*tensor));
+    CATCH_TENSOR(torch::gelu_(*tensor, approximate));
 }
 
 Tensor THSTensor_get1(const Tensor tensor, int64_t index)

src/Native/LibTorchSharp/THSTensor.h

@@ -600,8 +600,8 @@ EXPORT_API(Tensor) THSTensor_ge_scalar(const Tensor left, const Scalar right);
 
 EXPORT_API(void) THSTensor_ge_scalar_(const Tensor left, const Scalar right);
 
-EXPORT_API(Tensor) THSTensor_gelu(const Tensor tensor);
-EXPORT_API(Tensor) THSTensor_gelu_(const Tensor tensor);
+EXPORT_API(Tensor) THSTensor_gelu(const Tensor tensor, const char* approximate);
+EXPORT_API(Tensor) THSTensor_gelu_(const Tensor tensor, const char* approximate);
 
 EXPORT_API(Tensor) THSTensor_glu(const Tensor tensor, const int64_t dim);
 

src/TorchSharp/NN/Activation/GELU.cs

@@ -14,17 +14,20 @@ namespace Modules
         /// </summary>
         public sealed class GELU : ParameterLessModule<Tensor, Tensor>
         {
-            internal GELU(bool inplace) : base(nameof(GELU))
+            internal GELU(bool inplace, GELUApproximate approximate = GELUApproximate.none) : base(nameof(GELU))
             {
                 this.inplace = inplace;
+                this.approximate = approximate;
             }
 
             public override Tensor forward(Tensor tensor)
             {
-                return torch.nn.functional.gelu(tensor, inplace);
+                return torch.nn.functional.gelu(tensor, approximate, inplace);
             }
 
             public bool inplace {get; set; }
+
+            public GELUApproximate approximate { get; set; }
         }
     }
 
@@ -49,6 +52,16 @@ public static GELU GELU(bool inplace)
                 return new GELU(inplace);
             }
 
+            /// <summary>
+            /// Gaussian Error Linear Units
+            /// </summary>
+            /// <param name="approximate">The approximation method to use. Default: none</param>
+            /// <param name="inplace">Do the operation in-place. Default: False</param>
+            public static GELU GELU(GELUApproximate approximate, bool inplace = false)
+            {
+                return new GELU(inplace, approximate);
+            }
+
             public static partial class functional
             {
                 /// <summary>
@@ -61,6 +74,17 @@ public static Tensor gelu(Tensor x, bool inplace)
                     return inplace ? x.gelu_().alias() : x.gelu();
                 }
 
+                /// <summary>
+                /// Gaussian Error Linear Units
+                /// </summary>
+                /// <param name="x">The input tensor</param>
+                /// <param name="approximate">The approximation method to use.</param>
+                /// <param name="inplace">Do the operation in-place. Default: False</param>
+                public static Tensor gelu(Tensor x, GELUApproximate approximate, bool inplace = false)
+                {
+                    return inplace ? x.gelu_(approximate).alias() : x.gelu(approximate);
+                }
+
                 /// <summary>
                 /// Gaussian Error Linear Units
                 /// </summary>

src/TorchSharp/PInvoke/LibTorchSharp.THSTensor.cs

@@ -706,11 +706,11 @@ internal static extern IntPtr THSTensor_upsample_nearest3d(IntPtr input,
         [DllImport("LibTorchSharp")]
         internal static extern void THSTensor_elu_(IntPtr tensor, IntPtr alpha, IntPtr scale, IntPtr input_scale);
 
-        [DllImport("LibTorchSharp")]
-        internal static extern IntPtr THSTensor_gelu(IntPtr tensor);
+        [DllImport("LibTorchSharp", CharSet = CharSet.Ansi, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+        internal static extern IntPtr THSTensor_gelu(IntPtr tensor, [MarshalAs(UnmanagedType.LPStr)] string approximate);
 
-        [DllImport("LibTorchSharp")]
-        internal static extern IntPtr THSTensor_gelu_(IntPtr tensor);
+        [DllImport("LibTorchSharp", CharSet = CharSet.Ansi, BestFitMapping = false, ThrowOnUnmappableChar = true)]
+        internal static extern IntPtr THSTensor_gelu_(IntPtr tensor, [MarshalAs(UnmanagedType.LPStr)] string approximate);
 
         [DllImport("LibTorchSharp")]
         internal static extern IntPtr THSTensor_glu(IntPtr tensor, long dim);

src/TorchSharp/Tensor/Enums/GELUApproximate.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation and Contributors.  All Rights Reserved.  See LICENSE in the project root for license information.
+namespace TorchSharp
+{
+    /// <summary>
+    /// Specifies the approximation method for the GELU activation function.
+    /// </summary>
+    public enum GELUApproximate
+    {
+        /// <summary>
+        /// Exact GELU computation.
+        /// </summary>
+        none,
+        /// <summary>
+        /// Tanh-based approximation.
+        /// </summary>
+        tanh
+    }
+}

src/TorchSharp/Tensor/Tensor.cs

@@ -2977,15 +2977,31 @@ public Tensor elu_(Scalar alpha, Scalar scale, Scalar input_scale)
 
             public Tensor gelu()
             {
-                var res = NativeMethods.THSTensor_gelu(Handle);
+                var res = NativeMethods.THSTensor_gelu(Handle, "none");
+                if (res == IntPtr.Zero)
+                    CheckForErrors();
+                return new Tensor(res);
+            }
+
+            public Tensor gelu(GELUApproximate approximate)
+            {
+                var res = NativeMethods.THSTensor_gelu(Handle, approximate == GELUApproximate.tanh ? "tanh" : "none");
                 if (res == IntPtr.Zero)
                     CheckForErrors();
                 return new Tensor(res);
             }
 
             public Tensor gelu_()
             {
-                var res = NativeMethods.THSTensor_gelu_(Handle);
+                var res = NativeMethods.THSTensor_gelu_(Handle, "none");
+                if (res == IntPtr.Zero)
+                    CheckForErrors();
+                return new Tensor(res);
+            }
+
+            public Tensor gelu_(GELUApproximate approximate)
+            {
+                var res = NativeMethods.THSTensor_gelu_(Handle, approximate == GELUApproximate.tanh ? "tanh" : "none");
                 if (res == IntPtr.Zero)
                     CheckForErrors();
                 return new Tensor(res);

test/TorchSharpTest/NN.cs

@@ -618,6 +618,28 @@ public void EvaluateGELU()
             }
         }
 
+        [Fact]
+        public void EvaluateGELUWithTanhApproximate()
+        {
+            var rel = GELU(GELUApproximate.tanh);
+
+            foreach (var device in TestUtils.AvailableDevices()) {
+                var input = torch.randn(new long[] { 64, 8 }, device: device) * 25.0;
+                var output = rel.call(input);
+                Assert.Equal(device.type, output.device_type);
+
+                var values = output.data<float>().ToArray();
+                Assert.Equal(input.shape, output.shape);
+                Assert.All(values, val => Assert.True(val >= -0.2));
+            }
+
+            // Verify that tanh approximate produces different results from exact
+            var x = torch.tensor(new float[] { -1.0f, 0.0f, 1.0f, 2.0f });
+            var exact = torch.nn.functional.gelu(x);
+            var approx = torch.nn.functional.gelu(x, GELUApproximate.tanh);
+            Assert.False(exact.allclose(approx, rtol: 1e-5, atol: 1e-5));
+        }
+
         [Fact]
         public void EvaluatePReLU()
         {