From a0a78ba4b9b3e11b9a0a9ec717484df6b1007c6f Mon Sep 17 00:00:00 2001 From: M-Daniyal-123 <55560086+M-Daniyal-123@users.noreply.github.com> Date: Fri, 24 Feb 2023 18:58:36 +0500 Subject: [PATCH] Update gpt2.py --- gpt2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gpt2.py b/gpt2.py index 508a3ed..3753be4 100644 --- a/gpt2.py +++ b/gpt2.py @@ -3,6 +3,8 @@ import numpy as np + + def gelu(x): return 0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3)))