diff --git a/gpt2.py b/gpt2.py index 508a3ed..3753be4 100644 --- a/gpt2.py +++ b/gpt2.py @@ -3,6 +3,8 @@ import numpy as np + + def gelu(x): return 0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3)))