Unified · Split
| @@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines | |||
| 38 | // ---------------------------------------------------------------------------- | 38 | // ---------------------------------------------------------------------------- |
| 39 | // CPU code reference | 39 | // CPU code reference |
| 40 | 40 | ||
| 41 | #define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | ||
| 42 | |||
| 41 | void gelu_forward_cpu(float* out, float* inp, int N) { | 43 | void gelu_forward_cpu(float* out, float* inp, int N) { |
| 42 | float s = sqrtf(2.0f / M_PI); | ||
| 43 | for (int i = 0; i < N; i++) { | 44 | for (int i = 0; i < N; i++) { |
| 44 | float x = inp[i]; | 45 | float x = inp[i]; |
| 45 | float cube = 0.044715f * x * x * x; | 46 | float cube = 0.044715f * x * x * x; |
| 46 | out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube))); | 47 | out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube))); |
| 47 | } | 48 | } |
| 48 | } | 49 | } |
| 49 | 50 | ||
| @@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines | |||
| 53 | // elementwise ops are nice and ez | 54 | // elementwise ops are nice and ez |
| 54 | __global__ void gelu_kernel(float* out, const float* inp, int N) { | 55 | __global__ void gelu_kernel(float* out, const float* inp, int N) { |
| 55 | int i = blockIdx.x * blockDim.x + threadIdx.x; | 56 | int i = blockIdx.x * blockDim.x + threadIdx.x; |
| 56 | float s = sqrtf(2.0f / M_PI); | ||
| 57 | if (i < N) { | 57 | if (i < N) { |
| 58 | float xi = inp[i]; | 58 | float xi = inp[i]; |
| 59 | float cube = 0.044715f * xi * xi * xi; | 59 | float cube = 0.044715f * xi * xi * xi; |
| 60 | out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube))); | 60 | out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube))); |
| 61 | } | 61 | } |
| 62 | } | 62 | } |
| 63 | 63 | ||
| @@ -335,25 +335,24 @@ void attention_backward(float* dinp, float* dpreatt, float* datt, · 334 unmodified lines | |||
| 335 | } | 335 | } |
| 336 | } | 336 | } |
| 337 | 337 | ||
| 338 | #define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | ||
| 338 | void gelu_forward(float* out, float* inp, int N) { | 339 | void gelu_forward(float* out, float* inp, int N) { |
| 339 | float s = sqrtf(2.0f / M_PI); | ||
| 340 | for (int i = 0; i < N; i++) { | 340 | for (int i = 0; i < N; i++) { |
| 341 | float x = inp[i]; | 341 | float x = inp[i]; |
| 342 | float cube = 0.044715f * x * x * x; | 342 | float cube = 0.044715f * x * x * x; |
| 343 | out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube))); | 343 | out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube))); |
| 344 | } | 344 | } |
| 345 | } | 345 | } |
| 346 | 346 | ||
| 347 | void gelu_backward(float* dinp, float* inp, float* dout, int N) { | 347 | void gelu_backward(float* dinp, float* inp, float* dout, int N) { |
| 348 | float s = sqrtf(2.0f / M_PI); | ||
| 349 | for (int i = 0; i < N; i++) { | 348 | for (int i = 0; i < N; i++) { |
| 350 | float x = inp[i]; | 349 | float x = inp[i]; |
| 351 | float cube = 0.044715f * x * x * x; | 350 | float cube = 0.044715f * x * x * x; |
| 352 | float tanh_arg = s * (x + cube); | 351 | float tanh_arg = GELU_SCALING_FACTOR * (x + cube); |
| 353 | float tanh_out = tanhf(tanh_arg); | 352 | float tanh_out = tanhf(tanh_arg); |
| 354 | float coshf_out = coshf(tanh_arg); | 353 | float coshf_out = coshf(tanh_arg); |
| 355 | float sech_out = 1.0f / (coshf_out * coshf_out); | 354 | float sech_out = 1.0f / (coshf_out * coshf_out); |
| 356 | float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * s * (1.0f + 3.0f * 0.044715f * x * x); | 355 | float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * GELU_SCALING_FACTOR * (1.0f + 3.0f * 0.044715f * x * x); |
| 357 | dinp[i] += local_grad * dout[i]; | 356 | dinp[i] += local_grad * dout[i]; |
| 358 | } | 357 | } |
| 359 | } | 358 | } |
| @@ -305,13 +305,13 @@ __global__ void residual_forward_kernel(float* out, float* inp1, float* inp2, in · 304 unmodified lines | |||
| 305 | } | 305 | } |
| 306 | } | 306 | } |
| 307 | 307 | ||
| 308 | #define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | ||
| 308 | __global__ void gelu_kernel(float* out, const float* inp, int N) { | 309 | __global__ void gelu_kernel(float* out, const float* inp, int N) { |
| 309 | int i = blockIdx.x * blockDim.x + threadIdx.x; | 310 | int i = blockIdx.x * blockDim.x + threadIdx.x; |
| 310 | float s = sqrtf(2.0f / M_PI); | ||
| 311 | if (i < N) { | 311 | if (i < N) { |
| 312 | float xi = inp[i]; | 312 | float xi = inp[i]; |
| 313 | float cube = 0.044715f * xi * xi * xi; | 313 | float cube = 0.044715f * xi * xi * xi; |
| 314 | out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube))); | 314 | out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube))); |
| 315 | } | 315 | } |
| 316 | } | 316 | } |
| 317 | 317 | ||