Unified · Split
| @@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines | ||
| 38 | 38 | // ---------------------------------------------------------------------------- |
| 39 | 39 | // CPU code reference |
| 40 | 40 | |
| 41 | +#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | |
| 42 | + | |
| 41 | 43 | void gelu_forward_cpu(float* out, float* inp, int N) { |
| 42 | − float s = sqrtf(2.0f / M_PI); | |
| 43 | 44 | for (int i = 0; i < N; i++) { |
| 44 | 45 | float x = inp[i]; |
| 45 | 46 | float cube = 0.044715f * x * x * x; |
| 46 | − out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube))); | |
| 47 | + out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube))); | |
| 47 | 48 | } |
| 48 | 49 | } |
| 49 | 50 | |
| @@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines | ||
| 53 | 54 | // elementwise ops are nice and ez |
| 54 | 55 | __global__ void gelu_kernel(float* out, const float* inp, int N) { |
| 55 | 56 | int i = blockIdx.x * blockDim.x + threadIdx.x; |
| 56 | − float s = sqrtf(2.0f / M_PI); | |
| 57 | 57 | if (i < N) { |
| 58 | 58 | float xi = inp[i]; |
| 59 | 59 | float cube = 0.044715f * xi * xi * xi; |
| 60 | − out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube))); | |
| 60 | + out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube))); | |
| 61 | 61 | } |
| 62 | 62 | } |
| 63 | 63 | |
| @@ -335,25 +335,24 @@ void attention_backward(float* dinp, float* dpreatt, float* datt, · 334 unmodified lines | ||
| 335 | 335 | } |
| 336 | 336 | } |
| 337 | 337 | |
| 338 | +#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | |
| 338 | 339 | void gelu_forward(float* out, float* inp, int N) { |
| 339 | − float s = sqrtf(2.0f / M_PI); | |
| 340 | 340 | for (int i = 0; i < N; i++) { |
| 341 | 341 | float x = inp[i]; |
| 342 | 342 | float cube = 0.044715f * x * x * x; |
| 343 | − out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube))); | |
| 343 | + out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube))); | |
| 344 | 344 | } |
| 345 | 345 | } |
| 346 | 346 | |
| 347 | 347 | void gelu_backward(float* dinp, float* inp, float* dout, int N) { |
| 348 | − float s = sqrtf(2.0f / M_PI); | |
| 349 | 348 | for (int i = 0; i < N; i++) { |
| 350 | 349 | float x = inp[i]; |
| 351 | 350 | float cube = 0.044715f * x * x * x; |
| 352 | − float tanh_arg = s * (x + cube); | |
| 351 | + float tanh_arg = GELU_SCALING_FACTOR * (x + cube); | |
| 353 | 352 | float tanh_out = tanhf(tanh_arg); |
| 354 | 353 | float coshf_out = coshf(tanh_arg); |
| 355 | 354 | float sech_out = 1.0f / (coshf_out * coshf_out); |
| 356 | − float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * s * (1.0f + 3.0f * 0.044715f * x * x); | |
| 355 | + float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * GELU_SCALING_FACTOR * (1.0f + 3.0f * 0.044715f * x * x); | |
| 357 | 356 | dinp[i] += local_grad * dout[i]; |
| 358 | 357 | } |
| 359 | 358 | } |
| @@ -305,13 +305,13 @@ __global__ void residual_forward_kernel(float* out, float* inp1, float* inp2, in · 304 unmodified lines | ||
| 305 | 305 | } |
| 306 | 306 | } |
| 307 | 307 | |
| 308 | +#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | |
| 308 | 309 | __global__ void gelu_kernel(float* out, const float* inp, int N) { |
| 309 | 310 | int i = blockIdx.x * blockDim.x + threadIdx.x; |
| 310 | − float s = sqrtf(2.0f / M_PI); | |
| 311 | 311 | if (i < N) { |
| 312 | 312 | float xi = inp[i]; |
| 313 | 313 | float cube = 0.044715f * xi * xi * xi; |
| 314 | − out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube))); | |
| 314 | + out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube))); | |
| 315 | 315 | } |
| 316 | 316 | } |
| 317 | 317 | |