Showing the changes to dev (1 of 3 files). Show all
Unified · Split
| @@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines | ||
| 38 | 38 | // ---------------------------------------------------------------------------- |
| 39 | 39 | // CPU code reference |
| 40 | 40 | |
| 41 | +#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | |
| 42 | + | |
| 41 | 43 | void gelu_forward_cpu(float* out, float* inp, int N) { |
| 42 | − float s = sqrtf(2.0f / M_PI); | |
| 43 | 44 | for (int i = 0; i < N; i++) { |
| 44 | 45 | float x = inp[i]; |
| 45 | 46 | float cube = 0.044715f * x * x * x; |
| 46 | − out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube))); | |
| 47 | + out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube))); | |
| 47 | 48 | } |
| 48 | 49 | } |
| 49 | 50 | |
| @@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines | ||
| 53 | 54 | // elementwise ops are nice and ez |
| 54 | 55 | __global__ void gelu_kernel(float* out, const float* inp, int N) { |
| 55 | 56 | int i = blockIdx.x * blockDim.x + threadIdx.x; |
| 56 | − float s = sqrtf(2.0f / M_PI); | |
| 57 | 57 | if (i < N) { |
| 58 | 58 | float xi = inp[i]; |
| 59 | 59 | float cube = 0.044715f * xi * xi * xi; |
| 60 | − out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube))); | |
| 60 | + out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube))); | |
| 61 | 61 | } |
| 62 | 62 | } |
| 63 | 63 | |