Showing the changes to dev (1 of 3 files). Show all
Unified · Split
| @@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines | |||
| 38 | // ---------------------------------------------------------------------------- | 38 | // ---------------------------------------------------------------------------- |
| 39 | // CPU code reference | 39 | // CPU code reference |
| 40 | 40 | ||
| 41 | #define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI) | ||
| 42 | |||
| 41 | void gelu_forward_cpu(float* out, float* inp, int N) { | 43 | void gelu_forward_cpu(float* out, float* inp, int N) { |
| 42 | float s = sqrtf(2.0f / M_PI); | ||
| 43 | for (int i = 0; i < N; i++) { | 44 | for (int i = 0; i < N; i++) { |
| 44 | float x = inp[i]; | 45 | float x = inp[i]; |
| 45 | float cube = 0.044715f * x * x * x; | 46 | float cube = 0.044715f * x * x * x; |
| 46 | out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube))); | 47 | out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube))); |
| 47 | } | 48 | } |
| 48 | } | 49 | } |
| 49 | 50 | ||
| @@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines | |||
| 53 | // elementwise ops are nice and ez | 54 | // elementwise ops are nice and ez |
| 54 | __global__ void gelu_kernel(float* out, const float* inp, int N) { | 55 | __global__ void gelu_kernel(float* out, const float* inp, int N) { |
| 55 | int i = blockIdx.x * blockDim.x + threadIdx.x; | 56 | int i = blockIdx.x * blockDim.x + threadIdx.x; |
| 56 | float s = sqrtf(2.0f / M_PI); | ||
| 57 | if (i < N) { | 57 | if (i < N) { |
| 58 | float xi = inp[i]; | 58 | float xi = inp[i]; |
| 59 | float cube = 0.044715f * xi * xi * xi; | 59 | float cube = 0.044715f * xi * xi * xi; |
| 60 | out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube))); | 60 | out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube))); |
| 61 | } | 61 | } |
| 62 | } | 62 | } |
| 63 | 63 | ||