History

make gelu constant be a #define, speeds up the kernel by 1% or so

Andrej Karpathy committed 2 years ago

Commit f26cf001 parent a08c11b3 files · +10 −11Browse files

Showing the changes to dev (1 of 3 files). Show all

Unified · Split

dev/cuda/gelu_forward.cu+4−4
@@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines
3838 // ----------------------------------------------------------------------------
3939 // CPU code reference
4040
41+#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
42+
4143 void gelu_forward_cpu(float* out, float* inp, int N) {
42− float s = sqrtf(2.0f / M_PI);
4344 for (int i = 0; i < N; i++) {
4445 float x = inp[i];
4546 float cube = 0.044715f * x * x * x;
46− out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube)));
47+ out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube)));
4748 }
4849 }
4950
@@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines
5354 // elementwise ops are nice and ez
5455 __global__ void gelu_kernel(float* out, const float* inp, int N) {
5556 int i = blockIdx.x * blockDim.x + threadIdx.x;
56− float s = sqrtf(2.0f / M_PI);
5757 if (i < N) {
5858 float xi = inp[i];
5959 float cube = 0.044715f * xi * xi * xi;
60− out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube)));
60+ out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube)));
6161 }
6262 }
6363

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t