History

make gelu constant be a #define, speeds up the kernel by 1% or so

Andrej Karpathy committed 2 years ago

Commit f26cf001 parent a08c11b3 files · +10 −11Browse files

Unified · Split

3 files
  1. dev/cuda/gelu_forward.cu +4 −4
  2. train_gpt2.c +4 −5
  3. train_gpt2.cu +2 −2
dev/cuda/gelu_forward.cu+4−4
@@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines
3838 // ----------------------------------------------------------------------------
3939 // CPU code reference
4040
41+#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
42+
4143 void gelu_forward_cpu(float* out, float* inp, int N) {
42− float s = sqrtf(2.0f / M_PI);
4344 for (int i = 0; i < N; i++) {
4445 float x = inp[i];
4546 float cube = 0.044715f * x * x * x;
46− out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube)));
47+ out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube)));
4748 }
4849 }
4950
@@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines
5354 // elementwise ops are nice and ez
5455 __global__ void gelu_kernel(float* out, const float* inp, int N) {
5556 int i = blockIdx.x * blockDim.x + threadIdx.x;
56− float s = sqrtf(2.0f / M_PI);
5757 if (i < N) {
5858 float xi = inp[i];
5959 float cube = 0.044715f * xi * xi * xi;
60− out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube)));
60+ out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube)));
6161 }
6262 }
6363
train_gpt2.c+4−5
@@ -335,25 +335,24 @@ void attention_backward(float* dinp, float* dpreatt, float* datt, · 334 unmodified lines
335335 }
336336 }
337337
338+#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
338339 void gelu_forward(float* out, float* inp, int N) {
339− float s = sqrtf(2.0f / M_PI);
340340 for (int i = 0; i < N; i++) {
341341 float x = inp[i];
342342 float cube = 0.044715f * x * x * x;
343− out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube)));
343+ out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube)));
344344 }
345345 }
346346
347347 void gelu_backward(float* dinp, float* inp, float* dout, int N) {
348− float s = sqrtf(2.0f / M_PI);
349348 for (int i = 0; i < N; i++) {
350349 float x = inp[i];
351350 float cube = 0.044715f * x * x * x;
352− float tanh_arg = s * (x + cube);
351+ float tanh_arg = GELU_SCALING_FACTOR * (x + cube);
353352 float tanh_out = tanhf(tanh_arg);
354353 float coshf_out = coshf(tanh_arg);
355354 float sech_out = 1.0f / (coshf_out * coshf_out);
356− float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * s * (1.0f + 3.0f * 0.044715f * x * x);
355+ float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * GELU_SCALING_FACTOR * (1.0f + 3.0f * 0.044715f * x * x);
357356 dinp[i] += local_grad * dout[i];
358357 }
359358 }
train_gpt2.cu+2−2
@@ -305,13 +305,13 @@ __global__ void residual_forward_kernel(float* out, float* inp1, float* inp2, in · 304 unmodified lines
305305 }
306306 }
307307
308+#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
308309 __global__ void gelu_kernel(float* out, const float* inp, int N) {
309310 int i = blockIdx.x * blockDim.x + threadIdx.x;
310− float s = sqrtf(2.0f / M_PI);
311311 if (i < N) {
312312 float xi = inp[i];
313313 float cube = 0.044715f * xi * xi * xi;
314− out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube)));
314+ out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube)));
315315 }
316316 }
317317

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t