History

make gelu constant be a #define, speeds up the kernel by 1% or so

Andrej Karpathy committed 2 years ago

Commit f26cf001 parent a08c11b3 files · +10 −11Browse files

Unified · Split

3 files
  1. dev/cuda/gelu_forward.cu +4 −4
  2. train_gpt2.c +4 −5
  3. train_gpt2.cu +2 −2
dev/cuda/gelu_forward.cu+4−4
@@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines
38// ----------------------------------------------------------------------------38// ----------------------------------------------------------------------------
39// CPU code reference39// CPU code reference
40 40
41#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
42
41void gelu_forward_cpu(float* out, float* inp, int N) {43void gelu_forward_cpu(float* out, float* inp, int N) {
42 float s = sqrtf(2.0f / M_PI);
43 for (int i = 0; i < N; i++) {44 for (int i = 0; i < N; i++) {
44 float x = inp[i];45 float x = inp[i];
45 float cube = 0.044715f * x * x * x;46 float cube = 0.044715f * x * x * x;
46 out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube)));47 out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube)));
47 }48 }
48}49}
49 50
@@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines
53// elementwise ops are nice and ez54// elementwise ops are nice and ez
54__global__ void gelu_kernel(float* out, const float* inp, int N) {55__global__ void gelu_kernel(float* out, const float* inp, int N) {
55 int i = blockIdx.x * blockDim.x + threadIdx.x;56 int i = blockIdx.x * blockDim.x + threadIdx.x;
56 float s = sqrtf(2.0f / M_PI);
57 if (i < N) {57 if (i < N) {
58 float xi = inp[i];58 float xi = inp[i];
59 float cube = 0.044715f * xi * xi * xi;59 float cube = 0.044715f * xi * xi * xi;
60 out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube)));60 out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube)));
61 }61 }
62}62}
63 63
train_gpt2.c+4−5
@@ -335,25 +335,24 @@ void attention_backward(float* dinp, float* dpreatt, float* datt, · 334 unmodified lines
335 }335 }
336}336}
337 337
338#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
338void gelu_forward(float* out, float* inp, int N) {339void gelu_forward(float* out, float* inp, int N) {
339 float s = sqrtf(2.0f / M_PI);
340 for (int i = 0; i < N; i++) {340 for (int i = 0; i < N; i++) {
341 float x = inp[i];341 float x = inp[i];
342 float cube = 0.044715f * x * x * x;342 float cube = 0.044715f * x * x * x;
343 out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube)));343 out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube)));
344 }344 }
345}345}
346 346
347void gelu_backward(float* dinp, float* inp, float* dout, int N) {347void gelu_backward(float* dinp, float* inp, float* dout, int N) {
348 float s = sqrtf(2.0f / M_PI);
349 for (int i = 0; i < N; i++) {348 for (int i = 0; i < N; i++) {
350 float x = inp[i];349 float x = inp[i];
351 float cube = 0.044715f * x * x * x;350 float cube = 0.044715f * x * x * x;
352 float tanh_arg = s * (x + cube);351 float tanh_arg = GELU_SCALING_FACTOR * (x + cube);
353 float tanh_out = tanhf(tanh_arg);352 float tanh_out = tanhf(tanh_arg);
354 float coshf_out = coshf(tanh_arg);353 float coshf_out = coshf(tanh_arg);
355 float sech_out = 1.0f / (coshf_out * coshf_out);354 float sech_out = 1.0f / (coshf_out * coshf_out);
356 float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * s * (1.0f + 3.0f * 0.044715f * x * x);355 float local_grad = 0.5f * (1.0f + tanh_out) + x * 0.5f * sech_out * GELU_SCALING_FACTOR * (1.0f + 3.0f * 0.044715f * x * x);
357 dinp[i] += local_grad * dout[i];356 dinp[i] += local_grad * dout[i];
358 }357 }
359}358}
train_gpt2.cu+2−2
@@ -305,13 +305,13 @@ __global__ void residual_forward_kernel(float* out, float* inp1, float* inp2, in · 304 unmodified lines
305 }305 }
306}306}
307 307
308#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
308__global__ void gelu_kernel(float* out, const float* inp, int N) {309__global__ void gelu_kernel(float* out, const float* inp, int N) {
309 int i = blockIdx.x * blockDim.x + threadIdx.x;310 int i = blockIdx.x * blockDim.x + threadIdx.x;
310 float s = sqrtf(2.0f / M_PI);
311 if (i < N) {311 if (i < N) {
312 float xi = inp[i];312 float xi = inp[i];
313 float cube = 0.044715f * xi * xi * xi;313 float cube = 0.044715f * xi * xi * xi;
314 out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube)));314 out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube)));
315 }315 }
316}316}
317 317

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t