History

make gelu constant be a #define, speeds up the kernel by 1% or so

Andrej Karpathy committed 2 years ago

Commit f26cf001 parent a08c11b3 files · +10 −11Browse files

Showing the changes to dev (1 of 3 files). Show all

Unified · Split

dev/cuda/gelu_forward.cu+4−4
@@ -38,12 +38,13 @@ void cudaCheck(cudaError_t error, const char *file, int line) { · 37 unmodified lines
38// ----------------------------------------------------------------------------38// ----------------------------------------------------------------------------
39// CPU code reference39// CPU code reference
40 40
41#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
42
41void gelu_forward_cpu(float* out, float* inp, int N) {43void gelu_forward_cpu(float* out, float* inp, int N) {
42 float s = sqrtf(2.0f / M_PI);
43 for (int i = 0; i < N; i++) {44 for (int i = 0; i < N; i++) {
44 float x = inp[i];45 float x = inp[i];
45 float cube = 0.044715f * x * x * x;46 float cube = 0.044715f * x * x * x;
46 out[i] = 0.5f * x * (1.0f + tanhf(s * (x + cube)));47 out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube)));
47 }48 }
48}49}
49 50
@@ -53,11 +54,10 @@ void gelu_forward_cpu(float* out, float* inp, int N) { · 3 unmodified lines
53// elementwise ops are nice and ez54// elementwise ops are nice and ez
54__global__ void gelu_kernel(float* out, const float* inp, int N) {55__global__ void gelu_kernel(float* out, const float* inp, int N) {
55 int i = blockIdx.x * blockDim.x + threadIdx.x;56 int i = blockIdx.x * blockDim.x + threadIdx.x;
56 float s = sqrtf(2.0f / M_PI);
57 if (i < N) {57 if (i < N) {
58 float xi = inp[i];58 float xi = inp[i];
59 float cube = 0.044715f * xi * xi * xi;59 float cube = 0.044715f * xi * xi * xi;
60 out[i] = 0.5f * xi * (1.0f + tanhf(s * (xi + cube)));60 out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube)));
61 }61 }
62}62}
63 63

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t