/* Kernels for the positional encoder forward pass in GPT-2. Compile example: nvcc -O3 --use_fast_math encoder_forward.cu -o encoder_forward version 1 is naive port from CPU code to kernel: parallelizes over B,T, loops over C ./encoder_forward 1 version 2 is more optimized, parallelizes over all of B,T,C ./encoder_forward 2 version 3 is like version 2 but uses float4 reads/writes ./encoder_forward 3 */ #include #include #include #include "common.h" #include // ---------------------------------------------------------------------------- // CPU code reference // GPT-2 positional encoder forward pass void encoder_forward_cpu(float* out, const int* inp, const float* wte, const float* wpe, int B, int T, int C) { for (int b = 0; b < B; b++) { for (int t = 0; t < T; t++) { float* out_bt = out + b * T * C + t * C; int ix = inp[b * T + t]; const float* wte_ix = wte + ix * C; const float* wpe_t = wpe + t * C; for (int i = 0; i < C; i++) { out_bt[i] = wte_ix[i] + wpe_t[i]; } } } } // ---------------------------------------------------------------------------- // GPU kernels // naive implementation into kernel, parallelize over B,T, loop over C __global__ void encoder_forward_kernel1(float* out, const int* inp, const float* wte, const float* wpe, int B, int T, int C) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int N = B * T; if (idx < N) { int b = idx / T; int t = idx % T; float* out_bt = out + b * T * C + t * C; int ix = inp[b * T + t]; const float* wte_ix = wte + ix * C; const float* wpe_t = wpe + t * C; for (int i = 0; i < C; i++) { out_bt[i] = wte_ix[i] + wpe_t[i]; } } } // optimized implementation: parallelize over all of B,T,C __global__ void encoder_forward_kernel2(float* out, const int* inp, const float* wte, const float* wpe, int B, int T, int C) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int N = B * T * C; if (idx < N) { int bt = idx / C; int b = bt / T; int t = bt % T; int c = idx % C; int ix = inp[b * T + t]; float* out_btc = out + b * T * C + t * C + c; const float* wte_ix = wte + ix * C + c; const float* wpe_tc = wpe + t * C + c; *out_btc = *wte_ix + *wpe_tc; } } __device__ inline float4 add_float4(const float4& a, const float4& b) { return make_float4(a.x + b.x, a.y + b.y, a.z + b.z, a.w + b.w); } // use of float4 leads to using 128-bit LDG / STG instructions in SASS, // very helpful in memory-bound kernels like encoder_forward __global__ void encoder_forward_kernel3(float4* out, const int* inp, const float4* wte, const float4* wpe, int B, int T, int C) { int C4 = C / 4; int idx = blockIdx.x * blockDim.x + threadIdx.x; int N = B * T * C4; if (idx < N) { int bt = idx / C4; int b = bt / T; int t = bt % T; int c4 = idx % C4; int ix = inp[b * T + t]; out[b * T * C4 + t * C4 + c4] = add_float4(wte[ix * C4 + c4], wpe[t * C4 + c4]); } } // ---------------------------------------------------------------------------- // kernel launcher void encoder_forward1(float* out, const int* inp, const float* wte, const float* wpe, int B, int T, int C, const int block_size) { const int N = B * T; const int grid_size = ceil_div(N, block_size); encoder_forward_kernel1<<>>(out, inp, wte, wpe, B, T, C); cudaCheck(cudaGetLastError()); } void encoder_forward2(float* out, const int* inp, const float* wte, const float* wpe, int B, int T, int C, const int block_size) { const int N = B * T * C; const int grid_size = ceil_div(N, block_size); encoder_forward_kernel2<<>>(out, inp, wte, wpe, B, T, C); cudaCheck(cudaGetLastError()); } void encoder_forward3(float* out, const int* inp, const float* wte, const float* wpe, int B, int T, int C, const int block_size) { assert(C % 4 == 0); const int N = B * T * C; const int grid_size = ceil_div(N / 4, block_size); encoder_forward_kernel3<<>>((float4*) out, inp, (float4*) wte, (float4*) wpe, B, T, C); cudaCheck(cudaGetLastError()); } // kernel version dispatch void encoder_forward(int kernel_num, float* out, const int* inp, const float* wte, const float* wpe, int B, int T, int C, const int block_size) { switch (kernel_num) { case 1: encoder_forward1(out, inp, wte, wpe, B, T, C, block_size); break; case 2: encoder_forward2(out, inp, wte, wpe, B, T, C, block_size); break; case 3: encoder_forward3(out, inp, wte, wpe, B, T, C, block_size); break; default: printf("Invalid kernel number\n"); exit(1); } } // ---------------------------------------------------------------------------- int main(int argc, char **argv) { srand(0); int B = 8; int T = 1024; int C = 768; int V = 50257; int deviceIdx = 0; cudaCheck(cudaSetDevice(deviceIdx)); // create host memory of random numbers float* out = (float*)malloc(B * T * C * sizeof(float)); int* inp = make_random_int(B * T, V); float* wte = make_random_float(V * C); float* wpe = make_random_float(T * C); // move to GPU float* d_out; int* d_inp; float* d_wte; float* d_wpe; cudaCheck(cudaMalloc(&d_out, B * T * C * sizeof(float))); cudaCheck(cudaMalloc(&d_inp, B * T * sizeof(int))); cudaCheck(cudaMalloc(&d_wte, V * C * sizeof(float))); cudaCheck(cudaMalloc(&d_wpe, T * C * sizeof(float))); cudaCheck(cudaMemcpy(d_inp, inp, B * T * sizeof(int), cudaMemcpyHostToDevice)); cudaCheck(cudaMemcpy(d_wte, wte, V * C * sizeof(float), cudaMemcpyHostToDevice)); cudaCheck(cudaMemcpy(d_wpe, wpe, T * C * sizeof(float), cudaMemcpyHostToDevice)); // read kernel_num from command line int kernel_num = 2; if (argc > 1) { kernel_num = atoi(argv[1]); } printf("Using kernel %d\n", kernel_num); // first check the correctness of the kernel encoder_forward_cpu(out, inp, wte, wpe, B, T, C); // time the kernel at different block sizes int block_sizes[] = {32, 64, 128, 256, 512, 1024}; for (int j = 0; j < sizeof(block_sizes) / sizeof(int); j++) { int block_size = block_sizes[j]; printf("Checking block size %d.\n", block_size); encoder_forward(kernel_num, d_out, d_inp, d_wte, d_wpe, B, T, C, block_size); validate_result(d_out, out, "out", B * T * C, 1e-5f); } printf("All results match. Starting benchmarks.\n\n"); for (int j = 0; j < sizeof(block_sizes) / sizeof(int); j++) { int block_size = block_sizes[j]; int repeat_times = 1000; float elapsed_time = benchmark_kernel(repeat_times, encoder_forward, kernel_num, d_out, d_inp, d_wte, d_wpe, B, T, C, block_size ); // napkin math: estimate the memory bandwidth achieved // for each (B,T,C) output element, we do 3 reads and 1 write, 4 bytes each // and e.g. A100 40GB PCIe is advertised at 1,555GB/s long memory_ops = B * T * C * 4 * 4; float memory_bandwidth = memory_ops / elapsed_time / 1e6; printf("block_size %4d | time %.4f ms | bandwidth %.2f GB/s\n", block_size, elapsed_time, memory_bandwidth); } // free memory free(out); free(inp); free(wte); free(wpe); cudaCheck(cudaFree(d_out)); cudaCheck(cudaFree(d_inp)); cudaCheck(cudaFree(d_wte)); cudaCheck(cudaFree(d_wpe)); return 0; }