# Makefile for building dev/cuda kernels# Collects all the make commands in one file but each file also# has the compile and run commands in the header comments section. # Find nvcc (NVIDIA CUDA compiler)NVCC := $(shell which nvcc 2>/dev/null)ifeq ($(NVCC),)		$(error nvcc not found.)endif # Compiler flagsCFLAGS = -O3 --use_fast_mathNVCCFLAGS = -lcublas -lcublasLtMPI_PATHS = -I/usr/lib/x86_64-linux-gnu/openmpi/include -L/usr/lib/x86_64-linux-gnu/openmpi/lib/ # Default rule for our CUDA files%: %.cu	$(NVCC) $(CFLAGS) $(NVCCFLAGS) $< -o $@ # Build all targetsTARGETS = adamw attention_backward attention_forward classifier_fused crossentropy_forward crossentropy_softmax_backward encoder_backward encoder_forward gelu_backward gelu_forward layernorm_backward layernorm_forward matmul_backward matmul_backward_bias matmul_forward nccl_all_reduce residual_forward softmax_forward trimat_forward fused_residual_forward  global_normall: $(TARGETS) # Individual targets: forward passattention_forward: attention_forward.cuclassifier_fused: classifier_fused.cucrossentropy_forward: crossentropy_forward.cuencoder_forward: encoder_forward.cugelu_forward: gelu_forward.culayernorm_forward: layernorm_forward.cufused_residual_forward: fused_residual_forward.curesidual_forward: residual_forward.cusoftmax_forward: softmax_forward.cutrimat_forward: trimat_forward.cu# matmul fwd/bwd also uses OpenMP (optionally) and cuBLASLt libsmatmul_forward: matmul_forward.cu	$(NVCC) $(CFLAGS) $(NVCCFLAGS) -Xcompiler -fopenmp matmul_forward.cu -o matmul_forward # Individual targets: backward passattention_backward: attention_backward.cucrossentropy_softmax_backward: crossentropy_softmax_backward.cuencoder_backward: encoder_backward.cugelu_backward: gelu_backward.culayernorm_backward: layernorm_backward.cumatmul_backward_bias: matmul_backward_bias.cumatmul_backward: matmul_backward.cu	$(NVCC) $(CFLAGS) $(NVCCFLAGS) -Xcompiler -fopenmp matmul_backward.cu -o matmul_backward # Update kernelsadamw: adamw.cuglobal_norm: global_norm.cu # NCCL communication kernelsnccl_all_reduce: nccl_all_reduce.cu	$(NVCC) -lmpi -lnccl $(NVCCFLAGS) $(MPI_PATHS) nccl_all_reduce.cu -o nccl_all_reduce # Run all targetsrun_all: all	@for target in $(TARGETS); do \		echo "\n========================================"; \		echo "Running $$target ..."; \		echo "========================================\n"; \		./$$target; \	done # Clean upclean:	rm -f $(TARGETS)