/* Plugin for NVPTX execution.    Copyright (C) 2013-2026 Free Software Foundation, Inc.    Contributed by Mentor Embedded.    This file is part of the GNU Offloading and Multi Processing Library   (libgomp).    Libgomp is free software; you can redistribute it and/or modify it   under the terms of the GNU General Public License as published by   the Free Software Foundation; either version 3, or (at your option)   any later version.    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for   more details.    Under Section 7 of GPL version 3, you are granted additional   permissions described in the GCC Runtime Library Exception, version   3.1, as published by the Free Software Foundation.    You should have received a copy of the GNU General Public License and   a copy of the GCC Runtime Library Exception along with this program;   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see   <http://www.gnu.org/licenses/>.  */ /* Nvidia PTX-specific parts of OpenACC support.  The cuda driver   library appears to hold some implicit state, but the documentation   is not clear as to what that state might be.  Or how one might   propagate it from one thread to another.  */ #define _GNU_SOURCE#include "openacc.h"#include "config.h"#include "symcat.h"#define _LIBGOMP_PLUGIN_INCLUDE 1#include "libgomp-plugin.h"#undef _LIBGOMP_PLUGIN_INCLUDE#include "oacc-plugin.h"#include "gomp-constants.h"#include "oacc-int.h" /* For struct rev_offload + GOMP_REV_OFFLOAD_VAR. */#include "config/nvptx/libgomp-nvptx.h" #include <pthread.h>#ifndef PLUGIN_NVPTX_INCLUDE_SYSTEM_CUDA_H# include "cuda/cuda.h"#else# include <cuda.h>#endif#include <stdbool.h>#include <limits.h>#include <string.h>#include <stdio.h>#include <unistd.h>#include <assert.h>#include <errno.h>#include <stdlib.h> /* Create hash-table for declare target's indirect clause on the host;   see build-target-indirect-htab.h for details.  */#define USE_HASHTAB_LOOKUP_FOR_INDIRECT#ifdef USE_HASHTAB_LOOKUP_FOR_INDIRECTstatic void* create_target_indirect_map (size_t *, size_t,					 uint64_t *, uint64_t *);#endif /* An arbitrary fixed limit (128MB) for the size of the OpenMP soft stacks   block to cache between kernel invocations.  For soft-stacks blocks bigger   than this, we will free the block before attempting another GPU memory   allocation (i.e. in GOMP_OFFLOAD_alloc).  Otherwise, if an allocation fails,   we will free the cached soft-stacks block anyway then retry the   allocation.  If that fails too, we lose.  */ #define SOFTSTACK_CACHE_LIMIT 134217728 #if CUDA_VERSION < 6000extern CUresult cuGetErrorString (CUresult, const char **);#define CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR 82#endif #if CUDA_VERSION >= 6050#undef cuLinkCreate#undef cuLinkAddDataCUresult cuLinkAddData (CUlinkState, CUjitInputType, void *, size_t,			const char *, unsigned, CUjit_option *, void **);CUresult cuLinkCreate (unsigned, CUjit_option *, void **, CUlinkState *);#elsetypedef size_t (*CUoccupancyB2DSize)(int);CUresult cuLinkAddData_v2 (CUlinkState, CUjitInputType, void *, size_t,			   const char *, unsigned, CUjit_option *, void **);CUresult cuLinkCreate_v2 (unsigned, CUjit_option *, void **, CUlinkState *);CUresult cuOccupancyMaxPotentialBlockSize(int *, int *, CUfunction,					  CUoccupancyB2DSize, size_t, int);#endif #define DO_PRAGMA(x) _Pragma (#x) #ifndef PLUGIN_NVPTX_LINK_LIBCUDA# include <dlfcn.h> struct cuda_lib_s { # define CUDA_ONE_CALL(call)			\  __typeof (call) *call;# define CUDA_ONE_CALL_MAYBE_NULL(call)		\  CUDA_ONE_CALL (call)#include "cuda-lib.def"# undef CUDA_ONE_CALL# undef CUDA_ONE_CALL_MAYBE_NULL } cuda_lib; /* -1 if init_cuda_lib has not been called yet, false   if it has been and failed, true if it has been and succeeded.  */static signed char cuda_lib_inited = -1; /* Dynamically load the CUDA runtime library and initialize function   pointers, return false if unsuccessful, true if successful.  */static boolinit_cuda_lib (void){  if (cuda_lib_inited != -1)    return cuda_lib_inited;  const char *cuda_runtime_lib = "libcuda.so.1";  void *h = dlopen (cuda_runtime_lib, RTLD_LAZY);  cuda_lib_inited = false;  if (h == NULL)    return false; # define CUDA_ONE_CALL(call) CUDA_ONE_CALL_1 (call, false)# define CUDA_ONE_CALL_MAYBE_NULL(call) CUDA_ONE_CALL_1 (call, true)# define CUDA_ONE_CALL_1(call, allow_null)		\  cuda_lib.call = dlsym (h, #call);	\  if (!allow_null && cuda_lib.call == NULL)		\    GOMP_PLUGIN_fatal ("'%s' is missing '%s'", cuda_runtime_lib, #call);#include "cuda-lib.def"# undef CUDA_ONE_CALL# undef CUDA_ONE_CALL_1# undef CUDA_ONE_CALL_MAYBE_NULL   cuda_lib_inited = true;  return true;}# define CUDA_CALL_PREFIX cuda_lib.#else # define CUDA_ONE_CALL(call)# define CUDA_ONE_CALL_MAYBE_NULL(call) DO_PRAGMA (weak call)#include "cuda-lib.def"#undef CUDA_ONE_CALL_MAYBE_NULL#undef CUDA_ONE_CALL # define CUDA_CALL_PREFIX# define init_cuda_lib() true#endif #include "secure_getenv.h" static void notify_var (const char *, const char *); #undef MIN#undef MAX#define MIN(X,Y) ((X) < (Y) ? (X) : (Y))#define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) /* Convenience macros for the frequently used CUDA library call and   error handling sequence as well as CUDA library calls that   do the error checking themselves or don't do it at all.  */ #define CUDA_CALL_ERET(ERET, FN, ...)		\  do {						\    unsigned __r				\      = CUDA_CALL_PREFIX FN (__VA_ARGS__);	\    if (__r != CUDA_SUCCESS)			\      {						\	GOMP_PLUGIN_error (#FN " error: %s",	\			   cuda_error (__r));	\	return ERET;				\      }						\  } while (0) #define CUDA_CALL(FN, ...)			\  CUDA_CALL_ERET (false, FN, __VA_ARGS__) #define CUDA_CALL_ASSERT(FN, ...)		\  do {						\    unsigned __r				\      = CUDA_CALL_PREFIX FN (__VA_ARGS__);	\    if (__r != CUDA_SUCCESS)			\      {						\	GOMP_PLUGIN_fatal (#FN " error: %s",	\			   cuda_error (__r));	\      }						\  } while (0) #define CUDA_CALL_NOCHECK(FN, ...)		\  CUDA_CALL_PREFIX FN (__VA_ARGS__) #define CUDA_CALL_EXISTS(FN)			\  CUDA_CALL_PREFIX FN static const char *cuda_error (CUresult r){  const char *fallback = "unknown cuda error";  const char *desc;   if (!CUDA_CALL_EXISTS (cuGetErrorString))    return fallback;   r = CUDA_CALL_NOCHECK (cuGetErrorString, r, &desc);  if (r == CUDA_SUCCESS)    return desc;   return fallback;} /* Version of the CUDA Toolkit in the same MAJOR.MINOR format that is used by   Nvidia, such as in the 'deviceQuery' program (Nvidia's CUDA samples). */static char cuda_driver_version_s[30]; static unsigned int instantiated_devices = 0;static pthread_mutex_t ptx_dev_lock = PTHREAD_MUTEX_INITIALIZER; /* NVPTX/CUDA specific definition of asynchronous queues.  */struct goacc_asyncqueue{  CUstream cuda_stream;}; struct nvptx_callback{  void (*fn) (void *);  void *ptr;  struct goacc_asyncqueue *aq;  struct nvptx_callback *next;}; /* Thread-specific data for PTX.  */ struct nvptx_thread{  /* We currently have this embedded inside the plugin because libgomp manages     devices through integer target_ids.  This might be better if using an     opaque target-specific pointer directly from gomp_device_descr.  */  struct ptx_device *ptx_dev;}; /* Target data function launch information.  */ struct targ_fn_launch{  const char *fn;  unsigned short dim[GOMP_DIM_MAX];}; /* Target PTX object information.  */ struct targ_ptx_obj{  const char *code;  size_t size;}; /* Target data image information.  */ typedef struct nvptx_tdata{  const struct targ_ptx_obj *ptx_objs;  unsigned ptx_num;   const char *const *var_names;  unsigned var_num;   const struct targ_fn_launch *fn_descs;  unsigned fn_num;   unsigned ind_fn_num;} nvptx_tdata_t; /* Descriptor of a loaded function.  */ struct targ_fn_descriptor{  CUfunction fn;  const struct targ_fn_launch *launch;  int regs_per_thread;  int max_threads_per_block;}; /* A loaded PTX image.  */struct ptx_image_data{  const void *target_data;  CUmodule module;   struct targ_fn_descriptor *fns;  /* Array of functions.  */    struct ptx_image_data *next;}; struct ptx_free_block{  void *ptr;  struct ptx_free_block *next;}; struct ptx_device{  CUcontext ctx;  bool ctx_shared;  CUdevice dev;   int ord;  bool overlap;  bool map;  bool concur;  bool mkern;  int mode;  int clock_khz;  int num_sms;  int regs_per_block;  int regs_per_sm;  int warp_size;  int max_threads_per_block;  int max_threads_per_multiprocessor;  int default_dims[GOMP_DIM_MAX];   /* Length as used by the CUDA Runtime API ('struct cudaDeviceProp').  */  char name[256];   struct ptx_image_data *images;  /* Images loaded on device.  */  pthread_mutex_t image_lock;     /* Lock for above list.  */   struct ptx_free_block *free_blocks;  pthread_mutex_t free_blocks_lock;   /* OpenMP stacks, cached between kernel invocations.  */  struct    {      CUdeviceptr ptr;      size_t size;      pthread_mutex_t lock;    } omp_stacks;   struct rev_offload *rev_data;  struct ptx_device *next;}; static struct ptx_device **ptx_devices; static bool using_usm = false; /* "Native" GPU thread stack size.  */static unsigned native_gpu_thread_stack_size = 0; /* OpenMP kernels reserve a small amount of ".shared" space for use by   omp_alloc.  The size is configured using GOMP_NVPTX_LOWLAT_POOL, but the   default is set here.  */static unsigned lowlat_pool_size = 8 * 1024; static bool nvptx_do_global_cdtors (CUmodule, struct ptx_device *,				    const char *);static size_t nvptx_stacks_size ();static void *nvptx_stacks_acquire (struct ptx_device *, size_t, int); static inline struct nvptx_thread *nvptx_thread (void){  return (struct nvptx_thread *) GOMP_PLUGIN_acc_thread ();} /* Initialize the device.  Return TRUE on success, else FALSE.  PTX_DEV_LOCK   should be locked on entry and remains locked on exit.  */ static boolnvptx_init (void){  int ndevs;   if (instantiated_devices != 0)    return true;   if (!init_cuda_lib ())    return false;   CUDA_CALL (cuInit, 0);   int cuda_driver_version;  CUDA_CALL_ERET (NULL, cuDriverGetVersion, &cuda_driver_version);  snprintf (cuda_driver_version_s, sizeof cuda_driver_version_s,	    "CUDA Driver %u.%u",	    cuda_driver_version / 1000, cuda_driver_version % 1000 / 10);   CUDA_CALL (cuDeviceGetCount, &ndevs);  ptx_devices = GOMP_PLUGIN_malloc_cleared (sizeof (struct ptx_device *)					    * ndevs);   return true;} /* Select the N'th PTX device for the current host thread.  The device must   have been previously opened before calling this function.  */ static boolnvptx_attach_host_thread_to_device (int n){  CUdevice dev;  CUresult r;  struct ptx_device *ptx_dev;  CUcontext thd_ctx;   r = CUDA_CALL_NOCHECK (cuCtxGetDevice, &dev);  if (r == CUDA_ERROR_NOT_PERMITTED)    {      /* Assume we're in a CUDA callback, just return true.  */      return true;    }  if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT)    {      GOMP_PLUGIN_error ("cuCtxGetDevice error: %s", cuda_error (r));      return false;    }   if (r != CUDA_ERROR_INVALID_CONTEXT && dev == n)    return true;  else    {      CUcontext old_ctx;       ptx_dev = ptx_devices[n];      if (!ptx_dev)	{	  GOMP_PLUGIN_error ("device %d not found", n);	  return false;	}       CUDA_CALL (cuCtxGetCurrent, &thd_ctx);       /* We don't necessarily have a current context (e.g. if it has been         destroyed.  Pop it if we do though.  */      if (thd_ctx != NULL)	CUDA_CALL (cuCtxPopCurrent, &old_ctx);       CUDA_CALL (cuCtxPushCurrent, ptx_dev->ctx);    }  return true;} static struct ptx_device *nvptx_open_device (int n){  struct ptx_device *ptx_dev;  CUdevice dev, ctx_dev;  CUresult r;  int pi;   CUDA_CALL_ERET (NULL, cuDeviceGet, &dev, n);   ptx_dev = GOMP_PLUGIN_malloc (sizeof (struct ptx_device));   ptx_dev->ord = n;  ptx_dev->dev = dev;  ptx_dev->ctx_shared = false;   r = CUDA_CALL_NOCHECK (cuCtxGetDevice, &ctx_dev);  if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT)    {      GOMP_PLUGIN_error ("cuCtxGetDevice error: %s", cuda_error (r));      return NULL;    }    if (r != CUDA_ERROR_INVALID_CONTEXT && ctx_dev != dev)    {      /* The current host thread has an active context for a different device.         Detach it.  */      CUcontext old_ctx;      CUDA_CALL_ERET (NULL, cuCtxPopCurrent, &old_ctx);    }   CUDA_CALL_ERET (NULL, cuCtxGetCurrent, &ptx_dev->ctx);   if (!ptx_dev->ctx)    CUDA_CALL_ERET (NULL, cuCtxCreate, &ptx_dev->ctx, CU_CTX_SCHED_AUTO, dev);  else    ptx_dev->ctx_shared = true;   CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,		  &pi, CU_DEVICE_ATTRIBUTE_GPU_OVERLAP, dev);  ptx_dev->overlap = pi;   CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,		  &pi, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, dev);  ptx_dev->map = pi;   CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,

Lines 1–500 of 3066. Load more