/* Plugin for AMD GCN execution.    Copyright (C) 2013-2026 Free Software Foundation, Inc.    Contributed by Mentor Embedded    This file is part of the GNU Offloading and Multi Processing Library   (libgomp).    Libgomp is free software; you can redistribute it and/or modify it   under the terms of the GNU General Public License as published by   the Free Software Foundation; either version 3, or (at your option)   any later version.    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for   more details.    Under Section 7 of GPL version 3, you are granted additional   permissions described in the GCC Runtime Library Exception, version   3.1, as published by the Free Software Foundation.    You should have received a copy of the GNU General Public License and   a copy of the GCC Runtime Library Exception along with this program;   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see   <http://www.gnu.org/licenses/>.  */ /* {{{ Includes and defines  */ #include "config.h"#include "symcat.h"#include <stdio.h>#include <stdlib.h>#include <string.h>#include <pthread.h>#include <inttypes.h>#include <stdbool.h>#include <limits.h>#include <hsa.h>#include <hsa_ext_amd.h>#include <dlfcn.h>#include <signal.h>#include "alloc_cache.h"#define _LIBGOMP_PLUGIN_INCLUDE 1#include "libgomp-plugin.h"#undef _LIBGOMP_PLUGIN_INCLUDE#include "config/gcn/libgomp-gcn.h"  /* For struct output.  */#include "gomp-constants.h"#include <elf.h>#include "oacc-plugin.h"#include "oacc-int.h"#include <assert.h>#include <sys/mman.h>#include <unistd.h> /* Create hash-table for declare target's indirect clause on the host;   see build-target-indirect-htab.h for details.  */#define USE_HASHTAB_LOOKUP_FOR_INDIRECT#ifdef USE_HASHTAB_LOOKUP_FOR_INDIRECTstatic void* create_target_indirect_map (size_t *, size_t,					 uint64_t *, uint64_t *);#endif /* These probably won't be in elf.h for a while.  */#ifndef R_AMDGPU_NONE#define R_AMDGPU_NONE		0#define R_AMDGPU_ABS32_LO	1	/* (S + A) & 0xFFFFFFFF  */#define R_AMDGPU_ABS32_HI	2	/* (S + A) >> 32  */#define R_AMDGPU_ABS64		3	/* S + A  */#define R_AMDGPU_REL32		4	/* S + A - P  */#define R_AMDGPU_REL64		5	/* S + A - P  */#define R_AMDGPU_ABS32		6	/* S + A  */#define R_AMDGPU_GOTPCREL	7	/* G + GOT + A - P  */#define R_AMDGPU_GOTPCREL32_LO	8	/* (G + GOT + A - P) & 0xFFFFFFFF  */#define R_AMDGPU_GOTPCREL32_HI	9	/* (G + GOT + A - P) >> 32  */#define R_AMDGPU_REL32_LO	10	/* (S + A - P) & 0xFFFFFFFF  */#define R_AMDGPU_REL32_HI	11	/* (S + A - P) >> 32  */#define R_AMDGPU_RELATIVE64	13	/* B + A  */#endif #define ELFABIVERSION_AMDGPU_HSA_V6		4 #define EF_AMDGPU_GENERIC_VERSION_V		0xff000000  /* Mask.  */#define EF_AMDGPU_GENERIC_VERSION_OFFSET	24 #define GET_GENERIC_VERSION(VAR) ((VAR & EF_AMDGPU_GENERIC_VERSION_V) \				  >> EF_AMDGPU_GENERIC_VERSION_OFFSET) /* GCN specific definitions for asynchronous queues.  */ #define ASYNC_QUEUE_SIZE 64#define DRAIN_QUEUE_SYNCHRONOUS_P false#define DEBUG_QUEUES 0#define DEBUG_THREAD_SLEEP 0#define DEBUG_THREAD_SIGNAL 0 /* Defaults.  */#define DEFAULT_GCN_HEAP_SIZE (100*1024*1024)  /* 100MB.  */ /* Secure getenv() which returns NULL if running as SUID/SGID.  */#ifndef HAVE_SECURE_GETENV#ifdef HAVE___SECURE_GETENV#define secure_getenv __secure_getenv#elif defined (HAVE_UNISTD_H) && defined(HAVE_GETUID) && defined(HAVE_GETEUID) \  && defined(HAVE_GETGID) && defined(HAVE_GETEGID) #include <unistd.h> /* Implementation of secure_getenv() for targets where it is not provided but   we have at least means to test real and effective IDs. */ static char *secure_getenv (const char *name){  if ((getuid () == geteuid ()) && (getgid () == getegid ()))    return getenv (name);  else    return NULL;} #else#define secure_getenv getenv#endif#endif /* }}}  *//* {{{ Types  */ /* GCN-specific implementation of the GOMP_PLUGIN_acc_thread data.  */ struct gcn_thread{  /* The thread number from the async clause, or GOMP_ASYNC_SYNC.  */  int async;}; /* As an HSA runtime is dlopened, following structure defines function   pointers utilized by the HSA plug-in.  */ struct hsa_runtime_fn_info{  /* HSA runtime.  */  hsa_status_t (*hsa_status_string_fn) (hsa_status_t status,					const char **status_string);  hsa_status_t (*hsa_system_get_info_fn) (hsa_system_info_t attribute,					  void *value);  hsa_status_t (*hsa_agent_get_info_fn) (hsa_agent_t agent,					 hsa_agent_info_t attribute,					 void *value);  hsa_status_t (*hsa_isa_get_info_fn)(hsa_isa_t isa,				      hsa_isa_info_t attribute,				      uint32_t index,				      void *value);  hsa_status_t (*hsa_init_fn) (void);  hsa_status_t (*hsa_iterate_agents_fn)    (hsa_status_t (*callback)(hsa_agent_t agent, void *data), void *data);  hsa_status_t (*hsa_region_get_info_fn) (hsa_region_t region,					  hsa_region_info_t attribute,					  void *value);  hsa_status_t (*hsa_queue_create_fn)    (hsa_agent_t agent, uint32_t size, hsa_queue_type_t type,     void (*callback)(hsa_status_t status, hsa_queue_t *source, void *data),     void *data, uint32_t private_segment_size,     uint32_t group_segment_size, hsa_queue_t **queue);  hsa_status_t (*hsa_agent_iterate_regions_fn)    (hsa_agent_t agent,     hsa_status_t (*callback)(hsa_region_t region, void *data), void *data);  hsa_status_t (*hsa_executable_destroy_fn) (hsa_executable_t executable);  hsa_status_t (*hsa_executable_create_fn)    (hsa_profile_t profile, hsa_executable_state_t executable_state,     const char *options, hsa_executable_t *executable);  hsa_status_t (*hsa_executable_global_variable_define_fn)    (hsa_executable_t executable, const char *variable_name, void *address);  hsa_status_t (*hsa_executable_load_code_object_fn)    (hsa_executable_t executable, hsa_agent_t agent,     hsa_code_object_t code_object, const char *options);  hsa_status_t (*hsa_executable_freeze_fn)(hsa_executable_t executable,					   const char *options);  hsa_status_t (*hsa_signal_create_fn) (hsa_signal_value_t initial_value,					uint32_t num_consumers,					const hsa_agent_t *consumers,					hsa_signal_t *signal);  hsa_status_t (*hsa_memory_allocate_fn) (hsa_region_t region, size_t size,					  void **ptr);  hsa_status_t (*hsa_memory_assign_agent_fn) (void *ptr, hsa_agent_t agent,					      hsa_access_permission_t access);  hsa_status_t (*hsa_memory_copy_fn)(void *dst, const void *src, size_t size);  hsa_status_t (*hsa_memory_free_fn) (void *ptr);  hsa_status_t (*hsa_signal_destroy_fn) (hsa_signal_t signal);  hsa_status_t (*hsa_executable_get_symbol_fn)    (hsa_executable_t executable, const char *module_name,     const char *symbol_name, hsa_agent_t agent, int32_t call_convention,     hsa_executable_symbol_t *symbol);  hsa_status_t (*hsa_executable_symbol_get_info_fn)    (hsa_executable_symbol_t executable_symbol,     hsa_executable_symbol_info_t attribute, void *value);  hsa_status_t (*hsa_executable_iterate_symbols_fn)    (hsa_executable_t executable,     hsa_status_t (*callback)(hsa_executable_t executable,			      hsa_executable_symbol_t symbol, void *data),     void *data);  uint64_t (*hsa_queue_add_write_index_release_fn) (const hsa_queue_t *queue,						    uint64_t value);  uint64_t (*hsa_queue_load_read_index_acquire_fn) (const hsa_queue_t *queue);  uint64_t (*hsa_queue_load_read_index_relaxed_fn) (const hsa_queue_t *queue);  uint64_t (*hsa_queue_load_write_index_relaxed_fn) (const hsa_queue_t *queue);  void (*hsa_signal_store_relaxed_fn) (hsa_signal_t signal,				       hsa_signal_value_t value);  void (*hsa_signal_store_release_fn) (hsa_signal_t signal,				       hsa_signal_value_t value);  hsa_signal_value_t (*hsa_signal_wait_acquire_fn)    (hsa_signal_t signal, hsa_signal_condition_t condition,     hsa_signal_value_t compare_value, uint64_t timeout_hint,     hsa_wait_state_t wait_state_hint);  hsa_signal_value_t (*hsa_signal_load_acquire_fn) (hsa_signal_t signal);  hsa_status_t (*hsa_queue_destroy_fn) (hsa_queue_t *queue);   hsa_status_t (*hsa_code_object_deserialize_fn)    (void *serialized_code_object, size_t serialized_code_object_size,     const char *options, hsa_code_object_t *code_object);  hsa_status_t (*hsa_amd_memory_fill_fn)(void *ptr, uint32_t value,					 size_t count);  hsa_status_t (*hsa_amd_memory_lock_fn)    (void *host_ptr, size_t size, hsa_agent_t *agents, int num_agent,     void **agent_ptr);  hsa_status_t (*hsa_amd_memory_unlock_fn) (void *host_ptr);  hsa_status_t (*hsa_amd_memory_async_copy_rect_fn)    (const hsa_pitched_ptr_t *dst, const hsa_dim3_t *dst_offset,     const hsa_pitched_ptr_t *src, const hsa_dim3_t *src_offset,     const hsa_dim3_t *range, hsa_agent_t copy_agent,     hsa_amd_copy_direction_t dir, uint32_t num_dep_signals,     const hsa_signal_t *dep_signals, hsa_signal_t completion_signal);  hsa_status_t (*hsa_amd_svm_attributes_set_fn)    (void* ptr, size_t size, hsa_amd_svm_attribute_pair_t* attribute_list,     size_t attribute_count);  hsa_status_t (*hsa_amd_svm_attributes_get_fn)    (void* ptr, size_t size, hsa_amd_svm_attribute_pair_t* attribute_list,     size_t attribute_count);  hsa_status_t (*hsa_amd_pointer_info_fn)    (const void *, hsa_amd_pointer_info_t *, void *(*)(size_t),     uint32_t *, hsa_agent_t **); }; /* As an HIP runtime is dlopened, following structure defines function   pointers utilized by the interop feature of this plugin.   Add suffient type declarations to get this work.  */ typedef int hipError_t;  /* Actually an enum; 0 == success. */typedef void* hipCtx_t;struct hipStream_s;typedef struct hipStream_s* hipStream_t; struct hip_runtime_fn_info{  hipError_t (*hipStreamCreate_fn) (hipStream_t *);  hipError_t (*hipStreamDestroy_fn) (hipStream_t);  hipError_t (*hipStreamSynchronize_fn) (hipStream_t);  hipError_t (*hipCtxGetCurrent_fn) (hipCtx_t *ctx);  hipError_t (*hipSetDevice_fn) (int deviceId);  hipError_t (*hipGetDevice_fn) (int *deviceId);}; /* Structure describing the run-time and grid properties of an HSA kernel   lauch.  This needs to match the format passed to GOMP_OFFLOAD_run.  */ struct GOMP_kernel_launch_attributes{  /* Number of dimensions the workload has.  Maximum number is 3.  */  uint32_t ndim;  /* Size of the grid in the three respective dimensions.  */  uint32_t gdims[3];  /* Size of work-groups in the respective dimensions.  */  uint32_t wdims[3];}; /* Collection of information needed for a dispatch of a kernel from a   kernel.  */ struct kernel_dispatch{  struct agent_info *agent;  /* Pointer to a command queue associated with a kernel dispatch agent.  */  void *queue;  /* Pointer to a memory space used for kernel arguments passing, wrapped in a     node from the agent kernel argument cache.  */  struct alloc_cache_node *kernarg_cache_node;  /* Kernel object.  */  uint64_t object;  /* Synchronization signal used for dispatch synchronization.  */  uint64_t signal;  /* Private segment size.  */  uint32_t private_segment_size;  /* Group segment size.  */  uint32_t group_segment_size;}; /* Structure of the kernargs segment, supporting console output.    This needs to match the definitions in Newlib, and the expectations   in libgomp target code.  */ struct kernargs {  struct kernargs_abi abi;   /* Output data.  */  struct output output_data;}; /* A queue entry for a future asynchronous launch.  */ struct kernel_launch{  struct kernel_info *kernel;  void *vars;  struct GOMP_kernel_launch_attributes kla;}; /* A queue entry for a future callback.  */ struct callback{  void (*fn)(void *);  void *data;}; /* A data struct for the copy_data callback.  */ struct copy_data{  void *dst;  const void *src;  size_t len;  struct goacc_asyncqueue *aq;}; /* A queue entry for a placeholder.  These correspond to a wait event.  */ struct placeholder{  int executed;  pthread_cond_t cond;  pthread_mutex_t mutex;}; /* A queue entry for a wait directive.  */ struct asyncwait_info{  struct placeholder *placeholderp;}; /* Encode the type of an entry in an async queue.  */ enum entry_type{  KERNEL_LAUNCH,  CALLBACK,  ASYNC_WAIT,  ASYNC_PLACEHOLDER}; /* An entry in an async queue.  */ struct queue_entry{  enum entry_type type;  union {    struct kernel_launch launch;    struct callback callback;    struct asyncwait_info asyncwait;    struct placeholder placeholder;  } u;}; /* An async queue header.    OpenMP may create one of these.   OpenACC may create many.  */ struct goacc_asyncqueue{  struct agent_info *agent;  hsa_queue_t *hsa_queue;   pthread_t thread_drain_queue;  pthread_mutex_t mutex;  pthread_cond_t queue_cond_in;  pthread_cond_t queue_cond_out;  struct queue_entry queue[ASYNC_QUEUE_SIZE];  int queue_first;  int queue_n;  int drain_queue_stop;   int id;  struct goacc_asyncqueue *prev;  struct goacc_asyncqueue *next;}; /* Mkoffload uses this structure to describe a kernel.    OpenMP kernel dimensions are passed at runtime.   OpenACC kernel dimensions are passed at compile time, here.  */ struct hsa_kernel_description{  const char *name;  int oacc_dims[3];  /* Only present for GCN kernels.  */  int sgpr_count;  int vpgr_count;}; /* Mkoffload uses this structure to describe an offload variable.  */ struct global_var_info{  const char *name;  void *address;}; /* Mkoffload uses this structure to describe all the kernels in a   loadable module.  These are passed the libgomp via static constructors.  */ struct gcn_image_desc{  struct gcn_image {    size_t size;    void *image;  } *gcn_image;  const unsigned kernel_count;  struct hsa_kernel_description *kernel_infos;  const unsigned ind_func_count;  const unsigned global_variable_count;}; /* Enum values corresponding to the the ELF architecture codes.   Only 'special' values are actually referenced in this file, but having them   all may aid debugging.  */ typedef enum {  EF_AMDGPU_MACH_UNSUPPORTED = -1,#define GCN_DEVICE(name, NAME, ELF, ...) \  EF_AMDGPU_MACH_AMDGCN_ ## NAME = ELF,#include "../../gcc/config/gcn/gcn-devices.def"} EF_AMDGPU_MACH; const static int EF_AMDGPU_MACH_MASK = 0x000000ff;typedef EF_AMDGPU_MACH gcn_isa; /* Description of an HSA GPU agent (device) and the program associated with   it.  */ struct agent_info{  /* The HSA ID of the agent.  Assigned when hsa_context is initialized.  */  hsa_agent_t id;  /* The user-visible device number.  */  int device_id;  /* Whether the agent has been initialized.  The fields below are usable only     if it has been.  */  bool initialized;   /* The instruction set architecture of the device. */  gcn_isa device_isa;  /* Name of the agent. */  char name[64];  /* Name of the vendor of the agent. */  char vendor_name[64];  /* Command queues of the agent.  */  hsa_queue_t *sync_queue;  struct goacc_asyncqueue *async_queues, *omp_async_queue;  pthread_mutex_t async_queues_mutex;   /* The HSA memory region from which to allocate kernel arguments.  */  hsa_region_t kernarg_region;   /* A stack of allocations in kernarg_region of (sizeof (struct kernargs))     size each, used for ammortizing kernel argument allocation cost.  */  struct alloc_cache kernarg_cache;   /* The HSA memory region from which to allocate device data.  */  hsa_region_t data_region;   /* Allocated ephemeral memories (team arena and stack space).  */  struct ephemeral_memories_list *ephemeral_memories_list;  pthread_mutex_t ephemeral_memories_write_lock;   /* Read-write lock that protects kernels which are running or about to be run     from interference with loading and unloading of images.  Needs to be     locked for reading while a kernel is being run, and for writing if the     list of modules is manipulated (and thus the HSA program invalidated).  */  pthread_rwlock_t module_rwlock;   /* The module associated with this kernel.  */  struct module_info *module;   /* Mutex enforcing that only one thread will finalize the HSA program.  A     thread should have locked agent->module_rwlock for reading before     acquiring it.  */  pthread_mutex_t prog_mutex;

Lines 1–500 of 5704. Load more