/*
 * Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
 *
 * SPDX-License-Identifier: MIT
 */

/**
 *  @file  amd_detail/hip_runtime.h
 *  @brief Contains definitions of APIs for HIP runtime.
 */

// #pragma once
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H

#include <hip/amd_detail/amd_hip_common.h>

#if !defined(__HIPCC_RTC__)
#ifdef __cplusplus
#include <cstddef>
#else
#include <stddef.h>
#endif  // __cplusplus
#endif  // !defined(__HIPCC_RTC__)

#ifdef __cplusplus
extern "C" {
#endif

/**
 * @brief Query the installed library build name.
 *
 * This function can be used even when the library is not initialized.
 *
 * @returns Returns a string describing the build version of the library.  The
 * string is owned by the library.
 */
const char* amd_dbgapi_get_build_name();

/**
 * @brief Query the installed library git hash.
 *
 * This function can be used even when the library is not initialized.
 *
 * @returns Returns git hash of the library.
 */
const char* amd_dbgapi_get_git_hash();

/**
 * @brief Query the installed library build ID.
 *
 * This function can be used even when the library is not initialized.
 *
 * @returns Returns build ID of the library.
 */
size_t amd_dbgapi_get_build_id();

#ifdef __cplusplus
} /* extern "c" */
#endif

//---
// Top part of file can be compiled with any compiler

#if !defined(__HIPCC_RTC__)
#ifdef __cplusplus
#include <cmath>
#include <cstdint>
#include <tuple>
#include <array>
#include <utility>
#include <type_traits>
#else
#include <math.h>
#include <stdint.h>
#endif  // __cplusplus
#endif  // !defined(__HIPCC_RTC__)

#if __HIP_CLANG_ONLY__

#if !defined(__align__)
#define __align__(x) __attribute__((aligned(x)))
#endif

#define CUDA_SUCCESS hipSuccess

#if !defined(__HIPCC_RTC__)
#include <hip/hip_runtime_api.h>
#include <hip/amd_detail/amd_hip_atomic.h>
#include <hip/amd_detail/amd_device_functions.h>
#include <hip/amd_detail/amd_surface_functions.h>
#include <hip/amd_detail/texture_fetch_functions.h>
#include <hip/amd_detail/texture_indirect_functions.h>
extern int HIP_TRACE_API;
#endif  // !defined(__HIPCC_RTC__)

#ifdef __cplusplus
#include <hip/amd_detail/hip_ldg.h>
#endif

#include <hip/amd_detail/host_defines.h>

// TODO-HCC remove old definitions ; ~1602 hcc supports __HCC_ACCELERATOR__ define.
#if defined(__KALMAR_ACCELERATOR__) && !defined(__HCC_ACCELERATOR__)
#define __HCC_ACCELERATOR__ __KALMAR_ACCELERATOR__
#endif

// Feature tests:
#if (defined(__HCC_ACCELERATOR__) && (__HCC_ACCELERATOR__ != 0)) || __HIP_DEVICE_COMPILE__
// Device compile and not host compile:

// 32-bit Atomics:
#define __HIP_ARCH_HAS_GLOBAL_INT32_ATOMICS__ (1)
#define __HIP_ARCH_HAS_GLOBAL_FLOAT_ATOMIC_EXCH__ (1)
#define __HIP_ARCH_HAS_SHARED_INT32_ATOMICS__ (1)
#define __HIP_ARCH_HAS_SHARED_FLOAT_ATOMIC_EXCH__ (1)
#define __HIP_ARCH_HAS_FLOAT_ATOMIC_ADD__ (1)

// 64-bit Atomics:
#define __HIP_ARCH_HAS_GLOBAL_INT64_ATOMICS__ (1)
#define __HIP_ARCH_HAS_SHARED_INT64_ATOMICS__ (1)

// Doubles
#define __HIP_ARCH_HAS_DOUBLES__ (1)

// warp cross-lane operations:
#define __HIP_ARCH_HAS_WARP_VOTE__ (1)
#define __HIP_ARCH_HAS_WARP_BALLOT__ (1)
#define __HIP_ARCH_HAS_WARP_SHUFFLE__ (1)
#define __HIP_ARCH_HAS_WARP_FUNNEL_SHIFT__ (0)

// sync
#define __HIP_ARCH_HAS_THREAD_FENCE_SYSTEM__ (1)
#define __HIP_ARCH_HAS_SYNC_THREAD_EXT__ (0)

// misc
#define __HIP_ARCH_HAS_SURFACE_FUNCS__ (0)
#define __HIP_ARCH_HAS_3DGRID__ (1)
#define __HIP_ARCH_HAS_DYNAMIC_PARALLEL__ (0)

#endif /* Device feature flags */


#define launch_bounds_impl0(requiredMaxThreadsPerBlock)                                            \
  __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock)))
#define launch_bounds_impl1(requiredMaxThreadsPerBlock, minBlocksPerMultiprocessor)                \
  __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock),                       \
                 amdgpu_waves_per_eu(minBlocksPerMultiprocessor)))
#define select_impl_(_1, _2, impl_, ...) impl_
#define __launch_bounds__(...)                                                                     \
  select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0, )(__VA_ARGS__)

#if !defined(__HIPCC_RTC__)
__host__ inline void* __get_dynamicgroupbaseptr() { return nullptr; }
#endif  // !defined(__HIPCC_RTC__)

// End doxygen API:
/**
 *   @}
 */

//
// hip-clang functions
//
#if !defined(__HIPCC_RTC__)
#define HIP_KERNEL_NAME(...) __VA_ARGS__
#define HIP_SYMBOL(X) X

typedef int hipLaunchParm;

template <typename... Formals, typename... Actuals>
std::tuple<Formals...> validateArgsCountType(void (*kernel)(Formals...),
                                             Actuals... actuals) {
  static_assert(sizeof...(Formals) == sizeof...(Actuals), "Argument Count Mismatch");
  return {std::move(actuals)...};
}

template <typename T> constexpr bool validateArgType() {
  static_assert(!std::is_reference<T>{},
                "A __global__ function cannot have a reference as one of its "
                "arguments.");
#if defined(HIP_STRICT)
  static_assert(std::is_trivially_copyable<T>{},
                "Only TriviallyCopyable types can be arguments to a __global__ "
                "function");
#endif
  return !std::is_reference<T>{} && std::is_trivially_copyable<T>{};
}

template <typename... Ts> constexpr bool validateArgs(void (*)(Ts...)) {
  return (validateArgType<Ts>() && ... && true);
}

template <typename... Ts, size_t... Is>
std::array<void*, sizeof...(Ts)> pArgs(std::tuple<Ts...>& formals,
                                       __hip_internal::index_sequence<Is...>) {
  return {(static_cast<void*>(&std::get<Is>(formals)))...};
}

template <typename... Ts> std::array<void*, sizeof...(Ts)> pArgs(std::tuple<Ts...>& formals) {
  return pArgs(formals, __hip_internal::make_index_sequence<sizeof...(Ts)>());
}

#if defined(HIP_TEMPLATE_KERNEL_LAUNCH)
template <typename... Args, typename F = void (*)(Args...)>
void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
                        std::uint32_t sharedMemBytes, hipStream_t stream, Args... args) {
  validateArgs(kernel);
  auto k = reinterpret_cast<void*>(kernel);

  if constexpr (std::is_same_v<F, void (*)(Args...)>) {
    std::array<void*, sizeof...(Args)> ptrArgsArr{static_cast<void*>(&args)...};
    hipLaunchKernel(k, numBlocks, dimBlocks, ptrArgsArr.data(), sharedMemBytes, stream);
  } else {
    auto formals = validateArgsCountType(kernel, args...);
    auto ptrArgsArr = pArgs(formals);
    hipLaunchKernel(k, numBlocks, dimBlocks, ptrArgsArr.data(), sharedMemBytes, stream);
  }
}
#else
#define hipLaunchKernelGGLInternal(kernelName, numBlocks, numThreads, memPerBlock, streamId, ...)  \
  do {                                                                                             \
    kernelName<<<(numBlocks), (numThreads), (memPerBlock), (streamId)>>>(__VA_ARGS__);             \
  } while (0)

#define hipLaunchKernelGGL(kernelName, ...) hipLaunchKernelGGLInternal((kernelName), __VA_ARGS__)
#endif

#include <hip/hip_runtime_api.h>
#endif  // !defined(__HIPCC_RTC__)

#if defined(__HIPCC_RTC__)
typedef struct dim3 {
  __hip_uint32_t x;  ///< x
  __hip_uint32_t y;  ///< y
  __hip_uint32_t z;  ///< z
#ifdef __cplusplus
  constexpr __device__ dim3(__hip_uint32_t _x = 1, __hip_uint32_t _y = 1, __hip_uint32_t _z = 1)
      : x(_x), y(_y), z(_z) {};
#endif
} dim3;
#endif  // !defined(__HIPCC_RTC__)

#pragma push_macro("__DEVICE__")
#define __DEVICE__ static __device__ __forceinline__

extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_id(unsigned int);
__DEVICE__ unsigned int __hip_get_thread_idx_x() { return __ockl_get_local_id(0); }
__DEVICE__ unsigned int __hip_get_thread_idx_y() { return __ockl_get_local_id(1); }
__DEVICE__ unsigned int __hip_get_thread_idx_z() { return __ockl_get_local_id(2); }

extern "C" __device__ __attribute__((const)) size_t __ockl_get_group_id(unsigned int);
__DEVICE__ unsigned int __hip_get_block_idx_x() { return __ockl_get_group_id(0); }
__DEVICE__ unsigned int __hip_get_block_idx_y() { return __ockl_get_group_id(1); }
__DEVICE__ unsigned int __hip_get_block_idx_z() { return __ockl_get_group_id(2); }

extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_size(unsigned int);
__DEVICE__ unsigned int __hip_get_block_dim_x() { return __ockl_get_local_size(0); }
__DEVICE__ unsigned int __hip_get_block_dim_y() { return __ockl_get_local_size(1); }
__DEVICE__ unsigned int __hip_get_block_dim_z() { return __ockl_get_local_size(2); }

extern "C" __device__ __attribute__((const)) size_t __ockl_get_num_groups(unsigned int);
__DEVICE__ unsigned int __hip_get_grid_dim_x() { return __ockl_get_num_groups(0); }
__DEVICE__ unsigned int __hip_get_grid_dim_y() { return __ockl_get_num_groups(1); }
__DEVICE__ unsigned int __hip_get_grid_dim_z() { return __ockl_get_num_groups(2); }

#define __HIP_DEVICE_BUILTIN(DIMENSION, FUNCTION)                                                  \
  __declspec(property(get = __get_##DIMENSION)) unsigned int DIMENSION;                            \
  __DEVICE__ unsigned int __get_##DIMENSION(void) { return FUNCTION; }

struct __hip_builtin_threadIdx_t {
  __HIP_DEVICE_BUILTIN(x, __hip_get_thread_idx_x());
  __HIP_DEVICE_BUILTIN(y, __hip_get_thread_idx_y());
  __HIP_DEVICE_BUILTIN(z, __hip_get_thread_idx_z());
#ifdef __cplusplus
  __device__ operator dim3() const { return dim3(x, y, z); }
#endif
};

struct __hip_builtin_blockIdx_t {
  __HIP_DEVICE_BUILTIN(x, __hip_get_block_idx_x());
  __HIP_DEVICE_BUILTIN(y, __hip_get_block_idx_y());
  __HIP_DEVICE_BUILTIN(z, __hip_get_block_idx_z());
#ifdef __cplusplus
  __device__ operator dim3() const { return dim3(x, y, z); }
#endif
};

struct __hip_builtin_blockDim_t {
  __HIP_DEVICE_BUILTIN(x, __hip_get_block_dim_x());
  __HIP_DEVICE_BUILTIN(y, __hip_get_block_dim_y());
  __HIP_DEVICE_BUILTIN(z, __hip_get_block_dim_z());
#ifdef __cplusplus
  __device__ operator dim3() const { return dim3(x, y, z); }
#endif
};

struct __hip_builtin_gridDim_t {
  __HIP_DEVICE_BUILTIN(x, __hip_get_grid_dim_x());
  __HIP_DEVICE_BUILTIN(y, __hip_get_grid_dim_y());
  __HIP_DEVICE_BUILTIN(z, __hip_get_grid_dim_z());
#ifdef __cplusplus
  __device__ operator dim3() const { return dim3(x, y, z); }
#endif
};

#undef __HIP_DEVICE_BUILTIN
#pragma pop_macro("__DEVICE__")

extern const __device__ __attribute__((weak)) __hip_builtin_threadIdx_t threadIdx;
extern const __device__ __attribute__((weak)) __hip_builtin_blockIdx_t blockIdx;
extern const __device__ __attribute__((weak)) __hip_builtin_blockDim_t blockDim;
extern const __device__ __attribute__((weak)) __hip_builtin_gridDim_t gridDim;

#define hipThreadIdx_x threadIdx.x
#define hipThreadIdx_y threadIdx.y
#define hipThreadIdx_z threadIdx.z

#define hipBlockIdx_x blockIdx.x
#define hipBlockIdx_y blockIdx.y
#define hipBlockIdx_z blockIdx.z

#define hipBlockDim_x blockDim.x
#define hipBlockDim_y blockDim.y
#define hipBlockDim_z blockDim.z

#define hipGridDim_x gridDim.x
#define hipGridDim_y gridDim.y
#define hipGridDim_z gridDim.z

#if !defined(__HIPCC_RTC__)
#include <hip/amd_detail/amd_math_functions.h>
#endif

#if __HIP_HCC_COMPAT_MODE__
// Define HCC work item functions in terms of HIP builtin variables.
#pragma push_macro("__DEFINE_HCC_FUNC")
#define __DEFINE_HCC_FUNC(hc_fun, hip_var)                                                         \
  inline __device__ __attribute__((always_inline)) unsigned int hc_get_##hc_fun(unsigned int i) {  \
    if (i == 0)                                                                                    \
      return hip_var.x;                                                                            \
    else if (i == 1)                                                                               \
      return hip_var.y;                                                                            \
    else                                                                                           \
      return hip_var.z;                                                                            \
  }

__DEFINE_HCC_FUNC(workitem_id, threadIdx)
__DEFINE_HCC_FUNC(group_id, blockIdx)
__DEFINE_HCC_FUNC(group_size, blockDim)
__DEFINE_HCC_FUNC(num_groups, gridDim)
#pragma pop_macro("__DEFINE_HCC_FUNC")

extern "C" __device__ __attribute__((const)) size_t __ockl_get_global_id(unsigned int);
inline __device__ __attribute__((always_inline)) unsigned int hc_get_workitem_absolute_id(int dim) {
  return (unsigned int)__ockl_get_global_id(dim);
}

#endif

#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#if !defined(__HIPCC_RTC__)
// Support std::complex.
#if !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
#pragma push_macro("__CUDA__")
#define __CUDA__
#include <__clang_cuda_math_forward_declares.h>
#include <__clang_cuda_complex_builtins.h>
// Workaround for using libc++ with HIP-Clang.
// The following headers requires clang include path before standard C++ include path.
// However libc++ include path requires to be before clang include path.
// To workaround this, we pass -isystem with the parent directory of clang include
// path instead of the clang include path itself.
#include <include/cuda_wrappers/algorithm>
#include <include/cuda_wrappers/complex>
#include <include/cuda_wrappers/new>
#undef __CUDA__
#pragma pop_macro("__CUDA__")
#endif  // !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
#endif  // !defined(__HIPCC_RTC__)
#endif  // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#endif  // __HIP_CLANG_ONLY__

#endif  // HIP_AMD_DETAIL_RUNTIME_H
