# The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_COLOR_MAKEFILE ON)
set(VERBOSE_BUILD ON)

# Require C++14 and disable compiler-specific extensions
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
set(DEFAULT_BUILD_TYPE "Release")

include("./config/defaults.cmake")

message(CUDA_ARCH = ${CUDA_ARCH})

#-------------------------------------------------------------------------------
# Project name and version
#-------------------------------------------------------------------------------
project(DGTD LANGUAGES CXX)
set(${PROJECT_NAME}_VERSION 0.0.0)

# Prohibit in-source build
if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
    message(FATAL_ERROR
            "DGTD does not support in-source CMake builds at this time.")
endif (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})

# Enable languages.
enable_language(CXX)



if (CUDA_NON_HEAVY)
    message("CUDA_NON_HEAVY is ON")
else()
    if (NOT CUDA_GPU_ZERO_COPY)
        message("CUDA HEAVY MODE 1 is ON")
    else()
        message("CUDA HEAVY MODE 2 ZERO COPY is ON")
    endif()
endif()


message(CUDA_ARCH = ${CUDA_ARCH})


if (DGTD_USE_CUDA OR DGTD_USE_CUDA_OPENCL)
    # DGTD_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
    cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
    # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
    if (NOT CMAKE_CUDA_HOST_COMPILER)
        set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
    endif()
    enable_language(CUDA)
    set(CMAKE_CUDA_STANDARD 14)
    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
    set(CMAKE_CUDA_EXTENSIONS OFF)

    # Find CUDA ARCH VERSION
    include(FindCUDA/select_compute_arch)
    CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS)
    set(INSTALLED_GPU_CCS "60;61;70;75;80;86")
        
    list(GET INSTALLED_GPU_CCS 0 ARCH_NUMBER)

    set(CUDA_ARCH "sm_${ARCH_NUMBER}")
    set(CUDA_COMPUTE "compute_${ARCH_NUMBER}")

    message(--------------------------------------------)
    message(CUDA_ARCH = ${CUDA_ARCH})
    message(CUDA_COMPUTE = ${CUDA_COMPUTE})
    message(--------------------------------------------)

    set(CUDA_FLAGS "--expt-extended-lambda --compiler-options -fPIC")
    set(CUDA_FLAGS "-gencode code=${CUDA_ARCH},arch=${CUDA_COMPUTE} ${CUDA_FLAGS}")
    string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
    if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
        set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
    else()
        message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
    endif()
    message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")

    set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
    set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
    set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING "CUDA flags set for DGTD" FORCE)
    set(CUSPARSE_FOUND TRUE)
    set(CUSPARSE_LIBRARIES "cusparse")
endif()

# OpenCL
if(DGTD_USE_OPENCL OR DGTD_USE_CUDA_OPENCL)
    find_package(OpenCL REQUIRED)
    include_directories(${OpenCL_INCLUDE_DIRS})
    link_directories(${OpenCL_LIBRARY})
endif()

if(DGTD_USE_OPENCL)
    add_definitions(-DDGTD_USE_OPENCL)
endif()

# Suppress warnings about MACOSX_RPATH
set(CMAKE_MACOSX_RPATH OFF CACHE BOOL "")

# CMake needs to know where to find things
set(DGTD_CMAKE_PATH ${PROJECT_SOURCE_DIR}/config)
set(CMAKE_MODULE_PATH ${DGTD_CMAKE_PATH})

# Load DGTD CMake utilities.
include(CmakeUtilities)

#-------------------------------------------------------------------------------
# Process configuration options
#-------------------------------------------------------------------------------
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -no-pie -lpthread")

# DGTD_DEBUG
if (CMAKE_BUILD_TYPE MATCHES "Debug|debug|DEBUG")
    set(DGTD_DEBUG ON)
else()
    set(DGTD_DEBUG OFF)
endif()

# BLAS, LAPACK
#set(BLA_VENDOR OpenBLAS)  # OpenBLAS can also be used to substitute LAPACK
# find_package(BLAS REQUIRED)
# find_package(LAPACK REQUIRED)
# message(STATUS BLAS found: ${BLAS_LIBRARIES})

# MKL
set(ENV{MKLROOT} "/opt/intel/oneapi/mkl/latest")
find_package(MKL CONFIG REQUIRED)

if (DGTD_USE_CUDA)
    add_definitions(-DDGTD_USE_CUDA)
endif()

if (DGTD_USE_CUDA_OPENCL)
    add_definitions(-DDGTD_USE_CUDA_OPENCL)
endif()

if (DGTD_USE_DOUBLE)
    add_definitions(-DDGTD_USE_DOUBLE)
endif()

if (DGTD_USE_DOUBLE_TS)
    add_definitions(-DDGTD_USE_DOUBLE_TS)
endif()

if (DGTD_WRITE_SURF_FIELDS)
    add_definitions(-DDGTD_WRITE_SURF_FIELDS)
endif()

if (DGTD_INCIDENT_FIELDS)
    add_definitions(-DDGTD_INCIDENT_FIELDS)
endif()

if (CUDA_NON_HEAVY)
    add_definitions(-DCUDA_NON_HEAVY)
endif()

if (CUDA_GPU_ZERO_COPY)
    add_definitions(-DCUDA_GPU_ZERO_COPY)
endif()

if (DGTD_USE_LTS)
    add_definitions(-DDGTD_USE_LTS)
endif()


# OpenMP
if (DGTD_USE_OPENMP)
    find_package(OpenMP REQUIRED)
    set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})

    if (OPENMP_FOUND)
        message(STATUS "DGTD: using package OpenMP")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
        if (DGTD_USE_CUDA OR DGTD_USE_CUDA_OPENCL)
            set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
        endif()
    endif()

    add_definitions(-DDGTD_USE_OPENMP)
endif()


message(STATUS "DGTD build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
message(STATUS "DGTD version: v${DGTD_VERSION_STRING}")


#-------------------------------------------------------------------------------
# Define and configure the DGTD library
#-------------------------------------------------------------------------------

# Headers and sources
set(SOURCES "")
set(HEADERS "")
set(DGTD_SOURCE_DIRS fem utils)
foreach(DIR IN LISTS DGTD_SOURCE_DIRS)
    add_subdirectory(${DIR})
endforeach()

if (DGTD_USE_CUDA OR DGTD_USE_CUDA_OPENCL)
    set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
endif()

add_subdirectory(config)
set(MASTER_HEADERS
        ${PROJECT_SOURCE_DIR}/MaxwellTD.cpp
        ${PROJECT_SOURCE_DIR}/dgtd-performance.hpp)

set(_lib_path "${CMAKE_INSTALL_PREFIX}/lib")
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON CACHE BOOL "")
set(CMAKE_INSTALL_RPATH "${_lib_path}" CACHE PATH "")
set(CMAKE_INSTALL_NAME_DIR "${_lib_path}" CACHE PATH "")

set(DGTD_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "The DGTD source directory" FORCE)
set(DGTD_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH "The DGTD install directory" FORCE)

# add the directory as -I to the compilation command line
include_directories(${PROJECT_SOURCE_DIR}/utils)
# include_directories(${PROJECT_SOURCE_DIR}/utils/lapack-3.9.1/CBLAS/include)
include_directories("/opt/intel/oneapi/mkl/latest/include")
include_directories("/usr/local/cuda/include")
include_directories(${PROJECT_SOURCE_DIR}/fem)
link_directories(${PROJECT_SOURCE_DIR}/lib)
link_directories("/opt/intel/oneapi/mkl/latest/lib/intel64")
link_directories("/usr/local/cuda/lib64/")

# Declaring the library
set(DGTD_ALL ${SOURCES} ${HEADERS} ${MASTER_HEADERS})

foreach(FNAME IN LISTS DGTD_ALL)
    message(${FNAME})
endforeach()



# --------------------------------------
# Executable Name
set(EXE_NAME maxwelltd)
if (DGTD_USE_CUDA)
    set(EXE_NAME ${EXE_NAME}_CUDA)

if (DGTD_USE_CUDA_OPENCL)
    set(EXE_NAME ${EXE_NAME}_OPENCL)
endif()

if (NOT CUDA_NON_HEAVY)
if (CUDA_GPU_ZERO_COPY)
    set(EXE_NAME ${EXE_NAME}_HEAVYZEROCOPY)
else()
    set(EXE_NAME ${EXE_NAME}_HEAVY)
endif()
endif()

else()
    set(EXE_NAME ${EXE_NAME}_CPU) 
endif()


if (DGTD_USE_LTS)
    set(EXE_NAME ${EXE_NAME}_LTS)     
endif()

if (DGTD_USE_DOUBLE)
    set(EXE_NAME ${EXE_NAME}_DOUBLEpre) 
else()
    set(EXE_NAME ${EXE_NAME}_FLOATpre) 
endif()

if (DGTD_USE_DOUBLE_TS)
    set(EXE_NAME ${EXE_NAME}_DOUBLEpro) 
else()
    set(EXE_NAME ${EXE_NAME}_FLOATpro) 
endif()

# --------------------------------------






add_executable(${EXE_NAME} ${DGTD_ALL})
# lapack should be put before blas and cblas
# removed vtr quadrature tensor
target_link_libraries(${EXE_NAME} mkl_rt m register metis5 vtkIO vtkzlib vtkFiltering vtkCommon gfortran gomp dl pthread fftw3 cudart cublas cusolver)
# target_link_libraries(${EXE_NAME} m register lapack blas metis5 vtkIO vtkzlib vtkFiltering vtkCommon gfortran gomp dl pthread fftw3 cusolver cublas)
# EMoffice_Base -->  m vtr tensor quadrature register

if(DGTD_USE_OPENCL OR DGTD_USE_CUDA_OPENCL)
    target_link_libraries (${EXE_NAME} ${OpenCL_LIBRARY})
endif()

if(DGTD_USE_CUDA OR DGTD_USE_CUDA_OPENCL)
    include_directories(${EXE_NAME} ${CUDA_INCLUDE_DIRS})
    target_link_libraries (${EXE_NAME} ${CUDA_LIBRARIES})
endif()

set_target_properties(${EXE_NAME} PROPERTIES VERSION "${dgtd_VERSION}")
set_target_properties(${EXE_NAME} PROPERTIES SOVERSION "${dgtd_VERSION}")
set_target_properties(${EXE_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)
if (DGTD_USE_CUDA OR DGTD_USE_CUDA_OPENCL)
    set_target_properties(${EXE_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON CUDA_RESOLVE_DEVICE_SYMBOLS ON)
endif ()

# If building out-of-source, define DGTD_CONFIG_FILE to point to the config file inside the build directory.
if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
    target_compile_definitions(${EXE_NAME} PRIVATE "DGTD_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
endif()