aomp_common_vars

#!/bin/bash
#
#  aomp_common_vars: This file is sourced by all AOMP build scripts.
#
#     This file is stored in aomp/bin directory of the aomp repo.
#     You don't need to change this file unless you want to change
#     default values for all AOMP developers. To customize your build,
#     set environment variables defined in this file.
#
#     The bash syntax ENVNAME=${ENVNAME:-value} checks if ENVNAME is
#     already set. If not, it assigns it the value following the :-

# The AOMP_COMPILER_NAME is used whenever version is queried.
# FIXME: More work is needed to propogate the use of AOMP_COMPILER_NAME.
AOMP_COMPILER_NAME=${AOMP_COMPILER_NAME:-AOMP}

# ROCM_VERSION may be set by Jenkins, if not guess a good number
ROCM_VERSION=${ROCM_VERSION:-5.0.0}

# Set the AOMP VERSION STRING
AOMP_VERSION=${AOMP_VERSION:-"15.0"}
AOMP_VERSION_MOD=${AOMP_VERSION_MOD:-"1"}
AOMP_VERSION_STRING=${AOMP_VERSION_STRING:-"$AOMP_VERSION-$AOMP_VERSION_MOD"}
export AOMP_VERSION_STRING AOMP_VERSION AOMP_VERSION_MOD ROCM_VERSION

# Set install directory and the link directory.
# AOMP will be a symbolic link AOMP_INSTALL_DIR is the versioned dir name
AOMP=${AOMP:-$HOME/rocm/aomp}
AOMP_INSTALL_DIR=${AOMP}_${AOMP_VERSION_STRING}

# Set the directory location where all local AOMP git repos are stored.
# For example, if AOMP_REPOS=$HOME/git/aomp11, then the primary aomp repo
# is stored in $HOME/git/aomp11/aomp and the mono repo is stored in
# $HOME/aomp/llvm-project.
AOMP_MAJOR_VERSION=${AOMP_VERSION%.*}
export AOMP_MAJOR_VERSION
if [ "$AOMP_MAJOR_VERSION" == "12" ] ; then
  AOMP_REPOS=${AOMP_REPOS:-$HOME/git/aomp12}
elif [ "$AOMP_MAJOR_VERSION" == "13" ] ; then
  # AOMP 13.1 will use repo command with interal repos
  if [[ "$AOMP_VERSION" == "13.1" ]] ; then
    #The default for internal repos will be $HOME/git/aomp13.1
    AOMP_REPOS=${AOMP_REPOS:-${HOME}/git/aomp${AOMP_VERSION}}
    # Force build for 13.1 to 13.1-0, ignore AOMP_VERSION_MOD set above
    AOMP_VERSION_MOD=0
    AOMP_VERSION_STRING="13.1-0"
    AOMP_INSTALL_DIR=${AOMP}_${AOMP_VERSION_STRING}
    AOMP_PATCH_CONTROL_FILE="${AOMP_REPOS}/aomp/bin/patches/patch-control-file_13.1.txt"
    export AOMP_VERSION_STRING AOMP_VERSION AOMP_VERSION_MOD
  else
     AOMP_REPOS=${AOMP_REPOS:-$HOME/git/aomp13}
  fi
elif [ "$AOMP_MAJOR_VERSION" == "11" ] ; then
  AOMP_REPOS=${AOMP_REPOS:-$HOME/git/aomp11}
elif [ "$AOMP_MAJOR_VERSION" == "14" ] || [ "$AOMP_MAJOR_VERSION" == "15" ] ; then
    AOMP_REPOS=${AOMP_REPOS:-${HOME}/git/aomp${AOMP_VERSION}}
    AOMP_PATCH_CONTROL_FILE="${AOMP_REPOS}/aomp/bin/patches/patch-control-file_$AOMP_VERSION.txt"
else
  echo "ERROR:  Invalid AOMP_MAJOR_VERSION: $AOMP_MAJOR_VERSION"
  exit 1
fi

# Because the source for all repos are tarred for release, we store
# all the test repositories in a different directory.
AOMP_REPOS_TEST=${AOMP_REPOS_TEST:-$HOME/git/aomp-test}

# AOMP_SUPP is for SUPPlemental software components needed for testing.
# such as openmpi, hdf5, and silo. Applications that need supplemental
# components can get the latest version with $AOMP_SUPP/<component name>
# For example set OPENMPI_DIR=$AOMP_SUPP/openmpi in the run script.
# Supplemental components are built with either the ROCm or AOMP
# compiler using the build_supp.sh script. The build_supp.sh script
# will use the AOMP environment variable to identify which LLVM to use
# for building the supplemental component.
# The directory structure for a supplemental component includes:
#   1. a build directory for each component,
#   2. an install directory for each version of a component,
#   3. a link from AOMP_SUPP/<component name> to latest version.
# For example: The openmpi component is built in directory
# $AOMP_SUPP_BUILD/openmpi/ and installed in directory
# $AOMP_SUPP_INSTALL/openmpi/openmpi-4.1.1/ with a symbolic link from
# $AOMP_SUPP/openmpi to $AOMP_SUPP_INSTALL/openmpi/openmpi-4.1.1/
#
# The default AOMP_SUPP directory is $HOME/local but a sysadmin with root
# root authority could set AOMP_SUPP=/usr/local
AOMP_SUPP=${AOMP_SUPP:-$HOME/local}
#
# AOMP_SUPP_BUILD could be a temp directory but defaults to $AOMP_SUPP/build
AOMP_SUPP_BUILD=${AOMP_SUPP_BUILD:-$AOMP_SUPP/build}
#
# AOMP_SUPP_INSTALL must be permanent directory.
AOMP_SUPP_INSTALL=${AOMP_SUPP_INSTALL:-$AOMP_SUPP/install}

# To build aomp from the release source taball, set these values to 0
AOMP_APPLY_ROCM_PATCHES=${AOMP_APPLY_ROCM_PATCHES:-1}

CUDA=${CUDA:-/usr/local/cuda}
CUDAT=${CUDAT:-$CUDA/targets}
CUDAINCLUDE=${CUDAINCLUDE:-$CUDA/include}
CUDABIN=${CUDABIN:-$CUDA/bin}

# unless overridden with AOMP_CMAKE, get CMAKE from the
# prerequisite supplemental component cmake.
AOMP_CMAKE=${AOMP_CMAKE:-$AOMP_SUPP/cmake/bin/cmake}
if [ ! -f $AOMP_CMAKE ] ; then
  AOMP_CMAKE=`which cmake`
  if [ "$AOMP_CMAKE" == "" ] ; then
    echo "ERROR: finding cmake in aomp_common_vars."
    echo "       Missing prerequisite supplemental component cmake."
    echo "       Please run: ./build_prereq.sh cmake"
    echo "       OR set AOMP_CMAKE env variable to a valid cmake binary"
    echo "       OR build ALL prerequisite components with: ./build_prereq.sh"
    echo
    echo "NOTE:  All supplemental components are built and retrieved from "
    echo "       subdirectories of the directory defined by environment variable"
    echo "       AOMP_SUPP whose current value is: $AOMP_SUPP"
    exit 1
  else
    # FIXME: Add version check here for system cmake.
    echo "WARNING: Using system cmake $AOMP_CMAKE "
    echo "         consider building cmake with: ./build_prereq.sh cmake"
  fi
fi

AOMP_USE_NINJA=${AOMP_USE_NINJA:-0}
if [ "$AOMP_USE_NINJA" == 0 ] ; then
  AOMP_NINJA_BIN="make"
else
  AOMP_NINJA_BIN=`which ninja`
  if [ AOMP_NINJA_BIN == "" ] ; then
    echo "ERROR: AOMP_USE_NINJA is $AOMP_USE_NINJA but no ninja bin found in PATH"
    echo "       Either install ninja in PATH or set AOMP_USE_NINJA=0"
    echo
    exit 1
  fi
fi
AOMP_PROC=`uname -m`
if [ "$AOMP_PROC" == "unknown" ] ; then
   AOMP_PROC=`uname -a | awk '{print $(NF-1)}'`
fi

GCC=`which gcc`
GCCXX=`which g++`
AOMP_CC_COMPILER=${AOMP_CC_COMPILER:-$GCC}
AOMP_CXX_COMPILER=${AOMP_CXX_COMPILER:-$GCCXX}

# Non standalone builds are done with ROCm builds.
AOMP_STANDALONE_BUILD=${AOMP_STANDALONE_BUILD:-1}

#  The logic for turning on AOMP_BUILD_CUDA build is here
if [ "$AOMP_PROC" == "ppc64le" ] ; then
   AOMP_BUILD_CUDA=1
   AOMP_STANDALONE_BUILD=1
fi
if [ $AOMP_STANDALONE_BUILD == 1 ] ; then
   # Default is to build nvptx for STANDALONE build
   AOMP_BUILD_CUDA=${AOMP_BUILD_CUDA:-1}
   AOMP_BUILD_DEBUG=${AOMP_BUILD_DEBUG:-1}
   AOMP_ORIGIN_RPATH_NO_DTAGS="-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_RPATH=\$ORIGIN:\$ORIGIN/../lib:\$ORIGIN/../lib64:$AOMP_INSTALL_DIR/lib"
else
   # Default is to NOT build nvptx for ROCm integrated build
   AOMP_BUILD_CUDA=${AOMP_BUILD_CUDA:-0}
   AOMP_BUILD_DEBUG=${AOMP_BUILD_DEBUG:-0}
   AOMP_ORIGIN_RPATH_NO_DTAGS="-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_RPATH=\$ORIGIN:\$ORIGIN/../lib:\$ORIGIN/../../lib64:$AOMP_INSTALL_DIR/lib"
fi

# AOMP uses RPATH (not) RUNPATH because LD_LIBRARY_PATH is a user feature for
# user lib development. We do not want the compiler runtime to search user paths
# first because we want assurance to recreate reported compiler runtime fails.
# While it may appear to hamper users and developers from testing experimental
# compiler runtimes, this policy enforces replacement of installed compiler
# runtime for testing instead of path manipulation. AOMP uses $ORIGIN in the
# RPATH to allow relocation of the installation.
# Most build scripts use $AOMP_ORIGIN_RPATH, some components (build_ocl.sh) need to split
# the CMAKE arguments so
AOMP_ORIGIN_RPATH="-DCMAKE_SHARED_LINKER_FLAGS='-Wl,--disable-new-dtags' $AOMP_ORIGIN_RPATH_NO_DTAGS"

# Regardless, turn off AOMP_BUILD_CUDA if CUDA not available
if [ ! -d "$CUDABIN" ] ; then
   AOMP_BUILD_CUDA=0
fi
# Regardless, turn off AOMP_BUILD_CUDA for arm
if [ "$AOMP_PROC" == "aarch64" ] ; then
   AOMP_BUILD_CUDA=0
fi

# Set list of default nvptx subarchitectures to build
# Only Cuda 9 and above supports sm_70
if [ "$AOMP_BUILD_CUDA" == 1 ] ; then
   NVPTXGPUS_DEFAULT="35,37,50,52,53,60,61,62,70,72,75,80,86"
   #  Developers should override NVPTXGPUS to shorten build time.
   NVPTXGPUS=${NVPTXGPUS:-"${NVPTXGPUS_DEFAULT}"}
   AOMP_NVPTX_TARGET="NVPTX;"
else
   NVPTXGPUS_DEFAULT=""
   NVPTXGPUS=""
   AOMP_NVPTX_TARGET=""
fi

# Set list of default amdgcn subarchitectures to build
# Developers may override GFXLIST to shorten build time.
GFXLIST=${GFXLIST:-"gfx700 gfx701 gfx801 gfx803 gfx900 gfx902 gfx906 gfx908 gfx90a gfx1030 gfx1031"}
export GFXLIST

# Calculate the number of threads to use for make
COMP_THREADS=1
if [ ! -z `which "getconf"` ]; then
   COMP_THREADS=$(`which "getconf"` _NPROCESSORS_ONLN)
   if [ "$AOMP_PROC" == "ppc64le" ] ; then
      COMP_THREADS=$(( COMP_THREADS / 6))
   fi
   if [ "$AOMP_PROC" == "aarch64" ] ; then
      COMP_THREADS=$(( COMP_THREADS / 4))
   fi
fi
AOMP_JOB_THREADS=${AOMP_JOB_THREADS:-$COMP_THREADS}

# These are the web sites where the AOMP git repos are pulled from
GITROC="https://github.com/radeonopencompute"
GITROCINTERNAL="ssh://$USER@gerrit-git.amd.com:29418/lightning/ec"
GITGERRIT="ssh://$USER@gerrit-git.amd.com:29418"
GITROCDEV="https://github.com/ROCm-Developer-Tools"
GITROCLIB="https://github.com/AMDComputeLibraries"
GITROCSW="https://github.com/ROCmSoftwarePlatform"
GITKHRONOS="https://github.com/KhronosGroup"
GITHWLOC="https://github.com/open-mpi"
AOMP_GIT_INTERNAL_IP="gerrit-git.amd.com"
if [ "$AOMP_MAJOR_VERSION" == "12" ] ; then
  ping -c 1 $AOMP_GIT_INTERNAL_IP 2> /dev/null
  if [ $? != 0 ]; then
	  echo error: you must be internal to develop on 12.0
	  exit 1
  fi
  GITPROJECT=$GITROCINTERNAL
  AOMP_PROJECT_REPO_NAME=${AOMP_PROJECT_REPO_NAME:-llvm-project}
  AOMP_EXTRAS_REPO_NAME=${AOMP_EXTRAS_REPO_NAME:-aomp-extras}
  AOMP_FLANG_REPO_NAME=${AOMP_FLANG_REPO_NAME:-flang}
elif [ "$AOMP_MAJOR_VERSION" == "13" ] ; then
  GITPROJECT=$GITROCDEV
  AOMP_PROJECT_REPO_NAME=${AOMP_PROJECT_REPO_NAME:-llvm-project}
  AOMP_EXTRAS_REPO_NAME=${AOMP_EXTRAS_REPO_NAME:-aomp-extras}
  AOMP_FLANG_REPO_NAME=${AOMP_FLANG_REPO_NAME:-flang}
elif [ "$AOMP_MAJOR_VERSION" == "14" ] || [ "$AOMP_MAJOR_VERSION" == "15" ] ; then
  GITPROJECT=$GITROCDEV
  AOMP_PROJECT_REPO_NAME=${AOMP_PROJECT_REPO_NAME:-llvm-project}
  AOMP_EXTRAS_REPO_NAME=${AOMP_EXTRAS_REPO_NAME:-aomp-extras}
  AOMP_FLANG_REPO_NAME=${AOMP_FLANG_REPO_NAME:-flang}
else
  GITPROJECT=$GITROCDEV
  AOMP_PROJECT_REPO_NAME=${AOMP_PROJECT_REPO_NAME:-amd-llvm-project}
  AOMP_EXTRAS_REPO_NAME=${AOMP_EXTRAS_REPO_NAME:-aomp-extras}
  AOMP_FLANG_REPO_NAME=${AOMP_FLANG_REPO_NAME:-flang}
fi
#
# The difference between a _REPO_NAME and _REPO_GITNAME is that the
# _REPO_NAME is the directory name following $AOMP_REPOS.  The GITNAME
# Is the name of the git repository used in the clone_aomp script.
# The _REPO_GITNAME variables will also be removed with the conversion to
# the use of the repo tool.

# External component names for LLVM_EXTERNAL_PROJECTS may have no special
# characters in them.  Therefore, AOMP_xxx_REPO_NAME is different than
# AOMP_xxx_COMPONENT NAME for some components. Recommend that component
# names be short lowercase names.

#  These aomp development repositories
AOMP_REPO_NAME=${AOMP_REPO_NAME:-aomp}

#  These extra repositories are needed but we cannot update them
AOMP_ROCT_REPO_NAME=${AOMP_ROCT_REPO_NAME:-roct-thunk-interface}
AOMP_ROCT_COMPONENT_NAME=${AOMP_ROCT_COMPONENT_NAME:-roct}
AOMP_ROCR_REPO_NAME=${AOMP_ROCR_REPO_NAME:-rocr-runtime}
AOMP_ROCR_COMPONENT_NAME=${AOMP_ROCR_COMPONENT_NAME:-rocr}
AOMP_LIBDEVICE_REPO_NAME=${AOMP_LIBDEVICE_REPO_NAME:-rocm-device-libs}
AOMP_LIBDEVICE_COMPONENT_NAME=${AOMP_LIBDEVICE_COMPONENT_NAME:-rocdl}
DEVICELIBS_ROOT=${DEVICELIBS_ROOT:-$AOMP_REPOS/$AOMP_LIBDEVICE_REPO_NAME}
AOMP_COMGR_REPO_NAME=${AOMP_COMGR_REPO_NAME:-rocm-compilersupport}
# Set COMGR_REPO_SHA for aomp11
if [ $AOMP_MAJOR_VERSION -lt 12 ] ; then
  AOMP_COMGR_REPO_SHA=${AOMP_COMGR_REPO_SHA:-4b689d6}
fi
if [ "$AOMP_MAJOR_VERSION" == "13" ] || [ "$AOMP_MAJOR_VERSION" == "14" ] || [ "$AOMP_MAJOR_VERSION" == "15" ] ; then
  AOMP_COMGR_REPO_SHA=${AOMP_COMGR_REPO_SHA:-694f5a8}
fi
AOMP_RINFO_REPO_NAME=${AOMP_RINFO_REPO_NAME:-rocminfo}

# These are repos we will use when we switch to HIP VDI
AOMP_VDI_REPO_NAME=${AOMP_VDI_REPO_NAME:-vdi}
AOMP_VDI_REPO_GITNAME=${AOMP_VDI_REPO_GITNAME:-ROCclr}
AOMP_OCL_REPO_NAME=${AOMP_OCL_REPO_NAME:-opencl-on-vdi}
AOMP_OCL_REPO_GITNAME=${AOMP_OCL_REPO_GITNAME:-ROCm-OpenCL-Runtime}
AOMP_HIPVDI_REPO_NAME=${AOMP_HIPVDI_REPO_NAME:-hip-on-vdi}
AOMP_HIPVDI_REPO_GITNAME=${AOMP_HIPVDI_REPO_GITNAME:-hip}

AOMP_ROCMCMAKE_REPO_NAME=${AOMP_ROCMCMAKE_REPO_NAME:-rocm-cmake}

# These are needed when AOMP_BUILD_DEBUG=1
AOMP_GDB_REPO_NAME=${AOMP_GDB_REPO_NAME:-ROCgdb}
AOMP_DBGAPI_REPO_NAME=${AOMP_DBGAPI_REPO_NAME:-ROCdbgapi}
AOMP_PROF_REPO_NAME=${AOMP_PROF_REPO_NAME:-rocprofiler}
AOMP_TRACE_REPO_NAME=${AOMP_TRACE_REPO_NAME:-roctracer}

# These are test repositories
GITNEKBONE="https://github.com/AMDComputeLibraries"
AOMP_NEKBONE_REPO_NAME=${AOMP_NEKBONE_REPO_NAME:-Nekbone}
AOMP_APPS_REPO_NAME=${AOMP_APPS_REPO_NAME:-openmpapps}
GITSOLVV="https://github.com/SOLLVE"
AOMP_SOLVV_REPO_NAME=${AOMP_SOLVV_REPO_NAME:-sollve_vv}
AOMP_APPS_REPO_NAME=${AOMP_APPS_REPO_NAME:-openmpapps}
GITLLNL="https://github.com/llnl"
AOMP_RAJA_REPO_NAME=${AOMP_RAJA_REPO_NAME:-raja}
AOMP_RAJAPERF_REPO_NAME=${AOMP_RAJAPERF_REPO_NAME:-RAJAPerf}
GITKOKKOS="https://github.com/kokkos"
AOMP_KOKKOS_REPO_NAME=${AOMP_KOKKOS_REPO_NAME:-kokkos}
AOMP_INTERNAL_IP="gitlab1.amd.com"
GITINTERNAL="http://${AOMP_INTERNAL_IP}/dmcdouga"  # This will change soon
GITOVO="https://github.com/TApplencourt"
AOMP_OVO_REPO_NAME=${AOMP_OVO_REPO_NAME:-OvO}
GITOMPTESTS="https://github.com/clang-ykt"
AOMP_OMPTESTS_REPO_NAME=${AOMP_OMPTESTS_REPO_NAME:-omptests}
GITQMCPACK="https://github.com/QMCPACK"
AOMP_QMCPACK_REPO_NAME=${AOMP_QMCPACK_REPO_NAME:-qmcpack}
GITOPENLIBM="https://github.com/JuliaMath"
AOMP_OPENLIBM_REPO_NAME=${AOMP_OPENLIBM_REPO_NAME:-openlibm}
GITLLNLGOULASH="https://github.com/LLNL"
AOMP_GOULASH_REPO_NAME=${AOMP_GOULASH_REPO_NAME:-goulash}

#  We dont use this yet, waiting for trunk merge
AOMP_F18_REPO_NAME=${AOMP_F18_REPO_NAME:-f18}

if [ -d $AOMP_REPOS/$AOMP_TRACE_REPO_NAME ] ; then
   pushd $AOMP_REPOS/$AOMP_TRACE_REPO_NAME > /dev/null
   roctracer_branch=`git branch --list | grep "\*" | cut -d" " -f2`
   if [ "$roctracer_branch" == "rocm-4.0.x" ] ; then
      AOMP_PATCH_CONTROL_FILE="${AOMP_REPOS}/${AOMP_REPO_NAME}/bin/patches/patch-control-file_4.0.txt"
   fi
   popd > /dev/null
fi
AOMP_PATCH_CONTROL_FILE=${AOMP_PATCH_CONTROL_FILE:-${AOMP_REPOS}/${AOMP_REPO_NAME}/bin/patches/patch-control-file.txt}

STASH_BEFORE_PULL=${STASH_BEFORE_PULL:-YES}

# It is highly recommendded that developers set SUDO to disable and 
# this have update access to install directory $AOMP. In the future
# we may set this default to disable 
SUDO=${SUDO:-NO}
if [ "$SUDO" == "set" ]  || [ "$SUDO" == "yes" ] || [ "$SUDO" == "YES" ] ; then
   SUDO="sudo"
else
   SUDO=""
fi

# The default for BUILD_AOMP is AOMP_REPOS. 
# The cmake and make builds are actually done in 
# BUILD_AOMP/build, not the actual repos. 
BUILD_AOMP=${BUILD_AOMP:-$AOMP_REPOS}
BUILD_DIR=$BUILD_AOMP

# Here is where we define different behaviors for STANDALONE build
# and the new ROCm integrated build. The default is STANDALONE build.
if [ $AOMP_STANDALONE_BUILD == 1 ] ; then
   # Standalone build gets ROCm components from the AOMP Installation.
   # As a result their is an order to component builds.
   ROCM_DIR=${ROCM_DIR:-$AOMP_INSTALL_DIR}
   # We hardcode the ROCM_VERSION here for standalone builds
else
   # ROCm integrated build gets ROCm pieces from installed ROCm
   ROCM_DIR=${ROCM_DIR:-/opt/rocm}

   # For Integrated ROCm build we do not install to versioned dir and create 
   # a symbolic link.  Instead we install directly to /opt/rocm/aomp
   AOMP_INSTALL_DIR=${AOMP}
fi

#  TO use this function set variables patchdir and patchfile
function patchrepo(){
patchdir=$1
if [ "$AOMP_APPLY_ROCM_PATCHES" == 1 ] && [ -d "$patchdir" ] ; then
   patches=""
   cd $patchdir
   if [[ "$2" =~ "postinstall" ]]; then
     getpatchlist $2
   else
     getpatchlist
   fi

   #loop through list of patches to apply
   if [ "$patches" != "" ] ; then
   patchloc=${AOMP_PATCH_CONTROL_FILE%/*}
   echo patchloc=$patchloc
   for patch in $patches; do
     patchfile=$patchloc/$patch
     if [ ! -f $patchfile ] ; then
       echo
       echo "ERROR: Patch $patchfile does not exist"
       echo "       It is referenced in ${AOMP_PATCH_CONTROL_FILE}"
       exit 1
     fi

     echo "Testing patch $patchfile to $patchdir"

     applypatch="yes"
     patch -p1 -t -N --dry-run <$patchfile >/dev/null
     if [ $? != 0 ] ; then
        applypatch="no"
        # Check to see if reverse patch applies cleanly
        patch -p1 -R --dry-run -t <$patchfile >/dev/null
        if [ $? == 0 ] ; then
           echo "patch $patchfile was already applied to $patchdir"
        else
           echo
           echo "ERROR: Patch $patchfile will not apply"
           echo "       cleanly to directory $patchdir"
           echo "       Check if it was already applied."
           echo
           exit 1
        fi
     fi
     if [ "$applypatch" == "yes" ] ; then
        echo "Applying patch $patchfile to $patchdir"
        patch -p1 --no-backup-if-mismatch <$patchfile
     fi
   done
   fi
fi

}

function removepatch(){
patchdir=$1
if [ "$AOMP_APPLY_ROCM_PATCHES" == 1 ] && [ -d "$patchdir" ] ; then
   patches=""
   cd $patchdir
   getpatchlist
   if [ "$patches" != "" ] ; then
      echo "Patchdir $patchdir"
      echo "PATCHES TO REMOVE: $patches"
   fi
   patchloc=${AOMP_PATCH_CONTROL_FILE%/*}
   if [ "$patches" != "" ] ; then
   for patch in $patches; do
     patchfile=$patchloc/$patch
     echo "Testing reverse patch $patchfile to $patchdir"
     reversepatch="yes"
     # Check to see if reverse patch applies cleanly
     patch -p1 -R --dry-run -f <$patchfile  >/dev/null
     if [ $? != 0 ] ; then
        echo "patch $patchfile was NOT applied $patchdir, no patch to reverse"
        reversepatch="no"
     fi
     if [ "$reversepatch" == "yes" ] ; then
        echo "Reversing patch $patchfile to $patchdir"
        patch -p1 -R -f --no-backup-if-mismatch <$patchfile
     fi
   done
   fi
fi
}

function getpatchlist(){
   currdir=$(pwd)
   basedir=$(basename $currdir)
   if [[ "$1" =~ "postinstall" ]]; then
     reporegex="(^$1:\s)"
   else
     reporegex="(^$basedir:\s)"
   fi
   echo FILE: $AOMP_PATCH_CONTROL_FILE
   echo regex $reporegex
   #read patch control file and look for correct patches
   while read -r line; do
   if [[ "$line" =~ $reporegex ]]; then
     #remove basename from list of patches
     patches=${line/"${BASH_REMATCH[1]}"}
     echo "patches: $patches"
     break
   fi
   done < "$AOMP_PATCH_CONTROL_FILE"
}

function setaompgpu (){
  if [[ "$AOMP" =~ "opt" ]]; then
    echo Set AOMP_GPU with rocm_agent_enumerator.
    detected_gpu=$($AOMP/../bin/rocm_agent_enumerator | grep -m 1 -E gfx[^0]{1}.{2})
  else
    echo Set AOMP_GPU with mygpu.
    if [ -a $AOMP/bin/mygpu ]; then
      detected_gpu=$($AOMP/bin/mygpu)
    else
      detected_gpu=$($AOMP/../bin/mygpu)
    fi
  fi

  AOMP_GPU=${AOMP_GPU:-$detected_gpu}
  export AOMP_GPU
  echo AOMP_GPU = $AOMP_GPU
  echo AOMP     = $AOMP

  if [ "$AOMP_GPU" == "" ] || [ "$AOMP_GPU" == "unknown" ]; then
    echo Error: AOMP_GPU not properly set...exiting.
    exit 1
  fi
}

function help_build_aomp(){
   /bin/cat 2>&1 <<"EOF"

  The build scripts in this directory are used to build AOMP.
  
  Repositories:
     Many repositories are used to build AOMP.  The script clone_aomp.sh will
     clone all the necessary repositories as subdirectories of the directory
     $HOME/git/aomp11. The web repository locations and the required branches
     are set with the file aomp_common_vars
  
  Build all components:
     To build all components, run these commands:

        ./clone_aomp.sh
        unset LD_LIBRARY_PATH
        ./build_aomp.sh

  Component builds:
     Developers can rebuild individual components by running the build script for
     that component. Make sure dependent components are built first. You can run
     build_aomp.sh to build call components in the correct order orsee the README.md
     file in this directory for the required order.

     Each build script can run with no arguments or with a single argument
     "install" or "nocmake".  Running  with no options starts fresh with an empty
     component build directory.  It then runs cmake with the correct cmake options
     then it runs make with a proper -j option.
  
  Optional Arguments 'nocmake' and 'install' :
     The 'nocmake' or 'install' options can only be used after your initial build
     with no options. The 'nocmake' option is intended to restart make after 
     you fix code following a failed build. The 'install' option will run 'make' 
     and 'make install' causing installation into the directory $AOMP_INSTALL_DIR. 
     The 'install' option will also create a symbolic link to directory $AOMP.
  
  Environment Variables:
     You can set environment variables to override behavior of the build scripts
     NAME             DEFAULT             DESCRIPTION
     ----             -------             -----------
     AOMP             $HOME/rocm/aomp     Where the compiler will be installed
     AOMP_REPOS       $HOME/git/aomp11    Location of all aomp repos
     BUILD_TYPE       Release             The CMAKE build type
     BUILD_AOMP       same as AOMP_REPOS  Set a different build location than AOMP_REPOS
     AOMP_USE_NINJA   0                   Use ninja if possible
  
  To build a debug version of the compiler, run this command before the build:
     export BUILD_TYPE=debug
  
  The BUILD_AOMP Envronment Variable:
     The build scripts will always build with cmake and make outside your source git trees.
     By default (without BUILD_AOMP) the build will occur in the "build" subdirectory of
     AOMP_REPOS. For example build_llvm will build in $AOMP_REPOS/build/llvm
  
     The BUILD_AOMP environment variable enables source development outside your git
     repositories. By default, this feature is OFF.  The BUILD_AOMP environment variable 
     can be used if access to your git repositories is very slow or you want to test 
     changes outside of your local git repositories (specified by AOMP_REPOS env var). 
     If BUILD_AOMP is set, your git repositories (specifed by AOMP_REPOS) will be
     replicated to subdirectories of BUILD_AOMP using rsync.  The subsequent build 
     (cmake and make) will occur in subdirectory BUILD_AOMP/build/llvm.
     This replication only happens on your initial build, that is, if you specify no arguments.
     The option 'nocmake' skips replication and then restarts make in the build directory.
     The install option skips replication, skips cmake, runs 'make' and 'make install'. 
     Be careful to always use options nocmake or install if you made local changes in
     BUILD_AOMP or your changes will be lost by a new replica of your git repositories.
  
EOF
   exit 0
}