run_llama.sh

#!/usr/bin/env bash

#
#Copyright © Advanced Micro Devices, Inc., or its affiliates.
#
#SPDX-License-Identifier:  MIT
#

# Build script for LLaMA with HIP support using AOMP compiler

# shellcheck source=/dev/null
. aomp_common_vars

: "${AOMP_GPU:=gfx90a}"
: "${LLAMA_GPU:=$AOMP_GPU}"

: "${LLAMA_TLDIR:=$AOMP_REPOS_TEST/llama}"
: "${LLAMA_BUILD_DIR:=$LLAMA_TLDIR/build}"
: "${LLAMA_SRC_DIR:=$LLAMA_TLDIR/src}"
: "${LLAMA_BUILD_MODE:=Release}"
: "${LLAMA_TESTS_LOG_LOCATION:=$LLAMA_TLDIR/logs}"

# Model to use in benchmarks (default is a smaller model)
: "${LLAMA_BENCH_HF_ID:=ggml-org/gemma-3-1b-it-GGUF}"
: "${LLAMA_CACHE:=$HOME/.cache/llama.cpp}"

pushd "${AOMP_REPOS_TEST}" || exit
mkdir -p "${LLAMA_TLDIR}" && cd "${LLAMA_TLDIR}" || exit

# Run CMake configuration
DoConfigure='no'

# Run build command
DoCompile='no'

# Run ctest
DoCTest='no'

# Run benchmark (llama-bench)
DoBenchmark='no'

# Update llama sources
DoUpdate='no'

IsVerbose='no'

while getopts "j:cbtveu" opt; do
  case ${opt} in
  j) AOMP_BUILD_JOBS=${OPTARG} ;;
  c) DoConfigure='yes' ;;
  b) DoCompile='yes' ;;
  t) DoCTest='yes' ;;
  v) IsVerbose='yes' ;;
  e) DoBenchmark='yes' ;;
  u) DoUpdate='yes' ;;
  \?)
    echo "Usage: cmd [-j build_jobs] [-c configure] [-b build] [-t ctest] [-e benchmark] [-v verbose] [-u update_sources]"
    exit 1
    ;;
  esac
done

if [ "${IsVerbose}" == "yes" ]; then
  set -x
fi

if command -v ninja >/dev/null; then
  CmakeGenerator="-GNinja"
fi

if [ ! -d "${LLAMA_TESTS_LOG_LOCATION}" ]; then
  mkdir -p "${LLAMA_TESTS_LOG_LOCATION}"
fi

if [ ! -d "${LLAMA_SRC_DIR}" ]; then
  echo "Cloning llama.cpp repository..."
  git clone https://github.com/ggml-org/llama.cpp.git src
elif [ "${DoUpdate}" == "yes" ]; then
  echo "Updating llama.cpp repository..."
  cd "${LLAMA_SRC_DIR}" || exit
  git pull
  cd ..
fi

if ! command -v git-lfs >/dev/null; then
  echo "WARNING: git-lfs is not installed. Expect some tests to fail."
else
  # Ensure git-lfs is initialized and pulls any large files
  cd "${LLAMA_SRC_DIR}" || exit
  git lfs install
  git lfs pull
  cd ..
fi

echo "Configuring build with CMake..."
if [ "${DoConfigure}" == "yes" ]; then
  rm -rf "${LLAMA_BUILD_DIR}"
  cmake -B build \
    -S src \
    -DCMAKE_PREFIX_PATH="${AOMP}"/lib/cmake \
    -DGGML_HIP=On \
    -DCMAKE_BUILD_TYPE="${LLAMA_BUILD_MODE}" \
    -DGPU_TARGETS="${LLAMA_GPU}" \
    ${CmakeGenerator:+"${CmakeGenerator}"} \
    -DCMAKE_C_COMPILER="${AOMP}"/bin/clang \
    -DCMAKE_CXX_COMPILER="${AOMP}"/bin/clang++ \
    -DCMAKE_HIP_COMPILER="${AOMP}"/bin/clang++
fi

if [ "${DoCompile}" == "yes" ]; then
  echo "Building LLaMA..."
  cmake --build "${LLAMA_BUILD_DIR}" --parallel -j "${AOMP_BUILD_JOBS}"
fi

if [ "${DoCTest}" == "yes" ]; then
  echo "Running tests..."
  cd "${LLAMA_BUILD_DIR}" || exit
  echo "Log in ${LLAMA_TESTS_LOG_LOCATION}/ctest.log"

  # Some model files are git-lfs and come from huggingface. They will auto-download during test
  ctest --output-on-failure 2>&1 | tee "${LLAMA_TESTS_LOG_LOCATION}/ctest.log"
fi

run_llama_bench() {
  ./bin/llama-bench "$@" 2>&1 | tee -a "${LLAMA_TESTS_LOG_LOCATION}/llama-bench.log"
  BenchStatus=${PIPESTATUS[0]}

  if [ "${BenchStatus}" -ne 0 ]; then
    echo "ERROR: llama-bench failed with exit code ${BenchStatus}" | tee -a "${LLAMA_TESTS_LOG_LOCATION}/llama-bench.log"
    return "${BenchStatus}"
  fi
}

if [ "${DoBenchmark}" == "yes" ]; then
  echo "Running benchmark..."
  cd "${LLAMA_BUILD_DIR}" || exit

  # Get cache directory from llama-cli if supported by the local build.
  CacheListOutput=$(./bin/llama-cli --cache-list 2>&1 || true)
  CacheDir=$(echo "${CacheListOutput}" | grep "model cache directory:" | sed 's/.*: //')
  : "${CacheDir:=${LLAMA_CACHE}}"

  # Find requested model by converting HF ID to filename pattern (user/model -> user_model)
  SearchPattern="${LLAMA_BENCH_HF_ID//\//_}"
  LlamaModelPath=$(find "${CacheDir}" \( -type f -o -xtype f \) -name "${SearchPattern}*.gguf" 2>/dev/null | head -1)

  # Fallback: use all available .gguf files in cache
  if [ -z "${LlamaModelPath}" ]; then
    echo "Requested model not found, using all cached models"
    mapfile -t ModelPaths < <(find "${CacheDir}" \( -type f -o -xtype f \) -name "*.gguf" 2>/dev/null)
  else
    ModelPaths=("${LlamaModelPath}")
  fi

  # Marker for external scripts
  echo "LLAMA_BENCHMARK_BEGIN" | tee "${LLAMA_TESTS_LOG_LOCATION}/llama-bench.log"

  if [ ${#ModelPaths[@]} -eq 0 ] && ./bin/llama-bench --help 2>&1 | grep -q -- "--hf-repo"; then
    # Let llama-bench resolve/download the HF model directly.  Using llama-cli as
    # a prefetch step can hang in ROCm/KFD waits on cold cache.
    run_llama_bench -hf "${LLAMA_BENCH_HF_ID}" -ngl 999 -fa 1 -ub 2048 || exit $?
  else
    if [ ${#ModelPaths[@]} -eq 0 ]; then
      echo "ERROR: No model files found in cache directory: ${CacheDir}"
      exit 1
    fi

    # Run benchmark for each model
    for LlamaModelPath in "${ModelPaths[@]}"; do
      echo "Benchmarking: ${LlamaModelPath}" | tee -a "${LLAMA_TESTS_LOG_LOCATION}/llama-bench.log"
      run_llama_bench -ngl 999 -fa 1 -ub 2048 -m "${LlamaModelPath}" || exit $?
    done
  fi
fi

popd || exit