From fed0108491a3a3cbec6c6480dc8667ffff9d7659 Mon Sep 17 00:00:00 2001 From: Josh Ramer Date: Sat, 11 May 2024 12:26:35 -0500 Subject: [PATCH] Scripting & documenting debugging one test without anything else in the loop. (#7096) * A little documentation that shares my quick tips for working in the repository. * Update startup-testing-debugging.md * script that shows a menu of tests to pick from & run the debugger on * debug-test.sh: Refactor CLI help message * debug-test.sh: documentation update * debug-test.sh: CLI Help output corrections * debug-test.sh: minor doc fix --------- authored-by: Josh Ramer Assisted-by: brian khuu --- docs/debugging-tests.md | 88 ++++++++++++++++++++++++++++++ scripts/debug-test.sh | 117 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 docs/debugging-tests.md create mode 100755 scripts/debug-test.sh diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md new file mode 100644 index 000000000..51a125e19 --- /dev/null +++ b/docs/debugging-tests.md @@ -0,0 +1,88 @@ +# Debugging Tests Tips + +## How to run & debug a specific test without anything else to keep the feedback loop short? + +There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. + +For example, running the following command will output an interactive list from which you can select a test. It takes this form: + +`debug-test.sh [OPTION]... ` + +It will then build & run in the debugger for you. + +```bash +./scripts/debug-test.sh test-tokenizer + +# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows: +>>> b main +``` + +For further reference use `debug-test.sh -h` to print help. + +  + +### How does the script work? +If you want to be able to use the concepts contained in the script separately, the important ones are briefly outlined below. + +#### Step 1: Reset and Setup folder context + +From base of this repository, let's create `build-ci-debug` as our build context. + +```bash +rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug +``` + +#### Step 2: Setup Build Environment and Compile Test Binaries + +Setup and trigger a build under debug mode. You may adapt the arguments as needed, but in this case these are sane defaults. + +```bash +cmake -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON .. +make -j +``` + +#### Step 3.1: Identify Test Command for Debugging + +The output of this command will give you the command & arguments needed to run GDB. + +* `-R test-tokenizer` : looks for all the test files named `test-tokenizer*` (R=Regex) +* `-N` : "show-only" disables test execution & shows test commands that you can feed to GDB. +* `-V` : Verbose Mode + +```bash +ctest -R "test-tokenizer" -V -N +``` + +This may return output similar to below (focusing on key lines to pay attention to): + +```bash +... +1: Test command: ~/llama.cpp/build-ci-debug/bin/test-tokenizer-0 "~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf" +1: Working Directory: . +Labels: main + Test #1: test-tokenizer-0-llama-spm +... +4: Test command: ~/llama.cpp/build-ci-debug/bin/test-tokenizer-0 "~/llama.cpp/tests/../models/ggml-vocab-falcon.gguf" +4: Working Directory: . +Labels: main + Test #4: test-tokenizer-0-falcon +... +``` + +So for test #1 we can tell these two pieces of relevant information: +* Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0` +* Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf` + +#### Step 3.2: Run GDB on test command + +Based on the ctest 'test command' report above we can then run a gdb session via this command below: + +```bash +gdb --args ${Test Binary} ${Test GGUF Model} +``` + +Example: + +```bash +gdb --args ~/llama.cpp/build-ci-debug/bin/test-tokenizer-0 "~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf" +``` diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh new file mode 100755 index 000000000..231a23d69 --- /dev/null +++ b/scripts/debug-test.sh @@ -0,0 +1,117 @@ +#!/bin/bash +test_suite=${1:-} +test_number=${2:-} + +PROG=${0##*/} +build_dir="build-ci-debug" + +if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then + echo "Usage: $PROG [OPTION]... (test_number)" + echo "Debug specific ctest program." + echo + echo "Options:" + echo " -h, --help Display this help and exit" + echo + echo "Arguments:" + echo " (Mandatory) Supply one regex to the script to filter tests" + echo " (test_number) (Optional) Test number to run a specific test" + echo + echo "Example:" + echo " $PROG test-tokenizer" + echo " $PROG test-tokenizer 3" + echo + exit 0 +fi + +# Function to select and debug a test +function select_test() { + test_suite=${1:-test} + test_number=${2:-} + + # Sanity Check If Tests Is Detected + printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" + tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1')) + if [ ${#tests[@]} -eq 0 ] + then + echo "No tests avaliable... check your compliation process..." + echo "Exiting." + exit 1 + fi + + if [ -z $test_number ] + then + # List out avaliable tests + printf "Which test would you like to debug?\n" + id=0 + for s in "${tests[@]}" + do + echo "Test# ${id}" + echo " $s" + ((id++)) + done + + # Prompt user which test they wanted to run + printf "\nRun test#? " + read test_number + else + printf "\nUser Already Requested #${test_number}" + fi + + # Start GDB with the requested test binary and arguments + printf "Debugging(GDB) test: ${tests[test_number]}\n" + # Change IFS (Internal Field Separator) + sIFS=$IFS + IFS=$'\n' + + # Get test args + gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) + IFS=$sIFS + printf "Debug arguments: ${gdb_args[test_number]}\n\n" + + # Expand paths if needed + args=() + for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\"//') + do + args+=($(echo $x | sed -e 's/.*\/..\//..\//')) + done + + # Execute debugger + echo "gdb args: ${args[@]}" + gdb --args ${args[@]} +} + +# Step 0: Check the args +if [ -z "$test_suite" ] +then + echo "Usage: $PROG [OPTION]... (test_number)" + echo "Supply one regex to the script to filter tests," + echo "and optionally a test number to run a specific test." + echo "Use --help flag for full instructions" + exit 1 +fi + +# Step 1: Reset and Setup folder context +## Sanity check that we are actually in a git repo +repo_root=$(git rev-parse --show-toplevel) +if [ ! -d "$repo_root" ]; then + echo "Error: Not in a Git repository." + exit 1 +fi + +## Reset folder to root context of git repo +pushd "$repo_root" || exit 1 + +## Create and enter build directory +rm -rf "$build_dir" && mkdir "$build_dir" || exit 1 + +# Step 2: Setup Build Environment and Compile Test Binaries +cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1 +pushd "$build_dir" && make -j || exit 1 + +# Step 3: Debug the Test +select_test "$test_suite" "$test_number" + +# Step 4: Return to the directory from which the user ran the command. +popd || exit 1 +popd || exit 1 +popd || exit 1