diff --git a/llama.cpp b/llama.cpp
index f7c6f26d2..ad431a27e 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -765,8 +765,10 @@ struct llama_buffer {
     ~llama_buffer() {
         if (data) {
 #ifdef GGML_USE_KOMPUTE
-            if (ggml_vk_has_device()) {
-                ggml_vk_free_memory(memory);
+            if (memory.data) {
+                if (ggml_vk_has_device()) {
+                    ggml_vk_free_memory(memory);
+                }
                 data = NULL;
                 return;
             }