From e1b1db9f09fe73ca8460890b0c93c349685c54d7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 15 Jan 2024 16:42:16 +0200 Subject: [PATCH] simple : do not perform tensor data copy if not needed --- examples/simple/simple.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp index ce3497345..8db37eef7 100644 --- a/examples/simple/simple.cpp +++ b/examples/simple/simple.cpp @@ -22,12 +22,20 @@ static bool observe_compute(struct ggml_tensor * t, bool ask, void * user_data) __func__, t->name, ggml_op_name(t->op), (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]); // this will copy the data to host memory (if needed) - std::vector t_data(ggml_nelements(t)); - ggml_backend_tensor_get(t, t_data.data(), 0, ggml_nbytes(t)); + static std::vector t_data; + + const bool is_host = ggml_backend_buffer_is_host(t->buffer); + + if (!is_host || ggml_is_contiguous(t)) { + t_data.resize(ggml_nelements(t)); + ggml_backend_tensor_get(t, t_data.data(), 0, ggml_nbytes(t)); + } + + const float * data = is_host ? (const float *) t->data : t_data.data(); // print first row for (int i = 0; i < t->ne[0]; i++) { - printf("%8.4f ", t_data[i]); + printf("%8.4f ", data[i]); } printf("\n");