diff --git a/CMakeLists.txt b/CMakeLists.txt
index 57cce9bb05d76..cea51078dcdd9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -170,27 +170,23 @@ if (LLAMA_CUBLAS)
 endif()
 
 if (LLAMA_HIPBLAS)
-    cmake_minimum_required(VERSION 3.21)
-
     find_package(hip)
     find_package(hipblas)
 
-    if (hipblas_FOUND)
+    if (${hipblas_FOUND} AND ${hip_FOUND})
         message(STATUS "hipBLAS found")
+        add_compile_definitions(GGML_USE_HIPBLAS)
+        enable_language(HIP)
+        add_library(ggml-hip OBJECT ggml-cuda.cu ggml-cuda.h)
+        set_source_files_properties(ggml-cuda.cu PROPERTIES LANGUAGE HIP)
+        target_link_libraries(ggml-hip PRIVATE hip::device)
 
-        set(LLAMA_HIPBLAS_PLATFORM "AMD" CACHE STRING "hip device type" FORCE)
-        set_property(CACHE LLAMA_HIPBLAS_PLATFORM PROPERTY STRINGS "AMD" "NVIDIA")
-
-        add_compile_definitions(GGML_USE_HIPBLAS "__HIP_PLATFORM_${LLAMA_HIPBLAS_PLATFORM}__")
-
-        add_library(ggml-hip OBJECT ggml-cuda.cu)
-        set_source_files_properties(ggml-cuda.cu PROPERTIES  LANGUAGE  CXX)
-        target_link_libraries(ggml-hip hip::device)
-
+        if (LLAMA_STATIC)
+            message(FATAL_ERROR "Static linking not supported for HIP/ROCm")
+        endif()
         set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} hip::host roc::hipblas ggml-hip)
-
     else()
-        message(WARNING "hipBLAS not found")
+        message(WARNING "hipBLAS or HIP not found. Try setting CMAKE_PREFIX_PATH=/opt/rocm")
     endif()
 endif()
 
diff --git a/Makefile b/Makefile
index 8e0ada581e698..5b856a3677b96 100644
--- a/Makefile
+++ b/Makefile
@@ -108,13 +108,12 @@ ggml-cuda.o: ggml-cuda.cu ggml-cuda.h
 	nvcc -arch=native -c -o $@ $<
 endif
 ifdef LLAMA_HIPBLAS
-	ROCMPATH?= /opt/rocm
-	CFLAGS  += -DGGML_USE_HIPBLAS -D__HIP_PLATFORM_AMD__ -I$(ROCMPATH)/include
-	CXXFLAGS+= -D__HIP_PLATFORM_AMD__ -I$(ROCMPATH)/include
-	HIPFLAGS?= -amdgpu-early-inline-all=true -amdgpu-function-calls=false -march=native
-	LDFLAGS += -lhipblas -lamdhip64 -L$(ROCMPATH)/lib
-	HIPCC   ?= $(ROCMPATH)/bin/hipcc
-	OBJS	+= ggml-cuda.o
+	ROCM_PATH ?= /opt/rocm
+	LDFLAGS   += -lhipblas -lamdhip64 -L$(ROCM_PATH)/lib
+	HIPCC     ?= $(ROCM_PATH)/bin/hipcc
+	OBJS	  += ggml-cuda.o
+ggml.o:      CFLAGS    += -DGGML_USE_HIPBLAS -D__HIP_PLATFORM_AMD__ -I$(ROCM_PATH)/include
+ggml-cuda.o: CXXFLAGS += -march=native -D__HIP_PLATFORM_AMD__ -I$(ROCMPATH)/include
 ggml-cuda.o: ggml-cuda.cu ggml-cuda.h
 	$(HIPCC) $(CXXFLAGS) -x hip $(HIPFLAGS) -c -o $@ $<
 endif