diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
index 885019ff3049f..6365fcfacfc38 100755
--- a/ci/scripts/python_wheel_manylinux_build.sh
+++ b/ci/scripts/python_wheel_manylinux_build.sh
@@ -55,7 +55,6 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_GANDIVA:=OFF}
 : ${ARROW_GCS:=ON}
 : ${ARROW_HDFS:=ON}
-: ${ARROW_JEMALLOC:=ON}
 : ${ARROW_MIMALLOC:=ON}
 : ${ARROW_ORC:=ON}
 : ${ARROW_PARQUET:=ON}
@@ -81,6 +80,9 @@ if [[ "$(uname -m)" == arm* ]] || [[ "$(uname -m)" == aarch* ]]; then
     # 4k and 64k page arm64 systems. For more context see
     # https://github.com/apache/arrow/issues/10929
     export ARROW_EXTRA_CMAKE_FLAGS="-DARROW_JEMALLOC_LG_PAGE=16"
+    : ${ARROW_JEMALLOC:=OFF}
+else
+    : ${ARROW_JEMALLOC:=ON}
 fi
 
 mkdir /tmp/arrow-build
diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake
index 755887314d110..d823444cf790f 100644
--- a/cpp/cmake_modules/DefineOptions.cmake
+++ b/cpp/cmake_modules/DefineOptions.cmake
@@ -364,7 +364,8 @@ takes precedence over ccache if a storage backend is configured" ON)
 
   set(ARROW_JEMALLOC_DESCRIPTION "Build the Arrow jemalloc-based allocator")
   if(WIN32
-     OR "${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD"
+     OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD"
+     OR CMAKE_SYSTEM_PROCESSOR MATCHES "aarch|ARM|arm"
      OR NOT ARROW_ENABLE_THREADING)
     # jemalloc is not supported on Windows.
     #
@@ -372,6 +373,9 @@ takes precedence over ccache if a storage backend is configured" ON)
     # be built with --disable-libdl on FreeBSD. Because lazy-lock feature
     # is required on FreeBSD. Lazy-lock feature requires libdl.
     #
+    # jemalloc may have a problem on ARM.
+    # See also: https://github.com/apache/arrow/issues/44342
+    #
     # jemalloc requires thread.
     define_option(ARROW_JEMALLOC ${ARROW_JEMALLOC_DESCRIPTION} OFF)
   else()
diff --git a/python/pyarrow/tests/test_memory.py b/python/pyarrow/tests/test_memory.py
index 53c25f3b3ef20..b1eef176665af 100644
--- a/python/pyarrow/tests/test_memory.py
+++ b/python/pyarrow/tests/test_memory.py
@@ -17,6 +17,7 @@
 
 import contextlib
 import os
+import platform
 import signal
 import subprocess
 import sys
@@ -30,7 +31,7 @@
 
 possible_backends = ["system", "jemalloc", "mimalloc"]
 
-should_have_jemalloc = sys.platform == "linux"
+should_have_jemalloc = (sys.platform == "linux" and platform.machine() == 'x86_64')
 should_have_mimalloc = sys.platform == "win32"