diff --git a/CHANGELOG.md b/CHANGELOG.md
index 41940debbc2..3425cf600f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Update `num-complex` optional dependency to 0.4. [#1482](https://github.com/PyO3/pyo3/pull/1482)
 - Extend `hashbrown` optional dependency supported versions to include 0.11. [#1496](https://github.com/PyO3/pyo3/pull/1496)
 - Support PyPy 3.7. [#1538](https://github.com/PyO3/pyo3/pull/1538)
+- Try harder to filter sysconfigdata candidates on arch.config
 
 ### Added
 - Add conversions for `[T; N]` for all `N` on Rust 1.51 and up. [#1128](https://github.com/PyO3/pyo3/pull/1128)
diff --git a/build.rs b/build.rs
index 01731aed9d5..3ffa71ca90c 100644
--- a/build.rs
+++ b/build.rs
@@ -456,6 +456,21 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
         };
         sysconfig_paths.extend(sysc);
     }
+    // If we got more then one file, only take those that contain the arch name.
+    // For ubuntu 20.04 with host architecture x86_64 and a foreign architecture of armhf
+    // this reduces the number of candidates to 1:
+    //
+    // $ find /usr/lib/python3.8/ -name '_sysconfigdata*.py' -not -lname '*'
+    //  /usr/lib/python3.8/_sysconfigdata__x86_64-linux-gnu.py
+    //  /usr/lib/python3.8/_sysconfigdata__arm-linux-gnueabihf.py
+    if sysconfig_paths.len() > 1 {
+        sysconfig_paths = sysconfig_paths
+            .iter()
+            .filter(|p| p.to_string_lossy().contains(&cross.arch))
+            .map(|p| p.clone())
+            .collect::<Vec<PathBuf>>();
+    }
+
     sysconfig_paths
 }