diff --git a/bugsnag-android-core/src/androidTest/java/com/bugsnag/android/FileStoreTest.kt b/bugsnag-android-core/src/androidTest/java/com/bugsnag/android/FileStoreTest.kt
new file mode 100644
index 0000000000..02a8ef0927
--- /dev/null
+++ b/bugsnag-android-core/src/androidTest/java/com/bugsnag/android/FileStoreTest.kt
@@ -0,0 +1,73 @@
+package com.bugsnag.android
+
+import android.content.Context
+import androidx.test.core.app.ApplicationProvider
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import java.io.File
+import java.io.FileNotFoundException
+import java.lang.Exception
+import java.lang.RuntimeException
+
+class FileStoreTest {
+    val appContext = ApplicationProvider.getApplicationContext<Context>()
+    val config = Configuration("api-key")
+
+    @Test
+    fun sendsInternalErrorReport() {
+        val delegate = CustomDelegate()
+        val dir = File(appContext.dataDir, "custom-store")
+        dir.mkdir()
+
+        val store = CustomFileStore(config, appContext, dir.absolutePath, 1, null, delegate)
+        val exc = RuntimeException("Whoops")
+        store.write(CustomStreamable(exc))
+
+        assertEquals("Crash report serialization", delegate.context)
+        assertEquals(File(dir, "foo.json"), delegate.errorFile)
+        assertEquals(exc, delegate.exception)
+        assertEquals(0, dir.listFiles().size)
+    }
+
+    @Test
+    fun sendsInternalErrorReportNdk() {
+        val delegate = CustomDelegate()
+        val dir = File(appContext.dataDir, "custom-store")
+        dir.mkdir()
+
+        val store = CustomFileStore(config, appContext, "", 1, null, delegate)
+        store.enqueueContentForDelivery("foo")
+
+        assertEquals("NDK Crash report copy", delegate.context)
+        assertEquals(File("/foo.json"), delegate.errorFile)
+        assertTrue(delegate.exception is FileNotFoundException)
+    }
+}
+
+class CustomDelegate: FileStore.Delegate {
+    var exception: Exception? = null
+    var errorFile: File? = null
+    var context: String? = null
+
+    override fun onErrorIOFailure(exception: Exception?, errorFile: File?, context: String?) {
+        this.exception = exception
+        this.errorFile = errorFile
+        this.context = context
+    }
+}
+
+class CustomStreamable(val exc: Throwable) : JsonStream.Streamable {
+    override fun toStream(stream: JsonStream) = throw exc
+}
+
+internal class CustomFileStore(
+    config: Configuration,
+    appContext: Context,
+    val folder: String?,
+    maxStoreCount: Int,
+    comparator: java.util.Comparator<File>?,
+    delegate: Delegate?
+) : FileStore<CustomStreamable>(config, appContext, folder, maxStoreCount, comparator, delegate) {
+    override fun getFilename(`object`: Any?) = "$folder/foo.json"
+}
diff --git a/bugsnag-android-core/src/main/java/com/bugsnag/android/FileStore.java b/bugsnag-android-core/src/main/java/com/bugsnag/android/FileStore.java
index a6ed082c4a..65d407c824 100644
--- a/bugsnag-android-core/src/main/java/com/bugsnag/android/FileStore.java
+++ b/bugsnag-android-core/src/main/java/com/bugsnag/android/FileStore.java
@@ -83,8 +83,18 @@ void enqueueContentForDelivery(String content) {
             out = new BufferedWriter(new OutputStreamWriter(fos, "UTF-8"));
             out.write(content);
         } catch (Exception exc) {
+            File errorFile = new File(filename);
+
             if (delegate != null) {
-                delegate.onErrorIOFailure(exc, new File(filename), "NDK Crash report copy");
+                delegate.onErrorIOFailure(exc, errorFile, "NDK Crash report copy");
+            }
+
+            try {
+                if (!errorFile.delete()) {
+                    errorFile.deleteOnExit();
+                }
+            } catch (Exception ex) {
+                Logger.warn("Failed to delete partially written file", ex);
             }
         } finally {
             try {
@@ -118,8 +128,18 @@ String write(@NonNull JsonStream.Streamable streamable) {
             Logger.info(String.format("Saved unsent payload to disk (%s) ", filename));
             return filename;
         } catch (Exception exc) {
+            File errorFile = new File(filename);
+
             if (delegate != null) {
-                delegate.onErrorIOFailure(exc, new File(filename), "Crash report serialization");
+                delegate.onErrorIOFailure(exc, errorFile, "Crash report serialization");
+            }
+
+            try {
+                if (!errorFile.delete()) {
+                    errorFile.deleteOnExit();
+                }
+            } catch (Exception ex) {
+                Logger.warn("Failed to delete partially written file", ex);
             }
         } finally {
             IOUtils.closeQuietly(stream);
diff --git a/tests/features/cached_error_reports.feature b/tests/features/cached_error_reports.feature
index 20b40e1709..a0f6973713 100644
--- a/tests/features/cached_error_reports.feature
+++ b/tests/features/cached_error_reports.feature
@@ -21,7 +21,7 @@ Scenario: Sending internal error reports on API <26
     And the event "device.osName" equals "android"
     And the event "metaData.BugsnagDiagnostics.filename" is not null
     And the event "metaData.BugsnagDiagnostics.notifierName" equals "Android Bugsnag Notifier"
-    And the event "metaData.BugsnagDiagnostics.apiKey" equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
+    And the event "metaData.BugsnagDiagnostics.apiKey" equals "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"
     And the event "metaData.BugsnagDiagnostics.packageName" equals "com.bugsnag.android.mazerunner"
     And the event "metaData.BugsnagDiagnostics.notifierVersion" is not null
     And the event "metaData.BugsnagDiagnostics.fileLength" equals 4
@@ -43,7 +43,7 @@ Scenario: Sending internal error reports on API >=26
     And the event "device.osName" equals "android"
     And the event "metaData.BugsnagDiagnostics.filename" is not null
     And the event "metaData.BugsnagDiagnostics.notifierName" equals "Android Bugsnag Notifier"
-    And the event "metaData.BugsnagDiagnostics.apiKey" equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
+    And the event "metaData.BugsnagDiagnostics.apiKey" equals "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"
     And the event "metaData.BugsnagDiagnostics.packageName" equals "com.bugsnag.android.mazerunner"
     And the event "metaData.BugsnagDiagnostics.notifierVersion" is not null
     And the event "metaData.BugsnagDiagnostics.fileLength" equals 4
@@ -66,7 +66,7 @@ Scenario: Sending internal error reports with cache tombstone + groups enabled
     And the event "device.osName" equals "android"
     And the event "metaData.BugsnagDiagnostics.filename" is not null
     And the event "metaData.BugsnagDiagnostics.notifierName" equals "Android Bugsnag Notifier"
-    And the event "metaData.BugsnagDiagnostics.apiKey" equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
+    And the event "metaData.BugsnagDiagnostics.apiKey" equals "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"
     And the event "metaData.BugsnagDiagnostics.packageName" equals "com.bugsnag.android.mazerunner"
     And the event "metaData.BugsnagDiagnostics.notifierVersion" is not null
     And the event "metaData.BugsnagDiagnostics.fileLength" equals 4