[sql] Use recover virtual table in sql::Recovery.

Load the recover virtual table as part of sql::Recovery::Begin(), and
implement basic unit tests that it correctly recovers valid and
corrupt tables.

BUG=109482

Review URL: https://chromiumcodereview.appspot.com/20022006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@215764 0039d316-1c4b-4281-b951-d872f2087c98

sql/connection_unittest.cc

@@ -425,7 +425,7 @@ TEST_F(SQLConnectionTest, RazeEmptyDB) {
   db().Close();
 
   {
-    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "r+"));
+    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "rb+"));
     ASSERT_TRUE(file.get() != NULL);
     ASSERT_EQ(0, fseek(file.get(), 0, SEEK_SET));
     ASSERT_TRUE(file_util::TruncateFile(file.get()));
@@ -443,7 +443,7 @@ TEST_F(SQLConnectionTest, RazeNOTADB) {
   ASSERT_FALSE(base::PathExists(db_path()));
 
   {
-    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "w"));
+    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "wb"));
     ASSERT_TRUE(file.get() != NULL);
 
     const char* kJunk = "This is the hour of our discontent.";
@@ -476,7 +476,7 @@ TEST_F(SQLConnectionTest, RazeNOTADB2) {
   db().Close();
 
   {
-    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "r+"));
+    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "rb+"));
     ASSERT_TRUE(file.get() != NULL);
     ASSERT_EQ(0, fseek(file.get(), 0, SEEK_SET));
 
@@ -526,7 +526,7 @@ TEST_F(SQLConnectionTest, RazeCallbackReopen) {
 
   // Trim a single page from the end of the file.
   {
-    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "r+"));
+    file_util::ScopedFILE file(file_util::OpenFile(db_path(), "rb+"));
     ASSERT_TRUE(file.get() != NULL);
     ASSERT_EQ(0, fseek(file.get(), -page_size, SEEK_END));
     ASSERT_TRUE(file_util::TruncateFile(file.get()));

sql/recovery.cc

@@ -72,6 +72,24 @@ bool Recovery::Init(const base::FilePath& db_path) {
   if (!recover_db_.OpenTemporary())
     return false;
 
+  // TODO(shess): Figure out a story for USE_SYSTEM_SQLITE.  The
+  // virtual table implementation relies on SQLite internals for some
+  // types and functions, which could be copied inline to make it
+  // standalone.  Or an alternate implementation could try to read
+  // through errors entirely at the SQLite level.
+  //
+  // For now, defer to the caller.  The setup will succeed, but the
+  // later CREATE VIRTUAL TABLE call will fail, at which point the
+  // caller can fire Unrecoverable().
+#if !defined(USE_SYSTEM_SQLITE)
+  int rc = recoverVtableInit(recover_db_.db_);
+  if (rc != SQLITE_OK) {
+    LOG(ERROR) << "Failed to initialize recover module: "
+               << recover_db_.GetErrorMessage();
+    return false;
+  }
+#endif
+
   // Turn on |SQLITE_RecoveryMode| for the handle, which allows
   // reading certain broken databases.
   if (!recover_db_.Execute("PRAGMA writable_schema=1"))

sql/recovery_unittest.cc

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include "base/bind.h"
 #include "base/file_util.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/logging.h"
@@ -47,6 +48,49 @@ std::string GetSchema(sql::Connection* db) {
   return ExecuteWithResults(db, kSql, "|", "\n");
 }
 
+int GetPageSize(sql::Connection* db) {
+  sql::Statement s(db->GetUniqueStatement("PRAGMA page_size"));
+  EXPECT_TRUE(s.Step());
+  return s.ColumnInt(0);
+}
+
+// Get |name|'s root page number in the database.
+int GetRootPage(sql::Connection* db, const char* name) {
+  const char kPageSql[] = "SELECT rootpage FROM sqlite_master WHERE name = ?";
+  sql::Statement s(db->GetUniqueStatement(kPageSql));
+  s.BindString(0, name);
+  EXPECT_TRUE(s.Step());
+  return s.ColumnInt(0);
+}
+
+// Helper to read a SQLite page into a buffer.  |page_no| is 1-based
+// per SQLite usage.
+bool ReadPage(const base::FilePath& path, size_t page_no,
+              char* buf, size_t page_size) {
+  file_util::ScopedFILE file(file_util::OpenFile(path, "rb"));
+  if (!file.get())
+    return false;
+  if (0 != fseek(file.get(), (page_no - 1) * page_size, SEEK_SET))
+    return false;
+  if (1u != fread(buf, page_size, 1, file.get()))
+    return false;
+  return true;
+}
+
+// Helper to write a SQLite page into a buffer.  |page_no| is 1-based
+// per SQLite usage.
+bool WritePage(const base::FilePath& path, size_t page_no,
+               const char* buf, size_t page_size) {
+  file_util::ScopedFILE file(file_util::OpenFile(path, "rb+"));
+  if (!file.get())
+    return false;
+  if (0 != fseek(file.get(), (page_no - 1) * page_size, SEEK_SET))
+    return false;
+  if (1u != fwrite(buf, page_size, 1, file.get()))
+    return false;
+  return true;
+}
+
 class SQLRecoveryTest : public testing::Test {
  public:
   SQLRecoveryTest() {}
@@ -138,8 +182,244 @@ TEST_F(SQLRecoveryTest, RecoverBasic) {
   ASSERT_EQ("CREATE TABLE x (t TEXT)", GetSchema(&db()));
 
   const char* kXSql = "SELECT * FROM x ORDER BY 1";
-  ASSERT_EQ(ExecuteWithResults(&db(), kXSql, "|", "\n"),
-            "That was a test");
+  ASSERT_EQ("That was a test",
+            ExecuteWithResults(&db(), kXSql, "|", "\n"));
+}
+
+// The recovery virtual table is only supported for Chromium's SQLite.
+#if !defined(USE_SYSTEM_SQLITE)
+
+// Run recovery through its paces on a valid database.
+TEST_F(SQLRecoveryTest, VirtualTable) {
+  const char kCreateSql[] = "CREATE TABLE x (t TEXT)";
+  ASSERT_TRUE(db().Execute(kCreateSql));
+  ASSERT_TRUE(db().Execute("INSERT INTO x VALUES ('This is a test')"));
+  ASSERT_TRUE(db().Execute("INSERT INTO x VALUES ('That was a test')"));
+
+  // Successfully recover the database.
+  {
+    scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path());
+
+    // Tables to recover original DB, now at [corrupt].
+    const char kRecoveryCreateSql[] =
+        "CREATE VIRTUAL TABLE temp.recover_x using recover("
+        "  corrupt.x,"
+        "  t TEXT STRICT"
+        ")";
+    ASSERT_TRUE(recovery->db()->Execute(kRecoveryCreateSql));
+
+    // Re-create the original schema.
+    ASSERT_TRUE(recovery->db()->Execute(kCreateSql));
+
+    // Copy the data from the recovery tables to the new database.
+    const char kRecoveryCopySql[] =
+        "INSERT INTO x SELECT t FROM recover_x";
+    ASSERT_TRUE(recovery->db()->Execute(kRecoveryCopySql));
+
+    // Successfully recovered.
+    ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass()));
+  }
+
+  // Since the database was not corrupt, the entire schema and all
+  // data should be recovered.
+  ASSERT_TRUE(Reopen());
+  ASSERT_EQ("CREATE TABLE x (t TEXT)", GetSchema(&db()));
+
+  const char* kXSql = "SELECT * FROM x ORDER BY 1";
+  ASSERT_EQ("That was a test\nThis is a test",
+            ExecuteWithResults(&db(), kXSql, "|", "\n"));
+}
+
+void RecoveryCallback(sql::Connection* db, const base::FilePath& db_path,
+                      int* record_error, int error, sql::Statement* stmt) {
+  *record_error = error;
+
+  // Clear the error callback to prevent reentrancy.
+  db->reset_error_callback();
+
+  scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(db, db_path);
+  ASSERT_TRUE(recovery.get());
+
+  const char kRecoveryCreateSql[] =
+      "CREATE VIRTUAL TABLE temp.recover_x using recover("
+      "  corrupt.x,"
+      "  id INTEGER STRICT,"
+      "  v INTEGER STRICT"
+      ")";
+  const char kCreateTable[] = "CREATE TABLE x (id INTEGER, v INTEGER)";
+  const char kCreateIndex[] = "CREATE UNIQUE INDEX x_id ON x (id)";
+
+  // Replicate data over.
+  const char kRecoveryCopySql[] =
+      "INSERT OR REPLACE INTO x SELECT id, v FROM recover_x";
+
+  ASSERT_TRUE(recovery->db()->Execute(kRecoveryCreateSql));
+  ASSERT_TRUE(recovery->db()->Execute(kCreateTable));
+  ASSERT_TRUE(recovery->db()->Execute(kCreateIndex));
+  ASSERT_TRUE(recovery->db()->Execute(kRecoveryCopySql));
+
+  ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass()));
+}
+
+// Build a database, corrupt it by making an index reference to
+// deleted row, then recover when a query selects that row.
+TEST_F(SQLRecoveryTest, RecoverCorruptIndex) {
+  const char kCreateTable[] = "CREATE TABLE x (id INTEGER, v INTEGER)";
+  const char kCreateIndex[] = "CREATE UNIQUE INDEX x_id ON x (id)";
+  ASSERT_TRUE(db().Execute(kCreateTable));
+  ASSERT_TRUE(db().Execute(kCreateIndex));
+
+  // Insert a bit of data.
+  {
+    ASSERT_TRUE(db().BeginTransaction());
+
+    const char kInsertSql[] = "INSERT INTO x (id, v) VALUES (?, ?)";
+    sql::Statement s(db().GetUniqueStatement(kInsertSql));
+    for (int i = 0; i < 10; ++i) {
+      s.Reset(true);
+      s.BindInt(0, i);
+      s.BindInt(1, i);
+      EXPECT_FALSE(s.Step());
+      EXPECT_TRUE(s.Succeeded());
+    }
+
+    ASSERT_TRUE(db().CommitTransaction());
+  }
+
+
+  // Capture the index's root page into |buf|.
+  int index_page = GetRootPage(&db(), "x_id");
+  int page_size = GetPageSize(&db());
+  scoped_ptr<char[]> buf(new char[page_size]);
+  ASSERT_TRUE(ReadPage(db_path(), index_page, buf.get(), page_size));
+
+  // Delete the row from the table and index.
+  ASSERT_TRUE(db().Execute("DELETE FROM x WHERE id = 0"));
+
+  // Close to clear any cached data.
+  db().Close();
+
+  // Put the stale index page back.
+  ASSERT_TRUE(WritePage(db_path(), index_page, buf.get(), page_size));
+
+  // At this point, the index references a value not in the table.
+
+  ASSERT_TRUE(Reopen());
+
+  int error = SQLITE_OK;
+  db().set_error_callback(base::Bind(&RecoveryCallback,
+                                     &db(), db_path(), &error));
+
+  // This works before the callback is called.
+  const char kTrivialSql[] = "SELECT COUNT(*) FROM sqlite_master";
+  EXPECT_TRUE(db().IsSQLValid(kTrivialSql));
+
+  // TODO(shess): Could this be delete?  Anything which fails should work.
+  const char kSelectSql[] = "SELECT v FROM x WHERE id = 0";
+  ASSERT_FALSE(db().Execute(kSelectSql));
+  EXPECT_EQ(SQLITE_CORRUPT, error);
+
+  // Database handle has been poisoned.
+  EXPECT_FALSE(db().IsSQLValid(kTrivialSql));
+
+  ASSERT_TRUE(Reopen());
+
+  // The recovered table should reflect the deletion.
+  const char kSelectAllSql[] = "SELECT v FROM x ORDER BY id";
+  EXPECT_EQ("1,2,3,4,5,6,7,8,9",
+            ExecuteWithResults(&db(), kSelectAllSql, "|", ","));
+
+  // The failing statement should now succeed, with no results.
+  EXPECT_EQ("", ExecuteWithResults(&db(), kSelectSql, "|", ","));
+}
+
+// Build a database, corrupt it by making a table contain a row not
+// referenced by the index, then recover the database.
+TEST_F(SQLRecoveryTest, RecoverCorruptTable) {
+  const char kCreateTable[] = "CREATE TABLE x (id INTEGER, v INTEGER)";
+  const char kCreateIndex[] = "CREATE UNIQUE INDEX x_id ON x (id)";
+  ASSERT_TRUE(db().Execute(kCreateTable));
+  ASSERT_TRUE(db().Execute(kCreateIndex));
+
+  // Insert a bit of data.
+  {
+    ASSERT_TRUE(db().BeginTransaction());
+
+    const char kInsertSql[] = "INSERT INTO x (id, v) VALUES (?, ?)";
+    sql::Statement s(db().GetUniqueStatement(kInsertSql));
+    for (int i = 0; i < 10; ++i) {
+      s.Reset(true);
+      s.BindInt(0, i);
+      s.BindInt(1, i);
+      EXPECT_FALSE(s.Step());
+      EXPECT_TRUE(s.Succeeded());
+    }
+
+    ASSERT_TRUE(db().CommitTransaction());
+  }
+
+  // Capture the table's root page into |buf|.
+  // Find the page the table is stored on.
+  const int table_page = GetRootPage(&db(), "x");
+  const int page_size = GetPageSize(&db());
+  scoped_ptr<char[]> buf(new char[page_size]);
+  ASSERT_TRUE(ReadPage(db_path(), table_page, buf.get(), page_size));
+
+  // Delete the row from the table and index.
+  ASSERT_TRUE(db().Execute("DELETE FROM x WHERE id = 0"));
+
+  // Close to clear any cached data.
+  db().Close();
+
+  // Put the stale table page back.
+  ASSERT_TRUE(WritePage(db_path(), table_page, buf.get(), page_size));
+
+  // At this point, the table contains a value not referenced by the
+  // index.
+  // TODO(shess): Figure out a query which causes SQLite to notice
+  // this organically.  Meanwhile, just handle it manually.
+
+  ASSERT_TRUE(Reopen());
+
+  // Index shows one less than originally inserted.
+  const char kCountSql[] = "SELECT COUNT (*) FROM x";
+  EXPECT_EQ("9", ExecuteWithResults(&db(), kCountSql, "|", ","));
+
+  // A full table scan shows all of the original data.
+  const char kDistinctSql[] = "SELECT DISTINCT COUNT (id) FROM x";
+  EXPECT_EQ("10", ExecuteWithResults(&db(), kDistinctSql, "|", ","));
+
+  // Insert id 0 again.  Since it is not in the index, the insert
+  // succeeds, but results in a duplicate value in the table.
+  const char kInsertSql[] = "INSERT INTO x (id, v) VALUES (0, 100)";
+  ASSERT_TRUE(db().Execute(kInsertSql));
+
+  // Duplication is visible.
+  EXPECT_EQ("10", ExecuteWithResults(&db(), kCountSql, "|", ","));
+  EXPECT_EQ("11", ExecuteWithResults(&db(), kDistinctSql, "|", ","));
+
+  // This works before the callback is called.
+  const char kTrivialSql[] = "SELECT COUNT(*) FROM sqlite_master";
+  EXPECT_TRUE(db().IsSQLValid(kTrivialSql));
+
+  // Call the recovery callback manually.
+  int error = SQLITE_OK;
+  RecoveryCallback(&db(), db_path(), &error, SQLITE_CORRUPT, NULL);
+  EXPECT_EQ(SQLITE_CORRUPT, error);
+
+  // Database handle has been poisoned.
+  EXPECT_FALSE(db().IsSQLValid(kTrivialSql));
+
+  ASSERT_TRUE(Reopen());
+
+  // The recovered table has consistency between the index and the table.
+  EXPECT_EQ("10", ExecuteWithResults(&db(), kCountSql, "|", ","));
+  EXPECT_EQ("10", ExecuteWithResults(&db(), kDistinctSql, "|", ","));
+
+  // The expected value was retained.
+  const char kSelectSql[] = "SELECT v FROM x WHERE id = 0";
+  EXPECT_EQ("100", ExecuteWithResults(&db(), kSelectSql, "|", ","));
 }
+#endif  // !defined(USE_SYSTEM_SQLITE)
 
 }  // namespace

sql/sql.gyp

@@ -80,6 +80,7 @@
         'test_support_sql',
         '../base/base.gyp:test_support_base',
         '../testing/gtest.gyp:gtest',
+        '../third_party/sqlite/sqlite.gyp:sqlite',
       ],
       'sources': [
         'run_all_unittests.cc',