Skip to content

Commit

Permalink
[sql] SQLite patch to implement "smart" auto-vacuum.
Browse files Browse the repository at this point in the history 
SQLITE_FCNTL_CHUNK_SIZE can advise the VFS to resize files in quantum
amounts, to reduce fragmentation from tiny appends.  This change allows
a new PRAGMA auto_vacuum_slack_pages to provide auto_vacuum with a hint
to only rearrange pages when an entire quantum can be released.

BUG=698010

Review-Url: https://codereview.chromium.org/2732553002
Cr-Commit-Position: refs/heads/master@{#459847}
  • Loading branch information
@dshess
dshess authored and Commit bot committed Mar 27, 2017
1 parent c205b90 commit e812793
Show file tree
Hide file tree
Showing 8 changed files with 451 additions and 2 deletions.
92 changes: 92 additions & 0 deletions sql/sqlite_features_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,4 +361,96 @@ TEST_F(SQLiteFeaturesTest, DISABLED_TimeMachine) {
}
#endif

#if !defined(USE_SYSTEM_SQLITE)
// Test that Chromium's patch to make auto_vacuum integrate with
// SQLITE_FCNTL_CHUNK_SIZE is working.
TEST_F(SQLiteFeaturesTest, SmartAutoVacuum) {
// Turn on auto_vacuum, and set the page size low to make results obvious.
// These settings require re-writing the database, which VACUUM does.
ASSERT_TRUE(db().Execute("PRAGMA auto_vacuum = FULL"));
ASSERT_TRUE(db().Execute("PRAGMA page_size = 1024"));
ASSERT_TRUE(db().Execute("VACUUM"));

// Code-coverage of the PRAGMA set/get implementation.
const char kPragmaSql[] = "PRAGMA auto_vacuum_slack_pages";
ASSERT_EQ("0", sql::test::ExecuteWithResult(&db(), kPragmaSql));
ASSERT_TRUE(db().Execute("PRAGMA auto_vacuum_slack_pages = 4"));
ASSERT_EQ("4", sql::test::ExecuteWithResult(&db(), kPragmaSql));
// Max out at 255.
ASSERT_TRUE(db().Execute("PRAGMA auto_vacuum_slack_pages = 1000"));
ASSERT_EQ("255", sql::test::ExecuteWithResult(&db(), kPragmaSql));
ASSERT_TRUE(db().Execute("PRAGMA auto_vacuum_slack_pages = 0"));

// With page_size=1024, the following will insert rows which take up an
// overflow page, plus a small header in a b-tree node. An empty table takes
// a single page, so for small row counts each insert will add one page, and
// each delete will remove one page.
const char kCreateSql[] = "CREATE TABLE t (id INTEGER PRIMARY KEY, value)";
const char kInsertSql[] = "INSERT INTO t (value) VALUES (randomblob(980))";
#if !defined(OS_WIN)
const char kDeleteSql[] = "DELETE FROM t WHERE id = (SELECT MIN(id) FROM t)";
#endif

// This database will be 34 overflow pages plus the table's root page plus the
// SQLite header page plus the freelist page.
ASSERT_TRUE(db().Execute(kCreateSql));
{
sql::Statement s(db().GetUniqueStatement(kInsertSql));
for (int i = 0; i < 34; ++i) {
s.Reset(true);
ASSERT_TRUE(s.Run());
}
}
ASSERT_EQ("37", sql::test::ExecuteWithResult(&db(), "PRAGMA page_count"));

// http://sqlite.org/mmap.html indicates that Windows will silently fail when
// truncating a memory-mapped file. That pretty much invalidates these tests
// against the actual file size.
#if !defined(OS_WIN)
// Each delete will delete a single page, including crossing a
// multiple-of-four boundary.
{
sql::Statement s(db().GetUniqueStatement(kDeleteSql));
for (int i = 0; i < 5; ++i) {
int64_t file_size_before, file_size_after;
ASSERT_TRUE(base::GetFileSize(db_path(), &file_size_before));

s.Reset(true);
ASSERT_TRUE(s.Run());

ASSERT_TRUE(base::GetFileSize(db_path(), &file_size_after));
ASSERT_EQ(file_size_after, file_size_before - 1024);
}
}

// Turn on "smart" auto-vacuum to remove 4 pages at a time.
ASSERT_TRUE(db().Execute("PRAGMA auto_vacuum_slack_pages = 4"));

// No pages removed, then four deleted at once.
{
sql::Statement s(db().GetUniqueStatement(kDeleteSql));
for (int i = 0; i < 3; ++i) {
int64_t file_size_before, file_size_after;
ASSERT_TRUE(base::GetFileSize(db_path(), &file_size_before));

s.Reset(true);
ASSERT_TRUE(s.Run());

ASSERT_TRUE(base::GetFileSize(db_path(), &file_size_after));
ASSERT_EQ(file_size_after, file_size_before);
}

int64_t file_size_before, file_size_after;
ASSERT_TRUE(base::GetFileSize(db_path(), &file_size_before));

s.Reset(true);
ASSERT_TRUE(s.Run());

ASSERT_TRUE(base::GetFileSize(db_path(), &file_size_after));
ASSERT_EQ(file_size_after, file_size_before - 4096);
}
#endif
}
#endif // !defined(USE_SYSTEM_SQLITE)

} // namespace
88 changes: 87 additions & 1 deletion third_party/sqlite/amalgamation/sqlite3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12266,6 +12266,8 @@ SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree*);
SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p);
SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int);
SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *);
SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuumSlackPages(Btree *, int);
SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuumSlackPages(Btree *);
SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int);
SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster);
SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int);
Expand Down Expand Up @@ -58368,6 +58370,7 @@ struct BtShared {
u8 openFlags; /* Flags to sqlite3BtreeOpen() */
#ifndef SQLITE_OMIT_AUTOVACUUM
u8 autoVacuum; /* True if auto-vacuum is enabled */
u8 autoVacuumSlack; /* Optional pages of slack for auto-vacuum */
u8 incrVacuum; /* True if incr-vacuum is enabled */
u8 bDoTruncate; /* True to truncate db on commit */
#endif
Expand Down Expand Up @@ -61765,6 +61768,46 @@ SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){
#endif
}

/*
** Change the 'auto-vacuum-slack-pages' property of the database. If auto vacuum
** is enabled, this is the number of chunks of slack to allow before
** automatically running an incremental vacuum.
*/
SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuumSlackPages(Btree *p, int autoVacuumSlack){
#ifdef SQLITE_OMIT_AUTOVACUUM
return SQLITE_READONLY;
#else
BtShared *pBt = p->pBt;
int rc = SQLITE_OK;
u8 avs = (u8)autoVacuumSlack;
if( autoVacuumSlack>avs ){
avs = 0xFF;
}

sqlite3BtreeEnter(p);
pBt->autoVacuumSlack = avs;
sqlite3BtreeLeave(p);
return rc;
#endif
}

/*
** Return the value of the 'auto-vacuum-slack-pages' property.
*/
SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuumSlackPages(Btree *p){
#ifdef SQLITE_OMIT_AUTOVACUUM
return 0;
#else
int rc = 0;
sqlite3BtreeEnter(p);
if( p->pBt->autoVacuum!=0 ){
rc = p->pBt->autoVacuumSlack;
}
sqlite3BtreeLeave(p);
return rc;
#endif
}


/*
** Get a reference to pPage1 of the database file. This will
Expand Down Expand Up @@ -62606,13 +62649,27 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){
*/
static int autoVacuumCommit(BtShared *pBt){
int rc = SQLITE_OK;
int bShouldVacuum = pBt->autoVacuum && !pBt->incrVacuum;
Pager *pPager = pBt->pPager;
VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); )

assert( sqlite3_mutex_held(pBt->mutex) );
invalidateAllOverflowCache(pBt);
assert(pBt->autoVacuum);
if( !pBt->incrVacuum ){
if( bShouldVacuum && pBt->autoVacuumSlack ){
Pgno nOrig; /* Database size before freeing */
Pgno nFree; /* Number of pages on the freelist initially */

nOrig = btreePagecount(pBt);
nFree = get4byte(&pBt->pPage1->aData[36]);
bShouldVacuum =
(nOrig-nFree-1)/pBt->autoVacuumSlack < (nOrig-1)/pBt->autoVacuumSlack;
/* TODO: When integrating this test with the following code, contrive to
** trim to the integral chunk boundary, rather than trimming the entire free
** list.
*/
}
if( bShouldVacuum ){
Pgno nFin; /* Number of pages in database after autovacuuming */
Pgno nFree; /* Number of pages on the freelist initially */
Pgno iFree; /* The next page to be freed */
Expand Down Expand Up @@ -112205,6 +112262,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){
#define PragTyp_REKEY 40
#define PragTyp_LOCK_STATUS 41
#define PragTyp_PARSER_TRACE 42
#define PragTyp_AUTO_VACUUM_SLACK_PAGES 43

/* Property flags associated with various pragma. */
#define PragFlg_NeedSchema 0x01 /* Force schema load before running */
Expand Down Expand Up @@ -112302,6 +112360,13 @@ static const PragmaName aPragmaName[] = {
/* ColNames: */ 0, 0,
/* iArg: */ 0 },
#endif
#if !defined(SQLITE_OMIT_AUTOVACUUM)
{ /* zName: */ "auto_vacuum_slack_pages",
/* ePragTyp: */ PragTyp_AUTO_VACUUM_SLACK_PAGES,
/* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1,
/* ColNames: */ 0, 0,
/* iArg: */ 0 },
#endif
#if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
#if !defined(SQLITE_OMIT_AUTOMATIC_INDEX)
{/* zName: */ "automatic_index",
Expand Down Expand Up @@ -113472,6 +113537,27 @@ SQLITE_PRIVATE void sqlite3Pragma(
}
#endif

/*
** PRAGMA [schema.]auto_vacuum_slack_pages(N)
**
** Control chunk size of auto-vacuum.
*/
#ifndef SQLITE_OMIT_AUTOVACUUM
case PragTyp_AUTO_VACUUM_SLACK_PAGES: {
Btree *pBt = pDb->pBt;
assert( pBt!=0 );
if( !zRight ){
returnSingleInt(v, sqlite3BtreeGetAutoVacuumSlackPages(pBt));
}else{
int nPages = 8;
if( sqlite3GetInt32(zRight, &nPages) ){
sqlite3BtreeSetAutoVacuumSlackPages(pBt, nPages);
}
}
break;
}
#endif

#ifndef SQLITE_OMIT_PAGER_PRAGMAS
/*
** PRAGMA [schema.]cache_size
Expand Down
Loading

0 comments on commit e812793

Please sign in to comment.