diff --git a/autotest/ogr/ogr_gpkg.py b/autotest/ogr/ogr_gpkg.py index 822a46334459..2716c7aaba58 100755 --- a/autotest/ogr/ogr_gpkg.py +++ b/autotest/ogr/ogr_gpkg.py @@ -76,10 +76,9 @@ def gpkg_ds(gpkg_dsn): return ogr.GetDriverByName("GPKG").CreateDataSource(gpkg_dsn) -@pytest.fixture() -def tpoly(gpkg_ds, poly_feat): +def create_test_layer(ds, src_features, layer_name): - lyr = gpkg_ds.CreateLayer("tpoly") + lyr = ds.CreateLayer(layer_name) ogrtest.quick_create_layer_def( lyr, @@ -94,11 +93,21 @@ def tpoly(gpkg_ds, poly_feat): dst_feat = ogr.Feature(feature_def=lyr.GetLayerDefn()) - for feat in poly_feat: + for feat in src_features: dst_feat.SetFrom(feat) lyr.CreateFeature(dst_feat) +@pytest.fixture() +def tpoly(gpkg_ds, poly_feat): + create_test_layer(gpkg_ds, poly_feat, "tpoly") + + +@pytest.fixture() +def tpoly_dbl_quote(gpkg_ds, poly_feat): + create_test_layer(gpkg_ds, poly_feat, 'tpoly"dbl_quote') + + @pytest.fixture() def a_layer(gpkg_ds): gpkg_ds.CreateLayer("a_layer", options=["SPATIAL_INDEX=NO"]) @@ -400,6 +409,177 @@ def test_ogr_gpkg_5(gpkg_ds): assert gpkg_ds.GetLayerCount() == 0, "unexpected number of layers (not 0)" +############################################################################### +# Truncate a spatial layer + + +@pytest.mark.usefixtures("tpoly_dbl_quote") +@gdaltest.enable_exceptions() +def test_ogr_gpkg_truncate_spatial_layer(gpkg_ds, poly_feat): + + gpkg_ds.ExecuteSQL('DELETE FROM "tpoly""dbl_quote"') + + lyr = gpkg_ds.GetLayer(0) + assert lyr.GetFeatureCount() == 0 + + # Re-insert records + dst_feat = ogr.Feature(feature_def=lyr.GetLayerDefn()) + for feat in poly_feat: + dst_feat.SetFrom(feat) + lyr.CreateFeature(dst_feat) + + # Check that spatial index works fine + for feat in poly_feat: + minx, maxx, miny, maxy = feat.GetGeometryRef().GetEnvelope() + lyr.SetSpatialFilterRect(minx, miny, maxx, maxy) + assert lyr.GetFeatureCount() >= 1 + + +############################################################################### +# Truncate a non-spatial layer + + +@gdaltest.enable_exceptions() +def test_ogr_gpkg_truncate_non_spatial_layer(tmp_vsimem): + + filename = str(tmp_vsimem / "test.gpkg") + with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds: + lyr = ds.CreateLayer("test", geom_type=ogr.wkbNone) + lyr.CreateField(ogr.FieldDefn("foo")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "bar" + lyr.CreateFeature(f) + + # Test that the optimization doesn't trigger when there's a WHERE clause + with ogr.Open(filename, update=1) as ds: + ds.ExecuteSQL("DELETE FROM test WHERE 0") + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + + with ogr.Open(filename, update=1) as ds: + ds.ExecuteSQL("DELETE FROM test") + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 0 + + with ogr.Open(filename) as ds: + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 0 + + +############################################################################### +# Truncate a layer with a custom trigger + + +@gdaltest.enable_exceptions() +def test_ogr_gpkg_truncate_with_custom_trigger(tmp_vsimem): + + filename = str(tmp_vsimem / "test.gpkg") + with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds: + lyr = ds.CreateLayer("test", geom_type=ogr.wkbNone) + lyr.CreateField(ogr.FieldDefn("foo")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "bar" + lyr.CreateFeature(f) + ds.ExecuteSQL( + "CREATE TABLE counter(table_name TEXT UNIQUE NOT NULL, counter INT)" + ) + ds.ExecuteSQL("INSERT INTO counter VALUES ('test', 1)") + ds.ExecuteSQL( + "CREATE TRIGGER my_trigger AFTER DELETE ON test BEGIN UPDATE counter SET counter = counter - 1 WHERE table_name = 'test'; END" + ) + + with ogr.Open(filename, update=1) as ds: + with ds.ExecuteSQL( + "SELECT counter FROM counter WHERE table_name = 'test'" + ) as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f["counter"] == 1 + ds.ExecuteSQL('DELETE FROM "test"') + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 0 + with ds.ExecuteSQL( + "SELECT counter FROM counter WHERE table_name = 'test'" + ) as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f["counter"] == 0 + + with ogr.Open(filename) as ds: + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 0 + + +############################################################################### +# Truncate a layer with a custom trigger that causes an error + + +@gdaltest.enable_exceptions() +def test_ogr_gpkg_truncate_with_custom_trigger_error(tmp_vsimem): + + filename = str(tmp_vsimem / "test.gpkg") + with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds: + lyr = ds.CreateLayer("test", geom_type=ogr.wkbNone) + lyr.CreateField(ogr.FieldDefn("foo")) + f = ogr.Feature(lyr.GetLayerDefn()) + f["foo"] = "bar" + lyr.CreateFeature(f) + ds.ExecuteSQL( + "CREATE TABLE counter(table_name TEXT UNIQUE NOT NULL, counter INT)" + ) + ds.ExecuteSQL("INSERT INTO counter VALUES ('test', 1)") + ds.ExecuteSQL( + "CREATE TRIGGER my_trigger AFTER DELETE ON test BEGIN SELECT RAISE(ABORT, 'error from trigger'); END" + ) + + with ogr.Open(filename, update=1) as ds: + with ds.ExecuteSQL( + "SELECT counter FROM counter WHERE table_name = 'test'" + ) as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f["counter"] == 1 + with pytest.raises(Exception, match="error from trigger"): + ds.ExecuteSQL('DELETE FROM "test"') + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + with ds.ExecuteSQL( + "SELECT counter FROM counter WHERE table_name = 'test'" + ) as sql_lyr: + f = sql_lyr.GetNextFeature() + assert f["counter"] == 1 + + with ogr.Open(filename) as ds: + lyr = ds.GetLayer(0) + assert lyr.GetFeatureCount() == 1 + + +############################################################################### +# Truncate a layer in read-only mode (error) + + +@gdaltest.enable_exceptions() +def test_ogr_gpkg_truncate_read_only_mode(tmp_vsimem): + + filename = str(tmp_vsimem / "test.gpkg") + with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds: + ds.CreateLayer("test") + + with ogr.Open(filename) as ds: + with pytest.raises(Exception, match="read-only"): + ds.ExecuteSQL('DELETE FROM "test"') + + +############################################################################### +# Truncate a layer that doe not exist + + +@gdaltest.enable_exceptions() +def test_ogr_gpkg_truncate_non_existing_table(tmp_vsimem): + + filename = str(tmp_vsimem / "test.gpkg") + with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds: + with pytest.raises(Exception, match="no such table"): + ds.ExecuteSQL('DELETE FROM "i_do_not_exist"') + + ############################################################################### # Add fields diff --git a/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h b/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h index b42cf82da80e..ec7432de933c 100644 --- a/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h +++ b/ogr/ogrsf_frmts/gpkg/ogr_geopackage.h @@ -961,6 +961,8 @@ class OGRGeoPackageTableLayer final : public OGRGeoPackageLayer virtual CPLString GetSpatialWhere(int iGeomCol, OGRGeometry *poFilterGeom) override; + OGRErr Truncate(); + bool HasSpatialIndex(); void SetPrecisionFlag(int bFlag) diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp index f40b1aabd98c..4b475fae793a 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp @@ -7712,6 +7712,25 @@ OGRLayer *GDALGeoPackageDataset::ExecuteSQL(const char *pszSQLCommand, return nullptr; } + else if (STARTS_WITH_CI(osSQLCommand, "DELETE FROM ")) + { + // Optimize truncation of a table, especially if it has a spatial + // index. + const CPLStringList aosTokens(SQLTokenize(osSQLCommand)); + if (aosTokens.size() == 3) + { + const char *pszTableName = aosTokens[2]; + OGRGeoPackageTableLayer *poLayer = + dynamic_cast( + GetLayerByName(SQLUnescape(pszTableName))); + if (poLayer) + { + poLayer->Truncate(); + return nullptr; + } + } + } + else if (pszDialect != nullptr && EQUAL(pszDialect, "INDIRECT_SQLITE")) return GDALDataset::ExecuteSQL(osSQLCommand, poSpatialFilter, "SQLITE"); else if (pszDialect != nullptr && !EQUAL(pszDialect, "") && diff --git a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp index 6acc0ff19f49..6549676573f3 100644 --- a/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp +++ b/ogr/ogrsf_frmts/gpkg/ogrgeopackagetablelayer.cpp @@ -9140,3 +9140,179 @@ OGRErr OGRGeoPackageTableLayer::GetExtent3D(int iGeomField, return OGRERR_NONE; } + +/************************************************************************/ +/* Truncate() */ +/************************************************************************/ + +/** Implements "DELETE FROM {table_name}" in an optimzed way. + * + * Disable triggers if we detect that the only triggers on the table are ones + * under our control (i.e. the ones for the gpkg_ogr_contents table and the + * ones updating the RTree) + * And even if we cannot disable triggers, truncate the RTree before the main + * table, as this dramatically speeds up truncating the main table. + */ +OGRErr OGRGeoPackageTableLayer::Truncate() +{ + if (!m_poDS->GetUpdate()) + { + CPLError(CE_Failure, CPLE_NotSupported, UNSUPPORTED_OP_READ_ONLY, + "Truncate"); + return OGRERR_FAILURE; + } + + ResetReading(); + SyncToDisk(); + + bool bOK = (m_poDS->SoftStartTransaction() == OGRERR_NONE); + + struct ReenableTriggers + { + sqlite3 *m_hDB = nullptr; + + explicit ReenableTriggers(sqlite3 *hDB) : m_hDB(hDB) + { + } + + ~ReenableTriggers() + { + sqlite3_db_config(m_hDB, SQLITE_DBCONFIG_ENABLE_TRIGGER, 1, + nullptr); + } + CPL_DISALLOW_COPY_ASSIGN(ReenableTriggers) + }; + + // to keep in top level scope! + std::unique_ptr reenableTriggers; + + // Temporarily disable triggers for greater speed if we detect that the + // only triggers on the table are the RTree ones and the ones for the + // gpkg_ogr_contents table + if (bOK && m_bIsTable) + { + char *pszSQL = sqlite3_mprintf( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' " + "AND tbl_name = '%q' " + "AND name NOT IN ('trigger_insert_feature_count_%q'," + "'trigger_delete_feature_count_%q') " + "AND name NOT LIKE 'rtree_%q_%%'", + m_pszTableName, m_pszTableName, m_pszTableName, m_pszTableName); + OGRErr eErr = OGRERR_NONE; + if (SQLGetInteger(m_poDS->GetDB(), pszSQL, &eErr) == 0 && + eErr == OGRERR_NONE) + { + int nEnableTriggerOldVal = -1; + sqlite3_db_config(m_poDS->GetDB(), SQLITE_DBCONFIG_ENABLE_TRIGGER, + -1, &nEnableTriggerOldVal); + if (nEnableTriggerOldVal == 1) + { + int nNewVal = -1; + sqlite3_db_config(m_poDS->GetDB(), + SQLITE_DBCONFIG_ENABLE_TRIGGER, 0, &nNewVal); + if (nNewVal == 0) + { + CPLDebugOnly("GPKG", + "Disabling triggers during truncation of %s", + m_pszTableName); + reenableTriggers = + std::make_unique(m_poDS->GetDB()); + CPL_IGNORE_RET_VAL(reenableTriggers); // to please cppcheck + } + } + } + sqlite3_free(pszSQL); + } + + char *pszErrMsg = nullptr; + if (bOK && m_bIsTable && HasSpatialIndex()) + { + // Manually clean the 3 tables that are used by the RTree: + // - rtree_{tablename}_{geom}_node: all rows, but nodeno = 1 for which + // we reset the 'data' field to a zero blob of the same size + // - rtree_{tablename}_{geom}_parent: all rows + // - rtree_{tablename}_{geom}_rowid: all rows + + const char *pszT = m_pszTableName; + const char *pszC = m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef(); + + m_osRTreeName = "rtree_"; + m_osRTreeName += pszT; + m_osRTreeName += "_"; + m_osRTreeName += pszC; + + { + char *pszSQL = + sqlite3_mprintf("DELETE FROM \"%w_node\" WHERE nodeno > 1;" + "DELETE FROM \"%w_parent\"; " + "DELETE FROM \"%w_rowid\"", + m_osRTreeName.c_str(), m_osRTreeName.c_str(), + m_osRTreeName.c_str()); + bOK = sqlite3_exec(m_poDS->hDB, pszSQL, nullptr, nullptr, + &pszErrMsg) == SQLITE_OK; + sqlite3_free(pszSQL); + } + + if (bOK) + { + char *pszSQL = sqlite3_mprintf( + "SELECT length(data) FROM \"%w_node\" WHERE nodeno = 1", + m_osRTreeName.c_str()); + const int nBlobSize = + SQLGetInteger(m_poDS->GetDB(), pszSQL, nullptr); + sqlite3_free(pszSQL); + + pszSQL = sqlite3_mprintf( + "UPDATE \"%w_node\" SET data = zeroblob(%d) WHERE nodeno = 1", + m_osRTreeName.c_str(), nBlobSize); + bOK = sqlite3_exec(m_poDS->hDB, pszSQL, nullptr, nullptr, + &pszErrMsg) == SQLITE_OK; + sqlite3_free(pszSQL); + } + } + + if (bOK) + { + // Truncate main table + char *pszSQL = sqlite3_mprintf("DELETE FROM \"%w\"", m_pszTableName); + bOK = sqlite3_exec(m_poDS->hDB, pszSQL, nullptr, nullptr, &pszErrMsg) == + SQLITE_OK; + sqlite3_free(pszSQL); + } + +#ifdef ENABLE_GPKG_OGR_CONTENTS + // Reset feature count + if (bOK && m_poDS->m_bHasGPKGOGRContents) + { + char *pszSQL = + sqlite3_mprintf("UPDATE gpkg_ogr_contents SET feature_count = 0 " + "WHERE lower(table_name) = lower('%q')", + m_pszTableName); + bOK = sqlite3_exec(m_poDS->hDB, pszSQL, nullptr, nullptr, &pszErrMsg) == + SQLITE_OK; + sqlite3_free(pszSQL); + } + + if (bOK) + { + m_nTotalFeatureCount = 0; + } +#endif + + if (bOK) + { + m_poDS->SoftCommitTransaction(); + } + else + { + m_poDS->SoftRollbackTransaction(); +#ifdef ENABLE_GPKG_OGR_CONTENTS + DisableFeatureCount(); +#endif + CPLError(CE_Failure, CPLE_AppDefined, "Truncate(%s) failed: %s", + m_pszTableName, pszErrMsg ? pszErrMsg : "(unknown reason)"); + } + sqlite3_free(pszErrMsg); + + return bOK ? OGRERR_NONE : OGRERR_FAILURE; +}