Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GPKG: optimize speed of 'DELETE FROM table_name', especially on ones with RTree #11139

Merged
merged 1 commit into from
Oct 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 184 additions & 4 deletions autotest/ogr/ogr_gpkg.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,9 @@ def gpkg_ds(gpkg_dsn):
return ogr.GetDriverByName("GPKG").CreateDataSource(gpkg_dsn)


@pytest.fixture()
def tpoly(gpkg_ds, poly_feat):
def create_test_layer(ds, src_features, layer_name):

lyr = gpkg_ds.CreateLayer("tpoly")
lyr = ds.CreateLayer(layer_name)

ogrtest.quick_create_layer_def(
lyr,
Expand All @@ -94,11 +93,21 @@ def tpoly(gpkg_ds, poly_feat):

dst_feat = ogr.Feature(feature_def=lyr.GetLayerDefn())

for feat in poly_feat:
for feat in src_features:
dst_feat.SetFrom(feat)
lyr.CreateFeature(dst_feat)


@pytest.fixture()
def tpoly(gpkg_ds, poly_feat):
create_test_layer(gpkg_ds, poly_feat, "tpoly")


@pytest.fixture()
def tpoly_dbl_quote(gpkg_ds, poly_feat):
create_test_layer(gpkg_ds, poly_feat, 'tpoly"dbl_quote')


@pytest.fixture()
def a_layer(gpkg_ds):
gpkg_ds.CreateLayer("a_layer", options=["SPATIAL_INDEX=NO"])
Expand Down Expand Up @@ -400,6 +409,177 @@ def test_ogr_gpkg_5(gpkg_ds):
assert gpkg_ds.GetLayerCount() == 0, "unexpected number of layers (not 0)"


###############################################################################
# Truncate a spatial layer


@pytest.mark.usefixtures("tpoly_dbl_quote")
@gdaltest.enable_exceptions()
def test_ogr_gpkg_truncate_spatial_layer(gpkg_ds, poly_feat):

gpkg_ds.ExecuteSQL('DELETE FROM "tpoly""dbl_quote"')

lyr = gpkg_ds.GetLayer(0)
assert lyr.GetFeatureCount() == 0

# Re-insert records
dst_feat = ogr.Feature(feature_def=lyr.GetLayerDefn())
for feat in poly_feat:
dst_feat.SetFrom(feat)
lyr.CreateFeature(dst_feat)

# Check that spatial index works fine
for feat in poly_feat:
minx, maxx, miny, maxy = feat.GetGeometryRef().GetEnvelope()
lyr.SetSpatialFilterRect(minx, miny, maxx, maxy)
assert lyr.GetFeatureCount() >= 1


###############################################################################
# Truncate a non-spatial layer


@gdaltest.enable_exceptions()
def test_ogr_gpkg_truncate_non_spatial_layer(tmp_vsimem):

filename = str(tmp_vsimem / "test.gpkg")
with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds:
lyr = ds.CreateLayer("test", geom_type=ogr.wkbNone)
lyr.CreateField(ogr.FieldDefn("foo"))
f = ogr.Feature(lyr.GetLayerDefn())
f["foo"] = "bar"
lyr.CreateFeature(f)

# Test that the optimization doesn't trigger when there's a WHERE clause
with ogr.Open(filename, update=1) as ds:
ds.ExecuteSQL("DELETE FROM test WHERE 0")
lyr = ds.GetLayer(0)
assert lyr.GetFeatureCount() == 1

with ogr.Open(filename, update=1) as ds:
ds.ExecuteSQL("DELETE FROM test")
lyr = ds.GetLayer(0)
assert lyr.GetFeatureCount() == 0

with ogr.Open(filename) as ds:
lyr = ds.GetLayer(0)
assert lyr.GetFeatureCount() == 0


###############################################################################
# Truncate a layer with a custom trigger


@gdaltest.enable_exceptions()
def test_ogr_gpkg_truncate_with_custom_trigger(tmp_vsimem):

filename = str(tmp_vsimem / "test.gpkg")
with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds:
lyr = ds.CreateLayer("test", geom_type=ogr.wkbNone)
lyr.CreateField(ogr.FieldDefn("foo"))
f = ogr.Feature(lyr.GetLayerDefn())
f["foo"] = "bar"
lyr.CreateFeature(f)
ds.ExecuteSQL(
"CREATE TABLE counter(table_name TEXT UNIQUE NOT NULL, counter INT)"
)
ds.ExecuteSQL("INSERT INTO counter VALUES ('test', 1)")
ds.ExecuteSQL(
"CREATE TRIGGER my_trigger AFTER DELETE ON test BEGIN UPDATE counter SET counter = counter - 1 WHERE table_name = 'test'; END"
)

with ogr.Open(filename, update=1) as ds:
with ds.ExecuteSQL(
"SELECT counter FROM counter WHERE table_name = 'test'"
) as sql_lyr:
f = sql_lyr.GetNextFeature()
assert f["counter"] == 1
ds.ExecuteSQL('DELETE FROM "test"')
lyr = ds.GetLayer(0)
assert lyr.GetFeatureCount() == 0
with ds.ExecuteSQL(
"SELECT counter FROM counter WHERE table_name = 'test'"
) as sql_lyr:
f = sql_lyr.GetNextFeature()
assert f["counter"] == 0

with ogr.Open(filename) as ds:
lyr = ds.GetLayer(0)
assert lyr.GetFeatureCount() == 0


###############################################################################
# Truncate a layer with a custom trigger that causes an error


@gdaltest.enable_exceptions()
def test_ogr_gpkg_truncate_with_custom_trigger_error(tmp_vsimem):

filename = str(tmp_vsimem / "test.gpkg")
with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds:
lyr = ds.CreateLayer("test", geom_type=ogr.wkbNone)
lyr.CreateField(ogr.FieldDefn("foo"))
f = ogr.Feature(lyr.GetLayerDefn())
f["foo"] = "bar"
lyr.CreateFeature(f)
ds.ExecuteSQL(
"CREATE TABLE counter(table_name TEXT UNIQUE NOT NULL, counter INT)"
)
ds.ExecuteSQL("INSERT INTO counter VALUES ('test', 1)")
ds.ExecuteSQL(
"CREATE TRIGGER my_trigger AFTER DELETE ON test BEGIN SELECT RAISE(ABORT, 'error from trigger'); END"
)

with ogr.Open(filename, update=1) as ds:
with ds.ExecuteSQL(
"SELECT counter FROM counter WHERE table_name = 'test'"
) as sql_lyr:
f = sql_lyr.GetNextFeature()
assert f["counter"] == 1
with pytest.raises(Exception, match="error from trigger"):
ds.ExecuteSQL('DELETE FROM "test"')
lyr = ds.GetLayer(0)
assert lyr.GetFeatureCount() == 1
with ds.ExecuteSQL(
"SELECT counter FROM counter WHERE table_name = 'test'"
) as sql_lyr:
f = sql_lyr.GetNextFeature()
assert f["counter"] == 1

with ogr.Open(filename) as ds:
lyr = ds.GetLayer(0)
assert lyr.GetFeatureCount() == 1


###############################################################################
# Truncate a layer in read-only mode (error)


@gdaltest.enable_exceptions()
def test_ogr_gpkg_truncate_read_only_mode(tmp_vsimem):

filename = str(tmp_vsimem / "test.gpkg")
with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds:
ds.CreateLayer("test")

with ogr.Open(filename) as ds:
with pytest.raises(Exception, match="read-only"):
ds.ExecuteSQL('DELETE FROM "test"')


###############################################################################
# Truncate a layer that doe not exist


@gdaltest.enable_exceptions()
def test_ogr_gpkg_truncate_non_existing_table(tmp_vsimem):

filename = str(tmp_vsimem / "test.gpkg")
with ogr.GetDriverByName("GPKG").CreateDataSource(filename) as ds:
with pytest.raises(Exception, match="no such table"):
ds.ExecuteSQL('DELETE FROM "i_do_not_exist"')


###############################################################################
# Add fields

Expand Down
2 changes: 2 additions & 0 deletions ogr/ogrsf_frmts/gpkg/ogr_geopackage.h
Original file line number Diff line number Diff line change
Expand Up @@ -961,6 +961,8 @@ class OGRGeoPackageTableLayer final : public OGRGeoPackageLayer
virtual CPLString GetSpatialWhere(int iGeomCol,
OGRGeometry *poFilterGeom) override;

OGRErr Truncate();

bool HasSpatialIndex();

void SetPrecisionFlag(int bFlag)
Expand Down
19 changes: 19 additions & 0 deletions ogr/ogrsf_frmts/gpkg/ogrgeopackagedatasource.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7712,6 +7712,25 @@ OGRLayer *GDALGeoPackageDataset::ExecuteSQL(const char *pszSQLCommand,
return nullptr;
}

else if (STARTS_WITH_CI(osSQLCommand, "DELETE FROM "))
{
// Optimize truncation of a table, especially if it has a spatial
// index.
const CPLStringList aosTokens(SQLTokenize(osSQLCommand));
if (aosTokens.size() == 3)
{
const char *pszTableName = aosTokens[2];
OGRGeoPackageTableLayer *poLayer =
dynamic_cast<OGRGeoPackageTableLayer *>(
GetLayerByName(SQLUnescape(pszTableName)));
if (poLayer)
{
poLayer->Truncate();
return nullptr;
}
}
}

else if (pszDialect != nullptr && EQUAL(pszDialect, "INDIRECT_SQLITE"))
return GDALDataset::ExecuteSQL(osSQLCommand, poSpatialFilter, "SQLITE");
else if (pszDialect != nullptr && !EQUAL(pszDialect, "") &&
Expand Down
Loading
Loading