Skip to content

Commit

Permalink
Merge pull request #10730 from rouault/no_cache
Browse files Browse the repository at this point in the history
/vsicurl/: honor 'Cache-Control: no-cache' header
  • Loading branch information
rouault authored Sep 5, 2024
2 parents 99c7be4 + 595cc85 commit 0cd4fba
Show file tree
Hide file tree
Showing 3 changed files with 203 additions and 2 deletions.
86 changes: 86 additions & 0 deletions autotest/gcore/vsicurl.py
Original file line number Diff line number Diff line change
Expand Up @@ -1288,3 +1288,89 @@ def test_vsicurl_404_repeated_same_resource(server):

with pytest.raises(Exception, match="404"):
gdal.Open("/vsicurl/http://localhost:%d/does/not/exist.bin" % server.port)


###############################################################################


@gdaltest.enable_exceptions()
def test_vsicurl_cache_control_not_set(server):

gdal.VSICurlClearCache()

handler = webserver.SequentialHandler()
handler.add("GET", "/", 404)
handler.add("HEAD", "/test.txt", 200, {"Content-Length": "3"})
handler.add("GET", "/test.txt", 200, {}, "foo")
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
"/vsicurl/http://localhost:%d/test.txt" % server.port,
"rb",
)
assert f is not None
data = gdal.VSIFReadL(1, 3, f).decode("ascii")
gdal.VSIFCloseL(f)
assert data == "foo"

with webserver.install_http_handler(webserver.SequentialHandler()):
f = gdal.VSIFOpenL(
"/vsicurl/http://localhost:%d/test.txt" % server.port,
"rb",
)
assert f is not None
data = gdal.VSIFReadL(1, 3, f).decode("ascii")
gdal.VSIFCloseL(f)
assert data == "foo"


###############################################################################


@gdaltest.enable_exceptions()
def test_vsicurl_cache_control_no_cache(server):

gdal.VSICurlClearCache()

handler = webserver.SequentialHandler()
handler.add("GET", "/", 404)
handler.add(
"HEAD", "/test.txt", 200, {"Content-Length": "3", "Cache-Control": "no-cache"}
)
handler.add(
"GET",
"/test.txt",
200,
{"Content-Length": "3", "Cache-Control": "no-cache"},
"foo",
)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
"/vsicurl/http://localhost:%d/test.txt" % server.port,
"rb",
)
assert f is not None
data = gdal.VSIFReadL(1, 3, f).decode("ascii")
gdal.VSIFCloseL(f)
assert data == "foo"

handler = webserver.SequentialHandler()
handler.add("GET", "/", 404)
handler.add(
"HEAD", "/test.txt", 200, {"Content-Length": "6", "Cache-Control": "no-cache"}
)
handler.add(
"GET",
"/test.txt",
200,
{"Content-Length": "6", "Cache-Control": "no-cache"},
"barbaz",
)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
"/vsicurl/http://localhost:%d/test.txt" % server.port,
"rb",
)
assert f is not None
data = gdal.VSIFReadL(1, 6, f).decode("ascii")
gdal.VSIFCloseL(f)
assert data == "barbaz"
106 changes: 106 additions & 0 deletions autotest/gcore/vsis3.py
Original file line number Diff line number Diff line change
Expand Up @@ -2199,6 +2199,112 @@ def method(request):
assert gdal.GetLastErrorMsg() == ""


###############################################################################
# Test that PUT invalidates cached data


def test_vsis3_put_invalidate(aws_test_config, webserver_port):

gdal.VSICurlClearCache()

handler = webserver.SequentialHandler()
handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")

with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
assert f is not None
try:
assert gdal.VSIFReadL(3, 1, f) == b"foo"
finally:
gdal.VSIFCloseL(f)

handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
assert f is not None
try:
assert gdal.VSIFReadL(3, 1, f) == b"foo"
finally:
gdal.VSIFCloseL(f)

handler = webserver.SequentialHandler()
handler.add("PUT", "/s3_fake_bucket3/test_put_invalidate.bin", 200)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "wb")
assert f is not None
try:
assert gdal.VSIFWriteL("barbaw", 1, 6, f) == 6
finally:
gdal.VSIFCloseL(f)

handler = webserver.SequentialHandler()
handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")

with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
assert f is not None
try:
assert gdal.VSIFReadL(6, 1, f) == b"barbaw"
finally:
gdal.VSIFCloseL(f)


###############################################################################
# Test that CopyFile invalidates cached data


def test_vsis3_copy_invalidate(aws_test_config, webserver_port, tmp_vsimem):

gdal.VSICurlClearCache()

handler = webserver.SequentialHandler()
handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo")

with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
assert f is not None
try:
assert gdal.VSIFReadL(3, 1, f) == b"foo"
finally:
gdal.VSIFCloseL(f)

handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
assert f is not None
try:
assert gdal.VSIFReadL(3, 1, f) == b"foo"
finally:
gdal.VSIFCloseL(f)

handler = webserver.SequentialHandler()
handler.add("PUT", "/s3_fake_bucket3/test_put_invalidate.bin", 200)
memfilename = str(tmp_vsimem / "tmp.bin")
with webserver.install_http_handler(handler), gdaltest.tempfile(
memfilename, b"barbaw"
):
gdal.CopyFile(memfilename, "/vsis3/s3_fake_bucket3/test_put_invalidate.bin")

handler = webserver.SequentialHandler()
handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200)
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")
handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw")

with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb")
assert f is not None
try:
assert gdal.VSIFReadL(6, 1, f) == b"barbaw"
finally:
gdal.VSIFCloseL(f)


###############################################################################
# Test simple PUT support with retry logic

Expand Down
13 changes: 11 additions & 2 deletions port/cpl_vsil_curl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1363,7 +1363,6 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError,
}
}

if (bGetHeaders)
{
char **papszHeaders =
CSLTokenizeString2(sWriteFuncHeaderData.pBuffer, "\r\n", 0);
Expand All @@ -1374,7 +1373,17 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError,
CPLParseNameValue(papszHeaders[i], &pszKey);
if (pszKey && pszValue)
{
m_aosHeaders.SetNameValue(pszKey, pszValue);
if (bGetHeaders)
{
m_aosHeaders.SetNameValue(pszKey, pszValue);
}
if (EQUAL(pszKey, "Cache-Control") &&
EQUAL(pszValue, "no-cache") &&
CPLTestBool(CPLGetConfigOption(
"CPL_VSIL_CURL_HONOR_CACHE_CONTROL", "YES")))
{
m_bCached = false;
}
}
CPLFree(pszKey);
}
Expand Down

0 comments on commit 0cd4fba

Please sign in to comment.