'test_new_bucket_w_ubla' flakes w/ 412 PreconditionFailed

[tseaver]

From this failed run:

____________________________ test_new_bucket_w_ubla ____________________________

storage_client = <google.cloud.storage.client.Client object at 0x7fea45c62510>
buckets_to_delete = [<Bucket: new-w-ubla-1630575370947>], blobs_to_delete = []

    def test_new_bucket_w_ubla(
        storage_client, buckets_to_delete, blobs_to_delete,
    ):
        bucket_name = _helpers.unique_name("new-w-ubla")
        bucket = storage_client.bucket(bucket_name)
        bucket.iam_configuration.uniform_bucket_level_access_enabled = True
        _helpers.retry_429_503(bucket.create)()
        buckets_to_delete.append(bucket)

        bucket_acl = bucket.acl
        with pytest.raises(exceptions.BadRequest):
            bucket_acl.reload()

        bucket_acl.loaded = True  # Fake that we somehow loaded the ACL
        bucket_acl.all().grant_read()
        with pytest.raises(exceptions.BadRequest):
>           bucket_acl.save()

tests/system/test_bucket.py:711:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
google/cloud/storage/acl.py:546: in save
    self._save(acl, None, client, timeout=timeout)
google/cloud/storage/acl.py:510: in _save
    retry=None,
google/cloud/storage/client.py:454: in _patch_resource
    _target_object=_target_object,
google/cloud/storage/_http.py:78: in api_request
    return call()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <google.cloud.storage._http.Connection object at 0x7fea478ae390>
method = 'PATCH', path = '/b/new-w-ubla-1630575370947'
query_params = {'projection': 'full'}
data = '{"acl": [{"role": "READER", "entity": "allUsers"}]}'
content_type = 'application/json', headers = None, api_base_url = None
api_version = None, expect_json = True, _target_object = None, timeout = 60

    def api_request(
        self,
        method,
        path,
        query_params=None,
        data=None,
        content_type=None,
        headers=None,
        api_base_url=None,
        api_version=None,
        expect_json=True,
        _target_object=None,
        timeout=_DEFAULT_TIMEOUT,
    ):
        """Make a request over the HTTP transport to the API.
       ...
        """
        url = self.build_api_url(
            path=path,
            query_params=query_params,
            api_base_url=api_base_url,
            api_version=api_version,
        )

        # Making the executive decision that any dictionary
        # data will be sent properly as JSON.
        if data and isinstance(data, dict):
            data = json.dumps(data)
            content_type = "application/json"

        response = self._make_request(
            method=method,
            url=url,
            data=data,
            content_type=content_type,
            headers=headers,
            target_object=_target_object,
            timeout=timeout,
        )

        if not 200 <= response.status_code < 300:
>           raise exceptions.from_http_response(response)
E           PreconditionFailed: 412 PATCH https://storage.googleapis.com/storage/v1/b/new-w-ubla-1630575370947?projection=full&prettyPrint=false: One or more users named in the policy do not belong to a permitted customer.

[tseaver]

@frankyn, @andrewsg, @cojenco I can't see any reason for the error: has something on the back-end changed?