I have this simple piece of code:
from google.cloud import datastore
import requests
ds_c = datastore.Client(_http=requests.Session)
for entity in ds_c.query(kind='Kind').fetch():
ds_c.delete(entity.key)
I am getting:
google.api_core.exceptions.ResourceExhausted: 429 Received message larger than max (4207799 vs. 4194304)
Full stack trace:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/dist-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
return callable_(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/grpc/_channel.py", line 826, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.7/dist-packages/grpc/_channel.py", line 729, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.RESOURCE_EXHAUSTED
details = "Received message larger than max (4207799 vs. 4194304)"
debug_error_string = "{"created":"@1589308786.798030838","description":"Received message larger than max (4207799 vs. 4194304)","file":"src/core/ext/filters/message_size/message_size_filter.cc","file_line":191,"grpc_status":8}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/vagrant/.local/bin/fab", line 8, in <module>
sys.exit(program.run())
File "/home/vagrant/.local/lib/python3.7/site-packages/invoke/program.py", line 384, in run
self.execute()
File "/home/vagrant/.local/lib/python3.7/site-packages/invoke/program.py", line 566, in execute
executor.execute(*self.tasks)
File "/home/vagrant/.local/lib/python3.7/site-packages/invoke/executor.py", line 129, in execute
result = call.task(*args, **call.kwargs)
File "/home/vagrant/.local/lib/python3.7/site-packages/invoke/tasks.py", line 127, in __call__
result = self.body(*args, **kwargs)
File "/vagrant/cyclone/fabfile.py", line 34, in clean_test_datastore
for entity in ds_c.query(kind=kind).fetch():
File "/usr/local/lib/python3.7/dist-packages/google/api_core/page_iterator.py", line 212, in _items_iter
for page in self._page_iter(increment=False):
File "/usr/local/lib/python3.7/dist-packages/google/api_core/page_iterator.py", line 249, in _page_iter
page = self._next_page()
File "/usr/local/lib/python3.7/dist-packages/google/cloud/datastore/query.py", line 537, in _next_page
self._query.project, partition_id, read_options, query=query_pb
File "/usr/local/lib/python3.7/dist-packages/google/cloud/datastore_v1/gapic/datastore_client.py", line 384, in run_query
request, retry=retry, timeout=timeout, metadata=metadata
File "/usr/local/lib/python3.7/dist-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
return wrapped_func(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/google/api_core/retry.py", line 286, in retry_wrapped_func
on_error=on_error,
File "/usr/local/lib/python3.7/dist-packages/google/api_core/retry.py", line 184, in retry_target
return target()
File "/usr/local/lib/python3.7/dist-packages/google/api_core/timeout.py", line 214, in func_with_timeout
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
six.raise_from(exceptions.from_grpc_error(exc), exc)
File "<string>", line 3, in raise_from
google.api_core.exceptions.ResourceExhausted: 429 Received message larger than max (4207799 vs. 4194304)
The easiest work around would be to fetch only the keys you are looking for, e.g. for entity in ds_c.query(kind='Kind').keys_only().fetch(): It's probably a good idea to add a limit to your fetch to reduce the size of the response, e.g. for entity in ds_c.query(kind='Kind').keys_only().fetch(limit=500):