I have 375000 items in my table. I am doing a loop to obtain all id of all items, with API limit set to 20000 items per api call. After 200000 I always start to get httpx.ReadTimeout: The read operation timed out sometime it may reach 240000 but never go ahead.
I have tried to have different wait time after each loop. I have tried to change api limit to 10000 as well as increase it to 30000 or 50000 make less calls but in all cases it get's stuck at around 150000 or 200000.
existing_search_result = supabase.table('vehicles').select('ref_id', count='exact').order('id', desc=False).execute()
existing_items = []
range_step = len(existing_search_result.data)
total_existing_items = existing_search_result.count
print(total_existing_items)
while len(existing_items) < total_existing_items:
try:
existing_items += (
supabase.table(
'vehicles'
).select('ref_id')
.order('id', desc=False)
.range(range_start, range_start + range_step)
.execute()
).data
range_start += range_step
except Exception as e:
logging.exception(e)
print(range_start, len(existing_items))
time.sleep(0.30)
Error log:
2022-10-23 21:04:14,168:ERROR - The read operation timed out
Traceback (most recent call last):
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_exceptions.py", line 8, in map_exceptions
yield
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/backends/sync.py", line 26, in read
return self._sock.recv(max_bytes)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/ssl.py", line 1226, in recv
return self.read(buflen)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/ssl.py", line 1101, in read
return self._sslobj.read(len)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 204, in handle_request
resp = self._pool.handle_request(req)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py", line 253, in handle_request
raise exc
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py", line 237, in handle_request
response = connection.handle_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/connection.py", line 90, in handle_request
return self._connection.handle_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 102, in handle_request
raise exc
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 81, in handle_request
) = self._receive_response_headers(**kwargs)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 143, in _receive_response_headers
event = self._receive_event(timeout=timeout)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 172, in _receive_event
data = self._network_stream.read(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/backends/sync.py", line 26, in read
return self._sock.recv(max_bytes)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py", line 135, in __exit__
self.gen.throw(type, value, traceback)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_exceptions.py", line 12, in map_exceptions
raise to_exc(exc)
httpcore.ReadTimeout: The read operation timed out
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/ak4zh/updater/main.py", line 278, in job
supabase.table(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/postgrest/_sync/request_builder.py", line 53, in execute
r = self.session.request(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 802, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 889, in send
response = self._send_handling_auth(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 917, in _send_handling_auth
response = self._send_handling_redirects(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 954, in _send_handling_redirects
response = self._send_single_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 990, in _send_single_request
response = transport.handle_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 204, in handle_request
resp = self._pool.handle_request(req)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py", line 135, in __exit__
self.gen.throw(type, value, traceback)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ReadTimeout: The read operation timed out
I had the same problem, so I create a function to get 1000 item and merge it! this is my code:
def count_data(tb_name: str, field_name: str):
return supabase.table(tb_name).select(field_name, count='exact').execute().count
def get_field_data(tb_name: str, src_field: str, len_record: int, id_field: str = 'id'):
if len_record <= 1000:
field_data = supabase.table(tb_name).select(id_field, src_field).order(
column=id_field).execute().data
else:
rnk = int(len_record / 1000)
field_data = []
for i in range(rnk):
min_rg = (i * 1000) + 1
max_rg = (i + 1) * 1000
field_data = field_data + supabase.table(tb_name).select(id_field, src_field).order(
column=id_field).range(min_rg - 1, max_rg).execute().data
field_data = field_data + supabase.table(tb_name).select(id_field, src_field).order(
column=id_field).range(max_rg, len_record).execute().data
return field_data