unfortunately I don't want to rely on additional packages except of the googleapiclient one and facing some issue on downloading objects from a storage bucket.
from googleapiclient.http import MediaIoBaseDownload
import googleapiclient.discovery
storage_service = googleapiclient.discovery.build(
serviceName='storage', version='v1', credentials=creds)
f = storage_service.objects()
results = f.list(bucket='MYBUCKET').execute()
for d in results['items']:
with open(d['name']), 'wb') as fh:
req = MediaIoBaseDownload(
fh,
f.get_media(bucket=d['bucket'], object=d['name'], generation=d['generation']),
chunksize=1024*1024
)
done = False
while done is False:
status, done = req.next_chunk()
Now this yieldds the following error:
---------------------------------------------------------------------------
HttpError Traceback (most recent call last)
<ipython-input-210-d66ce751dec5> in <module>()
4 done = False
5 while done is False:
----> 6 status, done = req.next_chunk()
path_to_my_env/lib/python2.7/site-packages/googleapiclient/_helpers.pyc in positional_wrapper(*args, **kwargs)
128 elif positional_parameters_enforcement == POSITIONAL_WARNING:
129 logger.warning(message)
--> 130 return wrapped(*args, **kwargs)
131 return positional_wrapper
132
path_to_my_env/lib/python2.7/site-packages/googleapiclient/http.pyc in next_chunk(self, num_retries)
703 return MediaDownloadProgress(self._progress, self._total_size), self._done
704 else:
--> 705 raise HttpError(resp, content, uri=self._uri)
706
707
HttpError: <HttpError 416 when requesting https://www.googleapis.com/storage/v1/b/MYBUCKET/o/MYOBJECT?generation=1234&alt=media returned "Requested range not satisfiable">
Is somebody aware of what I'm missing somewhere or what is best practice to download files from storage? Everything I found relies on storage specific libraries.
This seems to be related to this issue.
Some of the files (most) are 0-byte files.
Problem is fixed by the following code:
for d in results['items']:
request = f.get_media(bucket=d['bucket'], object=d['name'], generation=d['generation'])
response = request.execute()
if response != '':
with open(d['name']), 'wb') as fh:
fh.write(response)