I would like to shorten the following code such that it downloads the bucket subfolder from the Amazon server
import boto3
import botocore
import datetime
import xarray as xr
import matplotlib.pyplot as plt
import cfgrib
#cfs_bucket = 'noaa-cfs-pds'
cfs_bucket = 'noaa-gefs-pds'
# Want to get 5 days worth of forecast data = 5 days x 4 forecast cycles/day
cycles2get = 4*5
client = boto3.client('s3', config=botocore.client.Config(signature_version=botocore.UNSIGNED))
paginator = client.get_paginator('list_objects')
result = paginator.paginate(Bucket=cfs_bucket, Delimiter='/')
count = 0
for prefix in result.search('CommonPrefixes'):
print(prefix.get('Prefix'))
count += 1
if(count>49):
break # skip the for loop after reporting 50 values
gefs.20170101/
gefs.20170102/
gefs.20170103/
gefs.20170104/
gefs.20170105/
gefs.20170106/
gefs.20170107/
gefs.20170108/
gefs.20170109/
gefs.20170110/
gefs.20170111/
gefs.20170112/
gefs.20170113/
gefs.20170114/
gefs.20170115/
gefs.20170116/
gefs.20170117/
gefs.20170118/
gefs.20170119/
gefs.20170120/
gefs.20170121/
gefs.20170122/
gefs.20170123/
gefs.20170124/
gefs.20170125/
gefs.20170126/
gefs.20170127/
gefs.20170128/
gefs.20170129/
gefs.20170130/
gefs.20170131/
gefs.20170201/
gefs.20170202/
gefs.20170203/
gefs.20170204/
gefs.20170205/
gefs.20170206/
gefs.20170207/
gefs.20170208/
gefs.20170209/
gefs.20170210/
gefs.20170211/
gefs.20170212/
gefs.20170213/
gefs.20170214/
gefs.20170215/
gefs.20170216/
gefs.20170217/
gefs.20170218/
gefs.20170219/
keys = []
date = datetime.datetime(2017,11,28,hour=0)
prefix = date.strftime('gefs.%Y%m%d/00/')
print(prefix)
response = client.list_objects_v2(Bucket=cfs_bucket, Prefix=prefix)
response_meta = response.get('ResponseMetadata')
if response_meta.get('HTTPStatusCode') == 200:
contents = response.get('Contents')
if contents == None:
print("No objects are available for %s" % date.strftime('%B %d, %Y'))
else:
for obj in contents:
keys.append(obj.get('Key'))
print("There are %s objects available for %s\n--" % (len(keys), date.strftime('%B %d, %Y')))
count = 0
for k in keys:
print(k)
count += 1
if(count>1200):
break # skip the for loop after reporting 50 values
else:
print("There was an error with your request.")
gefs_filebase = 'gec'
sdate = date
ensnum = '00' # GEFS Ensemble number
tz = 't00z'
pgr = 'pgrb2af'
pga = 'pgrb2aanl'
fdate = sdate
flx_files = []
for i in range(cycles2get):
sdatestr = sdate.strftime('%Y%m%d')
fdatestr = fdate.strftime('%Y%m%d')
flx_file_A = gefs_filebase + ensnum + '.' + tz + '.' + pga
flx_file_0 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '000'
flx_file_6 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '006'
flx_file_12 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '012'
flx_file_18 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '018'
flx_file_24 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '024'
flx_file_30 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '030'
flx_file_36 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '036'
flx_file_42 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '042'
flx_file_48 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '048'
flx_file_54 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '054'
flx_file_60 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '060'
flx_file_66 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '066'
flx_file_72 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '072'
flx_file_78 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '078'
flx_file_84 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '084'
flx_file_90 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '090'
flx_file_96 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '096'
flx_file_102 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '102'
flx_file_108 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '108'
flx_file_114 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '114'
flx_file_120 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '120'
flx_file_126 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '126'
flx_file_132 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '132'
flx_file_138 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '138'
flx_file_144 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '144'
flx_file_150 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '150'
flx_file_156 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '156'
flx_file_162 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '162'
flx_file_168 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '168'
flx_file_174 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '174'
flx_file_180 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '180'
flx_file_186 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '186'
flx_file_192 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '192'
flx_file_198 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '198'
flx_file_204 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '204'
flx_file_210 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '210'
flx_file_216 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '216'
flx_file_210 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '210'
flx_file_216 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '216'
flx_file_222 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '222'
flx_file_228 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '228'
flx_file_234 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '234'
flx_file_240 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '240'
flx_file_246 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '246'
flx_file_252 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '252'
flx_file_258 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '258'
flx_file_264 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '264'
flx_file_270 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '270'
flx_file_276 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '276'
flx_file_282 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '282'
flx_file_288 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '288'
flx_file_288 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '288'
flx_file_294 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '294'
flx_file_300 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '300'
flx_file_306 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '306'
flx_file_312 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '312'
flx_file_318 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '318'
flx_file_324 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '324'
flx_file_330 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '330'
flx_file_336 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '336'
flx_file_342 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '342'
flx_file_348 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '348'
flx_file_354 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '354'
flx_file_360 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '360'
flx_file_366 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '366'
flx_file_372 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '372'
flx_file_378 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '378'
flx_file_384 = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '384'
# flx_file = gefs_filebase + ensnum + '.' + tz + '.' + pgr + f'{(fcst)}'
flx_files.append(flx_file)
# Update the valid datetime by adding six hours from the valid time just used
fdate = fdate + datetime.timedelta(hours=6)
print('Downloading '+flx_file_A)
client.download_file(cfs_bucket,prefix+flx_file_A,flx_file_A)
print('Downloading '+flx_file_0)
client.download_file(cfs_bucket,prefix+flx_file_0,flx_file_0)
print('Downloading '+flx_file_6)
client.download_file(cfs_bucket,prefix+flx_file_6,flx_file_6)
print('Downloading '+flx_file_12)
client.download_file(cfs_bucket,prefix+flx_file_12,flx_file_12)
print('Downloading '+flx_file_18)
client.download_file(cfs_bucket,prefix+flx_file_18,flx_file_18)
print('Downloading '+flx_file_24)
client.download_file(cfs_bucket,prefix+flx_file_24,flx_file_24)
print('Downloading '+flx_file_30)
client.download_file(cfs_bucket,prefix+flx_file_30,flx_file_30)
print('Downloading '+flx_file_36)
client.download_file(cfs_bucket,prefix+flx_file_36,flx_file_36)
print('Downloading '+flx_file_42)
client.download_file(cfs_bucket,prefix+flx_file_42,flx_file_42)
print('Downloading '+flx_file_48)
client.download_file(cfs_bucket,prefix+flx_file_48,flx_file_48)
print('Downloading '+flx_file_54)
client.download_file(cfs_bucket,prefix+flx_file_54,flx_file_54)
print('Downloading '+flx_file_60)
client.download_file(cfs_bucket,prefix+flx_file_60,flx_file_60)
print('Downloading '+flx_file_66)
client.download_file(cfs_bucket,prefix+flx_file_66,flx_file_66)
print('Downloading '+flx_file_72)
client.download_file(cfs_bucket,prefix+flx_file_72,flx_file_72)
print('Downloading '+flx_file_78)
client.download_file(cfs_bucket,prefix+flx_file_78,flx_file_78)
print('Downloading '+flx_file_84)
client.download_file(cfs_bucket,prefix+flx_file_84,flx_file_84)
print('Downloading '+flx_file_90)
client.download_file(cfs_bucket,prefix+flx_file_90,flx_file_90)
print('Downloading '+flx_file_96)
client.download_file(cfs_bucket,prefix+flx_file_96,flx_file_96)
print('Downloading '+flx_file_102)
client.download_file(cfs_bucket,prefix+flx_file_102,flx_file_102)
print('Downloading '+flx_file_108)
client.download_file(cfs_bucket,prefix+flx_file_108,flx_file_108)
print('Downloading '+flx_file_114)
client.download_file(cfs_bucket,prefix+flx_file_114,flx_file_114)
print('Downloading '+flx_file_120)
client.download_file(cfs_bucket,prefix+flx_file_120,flx_file_120)
print('Downloading '+flx_file_126)
client.download_file(cfs_bucket,prefix+flx_file_126,flx_file_126)
print('Downloading '+flx_file_132)
client.download_file(cfs_bucket,prefix+flx_file_132,flx_file_132)
print('Downloading '+flx_file_138)
client.download_file(cfs_bucket,prefix+flx_file_138,flx_file_138)
print('Downloading '+flx_file_144)
client.download_file(cfs_bucket,prefix+flx_file_144,flx_file_144)
print('Downloading '+flx_file_150)
client.download_file(cfs_bucket,prefix+flx_file_150,flx_file_150)
print('Downloading '+flx_file_156)
client.download_file(cfs_bucket,prefix+flx_file_156,flx_file_156)
print('Downloading '+flx_file_162)
client.download_file(cfs_bucket,prefix+flx_file_162,flx_file_162)
print('Downloading '+flx_file_168)
client.download_file(cfs_bucket,prefix+flx_file_168,flx_file_168)
print('Downloading '+flx_file_174)
client.download_file(cfs_bucket,prefix+flx_file_174,flx_file_174)
print('Downloading '+flx_file_180)
client.download_file(cfs_bucket,prefix+flx_file_180,flx_file_180)
print('Downloading '+flx_file_186)
client.download_file(cfs_bucket,prefix+flx_file_186,flx_file_186)
print('Downloading '+flx_file_192)
client.download_file(cfs_bucket,prefix+flx_file_192,flx_file_192)
print('Downloading '+flx_file_198)
client.download_file(cfs_bucket,prefix+flx_file_198,flx_file_198)
print('Downloading '+flx_file_204)
client.download_file(cfs_bucket,prefix+flx_file_204,flx_file_204)
print('Downloading '+flx_file_210)
client.download_file(cfs_bucket,prefix+flx_file_210,flx_file_210)
print('Downloading '+flx_file_216)
client.download_file(cfs_bucket,prefix+flx_file_216,flx_file_216)
print('Downloading '+flx_file_222)
client.download_file(cfs_bucket,prefix+flx_file_222,flx_file_222)
print('Downloading '+flx_file_228)
client.download_file(cfs_bucket,prefix+flx_file_228,flx_file_228)
print('Downloading '+flx_file_234)
client.download_file(cfs_bucket,prefix+flx_file_234,flx_file_234)
print('Downloading '+flx_file_240)
client.download_file(cfs_bucket,prefix+flx_file_240,flx_file_240)
print('Downloading '+flx_file_246)
client.download_file(cfs_bucket,prefix+flx_file_246,flx_file_246)
print('Downloading '+flx_file_252)
client.download_file(cfs_bucket,prefix+flx_file_252,flx_file_252)
print('Downloading '+flx_file_258)
client.download_file(cfs_bucket,prefix+flx_file_258,flx_file_258)
print('Downloading '+flx_file_264)
client.download_file(cfs_bucket,prefix+flx_file_264,flx_file_264)
print('Downloading '+flx_file_270)
client.download_file(cfs_bucket,prefix+flx_file_270,flx_file_270)
print('Downloading '+flx_file_276)
client.download_file(cfs_bucket,prefix+flx_file_276,flx_file_276)
print('Downloading '+flx_file_282)
client.download_file(cfs_bucket,prefix+flx_file_282,flx_file_282)
print('Downloading '+flx_file_288)
client.download_file(cfs_bucket,prefix+flx_file_288,flx_file_288)
print('Downloading '+flx_file_294)
client.download_file(cfs_bucket,prefix+flx_file_294,flx_file_294)
print('Downloading '+flx_file_300)
client.download_file(cfs_bucket,prefix+flx_file_300,flx_file_300)
print('Downloading '+flx_file_306)
client.download_file(cfs_bucket,prefix+flx_file_306,flx_file_306)
print('Downloading '+flx_file_312)
client.download_file(cfs_bucket,prefix+flx_file_312,flx_file_312)
print('Downloading '+flx_file_318)
client.download_file(cfs_bucket,prefix+flx_file_318,flx_file_318)
print('Downloading '+flx_file_324)
client.download_file(cfs_bucket,prefix+flx_file_324,flx_file_324)
print('Downloading '+flx_file_330)
client.download_file(cfs_bucket,prefix+flx_file_330,flx_file_330)
print('Downloading '+flx_file_336)
client.download_file(cfs_bucket,prefix+flx_file_336,flx_file_336)
print('Downloading '+flx_file_342)
client.download_file(cfs_bucket,prefix+flx_file_342,flx_file_342)
print('Downloading '+flx_file_348)
client.download_file(cfs_bucket,prefix+flx_file_348,flx_file_348)
print('Downloading '+flx_file_354)
client.download_file(cfs_bucket,prefix+flx_file_354,flx_file_354)
print('Downloading '+flx_file_360)
client.download_file(cfs_bucket,prefix+flx_file_360,flx_file_360)
print('Downloading '+flx_file_366)
client.download_file(cfs_bucket,prefix+flx_file_366,flx_file_366)
print('Downloading '+flx_file_372)
client.download_file(cfs_bucket,prefix+flx_file_372,flx_file_372)
print('Downloading '+flx_file_378)
client.download_file(cfs_bucket,prefix+flx_file_378,flx_file_378)
print('Downloading '+flx_file_384)
client.download_file(cfs_bucket,prefix+flx_file_384,flx_file_384)
The above Code works to download the specific files. But I would like to shorten this code so that it loops the forecast time from 000 to 384, how can I modify the code above? And in this case direct these files to a specific directory with the name of the BUCKET
You can use a for..loop structure:
for i in range(cycles2get):
sdatestr = sdate.strftime('%Y%m%d')
fdatestr = fdate.strftime('%Y%m%d')
for i in range(0,384,6):
flx_file = gefs_filebase + ensnum + '.' + tz + '.' + pgr + '{:0>3}'.format(i)
print('Downloading '+flx_file)
client.download_file(cfs_bucket,prefix+flx_file,flx_file)
This will work identically, except for the line of the pga variable, because it is different from the pattern.
And this line
flx_files.append(flx_file)
isn't necessary.