Problems downloading TEMPO data

[ztf5015]

For some reason I'm getting permissions errors trying to access TEMPO data in the cloud today. Any idea what's happening?

One example is using kerchunk to extract TEMPO metadata

def process_url(url, folder, inline_threshold=250, **open_kws):
filename = Path(folder).joinpath('metadata', f'{Path(url).name}.json')
print(filename)

if not filename.exists():
with fsspec.open(url, mode='rb', **open_kws) as in_file:
meta = kerchunk.hdf.SingleHdf5ToZarr(in_file, url, inline_threshold=inline_threshold)
with filename.open('wb') as out_file:
out_file.write( ujson.dumps(meta.translate()).encode() )
return filename.as_posix()

Where filename is.../explore/nobackup/people/zfasnach/metadata/TEMPO_RAD_L1_V03_20240315T000117Z_S015G06.nc.json

It returns a PermissionError: Forbidden

Or if I have the metadata and try to access the data in the file, I get the following permission error

>>> import xarray as xr
>>> import numpy as np
>>> import gc, h5py, scipy
>>> from scipy import interpolate, ndimage
>>> import time
>>> group = '/band_290_490_nm/'
>>> metadata = '/explore/nobackup/people/zfasnach/metadata/TEMPO_RAD_L1_V03_20240630T131406Z_S005G01.nc.json'
>>> data = xr.open_dataset('reference://', **{'engine': 'zarr','group': group,'backend_kwargs' : {'decode_times':False,'storage_options' : {'fo': metadata},'consolidated': False,}})
>>> data['longitude'].values
Traceback (most recent call last):
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/s3fs/core.py", line 113, in _error_wrapper
return await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/aiobotocore/client.py", line 411, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the GetObject operation: Access Denied

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/fsspec/asyn.py", line 245, in _run_coro
return await asyncio.wait_for(coro, timeout=timeout), i
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/asyncio/tasks.py", line 452, in wait_for
return await fut
^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/s3fs/core.py", line 1128, in _cat_file
return await _error_wrapper(_call_and_read, retries=self.retries)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/s3fs/core.py", line 145, in _error_wrapper
raise err
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/s3fs/core.py", line 113, in _error_wrapper
return await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/s3fs/core.py", line 1115, in _call_and_read
resp = await self._call_s3(
^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/s3fs/core.py", line 365, in _call_s3
return await _error_wrapper(
^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/s3fs/core.py", line 145, in _error_wrapper
raise err
PermissionError: Access Denied

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/dataarray.py", line 786, in values
return self.variable.values
^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/variable.py", line 540, in values
return _as_array_or_item(self._data)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/variable.py", line 338, in _as_array_or_item
data = np.asarray(data)
^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/indexing.py", line 833, in __array__
return np.asarray(self.get_duck_array(), dtype=dtype)
^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/indexing.py", line 836, in get_duck_array
self._ensure_cached()
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/indexing.py", line 830, in _ensure_cached
self.array = as_indexable(self.array.get_duck_array())
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/indexing.py", line 787, in get_duck_array
return self.array.get_duck_array()
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/indexing.py", line 657, in get_duck_array
array = array.get_duck_array()
^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/coding/variables.py", line 81, in get_duck_array
return self.func(self.array.get_duck_array())
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/indexing.py", line 650, in get_duck_array
array = self.array[self.key]
~~~~~~~~~~^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/backends/zarr.py", line 104, in __getitem__
return indexing.explicit_indexing_adapter(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/core/indexing.py", line 1014, in explicit_indexing_adapter
result = raw_indexing_method(raw_key.tuple)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/xarray/backends/zarr.py", line 94, in _getitem
return self._array[key]
~~~~~~~~~~~^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/zarr/core.py", line 798, in __getitem__
result = self.get_orthogonal_selection(pure_selection, fields=fields)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/zarr/core.py", line 1080, in get_orthogonal_selection
return self._get_selection(indexer=indexer, out=out, fields=fields)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/zarr/core.py", line 1343, in _get_selection
self._chunk_getitems(
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/zarr/core.py", line 2179, in _chunk_getitems
cdatas = self.chunk_store.getitems(ckeys, contexts=contexts)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/explore/nobackup/people/zfasnach/miniconda3/envs/gpu/lib/python3.11/site-packages/zarr/storage.py", line 1435, in getitems
raise v
fsspec.implementations.reference.ReferenceNotReachable: Reference "band_290_490_nm/longitude/0.0" failed to fetch target ['s3://asdc-prod-protected/TEMPO/TEMPO_RAD_L1_V03/2024.06.30/TEMPO_RAD_L1_V03_20240630T131406Z_S005G01.nc', 2147911467, 1081344]


My earthdata credentials are working ok as they work if I use the earthaccess library

[ASDC - ghayescrepps]

ESDIS data are located in Amazon region us-west-2. ESDIS policy requires that users be in-region to access the data. Additional guidance can be found in the ReadMe (https://data.asdc.earthdata.nasa.gov/s3credentialsREADME).