Errors accessing HLS data
Posted: Thu Feb 15, 2024 6:54 pm America/New_York
Hello, I am currently trying to access HLS data for a watershed boundary to make a cloudmask in python and am getting file not supported for the .tif files returned by the earthacess.search function. This is for a class and multiple students are having the same issue. The professor has old code that is successfully running but the environment she is using seems to have slight differences in versions and modules, so it appears this may be an issue with package versions. Would just like some feedback on what possibly might be wrong
This is the login codeblock:
eta.login(persist=True)
results = eta.search_data(
short_name="HLSL30",
cloud_hosted=True,
bounding_box=tuple(watershed.total_bounds),
temporal=("2023-05-01", "2023-10-01"),
)
This is the codeblock used to retrieve the url:
processed_data = {}
# Open the Fmask band and compute the quality mask
fmask_da = rioxarray.open_rasterio(granule_info['fmask_path'])
bits = np.unpackbits(fmask_da.astype(np.uint8)).reshape(fmask_da.shape + (-1,))
mask = np.prod(bits[..., mask_bits] == 0, axis=-1)
This is the error
{
"name": "RasterioIOError",
"message": "'/vsicurl/https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif' not recognized as a supported file format.",
"stack": "---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\file_manager.py:211, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
210 try:
--> 211 file = self._cache[self._key]
212 except KeyError:
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\lru_cache.py:56, in LRUCache.__getitem__(self, key)
55 with self._lock:
---> 56 value = self._cache[key]
57 self._cache.move_to_end(key)
KeyError: [<function open at 0x000001D4523AF910>, ('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif',), 'r', (('sharing', False),), 'd323ca66-cd72-462d-a349-cb24cecda6b5']
During handling of the above exception, another exception occurred:
CPLE_OpenFailedError Traceback (most recent call last)
File rasterio\\\\_base.pyx:310, in rasterio._base.DatasetBase.__init__()
File rasterio\\\\_base.pyx:221, in rasterio._base.open_dataset()
File rasterio\\\\_err.pyx:221, in rasterio._err.exc_wrap_pointer()
CPLE_OpenFailedError: '/vsicurl/https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif' not recognized as a supported file format.
During handling of the above exception, another exception occurred:
RasterioIOError Traceback (most recent call last)
Cell In[8], line 64
58 granule_info = {
59 'fmask_path': fmask_path,
60 'bands': band_paths
61 }
63 # Process the granule
---> 64 processed_data = process_granule(granule_info, mask_bits, scale_factor)
65 # ... (rest of your processing code)
66
67
68 # Example: Print the processed data for the first granule to check
69 if granule_id == group.iloc[0]['granule_id']:
Cell In[8], line 26, in process_granule(granule_info, mask_bits, scale_factor)
23 processed_data = {}
25 # Open the Fmask band and compute the quality mask
---> 26 fmask_da = rioxarray.open_rasterio(granule_info['fmask_path'])
27 bits = np.unpackbits(fmask_da.astype(np.uint8)).reshape(fmask_da.shape + (-1,))
28 mask = np.prod(bits[..., mask_bits] == 0, axis=-1)
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\rioxarray\\_io.py:1124, in open_rasterio(filename, parse_coordinates, chunks, cache, lock, masked, mask_and_scale, variable, group, default_name, decode_times, decode_timedelta, band_as_variable, **open_kwargs)
1122 else:
1123 manager = URIManager(file_opener, filename, mode=\"r\", kwargs=open_kwargs)
-> 1124 riods = manager.acquire()
1125 captured_warnings = rio_warnings.copy()
1127 # raise the NotGeoreferencedWarning if applicable
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\file_manager.py:193, in CachingFileManager.acquire(self, needs_lock)
178 def acquire(self, needs_lock=True):
179 \"\"\"Acquire a file object from the manager.
180
181 A new file is only opened if it has expired from the
(...)
191 An open file object, as returned by ``opener(*args, **kwargs)``.
192 \"\"\"
--> 193 file, _ = self._acquire_with_cache_info(needs_lock)
194 return file
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\file_manager.py:217, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
215 kwargs = kwargs.copy()
216 kwargs[\"mode\"] = self._mode
--> 217 file = self._opener(*self._args, **kwargs)
218 if self._mode == \"w\":
219 # ensure file doesn't get overridden when opened again
220 self._mode = \"a\"
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\rasterio\\env.py:451, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)
448 session = DummySession()
450 with env_ctor(session=session):
--> 451 return f(*args, **kwds)
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\rasterio\\__init__.py:304, in open(fp, mode, driver, width, height, count, crs, transform, dtype, nodata, sharing, **kwargs)
301 path = _parse_path(raw_dataset_path)
303 if mode == \"r\":
--> 304 dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
305 elif mode == \"r+\":
306 dataset = get_writer_for_path(path, driver=driver)(
307 path, mode, driver=driver, sharing=sharing, **kwargs
308 )
File rasterio\\\\_base.pyx:312, in rasterio._base.DatasetBase.__init__()
RasterioIOError: '/vsicurl/https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif' not recognized as a supported file format."
}
This is the login codeblock:
eta.login(persist=True)
results = eta.search_data(
short_name="HLSL30",
cloud_hosted=True,
bounding_box=tuple(watershed.total_bounds),
temporal=("2023-05-01", "2023-10-01"),
)
This is the codeblock used to retrieve the url:
processed_data = {}
# Open the Fmask band and compute the quality mask
fmask_da = rioxarray.open_rasterio(granule_info['fmask_path'])
bits = np.unpackbits(fmask_da.astype(np.uint8)).reshape(fmask_da.shape + (-1,))
mask = np.prod(bits[..., mask_bits] == 0, axis=-1)
This is the error
{
"name": "RasterioIOError",
"message": "'/vsicurl/https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif' not recognized as a supported file format.",
"stack": "---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\file_manager.py:211, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
210 try:
--> 211 file = self._cache[self._key]
212 except KeyError:
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\lru_cache.py:56, in LRUCache.__getitem__(self, key)
55 with self._lock:
---> 56 value = self._cache[key]
57 self._cache.move_to_end(key)
KeyError: [<function open at 0x000001D4523AF910>, ('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif',), 'r', (('sharing', False),), 'd323ca66-cd72-462d-a349-cb24cecda6b5']
During handling of the above exception, another exception occurred:
CPLE_OpenFailedError Traceback (most recent call last)
File rasterio\\\\_base.pyx:310, in rasterio._base.DatasetBase.__init__()
File rasterio\\\\_base.pyx:221, in rasterio._base.open_dataset()
File rasterio\\\\_err.pyx:221, in rasterio._err.exc_wrap_pointer()
CPLE_OpenFailedError: '/vsicurl/https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif' not recognized as a supported file format.
During handling of the above exception, another exception occurred:
RasterioIOError Traceback (most recent call last)
Cell In[8], line 64
58 granule_info = {
59 'fmask_path': fmask_path,
60 'bands': band_paths
61 }
63 # Process the granule
---> 64 processed_data = process_granule(granule_info, mask_bits, scale_factor)
65 # ... (rest of your processing code)
66
67
68 # Example: Print the processed data for the first granule to check
69 if granule_id == group.iloc[0]['granule_id']:
Cell In[8], line 26, in process_granule(granule_info, mask_bits, scale_factor)
23 processed_data = {}
25 # Open the Fmask band and compute the quality mask
---> 26 fmask_da = rioxarray.open_rasterio(granule_info['fmask_path'])
27 bits = np.unpackbits(fmask_da.astype(np.uint8)).reshape(fmask_da.shape + (-1,))
28 mask = np.prod(bits[..., mask_bits] == 0, axis=-1)
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\rioxarray\\_io.py:1124, in open_rasterio(filename, parse_coordinates, chunks, cache, lock, masked, mask_and_scale, variable, group, default_name, decode_times, decode_timedelta, band_as_variable, **open_kwargs)
1122 else:
1123 manager = URIManager(file_opener, filename, mode=\"r\", kwargs=open_kwargs)
-> 1124 riods = manager.acquire()
1125 captured_warnings = rio_warnings.copy()
1127 # raise the NotGeoreferencedWarning if applicable
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\file_manager.py:193, in CachingFileManager.acquire(self, needs_lock)
178 def acquire(self, needs_lock=True):
179 \"\"\"Acquire a file object from the manager.
180
181 A new file is only opened if it has expired from the
(...)
191 An open file object, as returned by ``opener(*args, **kwargs)``.
192 \"\"\"
--> 193 file, _ = self._acquire_with_cache_info(needs_lock)
194 return file
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\xarray\\backends\\file_manager.py:217, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
215 kwargs = kwargs.copy()
216 kwargs[\"mode\"] = self._mode
--> 217 file = self._opener(*self._args, **kwargs)
218 if self._mode == \"w\":
219 # ensure file doesn't get overridden when opened again
220 self._mode = \"a\"
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\rasterio\\env.py:451, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)
448 session = DummySession()
450 with env_ctor(session=session):
--> 451 return f(*args, **kwds)
File c:\\Users\\Degg\\miniconda3\\envs\\earth-analytics-python\\lib\\site-packages\\rasterio\\__init__.py:304, in open(fp, mode, driver, width, height, count, crs, transform, dtype, nodata, sharing, **kwargs)
301 path = _parse_path(raw_dataset_path)
303 if mode == \"r\":
--> 304 dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
305 elif mode == \"r+\":
306 dataset = get_writer_for_path(path, driver=driver)(
307 path, mode, driver=driver, sharing=sharing, **kwargs
308 )
File rasterio\\\\_base.pyx:312, in rasterio._base.DatasetBase.__init__()
RasterioIOError: '/vsicurl/https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T15RYN.2023124T163132.v2.0/HLS.L30.T15RYN.2023124T163132.v2.0.Fmask.tif' not recognized as a supported file format."
}