xhong/open-earthdata-cli
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

fix(hls processing): 完善栅格错误处理并跳过异常数据块

Browse Source 
为 open_hls 函数添加异常捕获逻辑，打开栅格失败时返回 None；在处理数据块时检查质量图层可用性，无法访问时跳过并记录警告；移除 common_utils.py 中冗余的 netrc 相关 GDAL 配置项；更新 HLS_PER.py 的最后更新日期
This commit is contained in:
谢泓 2026-05-18 17:23:58 +08:00
parent 339755a42c
commit 231d6f1d72
2 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
HLS_SuPER/HLS_PER.py
View File

@ -8,7 +8,7 @@ search results.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Authors: Cole Krehbiel, Mahsa Jami, and Erik Bolch Authors: Cole Krehbiel, Mahsa Jami, and Erik Bolch
Editor: Hong Xie Editor: Hong Xie
Last Updated: 2026-05-17 Last Updated: 2026-05-18
=============================================================================== ===============================================================================
""" """
@ -52,9 +52,13 @@ def open_hls(
Some HLS Landsat scenes have the metadata in the wrong location. Some HLS Landsat scenes have the metadata in the wrong location.
""" """
# Open using rioxarray # Open using rioxarray
da = rxr.open_rasterio(url, chunks=chunk_size, mask_and_scale=False).squeeze( try:
"band", drop=True da = rxr.open_rasterio(url, chunks=chunk_size, mask_and_scale=False).squeeze(
) "band", drop=True
)
except Exception as e:
logging.warning(f"Failed to open {url}: {e}")
return None
# (Add) 若未获取到数据, 则返回 None # (Add) 若未获取到数据, 则返回 None
if da is None: if da is None:
return None return None
@ -197,6 +201,12 @@ def process_granule(
if not os.path.isfile(quality_output_file): if not os.path.isfile(quality_output_file):
# Open Quality Layer # Open Quality Layer
qa_da = open_hls(quality_url, roi, clip, scale, chunk_size) qa_da = open_hls(quality_url, roi, clip, scale, chunk_size)
# (Add) 若返回的da为None, 则表示该url对应的文件不存在/无法访问, 跳过整个granule
if qa_da is None:
logging.warning(
f"Quality layer {quality_url} not accessible. Skipping granule."
)
return
# Write Output # Write Output
# (Add) 添加压缩选项参数 compress # (Add) 添加压缩选项参数 compress
# compress 参数是源自 rioxarray 继承的 rasterio 的选项, 可以参考 https://rasterio.readthedocs.io/en/latest/api/rasterio.enums.html#rasterio.enums.Compression # compress 参数是源自 rioxarray 继承的 rasterio 的选项, 可以参考 https://rasterio.readthedocs.io/en/latest/api/rasterio.enums.html#rasterio.enums.Compression
@ -207,6 +217,12 @@ def process_granule(
) )
else: else:
qa_da = open_hls(quality_output_file, roi, clip, scale, chunk_size) qa_da = open_hls(quality_output_file, roi, clip, scale, chunk_size)
# (Add) 若返回的da为None, 则表示本地文件无法访问, 跳过整个granule
if qa_da is None:
logging.warning(
f"Local quality file {quality_output_file} not accessible. Skipping granule."
)
return
logging.info( logging.info(
f"Existing quality file {quality_output_name} found in {output_dir}." f"Existing quality file {quality_output_name} found in {output_dir}."
) )

7
utils/common_utils.py
View File

@ -243,18 +243,11 @@ def setup_dask_environment():
os.path.expanduser("~/cookies.txt"), os.path.expanduser("~/cookies.txt"),
) )
netrc_path = os.environ.get("EARTHDATA_NETRC_FILE") or os.path.expanduser(
"~/.netrc"
)
enable_netrc = "YES" if os.path.exists(netrc_path) else "NO"
global env global env
gdal_config = { gdal_config = {
"GDAL_HTTP_UNSAFESSL": "YES", "GDAL_HTTP_UNSAFESSL": "YES",
"GDAL_HTTP_COOKIEFILE": cookie_file_path, "GDAL_HTTP_COOKIEFILE": cookie_file_path,
"GDAL_HTTP_COOKIEJAR": cookie_file_path, "GDAL_HTTP_COOKIEJAR": cookie_file_path,
"GDAL_HTTP_NETRC": enable_netrc,
"GDAL_HTTP_NETRC_FILE": netrc_path,
"GDAL_DISABLE_READDIR_ON_OPEN": "YES", "GDAL_DISABLE_READDIR_ON_OPEN": "YES",
"CPL_VSIL_CURL_ALLOWED_EXTENSIONS": "TIF", "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": "TIF",
"GDAL_HTTP_MAX_RETRY": "10", "GDAL_HTTP_MAX_RETRY": "10",