From 5e96d0e026e3afec6026d6629cd8161310f7b62f Mon Sep 17 00:00:00 2001 From: xhong Date: Mon, 18 May 2026 19:09:07 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E7=9B=AE=E5=BD=95=EF=BC=8C=E6=B7=BB=E5=8A=A0=E5=85=A8?= =?UTF-8?q?=E5=A5=97=E9=81=A5=E6=84=9F=E6=95=B0=E6=8D=AE=E5=A4=84=E7=90=86?= =?UTF-8?q?=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 重构项目目录结构，将零散代码整理到src目录下，创建各子包初始化文件 - 包括HLS、GPM、SMAP、MODIS等多源遥感数据的搜索、下载与预处理脚本 - 包括通用工具函数库，包含坐标参数定义、格式转换、影像处理等工具 - 同步更新README.md示例命令路径，修正脚本调用的目录错误 - 新增AGENTS.md文档说明开发环境配置 - 更新.gitignore规则，添加环境变量和构建缓存的忽略配置 --- .gitignore | 8 +++++++- AGENTS.md | 1 + README.md | 10 +++++----- {Basemap_SuPER => src/Basemap_SuPER}/DataV_SuPER.py | 0 {DATA_SuPER => src/DATA_SuPER}/DEM_SuPER.py | 7 ++++--- {DATA_SuPER => src/DATA_SuPER}/GPM_SuPER.py | 7 ++++--- {DATA_SuPER => src/DATA_SuPER}/MODIS_SuPER.py | 7 ++++--- {DATA_SuPER => src/DATA_SuPER}/S1_SAR_SuPER.py | 7 ++++--- {DATA_SuPER => src/DATA_SuPER}/SMAP_SuPER.py | 9 +++++---- {DATA_SuPER => src/DATA_SuPER}/__init__.py | 0 {DATA_SuPER => src/DATA_SuPER}/insitu_SuPER.py | 0 {HLS_SuPER => src/HLS_SuPER}/HLS_PER.py | 0 {HLS_SuPER => src/HLS_SuPER}/HLS_Su.py | 0 {HLS_SuPER => src/HLS_SuPER}/HLS_SuPER.py | 9 +++++---- {HLS_SuPER => src/HLS_SuPER}/README.md | 0 {HLS_SuPER => src/HLS_SuPER}/__init__.py | 0 src/__init__.py | 9 +++++++++ {utils => src/utils}/__init__.py | 0 {utils => src/utils}/common_params.py | 0 {utils => src/utils}/common_utils.py | 0 {utils => src/utils}/raw_to_cog.py | 6 +++--- {utils => src/utils}/raw_to_rgba.py | 0 {utils => src/utils}/sr2rgb_light.py | 6 +++--- 23 files changed, 54 insertions(+), 32 deletions(-) create mode 100644 AGENTS.md rename {Basemap_SuPER => src/Basemap_SuPER}/DataV_SuPER.py (100%) rename {DATA_SuPER => src/DATA_SuPER}/DEM_SuPER.py (97%) rename {DATA_SuPER => src/DATA_SuPER}/GPM_SuPER.py (95%) rename {DATA_SuPER => src/DATA_SuPER}/MODIS_SuPER.py (97%) rename {DATA_SuPER => src/DATA_SuPER}/S1_SAR_SuPER.py (97%) rename {DATA_SuPER => src/DATA_SuPER}/SMAP_SuPER.py (96%) rename {DATA_SuPER => src/DATA_SuPER}/__init__.py (100%) rename {DATA_SuPER => src/DATA_SuPER}/insitu_SuPER.py (100%) rename {HLS_SuPER => src/HLS_SuPER}/HLS_PER.py (100%) rename {HLS_SuPER => src/HLS_SuPER}/HLS_Su.py (100%) rename {HLS_SuPER => src/HLS_SuPER}/HLS_SuPER.py (98%) rename {HLS_SuPER => src/HLS_SuPER}/README.md (100%) rename {HLS_SuPER => src/HLS_SuPER}/__init__.py (100%) create mode 100644 src/__init__.py rename {utils => src/utils}/__init__.py (100%) rename {utils => src/utils}/common_params.py (100%) rename {utils => src/utils}/common_utils.py (100%) rename {utils => src/utils}/raw_to_cog.py (97%) rename {utils => src/utils}/raw_to_rgba.py (100%) rename {utils => src/utils}/sr2rgb_light.py (97%) diff --git a/.gitignore b/.gitignore index fa7a391..c01ebd0 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,10 @@ data/ *.7z *.tar* *.rar -.env* \ No newline at end of file +.sisyphus/ + +# 环境变量 +.env +.env.local +.env.*.local +!.env.example \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4a50a33 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +1、本项目算法部分在开发阶段可以使用 conda 管理 Python 环境，本机环境名为 `openearth`，Python 版本为 `3.12` \ No newline at end of file diff --git a/README.md b/README.md index 58d4f22..0b07aae 100644 --- a/README.md +++ b/README.md @@ -206,27 +206,27 @@ data/ - 爬取 L30 与 S30 的核心光谱波段：仅按照感兴趣区，瓦片 ID，起止时间以及产品名称筛选影像，不进行云量筛选影像，对影像进行去云掩膜处理 ```sh -python .\\HLS_SuPER\\HLS_SuPER.py -roi '112.9834,30.5286,114.32373,31.64448' -tile T49RGQ -dir .\\data\\HLS\\ALL -start 2024-01-01 -end 2024-01-31 -prod both -bands BLUE,GREEN,RED,NIR1,SWIR1,SWIR2,Fmask -scale True +python .\\src\\HLS_SuPER\\HLS_SuPER.py -roi '112.9834,30.5286,114.32373,31.64448' -tile T49RGQ -dir .\\data\\HLS\\ALL -start 2024-01-01 -end 2024-01-10 -prod both -bands BLUE,GREEN,RED,NIR1,SWIR1,SWIR2,Fmask -scale True ``` - 爬取 L30 的所有波段：按照感兴趣区，瓦片 ID，起止时间以及产品名称筛选影像，过滤云量小于 70% 的影像，对影像进行去云掩膜处理 ```sh -python .\\HLS_SuPER\\HLS_SuPER.py -roi '112.9834,30.5286,114.32373,31.64448' -tile T49RGQ -dir .\\data\\HLS\\L30\\subset -start 2024-01-01 -end 2024-01-31 -prod HLSL30 -bands COASTAL-AEROSOL,BLUE,GREEN,RED,NIR1,SWIR1,SWIR2,CIRRUS,TIR1,TIR2,Fmask -cc 70 -scale True +python .\\src\\HLS_SuPER\\HLS_SuPER.py -roi '112.9834,30.5286,114.32373,31.64448' -tile T49RGQ -dir .\\data\\HLS\\L30\\subset -start 2024-01-01 -end 2024-01-31 -prod HLSL30 -bands COASTAL-AEROSOL,BLUE,GREEN,RED,NIR1,SWIR1,SWIR2,CIRRUS,TIR1,TIR2,Fmask -cc 70 -scale True ``` - 仅爬取 L30 的热红外波段：仅按照感兴趣区，瓦片 ID，起止时间以及产品名称筛选影像，不进行云量筛选影像，对影像进行去云掩膜处理 ```sh -python .\\HLS_SuPER\\HLS_SuPER.py -roi .\\data\\vectors\\49REL.geojson -tile T49REL -dir .\\data\\HLS\\2024 -start 2024-06-01 -end 2024-09-01 -prod HLSL30 -bands TIR1,TIR2 +python .\\src\\HLS_SuPER\\HLS_SuPER.py -roi .\\data\\vectors\\49REL.geojson -tile T49REL -dir .\\data\\HLS\\2024 -start 2024-06-01 -end 2024-09-01 -prod HLSL30 -bands TIR1,TIR2 ``` - 【测试用】根据给定的范围文件 `*.geojson`，不进行云量筛选，直接爬取数据 ```sh -python .\\HLS_SuPER\\HLS_SuPER.py -roi .\\data\\vectors\\49REL.geojson -tile T49REL -dir .\\data\\HLS\\2024 -start 2024-03-01 -end 2024-11-01 -prod both -bands BLUE,GREEN,RED,NIR1,SWIR1,SWIR2,Fmask +python .\\src\\HLS_SuPER\\HLS_SuPER.py -roi .\\data\\vectors\\49REL.geojson -tile T49REL -dir .\\data\\HLS\\2024 -start 2024-03-01 -end 2024-11-01 -prod both -bands BLUE,GREEN,RED,NIR1,SWIR1,SWIR2,Fmask ``` ### 4.2 其他数据 -v1.0: 直接运行 `DATA_SuPER/` 目录下所需数据对应的 `*.py` 文件即可. +v1.0: 直接运行 `src/DATA_SuPER/` 目录下所需数据对应的 `*.py` 文件即可. diff --git a/Basemap_SuPER/DataV_SuPER.py b/src/Basemap_SuPER/DataV_SuPER.py similarity index 100% rename from Basemap_SuPER/DataV_SuPER.py rename to src/Basemap_SuPER/DataV_SuPER.py diff --git a/DATA_SuPER/DEM_SuPER.py b/src/DATA_SuPER/DEM_SuPER.py similarity index 97% rename from DATA_SuPER/DEM_SuPER.py rename to src/DATA_SuPER/DEM_SuPER.py index e743156..ea7c484 100644 --- a/DATA_SuPER/DEM_SuPER.py +++ b/src/DATA_SuPER/DEM_SuPER.py @@ -50,15 +50,16 @@ import earthaccess import numpy as np from rioxarray import open_rasterio -sys.path.append(str(Path(".").parent)) +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from utils.common_utils import ( +from src.utils.common_utils import ( setup_dask_environment, setup_logging, clip_image, mosaic_images, ) -from HLS_SuPER.HLS_Su import earthdata_search +from src.HLS_SuPER.HLS_Su import earthdata_search def reorganize_nasadem_urls(dem_results_urls: list): diff --git a/DATA_SuPER/GPM_SuPER.py b/src/DATA_SuPER/GPM_SuPER.py similarity index 95% rename from DATA_SuPER/GPM_SuPER.py rename to src/DATA_SuPER/GPM_SuPER.py index a5ae6a6..ed9fe4a 100644 --- a/DATA_SuPER/GPM_SuPER.py +++ b/src/DATA_SuPER/GPM_SuPER.py @@ -28,10 +28,11 @@ import logging import earthaccess from xarray import open_dataset -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from utils.common_utils import setup_dask_environment -from HLS_SuPER.HLS_Su import earthdata_search +from src.utils.common_utils import setup_dask_environment +from src.HLS_SuPER.HLS_Su import earthdata_search def convert(source_nc_path: str, target_tif_path: str) -> None: diff --git a/DATA_SuPER/MODIS_SuPER.py b/src/DATA_SuPER/MODIS_SuPER.py similarity index 97% rename from DATA_SuPER/MODIS_SuPER.py rename to src/DATA_SuPER/MODIS_SuPER.py index 855b294..a58fabf 100644 --- a/DATA_SuPER/MODIS_SuPER.py +++ b/src/DATA_SuPER/MODIS_SuPER.py @@ -21,15 +21,16 @@ import rioxarray as rxr import dask.distributed import geopandas as gpd -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from utils.common_utils import ( +from src.utils.common_utils import ( clip_image, reproject_image, setup_dask_environment, setup_logging, ) -from HLS_SuPER.HLS_Su import earthdata_search +from src.HLS_SuPER.HLS_Su import earthdata_search def open_mcd43a4(file_path): diff --git a/DATA_SuPER/S1_SAR_SuPER.py b/src/DATA_SuPER/S1_SAR_SuPER.py similarity index 97% rename from DATA_SuPER/S1_SAR_SuPER.py rename to src/DATA_SuPER/S1_SAR_SuPER.py index 4ec4534..95fed05 100644 --- a/DATA_SuPER/S1_SAR_SuPER.py +++ b/src/DATA_SuPER/S1_SAR_SuPER.py @@ -49,10 +49,11 @@ import numpy as np import xarray as xr from rioxarray import open_rasterio -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from utils.common_utils import setup_dask_environment, clip_image, mosaic_images -from HLS_SuPER.HLS_Su import earthdata_search +from src.utils.common_utils import setup_dask_environment, clip_image, mosaic_images +from src.HLS_SuPER.HLS_Su import earthdata_search def download_granule( diff --git a/DATA_SuPER/SMAP_SuPER.py b/src/DATA_SuPER/SMAP_SuPER.py similarity index 96% rename from DATA_SuPER/SMAP_SuPER.py rename to src/DATA_SuPER/SMAP_SuPER.py index 1507023..6c56738 100644 --- a/DATA_SuPER/SMAP_SuPER.py +++ b/src/DATA_SuPER/SMAP_SuPER.py @@ -28,16 +28,17 @@ import h5py from osgeo import gdal import xarray as xr -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from utils.common_params import EASE2_GRID_PARAMS, EPSG -from utils.common_utils import ( +from src.utils.common_params import EASE2_GRID_PARAMS, EPSG +from src.utils.common_utils import ( array_to_raster, load_band_as_arr, reproject_image, setup_dask_environment, ) -from HLS_SuPER.HLS_Su import earthdata_search +from src.HLS_SuPER.HLS_Su import earthdata_search def convert(source_h5_path: str, target_tif_path: str, date: str) -> None: diff --git a/DATA_SuPER/__init__.py b/src/DATA_SuPER/__init__.py similarity index 100% rename from DATA_SuPER/__init__.py rename to src/DATA_SuPER/__init__.py diff --git a/DATA_SuPER/insitu_SuPER.py b/src/DATA_SuPER/insitu_SuPER.py similarity index 100% rename from DATA_SuPER/insitu_SuPER.py rename to src/DATA_SuPER/insitu_SuPER.py diff --git a/HLS_SuPER/HLS_PER.py b/src/HLS_SuPER/HLS_PER.py similarity index 100% rename from HLS_SuPER/HLS_PER.py rename to src/HLS_SuPER/HLS_PER.py diff --git a/HLS_SuPER/HLS_Su.py b/src/HLS_SuPER/HLS_Su.py similarity index 100% rename from HLS_SuPER/HLS_Su.py rename to src/HLS_SuPER/HLS_Su.py diff --git a/HLS_SuPER/HLS_SuPER.py b/src/HLS_SuPER/HLS_SuPER.py similarity index 98% rename from HLS_SuPER/HLS_SuPER.py rename to src/HLS_SuPER/HLS_SuPER.py index 4267624..9de3a2c 100644 --- a/HLS_SuPER/HLS_SuPER.py +++ b/src/HLS_SuPER/HLS_SuPER.py @@ -26,12 +26,13 @@ from datetime import datetime as dt import dask.distributed import earthaccess -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from HLS_PER import create_timeseries_dataset, process_granule -from HLS_Su import format_roi, hls_search +from src.HLS_SuPER.HLS_PER import create_timeseries_dataset, process_granule +from src.HLS_SuPER.HLS_Su import format_roi, hls_search -from utils.common_utils import setup_dask_environment +from src.utils.common_utils import setup_dask_environment def parse_arguments(): diff --git a/HLS_SuPER/README.md b/src/HLS_SuPER/README.md similarity index 100% rename from HLS_SuPER/README.md rename to src/HLS_SuPER/README.md diff --git a/HLS_SuPER/__init__.py b/src/HLS_SuPER/__init__.py similarity index 100% rename from HLS_SuPER/__init__.py rename to src/HLS_SuPER/__init__.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..b37a260 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +"""算法源码包 + +包含所有遥感数据处理算法模块： +- HLS_SuPER: HLS 遥感数据处理 +- DATA_SuPER: 多源遥感数据处理（MODIS、SMAP、DEM、SAR、GPM） +- Basemap_SuPER: 底图数据处理 +- utils: 通用工具函数 +""" diff --git a/utils/__init__.py b/src/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to src/utils/__init__.py diff --git a/utils/common_params.py b/src/utils/common_params.py similarity index 100% rename from utils/common_params.py rename to src/utils/common_params.py diff --git a/utils/common_utils.py b/src/utils/common_utils.py similarity index 100% rename from utils/common_utils.py rename to src/utils/common_utils.py diff --git a/utils/raw_to_cog.py b/src/utils/raw_to_cog.py similarity index 97% rename from utils/raw_to_cog.py rename to src/utils/raw_to_cog.py index 33f9800..a4a3338 100644 --- a/utils/raw_to_cog.py +++ b/src/utils/raw_to_cog.py @@ -20,11 +20,11 @@ from pathlib import Path import numpy as np from osgeo import gdal -# 添加父目录到 sys.path 以导入 utils -BASE_DIR = Path(__file__).parent.parent +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +BASE_DIR = Path(__file__).parent.parent.parent sys.path.append(str(BASE_DIR)) -from utils.common_utils import setup_logging +from src.utils.common_utils import setup_logging gdal.UseExceptions() # 设置 GDAL 选项以优化性能 diff --git a/utils/raw_to_rgba.py b/src/utils/raw_to_rgba.py similarity index 100% rename from utils/raw_to_rgba.py rename to src/utils/raw_to_rgba.py diff --git a/utils/sr2rgb_light.py b/src/utils/sr2rgb_light.py similarity index 97% rename from utils/sr2rgb_light.py rename to src/utils/sr2rgb_light.py index a4d9d53..2a1bef9 100644 --- a/utils/sr2rgb_light.py +++ b/src/utils/sr2rgb_light.py @@ -13,11 +13,11 @@ from pathlib import Path from osgeo import gdal import numpy as np -# 添加父目录到 sys.path 以导入 utils -BASE_DIR = Path(__file__).parent.parent +# 添加项目根目录到 sys.path，确保作为独立脚本运行时也能正确导入 +BASE_DIR = Path(__file__).parent.parent.parent sys.path.append(str(BASE_DIR)) -from utils.common_utils import setup_logging +from src.utils.common_utils import setup_logging gdal.UseExceptions() # 设置 GDAL 选项以优化性能