Skip to content

Commit

Permalink
Remove high_confidence from SlideRule retrieval (#72)
Browse files Browse the repository at this point in the history
This commit removes all traces of "high_confidence" points from the codebase. For a discussion of this, see: SlideRuleEarth/sliderule#448.

The main point is that "high_confidence" points are not necessarily a given surface type. Instead, we want to focus on just getting the ground, canopy, and top of canopy, which we already do.

In addition, this commit speeds up the altimetry tests by skipping the actual external retrieval call. This means the `request_atl06sr_multi_processing` function is not tested, but that is mostly a wrapper for sliderule anyway.

Also, I fixed CI tests here, by dropping use of mamba in favor of conda, which seems to have fixed things.
  • Loading branch information
bpurinton authored Jan 16, 2025
1 parent bfbbf7f commit db6bd46
Show file tree
Hide file tree
Showing 22 changed files with 864 additions and 164 deletions.
3 changes: 1 addition & 2 deletions .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ jobs:
echo $CONDA/bin >> $GITHUB_PATH
- name: Install dependencies
run: |
conda install mamba -c conda-forge
mamba env update --file environment.yml --name asp_plot
conda env create -f environment.yml
source $CONDA/etc/profile.d/conda.sh
conda activate asp_plot
pip install .
Expand Down
33 changes: 14 additions & 19 deletions asp_plot/altimetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def __init__(

def request_atl06sr_multi_processing(
self,
processing_levels=["high_confidence", "ground", "canopy", "top_of_canopy"],
processing_levels=["ground", "canopy", "top_of_canopy"],
res=20,
len=40,
ats=20,
Expand All @@ -95,11 +95,10 @@ def request_atl06sr_multi_processing(
if not region:
region = Raster(self.dem_fn).get_bounds(latlon=True)

# See parameter discussion on: https://github.com/SlideRuleEarth/sliderule/issues/448
# "srt": -1 tells the server side code to look at the ATL03 confidence array for each photon
# and choose the confidence level that is highest across all five surface type entries.
parms_dict = {
"high_confidence": {
"cnf": 4,
"srt": -1,
},
"ground": {
"cnf": 0,
"srt": -1,
Expand Down Expand Up @@ -286,10 +285,8 @@ def generic_temporal_filter_atl06sr(

self.atl06sr_processing_levels_filtered[key] = atl06_filtered

def to_csv_for_pc_align(
self, key="high_confidence", filename_prefix="atl06sr_for_pc_align"
):
atl06sr = self.atl06sr_processing_levels_filtered[key]
def to_csv_for_pc_align(self, key="ground", filename_prefix="atl06sr_for_pc_align"):
atl06sr = self.atl06sr_processing_levels_filtered[key].to_crs("EPSG:4326")
csv_fn = f"{filename_prefix}_{key}.csv"
df = atl06sr[["geometry", "h_mean"]].copy()
df["lon"] = df["geometry"].x
Expand All @@ -301,12 +298,12 @@ def to_csv_for_pc_align(

def alignment_report(
self,
processing_level="high_confidence",
processing_level="ground",
minimum_points=500,
agreement_threshold=0.25,
write_out_aligned_dem=False,
min_translation_threshold=0.1,
key_for_aligned_dem="high_confidence",
key_for_aligned_dem="ground",
):
filtered_keys = [
key
Expand All @@ -324,10 +321,8 @@ def alignment_report(
)
continue

if not os.path.exists(
glob_file(
os.path.join(self.directory, "pc_align"), f"*{key}*transform.txt"
)
if not glob_file(
os.path.join(self.directory, "pc_align"), f"*{key}*transform.txt"
):
csv_fn = self.to_csv_for_pc_align(key=key)

Expand Down Expand Up @@ -396,7 +391,7 @@ def alignment_report(

def plot_atl06sr_time_stamps(
self,
key="high_confidence",
key="ground",
title="ICESat-2 ATL06-SR Time Stamps",
cmap="inferno",
map_crs="4326",
Expand Down Expand Up @@ -482,7 +477,7 @@ def plot_atl06sr_time_stamps(

def plot_atl06sr(
self,
key="high_confidence",
key="ground",
plot_beams=False,
plot_dem=False,
column_name="h_mean",
Expand Down Expand Up @@ -599,7 +594,7 @@ def atl06sr_to_dem_dh(self):

def mapview_plot_atl06sr_to_dem(
self,
key="high_confidence",
key="ground",
clim=None,
plot_aligned=False,
save_dir=None,
Expand Down Expand Up @@ -645,7 +640,7 @@ def mapview_plot_atl06sr_to_dem(

def histogram(
self,
key="high_confidence",
key="ground",
title="Histogram",
plot_aligned=False,
save_dir=None,
Expand Down
918 changes: 807 additions & 111 deletions notebooks/icesat2_plots.ipynb

Large diffs are not rendered by default.

18 changes: 14 additions & 4 deletions tests/test_altimetry.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import geopandas as gpd
import matplotlib
import pytest

Expand All @@ -13,9 +14,18 @@ def icesat(self):
directory="tests/test_data",
dem_fn="tests/test_data/stereo/date_time_left_right_1m-DEM.tif",
)
icesat.request_atl06sr_multi_processing(
filename="tests/test_data/icesat_data/atl06sr_",
)

# The actual request is fairly time consuming, thus better to be skipped for testing.
# icesat.request_atl06sr_multi_processing(
# filename="tests/test_data/icesat_data/atl06sr_",
# )
for key in ["ground", "canopy", "top_of_canopy"]:
icesat.atl06sr_processing_levels[key] = gpd.read_parquet(
f"tests/test_data/icesat_data/atl06sr_res20_len40_cnt10_ats20_maxi5_{key}.parquet"
)
icesat.atl06sr_processing_levels_filtered[key] = gpd.read_parquet(
f"tests/test_data/icesat_data/atl06sr_res20_len40_cnt10_ats20_maxi5_{key}_filtered.parquet"
)
return icesat

def test_filter_esa_worldcover(self, icesat):
Expand Down Expand Up @@ -74,7 +84,7 @@ def test_mapview_plot_atl06sr_to_dem(self, icesat):

def test_histogram(self, icesat):
try:
icesat.histogram(key="high_confidence_45_day_pad")
icesat.histogram(key="ground_45_day_pad")
except Exception as e:
pytest.fail(f"histogram() method raised an exception: {str(e)}")

Expand Down
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ ASP 3.5.0-alpha
Build ID: 3585be8
Build date: 2024-08-21

/Users/ben/asp/dev/libexec/pc_align --max-displacement 20 --max-num-source-points 10000000 --alignment-method point-to-point --csv-format "1:lon 2:lat 3:height_above_datum" --compute-translation-only --output-prefix /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif atl06sr_for_pc_align_high_confidence.csv
/Users/ben/asp/dev/libexec/pc_align --max-displacement 20 --max-num-source-points 10000000 --alignment-method point-to-point --csv-format "1:lon 2:lat 3:height_above_datum" --compute-translation-only --output-prefix /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif atl06sr_for_pc_align_ground.csv

uname -a
Darwin Bens-MacBook-Air-2.local 23.6.0 Darwin Kernel Version 23.6.0: Mon Jul 29 21:16:46 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T8112 x86_64
Expand Down Expand Up @@ -45,7 +45,7 @@ Vision Workbench log started at 2024-11-25 15:52:26.
2024-11-25 15:52:39 {0} [ console ] : Reading: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif
2024-11-25 15:53:26 {0} [ console ] : Loaded points: 82687133
2024-11-25 15:53:26 {0} [ console ] : Loading the reference point cloud took 47.7141 s
2024-11-25 15:53:26 {0} [ console ] : Reading: atl06sr_for_pc_align_high_confidence.csv
2024-11-25 15:53:26 {0} [ console ] : Reading: atl06sr_for_pc_align_ground.csv
2024-11-25 15:53:26 {0} [ console ] : Loaded points: 32711
2024-11-25 15:53:26 {0} [ console ] : Loading the source point cloud took 0.102127 s
2024-11-25 15:53:28 {0} [ console ] : Data shifted internally by subtracting: Vector3(-1.88009e+06,-813400,6.01997e+06)
Expand Down Expand Up @@ -83,9 +83,9 @@ Vision Workbench log started at 2024-11-25 15:52:26.
2024-11-25 15:54:23 {0} [ console ] : Euler angles (degrees): Vector3(0,-0,0)
2024-11-25 15:54:23 {0} [ console ] : Euler angles (North-East-Down, degrees): Vector3(7.9513867e-16,-0,1.4001392e-15)
2024-11-25 15:54:23 {0} [ console ] : Axis of rotation and angle (degrees): Vector3(nan,nan,nan) 0
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence-transform.txt
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence-inverse-transform.txt
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence-beg_errors.csv
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence-end_errors.csv
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence-iterationInfo.csv
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground-transform.txt
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground-inverse-transform.txt
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground-beg_errors.csv
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground-end_errors.csv
2024-11-25 15:54:23 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground-iterationInfo.csv
2024-11-25 15:54:23 {0} [ console ] : Saving to disk took 0.279139 s
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ ASP 3.5.0-alpha
Build ID: 3585be8
Build date: 2024-08-21

/Users/ben/asp/dev/libexec/pc_align --max-displacement 20 --max-num-source-points 10000000 --alignment-method point-to-point --csv-format "1:lon 2:lat 3:height_above_datum" --compute-translation-only --output-prefix /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_15_day_pad /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif atl06sr_for_pc_align_high_confidence_15_day_pad.csv
/Users/ben/asp/dev/libexec/pc_align --max-displacement 20 --max-num-source-points 10000000 --alignment-method point-to-point --csv-format "1:lon 2:lat 3:height_above_datum" --compute-translation-only --output-prefix /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_15_day_pad /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif atl06sr_for_pc_align_ground_15_day_pad.csv

uname -a
Darwin Bens-MacBook-Air-2.local 23.6.0 Darwin Kernel Version 23.6.0: Mon Jul 29 21:16:46 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T8112 x86_64
Expand Down Expand Up @@ -45,7 +45,7 @@ Vision Workbench log started at 2024-11-25 15:54:25.
2024-11-25 15:54:36 {0} [ console ] : Reading: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif
2024-11-25 15:55:10 {0} [ console ] : Loaded points: 68331153
2024-11-25 15:55:10 {0} [ console ] : Loading the reference point cloud took 33.5239 s
2024-11-25 15:55:10 {0} [ console ] : Reading: atl06sr_for_pc_align_high_confidence_15_day_pad.csv
2024-11-25 15:55:10 {0} [ console ] : Reading: atl06sr_for_pc_align_ground_15_day_pad.csv
2024-11-25 15:55:10 {0} [ console ] : Loaded points: 750
2024-11-25 15:55:10 {0} [ console ] : Loading the source point cloud took 0.002246 s
2024-11-25 15:55:10 {0} [ console ] : Data shifted internally by subtracting: Vector3(-1.88264e+06,-811445,6.01944e+06)
Expand Down Expand Up @@ -83,9 +83,9 @@ Vision Workbench log started at 2024-11-25 15:54:25.
2024-11-25 15:55:45 {0} [ console ] : Euler angles (degrees): Vector3(0,-0,0)
2024-11-25 15:55:45 {0} [ console ] : Euler angles (North-East-Down, degrees): Vector3(0,-0,-6.6413833e-16)
2024-11-25 15:55:45 {0} [ console ] : Axis of rotation and angle (degrees): Vector3(nan,nan,nan) 0
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_15_day_pad-transform.txt
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_15_day_pad-inverse-transform.txt
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_15_day_pad-beg_errors.csv
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_15_day_pad-end_errors.csv
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_15_day_pad-iterationInfo.csv
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_15_day_pad-transform.txt
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_15_day_pad-inverse-transform.txt
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_15_day_pad-beg_errors.csv
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_15_day_pad-end_errors.csv
2024-11-25 15:55:45 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_15_day_pad-iterationInfo.csv
2024-11-25 15:55:45 {0} [ console ] : Saving to disk took 0.005852 s
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ ASP 3.5.0-alpha
Build ID: 3585be8
Build date: 2024-08-21

/Users/ben/asp/dev/libexec/pc_align --max-displacement 20 --max-num-source-points 10000000 --alignment-method point-to-point --csv-format "1:lon 2:lat 3:height_above_datum" --compute-translation-only --output-prefix /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_45_day_pad /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif atl06sr_for_pc_align_high_confidence_45_day_pad.csv
/Users/ben/asp/dev/libexec/pc_align --max-displacement 20 --max-num-source-points 10000000 --alignment-method point-to-point --csv-format "1:lon 2:lat 3:height_above_datum" --compute-translation-only --output-prefix /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_45_day_pad /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif atl06sr_for_pc_align_ground_45_day_pad.csv

uname -a
Darwin Bens-MacBook-Air-2.local 23.6.0 Darwin Kernel Version 23.6.0: Mon Jul 29 21:16:46 PDT 2024; root:xnu-10063.141.2~1/RELEASE_ARM64_T8112 x86_64
Expand Down Expand Up @@ -45,7 +45,7 @@ Vision Workbench log started at 2024-11-25 15:55:47.
2024-11-25 15:55:58 {0} [ console ] : Reading: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/stereo/20220417_2252_1040010074793300_1040010075633C00-DEM_1m.tif
2024-11-25 15:56:33 {0} [ console ] : Loaded points: 73202534
2024-11-25 15:56:33 {0} [ console ] : Loading the reference point cloud took 35.1191 s
2024-11-25 15:56:33 {0} [ console ] : Reading: atl06sr_for_pc_align_high_confidence_45_day_pad.csv
2024-11-25 15:56:33 {0} [ console ] : Reading: atl06sr_for_pc_align_ground_45_day_pad.csv
2024-11-25 15:56:33 {0} [ console ] : Loaded points: 1261
2024-11-25 15:56:33 {0} [ console ] : Loading the source point cloud took 0.003867 s
2024-11-25 15:56:34 {0} [ console ] : Data shifted internally by subtracting: Vector3(-1.88283e+06,-811207,6.01941e+06)
Expand Down Expand Up @@ -83,9 +83,9 @@ Vision Workbench log started at 2024-11-25 15:55:47.
2024-11-25 15:57:11 {0} [ console ] : Euler angles (degrees): Vector3(0,-0,0)
2024-11-25 15:57:11 {0} [ console ] : Euler angles (North-East-Down, degrees): Vector3(0,3.1805547e-15,-1.677861e-15)
2024-11-25 15:57:11 {0} [ console ] : Axis of rotation and angle (degrees): Vector3(nan,nan,nan) 0
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_45_day_pad-transform.txt
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_45_day_pad-inverse-transform.txt
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_45_day_pad-beg_errors.csv
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_45_day_pad-end_errors.csv
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_high_confidence_45_day_pad-iterationInfo.csv
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_45_day_pad-transform.txt
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_45_day_pad-inverse-transform.txt
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_45_day_pad-beg_errors.csv
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_45_day_pad-end_errors.csv
2024-11-25 15:57:11 {0} [ console ] : Writing: /Users/ben/Dropbox/UW_Shean/WV/2022/WV03_20220417_1040010074793300_1040010075633C00/pc_align/pc_align_ground_45_day_pad-iterationInfo.csv
2024-11-25 15:57:11 {0} [ console ] : Saving to disk took 0.01264 s
Loading

0 comments on commit db6bd46

Please sign in to comment.