From ea6b15700b20ace9eca6f2713a5754f1dbfed51b Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Wed, 3 Apr 2024 16:43:52 -0800 Subject: [PATCH 01/27] loose the limit of only processing the NADIR scenes --- landsat/src/main.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/landsat/src/main.py b/landsat/src/main.py index a774d548..91948dd6 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -62,10 +62,6 @@ def _qualifies_for_processing( log.log(log_level, f'{item.id} disqualifies for processing because it has too much cloud cover') return False - if item.properties['view:off_nadir'] != 0: - log.log(log_level, f'{item.id} disqualifies for processing because it is off-nadir') - return False - log.log(log_level, f'{item.id} qualifies for processing') return True @@ -99,11 +95,17 @@ def get_landsat_pairs_for_reference_scene( query=[ f'landsat:wrs_path={reference.properties["landsat:wrs_path"]}', f'landsat:wrs_row={reference.properties["landsat:wrs_row"]}', + f'view:off_nadir={reference.properties["view:off_nadir"]}', ], datetime=[reference.datetime - max_pair_separation, reference.datetime - timedelta(seconds=1)], ) + items = [item for page in results.pages() for item in page if _qualifies_for_processing(item, max_cloud_cover)] + if len(items) == 0: + log.info(f'Did not find secondary candidate scenes for the reference scene {reference.id}') + return None + features = [] for item in items: feature = item.to_dict() @@ -192,6 +194,10 @@ def process_scene( return sdk.Batch() pairs = get_landsat_pairs_for_reference_scene(reference, max_pair_separation, max_cloud_cover) + + if pairs is None: + return None + log.info(f'Found {len(pairs)} pairs for {scene}') with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None): log.debug(pairs.loc[:, ['reference', 'secondary']]) From bf194be4f2b4d78383ba0532656f68172c5b4e88 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 4 Apr 2024 14:24:15 -0800 Subject: [PATCH 02/27] update the CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36802b66..c836fd45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.1] + +### Fixed +- Off-NADIR scene can be processed as long as the secondary scene has the same wrs_path, wrs_row, and vire:off_nadir properties. ## [0.1.0] From e4e54aa5a256708fdbaa629985c7af478204fc06 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 4 Apr 2024 15:39:35 -0800 Subject: [PATCH 03/27] add test_get_stac_item function to test_main.py --- tests/landsat/test_main.py | 74 +++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index aacfcb09..a7be1352 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -1,8 +1,8 @@ import unittest.mock -import main - +from landsat.src import main +import pdb def get_mock_pystac_item() -> unittest.mock.NonCallableMagicMock: item = unittest.mock.NonCallableMagicMock() item.collection_id = 'landsat-c2l1' @@ -73,6 +73,70 @@ def test_qualifies_for_processing(): item.properties['landsat:cloud_cover_land'] = main.MAX_CLOUD_COVER_PERCENT + 1 assert not main._qualifies_for_processing(item) - item = get_mock_pystac_item() - item.properties['view:off_nadir'] = 1 - assert not main._qualifies_for_processing(item) + +def test_get_stac_item(): + scene1 = 'LC08_L1TP_138041_20240128_20240207_02_T1' + item1_id = 'LC08_L1TP_138041_20240128_20240207_02_T1' + item1_properties = {'datetime': '2024-01-28T04:29:49.361022Z', + 'eo:cloud_cover': 11.59, + 'view:sun_azimuth': 148.43311105, + 'view:sun_elevation': 37.83753177, + 'platform': 'LANDSAT_8', + 'instruments': ['OLI', 'TIRS'], + 'view:off_nadir': 0, + 'landsat:cloud_cover_land': 11.59, + 'landsat:wrs_type': '2', + 'landsat:wrs_path': '138', + 'landsat:wrs_row': '041', + 'landsat:scene_id': 'LC81380412024028LGN00', + 'landsat:collection_category': 'T1', + 'landsat:collection_number': '02', + 'landsat:correction': 'L1TP', + 'accuracy:geometric_x_bias': 0, + 'accuracy:geometric_y_bias': 0, + 'accuracy:geometric_x_stddev': 3.926, + 'accuracy:geometric_y_stddev': 3.525, + 'accuracy:geometric_rmse': 5.277, + 'proj:epsg': 32645, + 'proj:shape': [7681, 7531], + 'proj:transform': [30, 0, 674985, 0, -30, 3152415], + 'created': '2024-02-07T18:27:27.558Z', + 'updated': '2024-02-07T18:27:27.558Z' + } + + assert (main._get_stac_item(scene1).id == item1_id + and main._get_stac_item(scene1).properties == item1_properties) + + scene2 = 'LC08_L1GT_208119_20190225_20200829_02_T2' + + item2_id = 'LC08_L1GT_208119_20190225_20200829_02_T2' + + item2_properties = {'datetime': '2019-02-25T12:13:15.140373Z', + 'eo:cloud_cover': 46.47, + 'view:sun_azimuth': 87.63621418, + 'view:sun_elevation': 9.4970641, + 'platform': 'LANDSAT_8', + 'instruments': ['OLI', 'TIRS'], + 'view:off_nadir': 14.092, + 'landsat:cloud_cover_land': 46.47, + 'landsat:wrs_type': '2', + 'landsat:wrs_path': '208', + 'landsat:wrs_row': '119', + 'landsat:scene_id': 'LC82081192019056LGN00', + 'landsat:collection_category': 'T2', + 'landsat:collection_number': '02', + 'landsat:correction': 'L1GT', + 'proj:epsg': 3031, + 'proj:shape': [8901, 9151], + 'proj:transform': [30, 0, -980415, 0, -30, 188115], + 'created': '2022-07-06T18:15:03.664Z', + 'updated': '2022-07-06T18:15:03.664Z' + } + + assert (main._get_stac_item(scene2).id == item2_id and + main._get_stac_item(scene2).properties == item2_properties) + + + + + From 2bbf4f284e87287b955ffffd47006f0112a7d40a Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Fri, 5 Apr 2024 14:22:46 -0800 Subject: [PATCH 04/27] add test function and test_get_landsat_pairs_for_reference_scene and its test data --- landsat/src/__init__.py | 0 ...T_208119_20190225_20200829_02_T2_pairs.pkl | Bin 0 -> 2507 bytes ...P_138041_20240128_20240207_02_T1_pairs.pkl | Bin 0 -> 21806 bytes tests/landsat/test_main.py | 18 ++++++++++++++++-- 4 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 landsat/src/__init__.py create mode 100644 tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl create mode 100644 tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl diff --git a/landsat/src/__init__.py b/landsat/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl b/tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2c0e4a2c8d9d788b4c7a9b76a0895046f5a77b40 GIT binary patch literal 2507 zcmai0O>7%Q6t*2Zv6D9GU(%FT1+^f8)SC6u7B?y-siRh{6Zg*rsZuo>ugBgc>s@zu zHmRdV0-=T$jkt8m0VFtb0r;=Y#qY+coCrqP4sZnqx|Z*jB~w>PRs{0;g#i^EM+D zR03(Sd6h8rJlX=II|ymEE_KCAGt;SX_#P1J)CKNs3`SF5s4(iZbTYEMT|N@quI)ad20kVw?Z!=U&LdE>-gLY4343sLP048oN6R;T^3_kWFcOW z=j@j(SB0I56M!gbmAU-!b>p*6)d2ykusMnlFQ1qLOOl( z{p{%vRz`8T4E8G`T?s7k4&zGswq;<<#j)bnVjQg(6L>X;%w6Vl=#}|=$EUu~r#&K3 z>w3~>Y7h7ko;v>h+o#4q+w+rbheO(CGfhI%WHaW_E<^N zpiRmkn9$&ooXs68qouiTcFldhn*mqcS&W^S9=vl8$gkKspZuJNF)^-dy1rL?d9QY$ ztZS3|^hsUMzO+AgUOsMXVz!{=2_8ZsM}XuGf~Eby=$e+-4-!o$=+By>c`c{sLeZR- zJD^0_n*A{X{Q-5jlu~)`S5zTgY*4LLF=Ol0dcC5H%L5c-mmxYjv;(5ai(|k3$giQ0T&iICgOy{us|Qch=q9a`-D@F00Ju z6wxAvUo?CL{=phgbcA2qs>8$iA_*bWHOe)v$~7UL9730@M!-!eHr>C)Y_>qT1hsTdNU=RxPkNZo~BUDepap;5|63v$g2?B%E8TZ#5~NKhi7Zxr;q|zQG(u zEHJgbBV@R?%?xys7%=JTuGfOuxyCMxuL?%Isk`UFh>OC@G(Go?N!Il1kol1JO$u{W z42KKuQOA#>A22)y=YR_XK@$@TT*hNH<}n9`k{G?u6KY%xEZNd%5VNwf+5JM$Dyf3(&ugyCMYrtnb0Of!|-+lX$A+e zUDSJ2hCo};uzxOqyQI@6&=GdfaG;Ff7ZoYWIlMP5OVLkiHm!&3R%+YsqED(p6Gkuu T4~4BT+m}~;^I!AZppyL`mH&l` literal 0 HcmV?d00001 diff --git a/tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl b/tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..dcc269dba4a6a402018d0b440905d6c4c61fc442 GIT binary patch literal 21806 zcmeHP2Ut@{*9N4EG=X42L3G7Jy!WO@(V>eVO~A5h3?Wj4fFu!75qph_YgDYOuDxQ{ zUE8j0)z!t-wXMCoc3pe>&mDURZb#={*<%NavVgPL@lNA&XG?~XogyNPkxuTyk1G_b3(lUB+VN6O*cAgUa z>%f2v*9JFm=+xN$v2FZVeX?FxkLjABDK1c|xyAw*R{5p$h`=r2StW;7m95e1Y*c`5 zqZy>vx#I4$n(SPzW)S?<>+Cdxv>JucLz}MiR%!DJin#{e^0c{{Yy(TsOID~<}1nftRFwl*lw${ zZl{lrkMHyWehe?MRuI8(%+=s*tk!wRgetcdHUHV@8P%lT6Ndvw6cmp~}f zb&l!UY)Ay0LU!uhYz0oQ*aoo;W36!@V{PzjR_&}NS==U9r_78-pKX#}H zGnVdAXfbqudT$Ksh&vi9qT3}jKHpgHAVz)h&&^00C)$X=J{&#Av^Pg0<3bil_t(*g zQ{aOzjihbvG(UfYM>ELFg~ks@Eim!rL3Di^bo3YaiSsv>zqr!6xTC=a$5zJihWOC+ z9J=1w$g_Evv7aH4v|jB|AO8dDgnj9#SHR-b3D45e$>t$%bDAp9q&u;hu5k(!u;Sp_ z1}X)5)o#7l$}tL5(DD5bucj-ISU970Ww`=98ku&o(G&$bE-iStWq<-#^OaHA_auiX7$(}Qkk`kD?W&O7u(y}R8{3{Q}u_rX2apY!X3j%N9ATMzGp z0&Z^HCTt}~dmj(6_tMHy@(;5r7H*NFe!OP;etIrPqT*`~e~{_u+(?O=b#6M^sux_X zSd@-_Uvqg~+39o?pjdn~+7Wo>Z-3pNuR!WU`yRgUp+GHyYPpRrR-pS!tVSM&@g+_U zOl~<_f!5h>)=^OU*BP^G)C2{Jf26vS2>RGp>cg6KfU?ZxLynCCT~KGO{BuVi)M825 ztJgPsqpfFdgxSpQg_5TXjqE50Lv{(HhA*iIMYl@Fp6{0#ifn$mdZck`D6)!a63HDJ zikiQBvAg+&P?XW&+rB3Y!qCn1W-+74aMWeN^^OPXN1!#Fa7j|rNOZdPkv;DsqtNoY zGiJpXMWeZQ(zdK#*AdlSb@HnO~Y9dUy7l{_7*rN!Q*T?vIW@zd2t!yJ|!@`s(QO zE-O}qA;01ty^?Li(5fHDro`w&QLTG7@>}!@MPH3{9oD^1C~BIVHEq$HQ1pXGn++bW zVJM|&=%&>$54}&1dVY0yIC{OKVd`;h1ZrBld{6L%NTgP&mW`{3LW6{>+IPJjjm{AB z=MI_L868Z1bE&H^0d3wpY~lRt3Fymu^6yZ(VeuDX#HC8|FP+cHoGoaUby73jz7y z4f8w@yg-8oT7-vj(hV7g73_jW?l}x2b4S?f;W8&QcOM>zqp_b#C*;=yGj~J*czE~S z5biejR&8Xcp5#R9;)qIfc4(&gIv^kqfDQex{*u+S2tt*3r7@%+mr@jr+^m^uE{& z{XP&zFwU2@_oaDx({b}aFL63X_!#%0L3NQ~8|~(UmLG@}jsKC(CqH*|9I||9jysLV zK_GIB%Lp$zPM$PRA6j=`I!_zYac^kc4?ptzsP_qwyW92vBkoFn<9z_fpU%@p=+QDv zpb=`0C*HXRbl&+PLl}P_EQxFf@K$ahC8A47o4s^+CT zotNIUy{EB1Cr_i_X?L2Z8{KamT2D{Kxm`BYwaW;mVxTNu6 zm#g}t-M$q6yaJ=?&CHk5sP#XsgJDXxi*<+%jCK9=vkkYZXB!EOiK??6!g`?90YBa7 z1rHaU;zHvix`d|4Bzn`em~s98rJl)|dHr|s|0gj2E^hw>jyWbym0AgLF?yYCN=$MW zz4A-N7d`%K(?MrV>2m_%+5dm?i>?;QgGKMltC<1>p9D?5yfPHJ9UVV(CNi_!j38#p4;~QFVMuXc?b0^jqDY zWXxX^e~L#K`mJV9Nf`5&VNVGde3f6{{5tpZ!*^VrYd5O8GSyjIr~^B>(c{y0|Elk&`!%)x ztNof-nXXJ+;L60FzA~$$)i8NjLiu~@Ok?}sJVvGlGY@9xW_|dD)!;EY+0(JZEhhVDrjeJj@zu9AJ$J$DC3B8dQy}YE01@S=Bi5 zqfs?T!tBG0ZjG#J3@x*BL$>RD@$FeKK?YMq3L%yXNGV^;6%f20dPWm3g%nFEk(4jw zN=TB~MDSBcN=lJZQo!YlM64!4AQcFulz>Y}#B>ucn52R!VG6iV&?@0lJR!4*6r>P@ zRKSx8D6WuCF`Ecp3L%k_q?F)sMHB-N$V*ZPo|NQC2{D(?mzccBU{aJq2&DqR6mxkL z!D=D_kS`@ju8?5RAVCn^7ez@)f-B}TnuuT?nLXHeOcMQtZYKVx`C(3 zmz53HlP@dVKfkgGq&&zsn0jKyVh+FB{-%{JC@!23%ka|^nekD&%N zN(N8Navs6wVE&U|bXW_CB`t_5Q3zTKgQ+FiNUF+|$27HsO7Lk@xG*Yg8I?y;l*poK zHB>MqCD;`EP@h93N!pYYFikC?5lHi${T0kX8p^*T-_zWt{Cj(ak1ZJ34Mf2evrP;lDyCLu!MMd0;JY6#^I@7J)B5%@wwZ zwCTsi!}8qX5du1}hKo<5@*zzP1o*HvF-$FR%B~GtXBG!=gxIW)YAMpF<_-VNvFD%Mxl;v|u8B2DNH(!+*hk zE(BGR8>ZH0mJEWd$^jDQ9QVO)~Cycbjkd9vChr(ZEpH| z#dY9o+<8h(VR6-0p5cqj)jt)@g+XNE4^oErvc{ixhR+09;Q@6%Hu`Z&TpDVuEVy1e zupQbtp~=YCL>g)%XgqGvE-4bPc)RM>k~Fk&)6n|zK2nr=ZOwY=Pig4vn6?sigLa7R zuV>-M_^IAS0sig=;?=3e?NG?2)+6r$e$k$%Z4!4&(K4r|D~kaCkC@lgYrqex=yc(? zAJfos?^Z|d06yEFjeoOgy@^Kw|3F1+{T$$bc%)s+0c09lJvZvyOyECne#;lVmZqWZ zC398|1bn|AJ(g|-{v$T3_yYi+?a#(Pazea>0(||M)q4YgfA-r0#bL+aQxH1Pc5W($D-&kbHBZ~*_x z#cMjupNQ&IWC!@uch5Oo1o$?G95&|x{(GlEH!}gB?a#)KUwr2$b>}p+XKu_Z7vSF` zI(Lm1m4*smWXN9w{sKEvF$?suY4OdaWq_Y|?N*(KpwFG5ktY`eKHHy-|K`{aPuhY$ zl2%jNPXqp|$F=$O3h?jN?A1?$G5>kzJM0Jka}p0s$-w-jw%-H*eyiorv>AZU_Gjbo zFaFT)4Cphmw|$}n_-6!cT{Q^s9bf)1+aL7LYC9843JTu%tpGWoW{#*$BXYG5aeFOT0m1;)~0{o#f zo=@nyHx1Q4al=jv_-ubRzIEI0<8}i+(PMU02=L$jKF3`K_>IHY%?|J|TW1hC_TOb{KiK3*fW;+4v{g2h4sB z{1@L{m(&UPD`ytx=`sHWIj%zBKXClfYhIv#-igL*&H#S=_jk$*fq(p!8!t}*KHHy- z@AP5(=JUWmI_;3(D&YV2NLZ6noS%R2gBAmSt$Wuuy@3C@=A-Ya0Dqy&cUyh|{-fHR zRI31=?a#*d613S!#l!mWDs-0s_`96_b3-M>_jc;d%Oc=^?|sn8P7vQ-sB7~hfIs_2 zH`fgi-`XW}$@PHG_Gja_4E&4O4Em^AdDU$N_$y~d1f+mI_hQzKX%6`EMb~vFf&atg zrcYM@{tx^1ItoFbyj8fF zxpzSuLI1(W7suKG{=(iNV~PR)=P8rA_5=Le4vYWl1NdxzHvS-o`*Z67|ETqE&4K)R zKY0A9iNL?#h@f%FkUytZ{d#Ex@Gm^m>}6ZfKVZ)7F?fBRSYEH$RLGy^QBQA`13ufI zjlbkfEA1_aukW^^DR}*CeNh`SAL5%J9ez9z`1cOD{Gk!#zjYbW_zmFC@bCJU0^-|t zXGycSfY0`4t^3S4g6DfY0`4;~$he^m`2Y$K_hRXa@L)fGyK)}D3ee7u^;IsYN_}#n5Zgd6yjjrVM8wL854Ok+<{N1Ay z2bbXZj@fCu9Ppp)n*3}6)_?cQXYT>umgo9*BH*+A+4w0vru2Ob_#x{X1vkg}*>pj} z?;yT=3-6!s0RG$?MT-L=zL$@;Cr<-Ddwpa3v+?Z{c->T;($MvaT9p;Rf5Gv7kqz3w z`qC$o{|)euUR&>XEBO7}+VR!W2r0fA2i&@{n`nl$A*m1zu6!;vo27E=}h%E;JpY6}aU#(oz zQw{hkP54mE|6TCvG|d0)Z~7*6a3z~5y?-UungH{{Onu>#ncu#x|@fD4pRBJHc2TVzw z(izsD6*n^t`&+g@8~?@{qRu49pHnBbtFZpxADHuXTgac4&k}4o!2iMW1?_e~e*QIg z`?+5Me`#3LnRxyVK7O|l=Rez@jlWfu?Vk$x+y9L3`5o{tdM;Xu?=R;3Fw1s6@b73P ze1h*UmXx0wKML^IroKDd0q`@Z<7uM+pY6}aFKZ|0Y6a`_=*3$mc?18U711}AgFX^B z@7qrRKd@wEKoG3Y181D4RssGuEe5|T1$~ws*_4U*=WKsA{>mf!OLhVOS4}5RnhX4c zSJjTf`=6_WkDVO@_~OT-gF1o!&cBdBx2$z~9v>E-DxJ%PTy}C7{oigqW$AKii*;A0@R` zR{;Mbb^e^y2l&spFdzc&A5OYAsTB+SpC8HORziHU3z8Ba1Af@Uo>y9fKIP-a{`m;- z+5T+&eQgt3$N>LB;G`{he?Ij?(dZALPmdK}JK+6oHfWOc8R(ChxpZe1*{u7wLj_=RhSqu2J4^D2^1N7e# zcINAEfj`@yjXx(OJZ~!SUtj+Ne?9Pz>%8ls0QBi__O#m^;Gc8ztP}qIySL%odZmD0 z+<)i20ie%?UW%zx0H5v8#!s9T8+{D$+r1m0ZUOu&x#KtW2mB)B-PsHH_wFcc_yY8q z=n^5=2KbwM4GKyC{ADwu-{Aab`?K*gv(^SDLjL&ojh(Rt_%HgV&hf{9ufMsd*=FF+ zy|w((Rp1}K=C=<+0N?xm#%bY@KNFP~(~AI~?a#)K)ZTiE_YWS)ean{u{@GtNopJs* zPJOs-0pQ;sw(`m{;NSiHg%G@doB3+V>^i`|*0cJReF2~C&&EHp`fTzzh_CO4a}VX9 zPiCWozv26fGd=h0?hg7i%{~0!JBY8Jz3$;%!0*cY>Z}*Ucd{FQ&3V9Q`|I&DqWCfJ z#q)NtR``%Lvv0Y&W_g$noXhet9wC=i&v1HNRs)@5)k$@Fog+T5Zcx4o9|e|^15adh zPPisRnX8dQ6=1_3j6E9uX)S!s-FP${e9@hGAY8Sx!AfdA4zBF_H>KyEuHsZW>ss(j zS8yt9y2np-Ud95Dm5GdU$O0gE_UJ)4CsTKz4jW74b@A4Kyw-I`rr*<|7%{0of(|6N8+E6 ztQW`sZRw9m)@hu}jaw|(pzFeUd2nen7vCg~?@E5ber_BG&2QMU>Gv_5VNG^SD~XKe zy!`(CwF{LIoR{ZjrHFfmam02_PEEhnfwTNi;rfS#A)N96zbBpEOF52>FTPkhUd*Yq zFNvz5KVIIG1;JL+#!)=IqJ3c;0GLUC#2B^=557?8MrWBy zStN_WbvSb-;LNeBdJECk*+P!VEj$49t9k&abA(5Th6jd_g{FjbRpv_-x%t`*nIeCH zvQV#cACRpqlB%^TnS4-oo>r5k*Li-dQsycL$TivdD!?enm23Lu7v|}8wc&_6INmT% zsnWnvcdiEO{Jwo0ZKJzEYVwA5Biw>7Znr`BX-CF$4ovY!Ercka@8)M<1C{)6zK@bOR0S+@(ZQ;dC=QN3$wBq2o-Pp2@gmj^a4J>OqTeT*?Kp)G+7fQIWHXti8JRCg=p$D2ch3P{|on(-zazjq{0@AXo z+=VR7)#j-nwSb*mXD8Qa;80SSU-q)Tx$;cIB}bl{n_r}sWoQfF&_$2{k}X}CtJaTG P=Ntf9IWyozo6h|oRX?+O literal 0 HcmV?d00001 diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index a7be1352..4ac62147 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -1,8 +1,13 @@ import unittest.mock +import pickle from landsat.src import main +import geopandas as gpd + import pdb + + def get_mock_pystac_item() -> unittest.mock.NonCallableMagicMock: item = unittest.mock.NonCallableMagicMock() item.collection_id = 'landsat-c2l1' @@ -137,6 +142,15 @@ def test_get_stac_item(): main._get_stac_item(scene2).properties == item2_properties) +def test_get_landsat_pairs_for_reference_scene(): + item = main._get_stac_item('LC08_L1TP_138041_20240128_20240207_02_T1') + df = main.get_landsat_pairs_for_reference_scene(item) + with open("tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl", 'rb') as file: + df_expect = pickle.load(file) + assert df.equals(df_expect) - - + item = main._get_stac_item('LC08_L1GT_208119_20190225_20200829_02_T2') + df = main.get_landsat_pairs_for_reference_scene(item) + with open("tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl", 'rb') as file: + df_expect = pickle.load(file) + assert df.equals(df_expect) From cd637bd9aca6fca37ab04830deeb06ad1be811ef Mon Sep 17 00:00:00 2001 From: cirrusasf <62269400+cirrusasf@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:16:18 -0800 Subject: [PATCH 05/27] Update CHANGELOG.md Co-authored-by: Joseph H Kennedy --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77774fbd..4ab1e93a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.2.1] +## [0.3.0] ### Fixed -- Off-NADIR scene can be processed as long as the secondary scene has the same wrs_path, wrs_row, and vire:off_nadir properties. - +- Off-naider scenes will now be processed and will only be paired with other off-nadir scenes. ## [0.2.0] ### Added From 5cfe39d6cb6e396c59c1608a1aad9574d3d4a96c Mon Sep 17 00:00:00 2001 From: cirrusasf <62269400+cirrusasf@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:16:33 -0800 Subject: [PATCH 06/27] Update tests/landsat/test_main.py Co-authored-by: Joseph H Kennedy --- tests/landsat/test_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 4ac62147..b1130498 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -5,7 +5,6 @@ import geopandas as gpd -import pdb def get_mock_pystac_item() -> unittest.mock.NonCallableMagicMock: From 1c9b874d429de1b793b74f2d3ccd2af485097aae Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Mon, 8 Apr 2024 16:24:48 -0800 Subject: [PATCH 07/27] modify test function code to adopt unittest mock class --- landsat/src/main.py | 3 +- tests/data/job1.json | 1 + tests/data/job2.json | 1 + tests/data/scene1_pair.parquet | Bin 0 -> 33040 bytes tests/data/scene1_return_itemcollection.json | 101762 ++++++++++++++++ tests/landsat/test_main.py | 167 +- 6 files changed, 101859 insertions(+), 75 deletions(-) create mode 100644 tests/data/job1.json create mode 100644 tests/data/job2.json create mode 100644 tests/data/scene1_pair.parquet create mode 100644 tests/data/scene1_return_itemcollection.json diff --git a/landsat/src/main.py b/landsat/src/main.py index 91948dd6..1abfd28f 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -103,8 +103,7 @@ def get_landsat_pairs_for_reference_scene( items = [item for page in results.pages() for item in page if _qualifies_for_processing(item, max_cloud_cover)] if len(items) == 0: - log.info(f'Did not find secondary candidate scenes for the reference scene {reference.id}') - return None + return gpd.GeoDataFrame() features = [] for item in items: diff --git a/tests/data/job1.json b/tests/data/job1.json new file mode 100644 index 00000000..6897965d --- /dev/null +++ b/tests/data/job1.json @@ -0,0 +1 @@ +{"job_id": "5864010f-4f07-4030-beec-88e9c39d2baa", "job_type": "AUTORIFT", "request_time": "2024-04-08T00:23:21+00:00", "status_code": "FAILED", "user_id": "cirrusasf", "name": "LC08_L1TP_138041_20240128_20240207_02_T1", "job_parameters": {"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20230407_20230407_02_T1"], "parameter_file": "/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp", "publish_bucket": "\"\""}, "logs": ["https://hyp3-its-live-contentbucket-s10lg85g5sp4.s3.us-west-2.amazonaws.com/5864010f-4f07-4030-beec-88e9c39d2baa/5864010f-4f07-4030-beec-88e9c39d2baa.log"], "expiration_time": "2024-05-24T00:00:00+00:00", "processing_times": [699.518], "credit_cost": 1} \ No newline at end of file diff --git a/tests/data/job2.json b/tests/data/job2.json new file mode 100644 index 00000000..94b16336 --- /dev/null +++ b/tests/data/job2.json @@ -0,0 +1 @@ +{"job_id": "98cd33bf-4f32-499d-94db-20661f10f5f7", "job_type": "AUTORIFT", "request_time": "2024-04-08T00:23:21+00:00", "status_code": "FAILED", "user_id": "cirrusasf", "name": "LC08_L1TP_138041_20240128_20240207_02_T1", "job_parameters": {"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20221130_20230319_02_T1"], "parameter_file": "/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp", "publish_bucket": "\"\""}, "logs": ["https://hyp3-its-live-contentbucket-s10lg85g5sp4.s3.us-west-2.amazonaws.com/98cd33bf-4f32-499d-94db-20661f10f5f7/98cd33bf-4f32-499d-94db-20661f10f5f7.log"], "expiration_time": "2024-05-24T00:00:00+00:00", "processing_times": [752.675], "credit_cost": 1} \ No newline at end of file diff --git a/tests/data/scene1_pair.parquet b/tests/data/scene1_pair.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0497f63ea59300a9a3a38d431b72c74f71e299f7 GIT binary patch literal 33040 zcmeIb34Bvk`aXVdN!z5Pr9ji9&_Y`v7HQL)o25zFPSU+}H%;5fmZf`=hVCu0$Yxa# zR75r%Kz0yO5z$fXfGZ3PC?m@tg0dfIlj6fs5jGcKAaGKhGjL&;75v}74-q-{<|H`;dUk99+ zt(jkZ`Zq0F&l|S)0^npMI)Kg&4tsJR0DZVMcHuH0v}dO8gN+PpmV*z-W>g%=;Z zHg2sJl}uT-u^E6O{?>c#n?PvB>sDza0G)LL?fp!#h6bR8M>g+{0!EcLKWsSyj3&~1 z=m20umoHde0*nO9pUc<|K$j!*quv5WDFxRqtO20Cb|CWz^|PpMj21N?Up-_65c;_P zuep4{Id$Uex#X@#sMO3RUx0b;y=8lv0LJGN|2HfE^Bbr2Y`Keqa#mm1RhO+r?aMQM zkq?0EakBIqn-sio7>|5aSG@Z5BYW1WWb?bI6nFV;Il06!}%t1mm-n%OccNjTYt5| z z1OTC{heJLGpoAxsmP7zrz56upAOMjL1<{ulHSGi({+0ckUI0RQwnO8`15W>upHp*z z(CMMCDpJ;I(WVQZ1lQ>+6+suu*8$9>9O;7U4h7kU_c>MgKB)Ptl=c4{4s2wFb556o z8yXYI;+2ZB|j7Bls?~LSxD|$yrL>d4!UW6eBak0pc~6)17AJ@Fgb5uSX>Kqa!#H3 zK&BzM=*Bo8bL@fH7p;J@lKay3 zzXO?BBM#MBJ5a8m5Lx`|bPn{0UsXG0K!$tto7WD2hR+wCJ(&P(F8&sCC=)b14i!DH z7jTxIDGquKRNQmQa`CHx6XG=agM)se-TO$rm$TnezYu=~tbe zdk6?!E*Sjd1_1i&`@7k45Nmfxvf_CFde8~9BlYV^1AtNKyWYUI!ITKDc^}F959^KXOhB0DU}vUQsmwo%dh;(?kG@w*#45{Vy#K145~< z-q0rip(nDY_S^zEuZ^ZhGJFC7Lxn)RQfM}W`t@n~l| z@cFg^$(91p=Y{N0s!bQhhKZ459sGuym;Gt0CX#w`^v8X z^z7PUqecKw-f7*>djN>v7`GY=md|Qu?FXP3C(uWF|LX5SLs@3x+7JNxMF025RRHwl zL&aygAllAH-uiL?0DZS@_o^=eXk`C=TbcprV&(oH4*<|dcA%`1k}s;Xy! zWD6SCD9K&H!D)Gur$n8Jo%1ehJ)r!s^Qo&(M9osX_siAa0E8t8y7?r4*v&<0bo0x; z2aJ)g4v2d|waz?vMfgjg<=wTHKI;usL}!{-4+1TpJUB{xIP((AX=J}#NwGCki%xIt zabPPnz2abXO8cpu+ZEWzeokF9 zo?CLY5`cDWxtJ0My2u}(+ZPHzM?N&I4FUpf&z--vm%co<4sfjXNmI#P+*@&*wdAf9H$PqY!%INN^p0=4FVHDqO)125W=?C4 zssWS(=gfbb4Je!KDAec}e>4v)W4+#TDtF~8I~G}HB!Xc^UU>8|8IU=hGwR_F0cMf^ zys75_#{Z+&4#a>?%jk^4V;wp@zrZ36HbIBHesj%%r$ITlY8UDQpd(4Xsj6N8Y-a2^=u^Fs}rqGq+f{o%5Qhl61@ zUd?6kfy#I5SB!WEtozgQw~qY-FxILEKaah3^1*Lwv5%Z__{f{q%E&^1dFz{;u`dCa zrk@km20hA+UG~={tQUbxIwAi7uE(xf{Q2Xv0B1|#t)rs>$3!30&gu~93R*BiRzyH> zn7w-YJRaC->ejR~>p>=EFW&hd0B6vYnNcwi9hw&%r#Au4Geal;It^r6w|84P9+@m~ z8hqp452ow{Lcb1vYTj}n6t}5oDt<|QHF^KhIRK>ies)YIC};6!7cW%+(Efn;w!8#{ zp8oEK`ON^d(P8kASk>Avz-jxfCgRjp=@%dPef&O{>WqBD#qjr1FR`dKU+#Tj1sLZ| z{~M~mpStR3e?-Q!XM765IGlC8_dWydPaXUClpE0gzL+c3)1ZCj;;mI;NR;n4= z*Kh5;SP6n`&&_zcIcRmDL$9fcWZhP1esAD6OD2Lo7Jbo}9HeEhI20V+V+`2m=e^~k z17M#@TYld60HeM<_Eap0vAFHAZ?1qCsSYvTAD%mO0^ocxXx{dq_j<-Y{d?2w-$4%D zhKK!wZUs;&XB4Nu0-Tek!52RZ3jB5o(VkoJ-{qHyK2ZAKu2_L{g+<>ourpP{rwTZ;az(D*(5N}lcgsN zO@LEwr^8KYIQJvi<7DrGvc+ow=jh+-v$1~+EWEs91>jtozVXyL;8Sw^i;?)H`24SH zmIeZ!9#{JvCs@l?j_K3 zgx}oD-vUmNM0hj=^!!vW>C4B1zUifKtIs2N#k{exFuMun`SJY1yy5`Z+x9I+bcdl)sPELKQryvkDK@h0_F~tPTRoW>Qj>Q zbig|P^rwCxFFv=LPdv|`k`c9vUom!wde!E?@YC~`Z1_Fl1%C06(;IcWm+~X$&B%Ep zXEA?bCI72l$DZR;Ro9=WE`63y)dxNQ(Uk@K15ZwWf6yaO@hA3vMcM2wfackk%Slfro{r=y$n^`-TdmQu#iKlKki=`Z|HC46Ui_+2pQ z&Zk~ub^6rcfwRBO-*G~TmM$U5JmU40QbFmJ#I!_me39Rp^6 zt3an(L$;)y3P1tALkU4(WK0Hv%hnJP3_8iM!S4=2Pzh?La*DYnqfh}knsqQaFqjz4 zp}614(IisL+N$QTlY>*yYG~*1?hKTUAs#n+KpT(^^255^_YG)V)at?~t{Tvx2S(nk8f--KE{rJ;$}*y;4Igdk zZ#AM{N4y%cagGt$(trEwug@8gLcXZyfyG92Wv2GffcZvrFwu5-d!rEvUKpIbD94Dx zmw%nm&NHHD3v2wXJqDC@^gz`is{wryzi;RA?ZGeroGC)bOW*r$+jqt2xtB+OcHDm~8ee=VFDZ8d`YmqktH&ZHqJ0&iqS!|# zqNua4?~o7Equt*>;uq4OM+JXfvUTNlJ*t)rd4Jc>dX&)ewg11w26SwuvKMJKpf?xF zzS_FVfd2XN$yXK}HlQfu>V0W!;92w5^+{4As{7>q%fIQ2XlP82Ub9<_=+YWu<{pqQ z@2Nor4=y#LS6FY%rQtg=aL&$IPZ&|o73--y(2w7Nh3X-r;d=r1_bMMF`l7C4<2Ub2 zL_^ms`1ShP@#xJXXVgAR$9eHAC!&DulpR~t^X}byR=cLtvw|BdOPdnw8mtrat(BIB z`Ue9N=VZkXJmZxzg+nvyP4Bj4OvkFMFOLg%JEcD%Y=g?x5>wRd2l z3K1F6DWa(=^uVoa?>+E^3Yq%8S6fL&BHOWuQI-pE7-rkd2nKNfz}Vl7qH>_Rh** zR_3C$;|^3m`eq(Fz9o75iGKNLa?JDRMyBVZ`PMZLeRnJm6>t|+_yp%7(+S>VhrY~4 zIb+Ty9C$nnSw2c=3wb94)u882XKK^YYXdTWCa$HTsoyTWG3nJ5bSP;2=u5Mc(We1l zAKf$~2?_W8{LqFCY826;8&|+mqfLK%Op~!tg?d~(Q!{j&3JGThO)r_KLW2t`o?W#} zh5p(*{#pRDYosB*!xbbC?JQuyOd-}>3Pv?5^EvKXcwo~WWqMmo}-nV`k*Xdh* zI()0tT-vCwudK1Uc-N}2!o^zfF3(oG;o{Z1q&!6y9wI@Gzn_T%e6v(J$q%VC6O^tO z3JBccyvQ!&Wll#)LC2{A0D`3UaVQzMapR`K!Xd&J^)4e#n^|99R#R(X1ldJ~?jeQi zCkhWuz(dUr7^N$sSK0%3(N5aE6nNa8o0Y+WFHJ_lL%=nQr;g?AAUg;5Ec`&w14=<5 zVMrhx!#0MIngCpP2P|uZZS4F62xpl(C@XqaTI=d-8!V;P`Z|`mvaa4AtbwmUgdJq) z$7JX(QaIfx+-1NrxHROIcrY&{i|51Wn;i~qGaq2whZowZT>;zqr1K0L)$=ag;Y2$0 zJ(dvuRMt7x31YmbrmSp&Rd1@azw>}^J1IO}D%@5Ip5qDEO(R|7Tjd=d2Mm$%n zov}k;)>}<=`g+%4&jftW89`v|v(zH|xZ=Ma`vmZUoF45PjHl1EyE{X-AZYtrMQ8#Y{SJ?u-p36?45P{=4Nl+4;^#e-`Ep*&{|fK5|jY-Xo{9nG?kT>Ls6VCLJ<+h z64Mc3BAQ@P(ulAynk$jW#f}r1#QsDovwuqx?BBGK`6g)56as1zV#hZH-w`y6Ryw{F zj{DAUnP?1CUOIjZK}bZeFc*~0M6kpOa3sWZ6B)4xzEa2t6MXe031wj*OG3d}1Wiyw z`x4?&qkY&CDnaxy0bqn&*F+K&9ZLd@p|P2OA=N=GYG_I(@M8F3+#$>*CECE0TcD;1 zE-jHM3SRPI!*^*lL63?h;Z&-v?g3wt(rSQW(Q<~A?-YzB_MKumANDakrF^HPg1Ts> z<6Gt%7Dz{oVbK!Z6e1j+Gs*??yQiCYUacU2+lmh3(9Zf`w7X32v`y>uqK?j0XDqdr zPN=lUfa`rjfASZuog!S=g#GC5_;~d-0WaM7Qs&0umy$-z9C1MC#~aZ3euL(GG_=4+ zgMB}ca=gDHi5_GuA-wr0*-5>>vjZ4w%;r)f9s*4;!W_Q5#*wUQBSROE!qtzG!i&>M zEQb@=D?=_X*cmANLU6N`w_xQ%0XND$Ty37^!Yj-RFKdIv;Fw$1hdv@aK9dw~ooVOf z0QAZ!k~i3?wcrtW43j0+=@@W^b2pev?VlHCnkx|r3=h+=xEkC$P~bcVPe93#mdzr~WIju3wHI1qQv*TeKi zV?(Xp*pleTOja5vG*2*8>gyQ(?ReNF{Irb}u4*H_4P?4&_D_;+)x3WrTZ_AFPs}BS z$DaJ}Wo!54ZHMq*%nUeB`ur3`Hasg}X@i2$-jylaf4s#(gZ}mF)T(y^k(j+XTxaTu z79~AtK6j%R`XF*^ZN<}pXyTaz(f|6@58=BTzV}DlK5pFkm!-XtY5149^PUg1U-t_} z|G4(h*K3ybcBHcUqIu6-`^;?bjSRuX%55L_anuVz36HBLp2-VAevdwv^yI}5RK|Sz zqAv#HJ0a+`7cQ^+aBm;Pq=T-9ARrs}+6%oA9NC_;ysvQZQk=Ta|(n>C;7iJ05-IC1P)HkwxTPW`h2f7E9tna}b^ z?KQag;ArBA%LnG6B_kpw?{`5g_XK;OnJ=O$X4*M?U7=2Zd`DM=^AWa8 z=+RKbc>Gf#h;dje50RnL{Y|}57~@&Jz;Q@D(TgK!yT(ST*teRUre~oQ{o=td{g4Iw z{-@v>K_QPm=ZB8&Cl3;Vfov`Zq#%z(a_s5aEiBIB;7dN>qGWy~+wRF@{2@Eq6NNxx zwN*E}c`LwmbWAup@AC*Y(Ee*Y=y+gt} z5y;2)EwjK8Vfbh*M2D@lp$;eM171iD9n5p2{DTqsX2lCN!D!?6;H(RraWmKv7kPmc z(T8JWar$r++{3|49==y#Gprae(dXShjwOQLXvhPD$A+LGI7s&F8^=!WkEY4 z#2(g@BlJ^rCD`U|hDg#pn@ z>_nanuoHRSCA>OOm$As~X+R!65P-W(_aLM1c+zropNab8F;@E0y zsuD|Wb>&PPwWm+M*$ACsBbLJxteYmh%8|!Q&f*0yT_J{jEdnpWR0pzIJqP&qLIgR0 z5sT1A*uO!(M|ia^NiB#fjv@lKvk-3G^DY34>d1I)b7s7Hgtg6xZEK7Dct#tXJkb`* zo)J4p5#95(Nc`U_d~EX}gp0R!szVXLsH3hzZ!7hZ2|Mi*y5C2*aSJKj^D@?sCzEdY zdDYGpAKp)77Ei?JX9vj|K*&jgy)q@hw>Rsn-#H}vM*-y<%=$GA7PAKS;{>xl7!8Mi zX2Xh9LoytujDbU39vt#B;qX0zLj)|6v1S**VMqoXO7h{bAqx%{lyFdu;IKzzBn2=E z- zh0_!3Sv`EyBHJi7dkD9Onlp2HA}05Pkqz@oH3+s#Za!#jDX$Ksojy02cVjS>!S{wIei6y zw!>vK*Jl?z&4WuaF62S3k5A9tzK;bTf=6<)m%?6fHC)NKz*V?(hl>LQF%D2)tGCwS zB(A#?tY?KD@DE)GnWb%{aL#Mk2|Sh3jjFsl!GaLpF>aP)4|-dGSpdqzNsBm297j_c zN|7iNCrT9}870xt;e=G85i1hughZ)aq!f$c4rDnrVmgr)CyHevX+pxNa3YqHCCcQ9 zv`j=P6}`i=BynOoj+Sen3=L0|B3dHHHN`RwMJ384i85Lwm(m!Xl4vMpqF9_rNkj>> zNJAi?q0~^4M6o21Qi!BdWrPCmi4!!GJW&R<6(R{uL8CEZs3%RN#3DH*noDrSAOUF$ zr|Cp7B~nO5;lv0kP9oOO07WNC5=0WEXg$HD;>7Z}ykJ@^0v

tP#r-rBFsHqNUQX zxxth&PAm;n1dEj-nJmHv8mBd4Dv_3uH(63i#7P#Vh{_{xh65%oq9ApnVJYIGq+Brw zMrv46nQS#z5=Y5N1!OFujt5gREPglP|4H!Vp7VU!6X0V$Bg(P9miK!yj? za*-V0A}|ecLR4Nbm@z><(oBXAZ5K!qrId)0$u&d>rPR<=qKe{DA}|C@d1)1;T0ta` zCs}goIEqb4;-cCIgQ}%il~T#LS>Y5NM=3OvSe3;Ey(mWkIf<$QsH={!5)yzbsGpXS z;o&kmQK}RvJJu1ut)aAk=Si9Ver zSBP#xpwnMriRHeIKmpG6-IqDiH8f7#-blBX>c^Yq0Gy`=U;N{>J2rEqFhl+w9BvM8I~Q z;c=hyyz{P`%s-#%9Bm4C9WqbQ8z(hX)>U?Ffw0fS&YmQN+hK&=3Dea@p6_LIxQ5c3YfxdFV8$l`V+loOEUZ@1y<-Up9ZTdC<7KnJSUA9yx|j%-h-o@O z+i@;dlCh+Wpl1~06{It8Dsxto1QrvN{5X-hm0*i$87CHw%BcV-7W*D2B)%MivQ8wq z60w*HJWk3)vA86Qx!^c1D0;|TXpw?O;b}3!ro?j2agu{kC`IpB_-0c|F=v)9r;$0O zIJ5AL)YBv@O*5{YE1?whP&OBia^GTt9^dW*w-V@-i?GC|NIXrZvPm{2rvqn^oK|RD zPL&FRC6pYBFzpetxYNFzR%lHs&Zyvm5)}3aEGd;yVSgf##IfO0F6fau!iNP^cd#V% zjMFf<67L`+iEI*h6SR0Dtnz`9STq2^01rk_JwUmHI1t8U;u&*ED1wW(`#|wYKe1s6 z6K?9?V?*za8aXwKMNs&zn3l)*W}YVX-v$=9?tm3NF>U#YMTw_ggoQr)IgB$&HJb!f z@^={Sk(|Wigc?jhi^cgBEQuJZap9aUO(r>V#=<2FQzKdQNk3d9VHP|X+oM)bBk1jLPWd5E_#xIT2>QQ-QlqoQ-2*!AJ%Gpgg7YqJ-}~FtrFURBPDc-SR1F;)79Msp;{Vy7 ziwOUKpAQKCu&wICKXB`^@DJU}dhd6c-mgudf7~Wb;h(S}P@P9B-5W0U3{GMIN4&DDA83I4}86ohq!7m{6b%IBuCP!CV zO{LB7dr)(O#ab7g82Rwz=xVqJXVI{}U0xa;8yRiYTT0=EwZUwLtGar9Z9QC4%z2Fs zPNj0@ycC~{r;LLOJKzKcz0`r*6{f7R)NFzu<+wuIc`?6y0Ho*!Yo)QqWT((zv9)wO zs4-N*K3+@>ey^{G6o(#)PmYGXaE+B6PU6E{YN zmU@r|YQQe%f5!)_Yf3E8a>y9%OvQM%==#L1MdzdXR?lXg&*ajFt##w@^DgSZzwmCY zv0~GKI+8#cu57yg7TA-M=W{n^W#$@vyeT@|Zr{%RI(`(;`Ga`omtmeXZdcNckc;bgRfx-jJ0ayvXm(G=wS*gq|I#a* z;^G!?-t_hyoF(ob2l&C)?I}14+#d$cByjf>Tua;^4qkrs(>c+|?hWtmt#Ys)milZ(`t6?y}u)3 zu7&SEHRtNc9Z_>FeE+H8jplbm%USRqQ|j0y?n$D{Be!Nd-qu~6y5sik-j|&_vAb!L z*`-?2&f{%~+Cg?Tpa^1w~<|8jKTSZMQX(dEItZLwqf zi)Wi%9^I!_T{qp_hJkxAH#W|x+C87fb*xl+a_jQQjn9AJ1$vo{aCp9$ZR9N5ibs+521`(jrar|7RYROB8k8ff?vhlSFS4}gS4NWQ&Zn3l_v&m>N*OwG$%8KcnMnk%`-cXq# z&q&I&>WY;0y5fSC(uXqY;JV5{H;WBLvet~MnsTV;`dproURS9rl0lgQo55nN%&k;a zYN`69jLN3+3`~;GWMV^F6UOSSt5s5^#cDJ2JhiT}FilxkT4ZW8RA!p8l2nx? zmdwV|B&eHOC(TWgO0ruEB{@|k@SWY9(^@FbNop$3t}1EBZZ*<5$>q&CRq)qZ(v+K= z-3s+(*$+vZax_LsPI3mF(^^jDROPqiGIh!ua*~Zywx+NN?n!esCDQDwOjB7gh56CC z;>=ckk+K2Hno*U~kewuLa>}ABN~H{v43JTskYQ=Il~^+?jMi#KW*x0TqwOG%N;sXPD|GE=&_iFuZkDK=Ql*l)~s{eq7at7XNK0+}%l~zuKMq6OH)D>(X!TJdz-lL zSAZW=CD?|*kKt2nYcyEwcF}4JlCd9WCETJ4VqueObHWbsTa?rillvd8j=Ip1ge(S~u z`iHow)DiFOexs{1(5W~k*=2Kx+ zqokk(Y-l!E3mmcCTc2yTK{*5XlMQ^oTBo(k>!sf{mRM}&63N}$-azXt#k36irn$u) z6TvTwnyc|Wdn{H}I{dm(XVKv}T&;KXL2rHvF}>3VJ>vuD=kEFx*s(#Eo&$Z=t#6x^ zbvXWa$N7%xS-t7ptsi;H;1)Z&zC*h43x=x^`e&}y)Y1{}nemLLzFoLW;C=6muh^IV zC_i@R+XU~Bnmf=n!x#L*8TbAyzAXlcvAfvdE?Y-gZ~olT_v~@ROWajf>MhzT(_OqL zT-F*<35nRM52W`RgNsTb3R}z5i4QNoM|g5&SrewTi?DfW5DkDNQ)&fcT0o0 zLwDoBJFojcVHX%nRqN8Uba(Hu+u5{BU2lEda$m*`-S!uwMcmr`e6Cv`YcUk(+-8n< zUu1xskaAM4McZVgyL~5nl=0^O_j+t;OwVl8SJ_(iX{k7W;Qjq;Rks-E8hH0sRNx!} zp7(m=W=J!)no{fS^Mhm|IxjU0dIs|2pJ>xR}~tq&w$Fe<)wb&*7Yw z&M`*uGEc99cNHEt*vIbP`qKS)?_0G^IR9`PUSeY!ljHbg-?vZvkzW%p$y!j(z8;lF@^-}x$#>-f{GY6GZ zRbYVrV6yjxu5&QRaWJ{8Qj4+ybn3|CWmhu!*si&;Rv61d4yu63xq8hvCS`h*@y7pN z%oBr-!Cdd`5@tOe~;W{TD)(+HCHI&U!6{=Kucr74Fiqp#FIO(6#gj30d z_*#t9&zbqjP~M2w2Ga0#PClfglk)Lef`&;;H^FssA+BmvSiYF8WK>+)e`@$uF?E38+*Wqu8o6fcIr zhdE`=Qt@fJOW)EeVSJe?))(j4n6Wf)R^_%j+Rt{R((%%V8uG%~Dp?19Ap@GupmySe z)LrrMb&dw(?fT5ckG#A;RI1b}s6VXG$yZf@OM38S@E|Uj z4GbN%Dn5^q4?F`GPfl^xPsWa9Us-a<%T#6LXV$0EyS#5zsw~`oJmsu^XYD7IcWqx) zJq$pgUA3b=<8Lnd&cvh~@|%t6IaYiv&TF;n8!w;1rzj7M#`seLAA6~C z5?~Px`*ViM?1qTbuC zFb``rrNKD4xI|Ul=#al$Rp`K1tC}&0y{o$D53f0@Za02r=4hBoW`e>gzt*YWe7in9 zUY}COd zye@ehx7-HIF^e+IMjSt~t1zFGc<8^l;coB@@J!V`UT2;KqNp-HUa4}!!$HtBelY&& zg4da6!UbVb7}q!DNxH-nd+24VGKG>E|Le?`m?W94#S(b;G^uNFeVbc-rW3jNFXMlZ zt9RzF?Df?!_QQ%%sjRrZvI^vJiSHiv!X;ho<&rpGDmm(1X08X#2I`=^Fk?}oei8D9H)G8QAj0W{cC~ htu~R4Z6cBOO*Zgv6Tm;faX unittest.mock.NonCallableMagicMock: @@ -78,78 +84,93 @@ def test_qualifies_for_processing(): assert not main._qualifies_for_processing(item) +def get_expect_item(): + scene = 'LC08_L1TP_138041_20240128_20240207_02_T1' + expect_datetime = datetime.datetime(2024, 1, 28, 4, 29, 49, 361022) + expect_datetime = expect_datetime.replace(tzinfo=datetime.timezone.utc) + expect_collection_id = 'landsat-c2l1' + expect_properties = {'datetime': '2024-01-28T04:29:49.361022Z', + 'eo:cloud_cover': 11.59, + 'view:sun_azimuth': 148.43311105, + 'view:sun_elevation': 37.83753177, + 'platform': 'LANDSAT_8', + 'instruments': ['OLI', 'TIRS'], + 'view:off_nadir': 0, + 'landsat:cloud_cover_land': 11.59, + 'landsat:wrs_type': '2', + 'landsat:wrs_path': '138', + 'landsat:wrs_row': '041', + 'landsat:scene_id': 'LC81380412024028LGN00', + 'landsat:collection_category': 'T1', + 'landsat:collection_number': '02', + 'landsat:correction': 'L1TP', + 'accuracy:geometric_x_bias': 0, + 'accuracy:geometric_y_bias': 0, + 'accuracy:geometric_x_stddev': 3.926, + 'accuracy:geometric_y_stddev': 3.525, + 'accuracy:geometric_rmse': 5.277, + 'proj:epsg': 32645, + 'proj:shape': [7681, 7531], + 'proj:transform': [30, 0, 674985, 0, -30, 3152415], + } + expect_item = pystac.item.Item(id=scene, geometry=None, bbox=None, datetime=expect_datetime, + properties=expect_properties, collection=expect_collection_id) + return expect_item + + def test_get_stac_item(): - scene1 = 'LC08_L1TP_138041_20240128_20240207_02_T1' - item1_id = 'LC08_L1TP_138041_20240128_20240207_02_T1' - item1_properties = {'datetime': '2024-01-28T04:29:49.361022Z', - 'eo:cloud_cover': 11.59, - 'view:sun_azimuth': 148.43311105, - 'view:sun_elevation': 37.83753177, - 'platform': 'LANDSAT_8', - 'instruments': ['OLI', 'TIRS'], - 'view:off_nadir': 0, - 'landsat:cloud_cover_land': 11.59, - 'landsat:wrs_type': '2', - 'landsat:wrs_path': '138', - 'landsat:wrs_row': '041', - 'landsat:scene_id': 'LC81380412024028LGN00', - 'landsat:collection_category': 'T1', - 'landsat:collection_number': '02', - 'landsat:correction': 'L1TP', - 'accuracy:geometric_x_bias': 0, - 'accuracy:geometric_y_bias': 0, - 'accuracy:geometric_x_stddev': 3.926, - 'accuracy:geometric_y_stddev': 3.525, - 'accuracy:geometric_rmse': 5.277, - 'proj:epsg': 32645, - 'proj:shape': [7681, 7531], - 'proj:transform': [30, 0, 674985, 0, -30, 3152415], - 'created': '2024-02-07T18:27:27.558Z', - 'updated': '2024-02-07T18:27:27.558Z' - } - - assert (main._get_stac_item(scene1).id == item1_id - and main._get_stac_item(scene1).properties == item1_properties) - - scene2 = 'LC08_L1GT_208119_20190225_20200829_02_T2' - - item2_id = 'LC08_L1GT_208119_20190225_20200829_02_T2' - - item2_properties = {'datetime': '2019-02-25T12:13:15.140373Z', - 'eo:cloud_cover': 46.47, - 'view:sun_azimuth': 87.63621418, - 'view:sun_elevation': 9.4970641, - 'platform': 'LANDSAT_8', - 'instruments': ['OLI', 'TIRS'], - 'view:off_nadir': 14.092, - 'landsat:cloud_cover_land': 46.47, - 'landsat:wrs_type': '2', - 'landsat:wrs_path': '208', - 'landsat:wrs_row': '119', - 'landsat:scene_id': 'LC82081192019056LGN00', - 'landsat:collection_category': 'T2', - 'landsat:collection_number': '02', - 'landsat:correction': 'L1GT', - 'proj:epsg': 3031, - 'proj:shape': [8901, 9151], - 'proj:transform': [30, 0, -980415, 0, -30, 188115], - 'created': '2022-07-06T18:15:03.664Z', - 'updated': '2022-07-06T18:15:03.664Z' - } - - assert (main._get_stac_item(scene2).id == item2_id and - main._get_stac_item(scene2).properties == item2_properties) + scene = 'LC08_L1TP_138041_20240128_20240207_02_T1' + expect_item = get_expect_item() + + main.LANDSAT_CATALOG = MagicMock() + main.LANDSAT_CATALOG.get_collection().get_item.return_value = expect_item + + item = main._get_stac_item(scene) + + assert (item.collection_id == expect_item.collection_id) + assert (item.properties['instruments'] == expect_item.properties['instruments']) + assert (item.properties['landsat:wrs_path'] == expect_item.properties['landsat:wrs_path']) + assert (item.properties['landsat:wrs_row'] == expect_item.properties['landsat:wrs_row']) + assert (item.properties['view:off_nadir'] == expect_item.properties['view:off_nadir']) + assert (item.properties['landsat:cloud_cover_land'] == expect_item.properties['landsat:cloud_cover_land']) + assert (item.properties['landsat:collection_category'] == expect_item.properties['landsat:collection_category']) def test_get_landsat_pairs_for_reference_scene(): - item = main._get_stac_item('LC08_L1TP_138041_20240128_20240207_02_T1') - df = main.get_landsat_pairs_for_reference_scene(item) - with open("tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl", 'rb') as file: - df_expect = pickle.load(file) - assert df.equals(df_expect) - - item = main._get_stac_item('LC08_L1GT_208119_20190225_20200829_02_T2') - df = main.get_landsat_pairs_for_reference_scene(item) - with open("tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl", 'rb') as file: - df_expect = pickle.load(file) - assert df.equals(df_expect) + main.LANDSAT_CATALOG = MagicMock() + scene = 'LC08_L1TP_138041_20240128_20240207_02_T1' + reference_item = get_expect_item() + results_item_collection = pystac.item_collection.ItemCollection.from_file( + 'tests/data/scene1_return_itemcollection.json') + data_gen = (y for y in [results_item_collection]) + main.LANDSAT_CATALOG.search().pages.return_value = data_gen + + df = main.get_landsat_pairs_for_reference_scene(reference_item) + + assert (df['landsat:wrs_path'] == reference_item.properties['landsat:wrs_path']).all() + assert (df['landsat:wrs_row'] == reference_item.properties['landsat:wrs_row']).all() + assert (df['view:off_nadir'] == reference_item.properties['view:off_nadir']).all() + assert (df['instruments'].apply(lambda x: ''.join(x)) == ''.join(reference_item.properties['instruments'])).all() + assert (df['reference'] == reference_item.id).all() + + +def test_deduplicate_hyp3_pairs(): + pairs = gpd.read_parquet('tests/data/scene1_pair.parquet') + with open("tests/data/job1.json", "r") as f1: + job1 = sdk.jobs.Job.from_dict(json.load(f1)) + with open("tests/data/job2.json", "r") as f2: + job2 = sdk.jobs.Job.from_dict(json.load(f2)) + jobs = sdk.jobs.Batch([job1, job2]) + + main.HYP3 = MagicMock() + main.HYP3.find_jobs.return_value = jobs + + new_pairs = main.deduplicate_hyp3_pairs(pairs) + + p_idx = pairs.set_index(['reference', 'secondary']) + np_idx = new_pairs.set_index(['reference', 'secondary']) + assert np_idx.isin(p_idx).any().any() + assert len(p_idx) - 2 == len(np_idx) + + + From 3882759ae6c79b68ddd803172f30bd42b1e724d0 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Mon, 8 Apr 2024 21:23:27 -0800 Subject: [PATCH 08/27] remove unused data files --- ...1GT_208119_20190225_20200829_02_T2_pairs.pkl | Bin 2507 -> 0 bytes ...1TP_138041_20240128_20240207_02_T1_pairs.pkl | Bin 21806 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl delete mode 100644 tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl diff --git a/tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl b/tests/data/LC08_L1GT_208119_20190225_20200829_02_T2_pairs.pkl deleted file mode 100644 index 2c0e4a2c8d9d788b4c7a9b76a0895046f5a77b40..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2507 zcmai0O>7%Q6t*2Zv6D9GU(%FT1+^f8)SC6u7B?y-siRh{6Zg*rsZuo>ugBgc>s@zu zHmRdV0-=T$jkt8m0VFtb0r;=Y#qY+coCrqP4sZnqx|Z*jB~w>PRs{0;g#i^EM+D zR03(Sd6h8rJlX=II|ymEE_KCAGt;SX_#P1J)CKNs3`SF5s4(iZbTYEMT|N@quI)ad20kVw?Z!=U&LdE>-gLY4343sLP048oN6R;T^3_kWFcOW z=j@j(SB0I56M!gbmAU-!b>p*6)d2ykusMnlFQ1qLOOl( z{p{%vRz`8T4E8G`T?s7k4&zGswq;<<#j)bnVjQg(6L>X;%w6Vl=#}|=$EUu~r#&K3 z>w3~>Y7h7ko;v>h+o#4q+w+rbheO(CGfhI%WHaW_E<^N zpiRmkn9$&ooXs68qouiTcFldhn*mqcS&W^S9=vl8$gkKspZuJNF)^-dy1rL?d9QY$ ztZS3|^hsUMzO+AgUOsMXVz!{=2_8ZsM}XuGf~Eby=$e+-4-!o$=+By>c`c{sLeZR- zJD^0_n*A{X{Q-5jlu~)`S5zTgY*4LLF=Ol0dcC5H%L5c-mmxYjv;(5ai(|k3$giQ0T&iICgOy{us|Qch=q9a`-D@F00Ju z6wxAvUo?CL{=phgbcA2qs>8$iA_*bWHOe)v$~7UL9730@M!-!eHr>C)Y_>qT1hsTdNU=RxPkNZo~BUDepap;5|63v$g2?B%E8TZ#5~NKhi7Zxr;q|zQG(u zEHJgbBV@R?%?xys7%=JTuGfOuxyCMxuL?%Isk`UFh>OC@G(Go?N!Il1kol1JO$u{W z42KKuQOA#>A22)y=YR_XK@$@TT*hNH<}n9`k{G?u6KY%xEZNd%5VNwf+5JM$Dyf3(&ugyCMYrtnb0Of!|-+lX$A+e zUDSJ2hCo};uzxOqyQI@6&=GdfaG;Ff7ZoYWIlMP5OVLkiHm!&3R%+YsqED(p6Gkuu T4~4BT+m}~;^I!AZppyL`mH&l` diff --git a/tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl b/tests/data/LC08_L1TP_138041_20240128_20240207_02_T1_pairs.pkl deleted file mode 100644 index dcc269dba4a6a402018d0b440905d6c4c61fc442..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21806 zcmeHP2Ut@{*9N4EG=X42L3G7Jy!WO@(V>eVO~A5h3?Wj4fFu!75qph_YgDYOuDxQ{ zUE8j0)z!t-wXMCoc3pe>&mDURZb#={*<%NavVgPL@lNA&XG?~XogyNPkxuTyk1G_b3(lUB+VN6O*cAgUa z>%f2v*9JFm=+xN$v2FZVeX?FxkLjABDK1c|xyAw*R{5p$h`=r2StW;7m95e1Y*c`5 zqZy>vx#I4$n(SPzW)S?<>+Cdxv>JucLz}MiR%!DJin#{e^0c{{Yy(TsOID~<}1nftRFwl*lw${ zZl{lrkMHyWehe?MRuI8(%+=s*tk!wRgetcdHUHV@8P%lT6Ndvw6cmp~}f zb&l!UY)Ay0LU!uhYz0oQ*aoo;W36!@V{PzjR_&}NS==U9r_78-pKX#}H zGnVdAXfbqudT$Ksh&vi9qT3}jKHpgHAVz)h&&^00C)$X=J{&#Av^Pg0<3bil_t(*g zQ{aOzjihbvG(UfYM>ELFg~ks@Eim!rL3Di^bo3YaiSsv>zqr!6xTC=a$5zJihWOC+ z9J=1w$g_Evv7aH4v|jB|AO8dDgnj9#SHR-b3D45e$>t$%bDAp9q&u;hu5k(!u;Sp_ z1}X)5)o#7l$}tL5(DD5bucj-ISU970Ww`=98ku&o(G&$bE-iStWq<-#^OaHA_auiX7$(}Qkk`kD?W&O7u(y}R8{3{Q}u_rX2apY!X3j%N9ATMzGp z0&Z^HCTt}~dmj(6_tMHy@(;5r7H*NFe!OP;etIrPqT*`~e~{_u+(?O=b#6M^sux_X zSd@-_Uvqg~+39o?pjdn~+7Wo>Z-3pNuR!WU`yRgUp+GHyYPpRrR-pS!tVSM&@g+_U zOl~<_f!5h>)=^OU*BP^G)C2{Jf26vS2>RGp>cg6KfU?ZxLynCCT~KGO{BuVi)M825 ztJgPsqpfFdgxSpQg_5TXjqE50Lv{(HhA*iIMYl@Fp6{0#ifn$mdZck`D6)!a63HDJ zikiQBvAg+&P?XW&+rB3Y!qCn1W-+74aMWeN^^OPXN1!#Fa7j|rNOZdPkv;DsqtNoY zGiJpXMWeZQ(zdK#*AdlSb@HnO~Y9dUy7l{_7*rN!Q*T?vIW@zd2t!yJ|!@`s(QO zE-O}qA;01ty^?Li(5fHDro`w&QLTG7@>}!@MPH3{9oD^1C~BIVHEq$HQ1pXGn++bW zVJM|&=%&>$54}&1dVY0yIC{OKVd`;h1ZrBld{6L%NTgP&mW`{3LW6{>+IPJjjm{AB z=MI_L868Z1bE&H^0d3wpY~lRt3Fymu^6yZ(VeuDX#HC8|FP+cHoGoaUby73jz7y z4f8w@yg-8oT7-vj(hV7g73_jW?l}x2b4S?f;W8&QcOM>zqp_b#C*;=yGj~J*czE~S z5biejR&8Xcp5#R9;)qIfc4(&gIv^kqfDQex{*u+S2tt*3r7@%+mr@jr+^m^uE{& z{XP&zFwU2@_oaDx({b}aFL63X_!#%0L3NQ~8|~(UmLG@}jsKC(CqH*|9I||9jysLV zK_GIB%Lp$zPM$PRA6j=`I!_zYac^kc4?ptzsP_qwyW92vBkoFn<9z_fpU%@p=+QDv zpb=`0C*HXRbl&+PLl}P_EQxFf@K$ahC8A47o4s^+CT zotNIUy{EB1Cr_i_X?L2Z8{KamT2D{Kxm`BYwaW;mVxTNu6 zm#g}t-M$q6yaJ=?&CHk5sP#XsgJDXxi*<+%jCK9=vkkYZXB!EOiK??6!g`?90YBa7 z1rHaU;zHvix`d|4Bzn`em~s98rJl)|dHr|s|0gj2E^hw>jyWbym0AgLF?yYCN=$MW zz4A-N7d`%K(?MrV>2m_%+5dm?i>?;QgGKMltC<1>p9D?5yfPHJ9UVV(CNi_!j38#p4;~QFVMuXc?b0^jqDY zWXxX^e~L#K`mJV9Nf`5&VNVGde3f6{{5tpZ!*^VrYd5O8GSyjIr~^B>(c{y0|Elk&`!%)x ztNof-nXXJ+;L60FzA~$$)i8NjLiu~@Ok?}sJVvGlGY@9xW_|dD)!;EY+0(JZEhhVDrjeJj@zu9AJ$J$DC3B8dQy}YE01@S=Bi5 zqfs?T!tBG0ZjG#J3@x*BL$>RD@$FeKK?YMq3L%yXNGV^;6%f20dPWm3g%nFEk(4jw zN=TB~MDSBcN=lJZQo!YlM64!4AQcFulz>Y}#B>ucn52R!VG6iV&?@0lJR!4*6r>P@ zRKSx8D6WuCF`Ecp3L%k_q?F)sMHB-N$V*ZPo|NQC2{D(?mzccBU{aJq2&DqR6mxkL z!D=D_kS`@ju8?5RAVCn^7ez@)f-B}TnuuT?nLXHeOcMQtZYKVx`C(3 zmz53HlP@dVKfkgGq&&zsn0jKyVh+FB{-%{JC@!23%ka|^nekD&%N zN(N8Navs6wVE&U|bXW_CB`t_5Q3zTKgQ+FiNUF+|$27HsO7Lk@xG*Yg8I?y;l*poK zHB>MqCD;`EP@h93N!pYYFikC?5lHi${T0kX8p^*T-_zWt{Cj(ak1ZJ34Mf2evrP;lDyCLu!MMd0;JY6#^I@7J)B5%@wwZ zwCTsi!}8qX5du1}hKo<5@*zzP1o*HvF-$FR%B~GtXBG!=gxIW)YAMpF<_-VNvFD%Mxl;v|u8B2DNH(!+*hk zE(BGR8>ZH0mJEWd$^jDQ9QVO)~Cycbjkd9vChr(ZEpH| z#dY9o+<8h(VR6-0p5cqj)jt)@g+XNE4^oErvc{ixhR+09;Q@6%Hu`Z&TpDVuEVy1e zupQbtp~=YCL>g)%XgqGvE-4bPc)RM>k~Fk&)6n|zK2nr=ZOwY=Pig4vn6?sigLa7R zuV>-M_^IAS0sig=;?=3e?NG?2)+6r$e$k$%Z4!4&(K4r|D~kaCkC@lgYrqex=yc(? zAJfos?^Z|d06yEFjeoOgy@^Kw|3F1+{T$$bc%)s+0c09lJvZvyOyECne#;lVmZqWZ zC398|1bn|AJ(g|-{v$T3_yYi+?a#(Pazea>0(||M)q4YgfA-r0#bL+aQxH1Pc5W($D-&kbHBZ~*_x z#cMjupNQ&IWC!@uch5Oo1o$?G95&|x{(GlEH!}gB?a#)KUwr2$b>}p+XKu_Z7vSF` zI(Lm1m4*smWXN9w{sKEvF$?suY4OdaWq_Y|?N*(KpwFG5ktY`eKHHy-|K`{aPuhY$ zl2%jNPXqp|$F=$O3h?jN?A1?$G5>kzJM0Jka}p0s$-w-jw%-H*eyiorv>AZU_Gjbo zFaFT)4Cphmw|$}n_-6!cT{Q^s9bf)1+aL7LYC9843JTu%tpGWoW{#*$BXYG5aeFOT0m1;)~0{o#f zo=@nyHx1Q4al=jv_-ubRzIEI0<8}i+(PMU02=L$jKF3`K_>IHY%?|J|TW1hC_TOb{KiK3*fW;+4v{g2h4sB z{1@L{m(&UPD`ytx=`sHWIj%zBKXClfYhIv#-igL*&H#S=_jk$*fq(p!8!t}*KHHy- z@AP5(=JUWmI_;3(D&YV2NLZ6noS%R2gBAmSt$Wuuy@3C@=A-Ya0Dqy&cUyh|{-fHR zRI31=?a#*d613S!#l!mWDs-0s_`96_b3-M>_jc;d%Oc=^?|sn8P7vQ-sB7~hfIs_2 zH`fgi-`XW}$@PHG_Gja_4E&4O4Em^AdDU$N_$y~d1f+mI_hQzKX%6`EMb~vFf&atg zrcYM@{tx^1ItoFbyj8fF zxpzSuLI1(W7suKG{=(iNV~PR)=P8rA_5=Le4vYWl1NdxzHvS-o`*Z67|ETqE&4K)R zKY0A9iNL?#h@f%FkUytZ{d#Ex@Gm^m>}6ZfKVZ)7F?fBRSYEH$RLGy^QBQA`13ufI zjlbkfEA1_aukW^^DR}*CeNh`SAL5%J9ez9z`1cOD{Gk!#zjYbW_zmFC@bCJU0^-|t zXGycSfY0`4t^3S4g6DfY0`4;~$he^m`2Y$K_hRXa@L)fGyK)}D3ee7u^;IsYN_}#n5Zgd6yjjrVM8wL854Ok+<{N1Ay z2bbXZj@fCu9Ppp)n*3}6)_?cQXYT>umgo9*BH*+A+4w0vru2Ob_#x{X1vkg}*>pj} z?;yT=3-6!s0RG$?MT-L=zL$@;Cr<-Ddwpa3v+?Z{c->T;($MvaT9p;Rf5Gv7kqz3w z`qC$o{|)euUR&>XEBO7}+VR!W2r0fA2i&@{n`nl$A*m1zu6!;vo27E=}h%E;JpY6}aU#(oz zQw{hkP54mE|6TCvG|d0)Z~7*6a3z~5y?-UungH{{Onu>#ncu#x|@fD4pRBJHc2TVzw z(izsD6*n^t`&+g@8~?@{qRu49pHnBbtFZpxADHuXTgac4&k}4o!2iMW1?_e~e*QIg z`?+5Me`#3LnRxyVK7O|l=Rez@jlWfu?Vk$x+y9L3`5o{tdM;Xu?=R;3Fw1s6@b73P ze1h*UmXx0wKML^IroKDd0q`@Z<7uM+pY6}aFKZ|0Y6a`_=*3$mc?18U711}AgFX^B z@7qrRKd@wEKoG3Y181D4RssGuEe5|T1$~ws*_4U*=WKsA{>mf!OLhVOS4}5RnhX4c zSJjTf`=6_WkDVO@_~OT-gF1o!&cBdBx2$z~9v>E-DxJ%PTy}C7{oigqW$AKii*;A0@R` zR{;Mbb^e^y2l&spFdzc&A5OYAsTB+SpC8HORziHU3z8Ba1Af@Uo>y9fKIP-a{`m;- z+5T+&eQgt3$N>LB;G`{he?Ij?(dZALPmdK}JK+6oHfWOc8R(ChxpZe1*{u7wLj_=RhSqu2J4^D2^1N7e# zcINAEfj`@yjXx(OJZ~!SUtj+Ne?9Pz>%8ls0QBi__O#m^;Gc8ztP}qIySL%odZmD0 z+<)i20ie%?UW%zx0H5v8#!s9T8+{D$+r1m0ZUOu&x#KtW2mB)B-PsHH_wFcc_yY8q z=n^5=2KbwM4GKyC{ADwu-{Aab`?K*gv(^SDLjL&ojh(Rt_%HgV&hf{9ufMsd*=FF+ zy|w((Rp1}K=C=<+0N?xm#%bY@KNFP~(~AI~?a#)K)ZTiE_YWS)ean{u{@GtNopJs* zPJOs-0pQ;sw(`m{;NSiHg%G@doB3+V>^i`|*0cJReF2~C&&EHp`fTzzh_CO4a}VX9 zPiCWozv26fGd=h0?hg7i%{~0!JBY8Jz3$;%!0*cY>Z}*Ucd{FQ&3V9Q`|I&DqWCfJ z#q)NtR``%Lvv0Y&W_g$noXhet9wC=i&v1HNRs)@5)k$@Fog+T5Zcx4o9|e|^15adh zPPisRnX8dQ6=1_3j6E9uX)S!s-FP${e9@hGAY8Sx!AfdA4zBF_H>KyEuHsZW>ss(j zS8yt9y2np-Ud95Dm5GdU$O0gE_UJ)4CsTKz4jW74b@A4Kyw-I`rr*<|7%{0of(|6N8+E6 ztQW`sZRw9m)@hu}jaw|(pzFeUd2nen7vCg~?@E5ber_BG&2QMU>Gv_5VNG^SD~XKe zy!`(CwF{LIoR{ZjrHFfmam02_PEEhnfwTNi;rfS#A)N96zbBpEOF52>FTPkhUd*Yq zFNvz5KVIIG1;JL+#!)=IqJ3c;0GLUC#2B^=557?8MrWBy zStN_WbvSb-;LNeBdJECk*+P!VEj$49t9k&abA(5Th6jd_g{FjbRpv_-x%t`*nIeCH zvQV#cACRpqlB%^TnS4-oo>r5k*Li-dQsycL$TivdD!?enm23Lu7v|}8wc&_6INmT% zsnWnvcdiEO{Jwo0ZKJzEYVwA5Biw>7Znr`BX-CF$4ovY!Ercka@8)M<1C{)6zK@bOR0S+@(ZQ;dC=QN3$wBq2o-Pp2@gmj^a4J>OqTeT*?Kp)G+7fQIWHXti8JRCg=p$D2ch3P{|on(-zazjq{0@AXo z+=VR7)#j-nwSb*mXD8Qa;80SSU-q)Tx$;cIB}bl{n_r}sWoQfF&_$2{k}X}CtJaTG P=Ntf9IWyozo6h|oRX?+O From 9d4831cbe94d5263d54d1e9ae0fb5654ab4af80a Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 9 Apr 2024 13:54:17 -0500 Subject: [PATCH 09/27] modified test to use hyp3-sdk rather than json --- tests/landsat/test_main.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 3c0297a6..9e50c1e5 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -156,21 +156,30 @@ def test_get_landsat_pairs_for_reference_scene(): def test_deduplicate_hyp3_pairs(): pairs = gpd.read_parquet('tests/data/scene1_pair.parquet') - with open("tests/data/job1.json", "r") as f1: - job1 = sdk.jobs.Job.from_dict(json.load(f1)) - with open("tests/data/job2.json", "r") as f2: - job2 = sdk.jobs.Job.from_dict(json.load(f2)) - jobs = sdk.jobs.Batch([job1, job2]) + job1 = sdk.jobs.Job( + job_type='AUTORIFT', + job_id='job_id_1', + request_time=datetime.datetime(2024, 1, 1), + status_code=200, + user_id='hyp3', + job_parameters={"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20231101_20231101_02_T1"]} + ) + job2 = sdk.jobs.Job( + job_type='AUTORIFT', + job_id='job_id_2', + request_time=datetime.datetime(2024, 1, 2), + status_code=200, + user_id='hyp3', + job_parameters={"granules": ["not_in_pairs", "also_not_in_pairs"]} + ) + duplicate_jobs = sdk.jobs.Batch([job1, job2]) main.HYP3 = MagicMock() - main.HYP3.find_jobs.return_value = jobs + main.HYP3.find_jobs.return_value = duplicate_jobs new_pairs = main.deduplicate_hyp3_pairs(pairs) p_idx = pairs.set_index(['reference', 'secondary']) np_idx = new_pairs.set_index(['reference', 'secondary']) assert np_idx.isin(p_idx).any().any() - assert len(p_idx) - 2 == len(np_idx) - - - + assert len(p_idx) - 1 == len(np_idx) From 15ffb4f10982ee3de73145148aaa076a9abd3a19 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 9 Apr 2024 13:55:46 -0500 Subject: [PATCH 10/27] remove unnecessary json files --- tests/data/job1.json | 1 - tests/data/job2.json | 1 - 2 files changed, 2 deletions(-) delete mode 100644 tests/data/job1.json delete mode 100644 tests/data/job2.json diff --git a/tests/data/job1.json b/tests/data/job1.json deleted file mode 100644 index 6897965d..00000000 --- a/tests/data/job1.json +++ /dev/null @@ -1 +0,0 @@ -{"job_id": "5864010f-4f07-4030-beec-88e9c39d2baa", "job_type": "AUTORIFT", "request_time": "2024-04-08T00:23:21+00:00", "status_code": "FAILED", "user_id": "cirrusasf", "name": "LC08_L1TP_138041_20240128_20240207_02_T1", "job_parameters": {"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20230407_20230407_02_T1"], "parameter_file": "/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp", "publish_bucket": "\"\""}, "logs": ["https://hyp3-its-live-contentbucket-s10lg85g5sp4.s3.us-west-2.amazonaws.com/5864010f-4f07-4030-beec-88e9c39d2baa/5864010f-4f07-4030-beec-88e9c39d2baa.log"], "expiration_time": "2024-05-24T00:00:00+00:00", "processing_times": [699.518], "credit_cost": 1} \ No newline at end of file diff --git a/tests/data/job2.json b/tests/data/job2.json deleted file mode 100644 index 94b16336..00000000 --- a/tests/data/job2.json +++ /dev/null @@ -1 +0,0 @@ -{"job_id": "98cd33bf-4f32-499d-94db-20661f10f5f7", "job_type": "AUTORIFT", "request_time": "2024-04-08T00:23:21+00:00", "status_code": "FAILED", "user_id": "cirrusasf", "name": "LC08_L1TP_138041_20240128_20240207_02_T1", "job_parameters": {"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20221130_20230319_02_T1"], "parameter_file": "/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp", "publish_bucket": "\"\""}, "logs": ["https://hyp3-its-live-contentbucket-s10lg85g5sp4.s3.us-west-2.amazonaws.com/98cd33bf-4f32-499d-94db-20661f10f5f7/98cd33bf-4f32-499d-94db-20661f10f5f7.log"], "expiration_time": "2024-05-24T00:00:00+00:00", "processing_times": [752.675], "credit_cost": 1} \ No newline at end of file From 173a2a4b3de87723044770340fc70d9536be2d3e Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 9 Apr 2024 14:54:06 -0800 Subject: [PATCH 11/27] add test_submit_pairs_for_processing function to test_main.py --- tests/landsat/test_main.py | 51 ++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 3c0297a6..8d701826 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -8,9 +8,10 @@ import hyp3_sdk as sdk from landsat.src import main - +import pdb LANDSAT_CATALOG_real = main.LANDSAT_CATALOG +HYP3_real = main.HYP3 def get_mock_pystac_item() -> unittest.mock.NonCallableMagicMock: @@ -118,6 +119,35 @@ def get_expect_item(): return expect_item +def get_expect_pairs(): + pairs = gpd.read_parquet('tests/data/scene1_pair.parquet') + return pairs + + +def get_expect_jobs(): + job1 = sdk.jobs.Job.from_dict({'job_id': '88ea6109-8afa-483a-93d5-7f3231db7751', 'job_type': 'AUTORIFT', + 'request_time': '2024-04-09T18:13:41+00:00', 'status_code': 'PENDING', + 'user_id': 'cirrusasf', 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', + 'job_parameters': {'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', + 'LC09_L1TP_138041_20240120_20240120_02_T1'], + 'parameter_file': + '/vsicurl/http://its-live-data.s3.amazonaws.com/' + 'autorift_parameters/v001/autorift_landice_0120m.shp', + 'publish_bucket': '""'}, 'credit_cost': 1}) + job2 = sdk.jobs.Job.from_dict({'job_id': '4eea15af-167a-43b3-b292-aee55b3e893e', 'job_type': 'AUTORIFT', + 'request_time': '2024-04-09T18:15:06+00:00', 'status_code': 'PENDING', + 'user_id': 'cirrusasf', 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', + 'job_parameters': {'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', + 'LC08_L1TP_138041_20231227_20240104_02_T1'], + 'parameter_file': + '/vsicurl/http://its-live-data.s3.amazonaws.com/' + 'autorift_parameters/v001/autorift_landice_0120m.shp', + 'publish_bucket': '""'}, 'credit_cost': 1}) + + jobs_expect = sdk.jobs.Batch([job1, job2]) + return jobs_expect + + def test_get_stac_item(): scene = 'LC08_L1TP_138041_20240128_20240207_02_T1' expect_item = get_expect_item() @@ -155,17 +185,14 @@ def test_get_landsat_pairs_for_reference_scene(): def test_deduplicate_hyp3_pairs(): - pairs = gpd.read_parquet('tests/data/scene1_pair.parquet') - with open("tests/data/job1.json", "r") as f1: - job1 = sdk.jobs.Job.from_dict(json.load(f1)) - with open("tests/data/job2.json", "r") as f2: - job2 = sdk.jobs.Job.from_dict(json.load(f2)) - jobs = sdk.jobs.Batch([job1, job2]) + pairs = get_expect_pairs() + jobs = get_expect_jobs() main.HYP3 = MagicMock() main.HYP3.find_jobs.return_value = jobs new_pairs = main.deduplicate_hyp3_pairs(pairs) + main.HYP3 = HYP3_real p_idx = pairs.set_index(['reference', 'secondary']) np_idx = new_pairs.set_index(['reference', 'secondary']) @@ -173,4 +200,14 @@ def test_deduplicate_hyp3_pairs(): assert len(p_idx) - 2 == len(np_idx) +def test_submit_pairs_for_processing(): + pdb.set_trace() + pairs = get_expect_pairs() + jobs_expect = get_expect_jobs() + + main.HYP3.submit_prepared_jobs = MagicMock() + main.HYP3.submit_prepared_jobs.return_value = jobs_expect + + jobs = main.submit_pairs_for_processing(pairs) + assert jobs == jobs_expect From db9a3ddb65544bb6c7c84538837352e2a7d38b54 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 9 Apr 2024 15:03:07 -0800 Subject: [PATCH 12/27] remove code for debug --- tests/landsat/test_main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 8d701826..249839eb 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -1,6 +1,5 @@ import unittest.mock from unittest.mock import MagicMock -import json import pystac import datetime import geopandas as gpd @@ -8,7 +7,6 @@ import hyp3_sdk as sdk from landsat.src import main -import pdb LANDSAT_CATALOG_real = main.LANDSAT_CATALOG HYP3_real = main.HYP3 @@ -201,7 +199,6 @@ def test_deduplicate_hyp3_pairs(): def test_submit_pairs_for_processing(): - pdb.set_trace() pairs = get_expect_pairs() jobs_expect = get_expect_jobs() @@ -209,5 +206,6 @@ def test_submit_pairs_for_processing(): main.HYP3.submit_prepared_jobs.return_value = jobs_expect jobs = main.submit_pairs_for_processing(pairs) + main.HYP3 = HYP3_real assert jobs == jobs_expect From 697b880f33f62ae2ca710b1ce3c560600dc394d9 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 9 Apr 2024 15:13:46 -0800 Subject: [PATCH 13/27] modify main.py code --- landsat/src/__init__.py | 0 landsat/src/main.py | 3 --- tests/landsat/test_main.py | 8 ++++---- 3 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 landsat/src/__init__.py diff --git a/landsat/src/__init__.py b/landsat/src/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/landsat/src/main.py b/landsat/src/main.py index 1abfd28f..040d4d39 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -194,9 +194,6 @@ def process_scene( pairs = get_landsat_pairs_for_reference_scene(reference, max_pair_separation, max_cloud_cover) - if pairs is None: - return None - log.info(f'Found {len(pairs)} pairs for {scene}') with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None): log.debug(pairs.loc[:, ['reference', 'secondary']]) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 249839eb..a61ca169 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -1,13 +1,14 @@ +import datetime import unittest.mock from unittest.mock import MagicMock -import pystac -import datetime -import geopandas as gpd +import geopandas as gpd import hyp3_sdk as sdk +import pystac from landsat.src import main + LANDSAT_CATALOG_real = main.LANDSAT_CATALOG HYP3_real = main.HYP3 @@ -166,7 +167,6 @@ def test_get_stac_item(): def test_get_landsat_pairs_for_reference_scene(): main.LANDSAT_CATALOG = MagicMock() - scene = 'LC08_L1TP_138041_20240128_20240207_02_T1' reference_item = get_expect_item() results_item_collection = pystac.item_collection.ItemCollection.from_file( 'tests/data/scene1_return_itemcollection.json') From 2fc4010f1c6f279bd92822eb317b85d4bc869f0e Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 9 Apr 2024 15:24:19 -0800 Subject: [PATCH 14/27] ruff format test_main.py --- tests/landsat/test_main.py | 128 ++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 52 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index a61ca169..ba786c2e 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -89,32 +89,39 @@ def get_expect_item(): expect_datetime = datetime.datetime(2024, 1, 28, 4, 29, 49, 361022) expect_datetime = expect_datetime.replace(tzinfo=datetime.timezone.utc) expect_collection_id = 'landsat-c2l1' - expect_properties = {'datetime': '2024-01-28T04:29:49.361022Z', - 'eo:cloud_cover': 11.59, - 'view:sun_azimuth': 148.43311105, - 'view:sun_elevation': 37.83753177, - 'platform': 'LANDSAT_8', - 'instruments': ['OLI', 'TIRS'], - 'view:off_nadir': 0, - 'landsat:cloud_cover_land': 11.59, - 'landsat:wrs_type': '2', - 'landsat:wrs_path': '138', - 'landsat:wrs_row': '041', - 'landsat:scene_id': 'LC81380412024028LGN00', - 'landsat:collection_category': 'T1', - 'landsat:collection_number': '02', - 'landsat:correction': 'L1TP', - 'accuracy:geometric_x_bias': 0, - 'accuracy:geometric_y_bias': 0, - 'accuracy:geometric_x_stddev': 3.926, - 'accuracy:geometric_y_stddev': 3.525, - 'accuracy:geometric_rmse': 5.277, - 'proj:epsg': 32645, - 'proj:shape': [7681, 7531], - 'proj:transform': [30, 0, 674985, 0, -30, 3152415], - } - expect_item = pystac.item.Item(id=scene, geometry=None, bbox=None, datetime=expect_datetime, - properties=expect_properties, collection=expect_collection_id) + expect_properties = { + 'datetime': '2024-01-28T04:29:49.361022Z', + 'eo:cloud_cover': 11.59, + 'view:sun_azimuth': 148.43311105, + 'view:sun_elevation': 37.83753177, + 'platform': 'LANDSAT_8', + 'instruments': ['OLI', 'TIRS'], + 'view:off_nadir': 0, + 'landsat:cloud_cover_land': 11.59, + 'landsat:wrs_type': '2', + 'landsat:wrs_path': '138', + 'landsat:wrs_row': '041', + 'landsat:scene_id': 'LC81380412024028LGN00', + 'landsat:collection_category': 'T1', + 'landsat:collection_number': '02', + 'landsat:correction': 'L1TP', + 'accuracy:geometric_x_bias': 0, + 'accuracy:geometric_y_bias': 0, + 'accuracy:geometric_x_stddev': 3.926, + 'accuracy:geometric_y_stddev': 3.525, + 'accuracy:geometric_rmse': 5.277, + 'proj:epsg': 32645, + 'proj:shape': [7681, 7531], + 'proj:transform': [30, 0, 674985, 0, -30, 3152415], + } + expect_item = pystac.item.Item( + id=scene, + geometry=None, + bbox=None, + datetime=expect_datetime, + properties=expect_properties, + collection=expect_collection_id, + ) return expect_item @@ -124,24 +131,40 @@ def get_expect_pairs(): def get_expect_jobs(): - job1 = sdk.jobs.Job.from_dict({'job_id': '88ea6109-8afa-483a-93d5-7f3231db7751', 'job_type': 'AUTORIFT', - 'request_time': '2024-04-09T18:13:41+00:00', 'status_code': 'PENDING', - 'user_id': 'cirrusasf', 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', - 'job_parameters': {'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', - 'LC09_L1TP_138041_20240120_20240120_02_T1'], - 'parameter_file': - '/vsicurl/http://its-live-data.s3.amazonaws.com/' - 'autorift_parameters/v001/autorift_landice_0120m.shp', - 'publish_bucket': '""'}, 'credit_cost': 1}) - job2 = sdk.jobs.Job.from_dict({'job_id': '4eea15af-167a-43b3-b292-aee55b3e893e', 'job_type': 'AUTORIFT', - 'request_time': '2024-04-09T18:15:06+00:00', 'status_code': 'PENDING', - 'user_id': 'cirrusasf', 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', - 'job_parameters': {'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', - 'LC08_L1TP_138041_20231227_20240104_02_T1'], - 'parameter_file': - '/vsicurl/http://its-live-data.s3.amazonaws.com/' - 'autorift_parameters/v001/autorift_landice_0120m.shp', - 'publish_bucket': '""'}, 'credit_cost': 1}) + job1 = sdk.jobs.Job.from_dict( + { + 'job_id': '88ea6109-8afa-483a-93d5-7f3231db7751', + 'job_type': 'AUTORIFT', + 'request_time': '2024-04-09T18:13:41+00:00', + 'status_code': 'PENDING', + 'user_id': 'cirrusasf', + 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', + 'job_parameters': { + 'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', 'LC09_L1TP_138041_20240120_20240120_02_T1'], + 'parameter_file': '/vsicurl/http://its-live-data.s3.amazonaws.com/' + 'autorift_parameters/v001/autorift_landice_0120m.shp', + 'publish_bucket': '""', + }, + 'credit_cost': 1, + } + ) + job2 = sdk.jobs.Job.from_dict( + { + 'job_id': '4eea15af-167a-43b3-b292-aee55b3e893e', + 'job_type': 'AUTORIFT', + 'request_time': '2024-04-09T18:15:06+00:00', + 'status_code': 'PENDING', + 'user_id': 'cirrusasf', + 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', + 'job_parameters': { + 'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', 'LC08_L1TP_138041_20231227_20240104_02_T1'], + 'parameter_file': '/vsicurl/http://its-live-data.s3.amazonaws.com/' + 'autorift_parameters/v001/autorift_landice_0120m.shp', + 'publish_bucket': '""', + }, + 'credit_cost': 1, + } + ) jobs_expect = sdk.jobs.Batch([job1, job2]) return jobs_expect @@ -156,20 +179,21 @@ def test_get_stac_item(): item = main._get_stac_item(scene) - assert (item.collection_id == expect_item.collection_id) - assert (item.properties['instruments'] == expect_item.properties['instruments']) - assert (item.properties['landsat:wrs_path'] == expect_item.properties['landsat:wrs_path']) - assert (item.properties['landsat:wrs_row'] == expect_item.properties['landsat:wrs_row']) - assert (item.properties['view:off_nadir'] == expect_item.properties['view:off_nadir']) - assert (item.properties['landsat:cloud_cover_land'] == expect_item.properties['landsat:cloud_cover_land']) - assert (item.properties['landsat:collection_category'] == expect_item.properties['landsat:collection_category']) + assert item.collection_id == expect_item.collection_id + assert item.properties['instruments'] == expect_item.properties['instruments'] + assert item.properties['landsat:wrs_path'] == expect_item.properties['landsat:wrs_path'] + assert item.properties['landsat:wrs_row'] == expect_item.properties['landsat:wrs_row'] + assert item.properties['view:off_nadir'] == expect_item.properties['view:off_nadir'] + assert item.properties['landsat:cloud_cover_land'] == expect_item.properties['landsat:cloud_cover_land'] + assert item.properties['landsat:collection_category'] == expect_item.properties['landsat:collection_category'] def test_get_landsat_pairs_for_reference_scene(): main.LANDSAT_CATALOG = MagicMock() reference_item = get_expect_item() results_item_collection = pystac.item_collection.ItemCollection.from_file( - 'tests/data/scene1_return_itemcollection.json') + 'tests/data/scene1_return_itemcollection.json' + ) data_gen = (y for y in [results_item_collection]) main.LANDSAT_CATALOG.search().pages.return_value = data_gen From 423743ac8da151f5b7b68d0da47e446f4c8205fa Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Tue, 9 Apr 2024 15:35:20 -0800 Subject: [PATCH 15/27] make variable name more sense --- tests/landsat/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index ba786c2e..fb66ab36 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -208,10 +208,10 @@ def test_get_landsat_pairs_for_reference_scene(): def test_deduplicate_hyp3_pairs(): pairs = get_expect_pairs() - jobs = get_expect_jobs() + duplicate_jobs = get_expect_jobs() main.HYP3 = MagicMock() - main.HYP3.find_jobs.return_value = jobs + main.HYP3.find_jobs.return_value = duplicate_jobs new_pairs = main.deduplicate_hyp3_pairs(pairs) main.HYP3 = HYP3_real From 673457ca8ee8fc780c2e8ccf314f68cede10057b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 10 Apr 2024 14:11:02 -0500 Subject: [PATCH 16/27] formatting --- tests/landsat/test_main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 2230a983..1ba30617 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -214,7 +214,9 @@ def test_deduplicate_hyp3_pairs(): request_time=datetime.datetime(2024, 1, 1), status_code=200, user_id='hyp3', - job_parameters={"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20231101_20231101_02_T1"]} + job_parameters={ + 'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', 'LC09_L1TP_138041_20231101_20231101_02_T1'] + } ) job2 = sdk.jobs.Job( job_type='AUTORIFT', @@ -222,7 +224,7 @@ def test_deduplicate_hyp3_pairs(): request_time=datetime.datetime(2024, 1, 2), status_code=200, user_id='hyp3', - job_parameters={"granules": ["not_in_pairs", "also_not_in_pairs"]} + job_parameters={'granules': ['not_in_pairs', 'also_not_in_pairs']} ) duplicate_jobs = sdk.jobs.Batch([job1, job2]) From c49a67f610faa2b514900aa4fdda40f83790af8a Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 10 Apr 2024 14:12:11 -0500 Subject: [PATCH 17/27] formatting --- tests/landsat/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 1ba30617..391b4845 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -216,7 +216,7 @@ def test_deduplicate_hyp3_pairs(): user_id='hyp3', job_parameters={ 'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', 'LC09_L1TP_138041_20231101_20231101_02_T1'] - } + }, ) job2 = sdk.jobs.Job( job_type='AUTORIFT', @@ -224,7 +224,7 @@ def test_deduplicate_hyp3_pairs(): request_time=datetime.datetime(2024, 1, 2), status_code=200, user_id='hyp3', - job_parameters={'granules': ['not_in_pairs', 'also_not_in_pairs']} + job_parameters={'granules': ['not_in_pairs', 'also_not_in_pairs']}, ) duplicate_jobs = sdk.jobs.Batch([job1, job2]) From 35051b830adf313f744448dd227c0d80f43b26dc Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Wed, 10 Apr 2024 17:26:27 -0800 Subject: [PATCH 18/27] modify the main.py code --- landsat/src/main.py | 2 +- tests/data/job1.json | 1 - tests/data/job2.json | 1 - tests/landsat/test_main.py | 46 +++++++++++++++++--------------------- 4 files changed, 21 insertions(+), 29 deletions(-) delete mode 100644 tests/data/job1.json delete mode 100644 tests/data/job2.json diff --git a/landsat/src/main.py b/landsat/src/main.py index 040d4d39..e988e605 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -138,7 +138,7 @@ def deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame) -> gpd.GeoDataFrame: user_id=EARTHDATA_USERNAME, ) - df = pd.DataFrame([job.job_parameters['granules'] for job in jobs], columns=['reference', 'secondary']) + df = pd.DataFrame([[*job.job_parameters['granules'], *[job.job_id]] for job in jobs], columns=['reference', 'secondary', 'job_id']) df = df.set_index(['reference', 'secondary']) pairs = pairs.set_index(['reference', 'secondary']) diff --git a/tests/data/job1.json b/tests/data/job1.json deleted file mode 100644 index 6897965d..00000000 --- a/tests/data/job1.json +++ /dev/null @@ -1 +0,0 @@ -{"job_id": "5864010f-4f07-4030-beec-88e9c39d2baa", "job_type": "AUTORIFT", "request_time": "2024-04-08T00:23:21+00:00", "status_code": "FAILED", "user_id": "cirrusasf", "name": "LC08_L1TP_138041_20240128_20240207_02_T1", "job_parameters": {"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20230407_20230407_02_T1"], "parameter_file": "/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp", "publish_bucket": "\"\""}, "logs": ["https://hyp3-its-live-contentbucket-s10lg85g5sp4.s3.us-west-2.amazonaws.com/5864010f-4f07-4030-beec-88e9c39d2baa/5864010f-4f07-4030-beec-88e9c39d2baa.log"], "expiration_time": "2024-05-24T00:00:00+00:00", "processing_times": [699.518], "credit_cost": 1} \ No newline at end of file diff --git a/tests/data/job2.json b/tests/data/job2.json deleted file mode 100644 index 94b16336..00000000 --- a/tests/data/job2.json +++ /dev/null @@ -1 +0,0 @@ -{"job_id": "98cd33bf-4f32-499d-94db-20661f10f5f7", "job_type": "AUTORIFT", "request_time": "2024-04-08T00:23:21+00:00", "status_code": "FAILED", "user_id": "cirrusasf", "name": "LC08_L1TP_138041_20240128_20240207_02_T1", "job_parameters": {"granules": ["LC08_L1TP_138041_20240128_20240207_02_T1", "LC09_L1TP_138041_20221130_20230319_02_T1"], "parameter_file": "/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp", "publish_bucket": "\"\""}, "logs": ["https://hyp3-its-live-contentbucket-s10lg85g5sp4.s3.us-west-2.amazonaws.com/98cd33bf-4f32-499d-94db-20661f10f5f7/98cd33bf-4f32-499d-94db-20661f10f5f7.log"], "expiration_time": "2024-05-24T00:00:00+00:00", "processing_times": [752.675], "credit_cost": 1} \ No newline at end of file diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index fb66ab36..418680e4 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -11,6 +11,7 @@ LANDSAT_CATALOG_real = main.LANDSAT_CATALOG HYP3_real = main.HYP3 +SAMPLE_PAIRS = gpd.read_parquet('tests/data/scene1_pair.parquet') def get_mock_pystac_item() -> unittest.mock.NonCallableMagicMock: @@ -84,12 +85,12 @@ def test_qualifies_for_processing(): assert not main._qualifies_for_processing(item) -def get_expect_item(): +def get_expected_item(): scene = 'LC08_L1TP_138041_20240128_20240207_02_T1' - expect_datetime = datetime.datetime(2024, 1, 28, 4, 29, 49, 361022) - expect_datetime = expect_datetime.replace(tzinfo=datetime.timezone.utc) - expect_collection_id = 'landsat-c2l1' - expect_properties = { + expected_datetime = datetime.datetime(2024, 1, 28, 4, 29, 49, 361022) + expected_datetime = expected_datetime.replace(tzinfo=datetime.timezone.utc) + expected_collection_id = 'landsat-c2l1' + expected_properties = { 'datetime': '2024-01-28T04:29:49.361022Z', 'eo:cloud_cover': 11.59, 'view:sun_azimuth': 148.43311105, @@ -114,23 +115,18 @@ def get_expect_item(): 'proj:shape': [7681, 7531], 'proj:transform': [30, 0, 674985, 0, -30, 3152415], } - expect_item = pystac.item.Item( + expected_item = pystac.item.Item( id=scene, geometry=None, bbox=None, - datetime=expect_datetime, - properties=expect_properties, - collection=expect_collection_id, + datetime=expected_datetime, + properties=expected_properties, + collection=expected_collection_id, ) - return expect_item + return expected_item -def get_expect_pairs(): - pairs = gpd.read_parquet('tests/data/scene1_pair.parquet') - return pairs - - -def get_expect_jobs(): +def get_expected_jobs(): job1 = sdk.jobs.Job.from_dict( { 'job_id': '88ea6109-8afa-483a-93d5-7f3231db7751', @@ -166,13 +162,13 @@ def get_expect_jobs(): } ) - jobs_expect = sdk.jobs.Batch([job1, job2]) - return jobs_expect + jobs_expected = sdk.jobs.Batch([job1, job2]) + return jobs_expected def test_get_stac_item(): scene = 'LC08_L1TP_138041_20240128_20240207_02_T1' - expect_item = get_expect_item() + expect_item = get_expected_item() main.LANDSAT_CATALOG = MagicMock() main.LANDSAT_CATALOG.get_collection().get_item.return_value = expect_item @@ -190,7 +186,7 @@ def test_get_stac_item(): def test_get_landsat_pairs_for_reference_scene(): main.LANDSAT_CATALOG = MagicMock() - reference_item = get_expect_item() + reference_item = get_expected_item() results_item_collection = pystac.item_collection.ItemCollection.from_file( 'tests/data/scene1_return_itemcollection.json' ) @@ -206,9 +202,8 @@ def test_get_landsat_pairs_for_reference_scene(): assert (df['reference'] == reference_item.id).all() -def test_deduplicate_hyp3_pairs(): - pairs = get_expect_pairs() - duplicate_jobs = get_expect_jobs() +def test_deduplicate_hyp3_pairs(pairs=SAMPLE_PAIRS): + duplicate_jobs = get_expected_jobs() main.HYP3 = MagicMock() main.HYP3.find_jobs.return_value = duplicate_jobs @@ -222,9 +217,8 @@ def test_deduplicate_hyp3_pairs(): assert len(p_idx) - 2 == len(np_idx) -def test_submit_pairs_for_processing(): - pairs = get_expect_pairs() - jobs_expect = get_expect_jobs() +def test_submit_pairs_for_processing(pairs=SAMPLE_PAIRS): + jobs_expect = get_expected_jobs() main.HYP3.submit_prepared_jobs = MagicMock() main.HYP3.submit_prepared_jobs.return_value = jobs_expect From e88799ec7d84e5a99453ba937b8bd2977a11812f Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Wed, 10 Apr 2024 17:31:01 -0800 Subject: [PATCH 19/27] code style --- landsat/src/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/landsat/src/main.py b/landsat/src/main.py index e988e605..85076d70 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -138,7 +138,8 @@ def deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame) -> gpd.GeoDataFrame: user_id=EARTHDATA_USERNAME, ) - df = pd.DataFrame([[*job.job_parameters['granules'], *[job.job_id]] for job in jobs], columns=['reference', 'secondary', 'job_id']) + df = pd.DataFrame([[*job.job_parameters['granules'], *[job.job_id]] for job in jobs], + columns=['reference', 'secondary', 'job_id']) df = df.set_index(['reference', 'secondary']) pairs = pairs.set_index(['reference', 'secondary']) From 88dca4ec0f023b88a51744c819d45452ff900d32 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Wed, 10 Apr 2024 17:39:19 -0800 Subject: [PATCH 20/27] modify test_main.py --- tests/landsat/test_main.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 8b1d122c..418680e4 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -202,32 +202,8 @@ def test_get_landsat_pairs_for_reference_scene(): assert (df['reference'] == reference_item.id).all() -<<<<<<< HEAD def test_deduplicate_hyp3_pairs(pairs=SAMPLE_PAIRS): duplicate_jobs = get_expected_jobs() -======= -def test_deduplicate_hyp3_pairs(): - pairs = gpd.read_parquet('tests/data/scene1_pair.parquet') - job1 = sdk.jobs.Job( - job_type='AUTORIFT', - job_id='job_id_1', - request_time=datetime.datetime(2024, 1, 1), - status_code=200, - user_id='hyp3', - job_parameters={ - 'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', 'LC09_L1TP_138041_20231101_20231101_02_T1'] - }, - ) - job2 = sdk.jobs.Job( - job_type='AUTORIFT', - job_id='job_id_2', - request_time=datetime.datetime(2024, 1, 2), - status_code=200, - user_id='hyp3', - job_parameters={'granules': ['not_in_pairs', 'also_not_in_pairs']}, - ) - duplicate_jobs = sdk.jobs.Batch([job1, job2]) ->>>>>>> 55497efc680087655632c04a7e739fca4258383f main.HYP3 = MagicMock() main.HYP3.find_jobs.return_value = duplicate_jobs @@ -238,7 +214,7 @@ def test_deduplicate_hyp3_pairs(): p_idx = pairs.set_index(['reference', 'secondary']) np_idx = new_pairs.set_index(['reference', 'secondary']) assert np_idx.isin(p_idx).any().any() - assert len(p_idx) - 1 == len(np_idx) + assert len(p_idx) - 2 == len(np_idx) def test_submit_pairs_for_processing(pairs=SAMPLE_PAIRS): From a34baa2085b48bb5e18f4bedaf59cc312e131e6e Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 11 Apr 2024 08:25:23 -0800 Subject: [PATCH 21/27] modify the duplicate_hyp3_pairs function in the main.py --- landsat/src/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/landsat/src/main.py b/landsat/src/main.py index 85076d70..f5e0fb36 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -138,9 +138,9 @@ def deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame) -> gpd.GeoDataFrame: user_id=EARTHDATA_USERNAME, ) - df = pd.DataFrame([[*job.job_parameters['granules'], *[job.job_id]] for job in jobs], - columns=['reference', 'secondary', 'job_id']) - + df = pd.DataFrame([[*job.job_parameters['granules'], *[job.job_id, job.status_code]] for job in jobs], + columns=['reference', 'secondary', 'job_id', 'status_code']) + df = df[df.status_code == 'SUCCESS'] df = df.set_index(['reference', 'secondary']) pairs = pairs.set_index(['reference', 'secondary']) From 51405ba99adfd8f1737207537d13f21c49bcca15 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 11 Apr 2024 09:45:16 -0800 Subject: [PATCH 22/27] modify duplicates function --- tests/landsat/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 418680e4..ccb618c7 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -214,7 +214,7 @@ def test_deduplicate_hyp3_pairs(pairs=SAMPLE_PAIRS): p_idx = pairs.set_index(['reference', 'secondary']) np_idx = new_pairs.set_index(['reference', 'secondary']) assert np_idx.isin(p_idx).any().any() - assert len(p_idx) - 2 == len(np_idx) + assert len(p_idx) >= len(np_idx) def test_submit_pairs_for_processing(pairs=SAMPLE_PAIRS): From 2f3abcff91a319e72ad44fbc0541e14a7712adfa Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 11 Apr 2024 10:00:50 -0800 Subject: [PATCH 23/27] modify to satisfy ruff check --- tests/landsat/test_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index ccb618c7..d8c81f07 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -202,7 +202,7 @@ def test_get_landsat_pairs_for_reference_scene(): assert (df['reference'] == reference_item.id).all() -def test_deduplicate_hyp3_pairs(pairs=SAMPLE_PAIRS): +def test_deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame = SAMPLE_PAIRS): duplicate_jobs = get_expected_jobs() main.HYP3 = MagicMock() @@ -217,7 +217,7 @@ def test_deduplicate_hyp3_pairs(pairs=SAMPLE_PAIRS): assert len(p_idx) >= len(np_idx) -def test_submit_pairs_for_processing(pairs=SAMPLE_PAIRS): +def test_submit_pairs_for_processing(pairs: gpd.GeoDataFrame = SAMPLE_PAIRS): jobs_expect = get_expected_jobs() main.HYP3.submit_prepared_jobs = MagicMock() From fe4c1ea31929810f41210fc9ef15a7a4e7bb12f8 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 11 Apr 2024 10:05:53 -0800 Subject: [PATCH 24/27] modify code to satisfy ruff format --- landsat/src/main.py | 6 ++++-- tests/landsat/test_main.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/landsat/src/main.py b/landsat/src/main.py index f5e0fb36..84f2dbd7 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -138,8 +138,10 @@ def deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame) -> gpd.GeoDataFrame: user_id=EARTHDATA_USERNAME, ) - df = pd.DataFrame([[*job.job_parameters['granules'], *[job.job_id, job.status_code]] for job in jobs], - columns=['reference', 'secondary', 'job_id', 'status_code']) + df = pd.DataFrame( + [[*job.job_parameters['granules'], *[job.job_id, job.status_code]] for job in jobs], + columns=['reference', 'secondary', 'job_id', 'status_code'], + ) df = df[df.status_code == 'SUCCESS'] df = df.set_index(['reference', 'secondary']) pairs = pairs.set_index(['reference', 'secondary']) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index d8c81f07..064debf3 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -214,7 +214,7 @@ def test_deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame = SAMPLE_PAIRS): p_idx = pairs.set_index(['reference', 'secondary']) np_idx = new_pairs.set_index(['reference', 'secondary']) assert np_idx.isin(p_idx).any().any() - assert len(p_idx) >= len(np_idx) + assert len(p_idx) >= len(np_idx) def test_submit_pairs_for_processing(pairs: gpd.GeoDataFrame = SAMPLE_PAIRS): From 34e74cc71f5217e25d62e9210fe1d3e6d3da3bbf Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 11 Apr 2024 10:31:19 -0800 Subject: [PATCH 25/27] add sample jobs with job_status=SUCCESS to test the test_deduplicate_hyp3_pairs function --- tests/landsat/test_main.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 064debf3..08ba9333 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -161,8 +161,26 @@ def get_expected_jobs(): 'credit_cost': 1, } ) + job3 = sdk.jobs.Job.from_dict( + { + 'job_id': '88ea6109-8afa-483a-93d5-7f3231db7751', + 'job_type': 'AUTORIFT', + 'request_time': '2024-04-09T18:13:41+00:00', + 'status_code': 'SUCCESS', + 'user_id': 'cirrusasf', + 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', + 'job_parameters': { + 'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', 'LC09_L1TP_138041_20231101_20231101_02_T1'], + 'parameter_file': '/vsicurl/http://its-live-data.s3.amazonaws.com/' + 'autorift_parameters/v001/autorift_landice_0120m.shp', + 'publish_bucket': '""', + }, + 'credit_cost': 1, + } + ) + - jobs_expected = sdk.jobs.Batch([job1, job2]) + jobs_expected = sdk.jobs.Batch([job1, job2, job3]) return jobs_expected @@ -214,7 +232,7 @@ def test_deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame = SAMPLE_PAIRS): p_idx = pairs.set_index(['reference', 'secondary']) np_idx = new_pairs.set_index(['reference', 'secondary']) assert np_idx.isin(p_idx).any().any() - assert len(p_idx) >= len(np_idx) + assert len(p_idx) - 1 == len(np_idx) def test_submit_pairs_for_processing(pairs: gpd.GeoDataFrame = SAMPLE_PAIRS): From d0dafb2eca9e4cedfd8ef8a401f396cf6167cbe0 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 11 Apr 2024 10:33:42 -0800 Subject: [PATCH 26/27] fix ruff format --- tests/landsat/test_main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 08ba9333..547a7e47 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -172,14 +172,13 @@ def get_expected_jobs(): 'job_parameters': { 'granules': ['LC08_L1TP_138041_20240128_20240207_02_T1', 'LC09_L1TP_138041_20231101_20231101_02_T1'], 'parameter_file': '/vsicurl/http://its-live-data.s3.amazonaws.com/' - 'autorift_parameters/v001/autorift_landice_0120m.shp', + 'autorift_parameters/v001/autorift_landice_0120m.shp', 'publish_bucket': '""', }, 'credit_cost': 1, } ) - jobs_expected = sdk.jobs.Batch([job1, job2, job3]) return jobs_expected From 4903c6031604e39bc4eac0887ddd49d47f93b954 Mon Sep 17 00:00:00 2001 From: jiangzhu Date: Thu, 11 Apr 2024 11:09:32 -0800 Subject: [PATCH 27/27] fix job_status value to be SUCCEEDED --- landsat/src/main.py | 2 +- tests/landsat/test_main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/landsat/src/main.py b/landsat/src/main.py index ef074320..2641d76a 100644 --- a/landsat/src/main.py +++ b/landsat/src/main.py @@ -142,7 +142,7 @@ def deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame) -> gpd.GeoDataFrame: [[*job.job_parameters['granules'], *[job.job_id, job.status_code]] for job in jobs], columns=['reference', 'secondary', 'job_id', 'status_code'], ) - df = df[df.status_code == 'SUCCESS'] + df = df[df.status_code == 'SUCCEEDED'] df = df.set_index(['reference', 'secondary']) pairs = pairs.set_index(['reference', 'secondary']) diff --git a/tests/landsat/test_main.py b/tests/landsat/test_main.py index 547a7e47..5154576f 100644 --- a/tests/landsat/test_main.py +++ b/tests/landsat/test_main.py @@ -166,7 +166,7 @@ def get_expected_jobs(): 'job_id': '88ea6109-8afa-483a-93d5-7f3231db7751', 'job_type': 'AUTORIFT', 'request_time': '2024-04-09T18:13:41+00:00', - 'status_code': 'SUCCESS', + 'status_code': 'SUCCEEDED', 'user_id': 'cirrusasf', 'name': 'LC08_L1TP_138041_20240128_20240207_02_T1', 'job_parameters': {