From 75c1772bb9c63fdfb7417409cc376e07d50fb84d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 26 Jan 2017 20:39:09 -0500 Subject: [PATCH 01/15] Task creation via file upload unit test --- app/api/tasks.py | 3 ++ app/fixtures/tiny_drone_image.jpg | Bin 0 -> 17389 bytes app/fixtures/tiny_drone_image_2.jpg | Bin 0 -> 17360 bytes app/tests/test_api.py | 79 ++++++++++++++++++++++++++-- 4 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 app/fixtures/tiny_drone_image.jpg create mode 100644 app/fixtures/tiny_drone_image_2.jpg diff --git a/app/api/tasks.py b/app/api/tasks.py index 8e11008b..f5c23842 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -121,6 +121,9 @@ class TaskViewSet(viewsets.ViewSet): [keys for keys in request.FILES]) for file in filesList] + if len(files) <= 1: + raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images") + task = models.Task.create_from_images(files, project) if task is not None: return Response({"id": task.id}, status=status.HTTP_201_CREATED) diff --git a/app/fixtures/tiny_drone_image.jpg b/app/fixtures/tiny_drone_image.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ebec314ff72e6784ec613fa09825927a5e65ffb7 GIT binary patch literal 17389 zcmeG@30xD$`m?*aK!t#a2Pg}QC@3W1M(#sJ1rhK-pGSZQsgPh26sfjqz3@b;sCc1N zYg_e6TU)PM52{wHqP0G0wW6h7tygWWRv_=2-DC*_thW8X|LgBDFuUJ;*L?HMH#0kv zUHv6}H4+c%AKM=h1c5$;Kcv6rmKc+#oQP0ttUD4RgxVo1feUXqh*pHi7d!lin^ha*=$SaK$F%R;KcyDEnqD##HJSt5f5$>nYV;0 zWLvrdcz6bQ8^FYJmWRMCF0y2-H~@IEh}B<#`3?9l0OlL;7XTI(v32pG4hsV;2H4WT za~XK17YWh!?g-TYJP%;pmxb74r3Txi5(!b-;BC?XgHsS?fO`OJwVAbNHON{U;8K9a zn^{>fZ)B1txebgI!rE1X%+NJ>nZA@rRC} zr9MI4vLLAs?j4IT}C}!0D*l+Lk$#?0rU!V7X^Q~*UrF&2e1pk7XcPP z{aB(4hzJ3;2iON-5pZD(Z(E9Cl<-hTP(KLpVgnu_KdQY!=hk)zb;8{6zZ2ljfXA{p zI~rgBQCEO*Pwuor5NoV{M3RIP;9~%P2Jqbhk6SXz8lj$mmjXNiU^jq4H97)tFVKwj zTn4zWQ4f?ofyWZ=KLXwhU>krjqZD8PfGz;bfFGQWJOTCw82e-pz`h1}G{AnqkH_*W z0G9wBnvG@yTnR946He6z{l$PM!a*Yp_&or30QeFx?dietr8BL;wcx}M`Zy5t?7(VB$~i*km$E2^w_uoK?9W;nRL2hqEe;QDb*@knJv#yp#Dlt_Efn> zL61;qv=}Y*l*kxAjfW0<)}+Q0pqGYbr#I1$i{Ze(3i4sSv6VOu*@(vUHjr`Im~emL zeq97I1%{0E%V8L9G0|b98Rf7^z|P1TanHMt#YEgykdJ}hu(}5On#z?K5MjV~?>T%E zLDwCy^EJ4y$AJXND02$nPCz3p5DI}tV%Rj{Hvf@?+fBp80`y^@;l9Csg5iZ`LdJAs ziiQIAbs*ehA&-Wvggg!M49I`C)YB88U=L`oBSbmOlI1XIl;DZuZ(Uc|6S8$7!w7M_dxy%@)O9WDXdA^l=dQU z2*~4Er+*VYcun38IByuw{{%(dVQkmJ{Q=}S$j@MPz!{6-jGu^pFX6TIjA5PmJ>FLO zAM?PuDf@><%-uhnlg6DhW&Y4pbKqeCp&<;uX5LU##6WC$`rP;h?(p=otvG&iKitCx zK8B^y1@ptoVo3F?hJL-5ne13S$@7OInxU~}Bj11V-4$%Z>(u9k+Dre%X8uP_qnQy+ z!m|wiGfxW{I`4(Xyt(vWi80PPsbHrCSQReV#VfVm)27nQY)3Q9i`an&51a6_5&Lw8 zcP|ZzKmqpa=e%3}#=}Yb3QOFE7af3w6B3rh&&{wQfTwR4rp(TsdNhmSU@w_K!MttZ z3Xe=of=3AhKQhyP!7Zxb%HW3D@TeApR6W!LOCFa>`2jrKWc(Dr2Oe-uVIz|Z3K#pd z#r4mGeUOv*>ozli&q<7*f7w*0KSR}$wPbheB0(itNhS;57CBK)R9D_)@&@revDZSw zA4EoxPxu<)2C{%0N7Lv`M z!l)CXT)PIST;)|i5rx~oDAb_kMWHRCo8(RMxKQ7Pu;@f}qC$i#yN1_Q)mI5zsTJ;D)K%8^6eLq( zxAXOf>fQODP+9a^{Rj0WJQv<52VbK~iTWCHrf_E4)6{hR zXQZQOPn#g#$NDYgSH%{2V6rPgb!2ma?hfRA)|ZkEaRGy1%U zFSP6tP{T7+6QO+SDz%n(fjq{4N*G7mucSqtH#)7KM65)7;u*S9-<>F<#?~6=S3Kw5 z(z~Msv=q*&dLlj13$@DpF(34yF4Q`r8vQt8Gund&61Mt#sFa#eyB5~5wXmL9p>Oq% z^*vBL%!GjkU(cbY*N#N*=wBg>MJeHr9@TlRO=<7zv5r(7AdqVlcJ4 z7RxM#EI{Y-tn!>3B!v=2&4d*P(&_ zv|AY;0o@VkNkCucP+{?)3!LCG@*nE&ku^?Qbc^~4uGe{p9)vFQE$9R2qJXBqhc{P( z=~vN7(O`PJLBfZ)M^2`f=xU;dgvk{75L{w^lIT1@HoZv6di!tqpJ_ zz&Q1WjA{5+BYf}mo&a5yAT-QSb~oIgsz{XUgizEx0@VPndruy^0u|`5*;QxibU8s@ zUMj7pJYAit@Jv%@d*#Vmm^P`q|ni+iVUSH#O?n6Pu*x`dWhRd-#AHJ zPLv{3IWS+NNXm~-PRpN^7MSi97HS!s7nGNsldaIn>AdVLl{P3Z#EnrN1TaQ>xdD?V zePU2zbbkX2ltSDLwoIKm)pM%1r&^QYB@GM=^peQDWU{`%(N{Z7rIY9NRcX6}21}aJ z5v|asX_Pt4%PwqnQSxksM$V~&rfC%_tzu%&QIo&OI z$jk{w_Y6o#8kG`}mX@oLr%em>g?GdJVc-S$`v=Mb{Q?5GAZ^TNGIoG4nv?hO43K$a z`2eZEkB@f%^qwKe8?l*)Cn$AknV~XIZ$F8|&nGaz+uIMS2B#b3Sh|ToqB<)p)X&o| zFd)#|*H;qYFAD&hIevz0!p`-4L{?6w{5jq4=M5^G@9UQ$HQxc|W*9qwqeQ~=_#FGB z0p8Hc=4POmp!|+;Mib0z8z9flex7kMPnoQF<6eUDJH~M~7_DoVbNY`Q822341-3AT z;ce^Yjcy{)ykX|m8x2d5PYsp%`*;R=OJy(!0{wgfe7yY|#sa3BgsA!I6DiMNn!`;( zf8pw5(9I1P)7p&*Fvb4_+o&)HRJ4ST2-_fPz>XfNgo zfuY68Cn-Xi55|JM8W7l2wxmv1WU=%npOP8b83vAjRH8zw&dSBR=1^$>G?CF_ise{S znq@V`a;y!U!{PIj&{$Pkra^NP3MY}Io~Xkgri3aY6|hZatl)4QSF~KGNLIquyqRyR z3_=_WClsO4DYZINAErd5Mrjh%ct5RH;eN-fwnBq9^-RTPbWT^4I%k>&zO2!Oj)_;R z6l3XL3Gk_&IzuDR$y8{1(|u_pJ|Q6m(%!%Bq;PAE0_C{q6oih@shal%&;Cmw~bd>)VD3HSm5lZ6%{ zfzU!I5Qy4{EG%&XsBIg|wzzVqX-Nk_+&=^0oBo~AD`bpBgMsK z-jtokD&jvGX0x9d8RrsnY}NX)SN?T+z@)dT3&KC!KYjH2SzW}#Ryhq!TjsE8&0B$< zv!-6JUNPeAzI~4Za-;VK7lqU&joesOa@hXt){2{7jXU>feML&Dx;XT#$1rs1l;6;$ zAI->l9h(LoBe8J}i6Cv9rIgr~W=sP=g%R3=o4@X|ebqzV+&7Q!u->QZ?pjneV`snM zWAlqc7B7s}jq3xdTtF3kw8_Iq@oa)`yeS)Jo~?_teS~b-WJVvvN*J+oZvUz>*Y}2R zOz5|N`s>>koIG$$UEW4;!|}vdTW=L7Tl^y4x6@yeqH&b&{Tvj+cj9Et@=fK}9vFH;5JvW2@m8B0^wag_hgCm! zeUGkiO!KWu`e)}6H*9xo@^;$cv7Oi*uw+n$=uv#ev&~zJE3Y|NuFhP(@Wvc?DB1Px9cv!%^xQal^_Oe+_`CM*enaB% z(JyIbi(=}|`W-oV_Tq{9-n#{rpq*~NKxPuR4-Kj3=Shqq&|xplPrh#I$0dfDg| zhpJK)GG27;J6%SqzjbxHA5pqqmK<@z&CTx1T@G2lF4oj~WDNL#{`J(fEy{Io9Em6& z=knp*-q(}zzn9S0wLhFJnZ9nt8tPHvxFJ7&5pU&Dc<7kN(qHqhd*Ch?!&rfzS0I9n zU2fxyq!Gj9lV>g8boksuV|0f(Z<)g{agCo*op{MApT9Z!(T0cod+|f3udID7rt^`~ zoNm(LcW+wHrHLyQs&f%931oAihVYD3PKCfA&*Dt+ww+K~DsLsL2*v-P3A0ADb33-kMvxa#cZd)h7t zd2`_1+jty0*f8^v$v`BKjk8!fEP`<%9);rg^owDO)0{q59a^_k^N$)YxliVAyQUD{ zHOg;Z4QaQ)*JAS0+_Y)c{fg2id{=#R!FbFAh)XGih0mNabtc9->XspT{4LN*Fe8r43b4cK>nj z&PmR_xATtVFKV;$@WNc)pvz9kYT}R$btN0d{A9oJR6(ZugQydO*3yF1Z9PYbu6;1P z;H#YcHnWT8y4F638)&t|p>|=)Hfv(&ut{}M1>xs!7nE$ekP>}^SMZ(FL}wqb3yWs* z)gPz7_sf&E>L4O1Q{KJ9 zbNXM)f@f}fFrenjwfsYgK^+S%E8<1hx2_W1No?B>X_t~=pl6Uh0 zdy$f-b+;1Y=cIc7e6DM8_y)I&Z$xZZvQeIO-RTTo0V%8ex(;@{&}sKy*H`@;1zt%b literal 0 HcmV?d00001 diff --git a/app/fixtures/tiny_drone_image_2.jpg b/app/fixtures/tiny_drone_image_2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1f3bb979d05f64bada9d0c886acaa621589c8ce7 GIT binary patch literal 17360 zcmeG^2|!cFwsV&(AZkEVK!gjZpw*CsH4za+MN|}5gt`zRKnf%nqM){~YANnM!SyLs z)LOJsx7JFnR_*JvYOUZ>yXbSPRjjR0+p0z6zBBhGArNrs>wo|Ce+K8?GiRGKXU@#r zncOS;6?+v)h7OJyj0l22@4+8pulbH1l&4KaC?=*C5+j7#qILv{7=Ye}+#X;d+)qN5 z0NfGom5|*C0V3dj6LJ?5d>3Fpg5vNbz;fVsfct&O3V@yAPJ&QhfUzu29Y~%Zm)Fi~ zz-#Bz98V`8bYH;3yL)gVhnL|gqzIiHQ%FSV&fQv zG*l`NMGBcbL?x4}6v2!_rjmuK0)mkTS~rhxX+AXR4HLWsVBZD2<;Ss6%SDKWn@kZP zqn7h6T?st23El=UQONTUxWz^GoD~NFPZsg|Kf(Ma{3(D1Cj5DTMMZpF0;t2z1WN$6 zH}PBoo|lV6sLThU8vqvojQesq_E?F@Hn~hjlr(soG{N8$M48}j0Jq!B+p`vA+neAL zfF+xGSx}EUTfu0Hh-(W4c;Zc-2S7wx!ITwDzsc+020DdScr1gXpi%^~CK!4oNF@tV z1;`nBpejJ2;(N&k7kUOFsY8RDS(jOl8z8Xn7^s0l3V>dM?xNri_u6T=&;WY_d;wq~ z)Q=_HKtu#^Cx8P176TWy@Oyg+j1n4k2K7S$FEQZ}3Z^=kbiUOVAs5UI|8@fG3V1Aw zv%3if5OoC@_vFrY2x5)bk4Td61pFYtp9Z`S;BiYPv`451;N<`(0qhGfs74<5~$ zo=X7tG3$Y{6nO06{sZ9s0d@o!Gs*!L0_Y8}0{Fq{ND6QOz}P240S+?3i2w%!KOW03 z0lW_I&}=jh;7WjTn{c{n(!Uz;L2$wc;I}M;4I05avskJ*MX3apx;mg zn$l2#g~|g2`kO0415gsjL8ICx^qAOa)evo3I+LPF)#|iHtzO4yv(#xCG+1lM%2gXQ z%y^9<2czXunS%3E9q4-0q((ebFwpEKXs)&h=-@9QZ)MG`#4~Y2!u@7l#(m25E6Ds{ z%2@v=Fbu7Am}zEBxNczQWX-rI-Twl_)1VXQufVp^j~nbu2HMwkEQ{|rsreg<2foKv zd?vKzmI^+&1^VW}9VQqqnPonNk~h$9!Spo=LI}gw3Ag$8B;0NWE?J-t`vmt5_7e;* zGzT)Ko6|rj;IW~E`!L88A!k6I4H<&$PnS->gs1Hdedq!611#X$=sC@-b)PJ&66$7z26rrh*orWRQ6LKHOf7Ub&#>7I%<&eLC{1CEr z3U5-jracQB2J(2;8Qeqlz}j5}xE>fyZ;VN$ADHJroadl5k9a{1Pg3wir4eZ?ZB!C;cmL5eR-Qo8?)VQFwbHK z9z1-)&qn;yS>r{KK*~-#^54Gt+D{n=map?2S9A~_%pqY({M-y10(koN=F0r+$)X&N zgTE|;;e?;a9U#La6PMgg@|dHk_Vd0`3;#K+p*Gk8Kn*Ng4>g5h0aGdwgwnXl_$j`; z4gXwfhRrH=z&^-R@=cpL;73mh-mmcKF?*V-BkM??_C>-W(Q8hMlWj`+aNAQ(zUkq-q1(FSrMIhk@s-?A%U#aICf@O2S-12E307pnTVXa2sT zXeyeD6wmM29ONPDPxYsci>Es`Kzl^NR4_%0_jGQ6KBMQ7bIDz84z^7Z`B8pUkbUnC z4HD7f_EdW+Q@GJ1AE=&GKXF8-XN4NHJS#LuTuoM!q2lSTx5VC*H#JFA>mKp=>-w*S z-KmwmPCc%y?;#vZNqo=NAF1ypI7DSK>)Chf*U{eegf2m7cYPQ=o6hcRK>O=m=@R;C z$J6L&{cY+p9p^lZsHopbg$X7)eM(%apGj>Kq)VodKh{T3Lxf+o`zPtjI#K6@Wo=Wb z(d-R!j%bd<=hVyWKS+0Rc^ehIpWRAUh->Ua1+SwiE62iRm zO??j{hT31Z2Nk2G5Zgj@j{SuVLYZg=JU{X>%}#2q$W{-udZ5(Tk1An7*WW~y?%#`j z-9NE=y-NfAH;*j>1axZ}xL&96k9d~^cFaL^ zLC7%Q!J8{#%xrX0Jd7zbNdyvi$!W||wu(w-USO}&F>t*k@Mi|H7lbZM7q(JV=b6Qx z6o2peItz!OTrJ1Z0ng>cZjTA%>aGk`>|RCf>9U(X;g%=3;~F89x{5`GE|sF~o!5%P zIwsk@+s+S;eZPe41-m^4_Hqnb33)B#b^i<52^@{zF6e=V!gG2Fs(?4X0tqd#f%uYW zM-CxhBR?V^k$tIGsDtqGO%%P5K0~_-k_Ed2wSpnS)xz6CC3HuGELR*kCMpL1&hyTd zHn59@bLkio!67jsOc~e!rr}D$?|uYC06t3|fy?lA3ZRg^FMh{|$Hzk%zqNw#-hdYZ zj5VnMZf}AI0E|;l$e4!Tx4`$Ar3AVSRgW^2dztQiZxiSUyc{I{2Z`PQ#=k}o$P4c5 zHRr48Mq{?h-(Qy_Rj25uYNW~fEdM-pwm-Ze;*Y}m=VhytGc-nKswPdV>*rhj;bC7! zo6^rWJ}6cen;oS|*AAIw(2SWiVr=rPjAUhsZ~t)nusl^>R(6)gsAlrAGIcqsyneo% zauvWB?e7arhLlv*=z)VxEKus_YqBLbH&>b)Ak`bv{N+le(qE?VS19@bN1vRTI-@$T zk1odtG}tqoj)9t-WP>)Fd)bArE=rxHF{mx-plLZ8U5+L-b0*h1p3F56v@lErzI5D8})s$7rT;1-_K24IiyZ3G>H>1B|&WglL0CqidctAalB=5rr?9bcgvjQ@gcA zW59HHlV^t3IP(dHDP#(%EKu2~9*c4E&|s-j8PozXI8dsP2R9s50ya*kOj#@fwl>G{aa>~!@Lx<6Q^2x?&zJr}9@4zM-D+yNHK z0GJ-1V4pl7G@$uVd=AP}j5C{HV_UR3E9*(dDWnQT^Ts^~dl5FsB^<1oTbX3Ab4F{7OD&l41g%L#5AT`g{bZ7GeDij zHODdu{ei2GNw>uq>)Oo?z_=#!J#UJQVgD1WPjePFvBqp@von7Srv6c@PgC=nwZUSb z&BWV+b^o~)!V*+11po7)8{5r~6dMoOIKk|m#;eNzpVh}^96i?(W9dO#spdPtvc^~z zL30OKDCViZrSSaYR-dNPVPk@gZI+q81>>G$jj_1SM&>D=v)Ev^&UX55!MY}kiN%8F zTYZ{(&9-{8Vb8Yu@XfW_K|Nttm1EjNwYaY`Gg$4c%qSH+$Hfc`m%}{K1cg_?Ca7k) zp2t$yd|Df6yRYT9eZZy~tu|_PY2h&2S(UI&R>h{o=P^YCU#vB63#M3k3Kvx|_*)ID z-eCN>Cj^cbtIp7bb03U_`8Oc2seDPFqRHgxO+FOapvb zV+?PxXg=zKPR}voOVs8963fqeZtTl_pJaZ3?1V?Ih>da^8Obe@xl7XPZ zsGX%rc?K;FycRW4`mAi|N1ZVzO0P@RroqOS8w$X{Pt7(*KM73KR3^&6^m(e|o}im| z;F$F3MhH8uug0t4cqm*hlPW`)LE{-eE+7KAfH239ExV-|k=Yu95n_qg0-+(}lyFgE z7H`aB5jDrz0IfQQ7nkwkmM-MOw1I9BjaBD0R@!{UHf#+FG|+q)f3Ywpg~zF}GvM%y zi=0?%5}fdHr)^eKA}mlm4vr4t5YgXq)CdL3VWma~CzJ+!6vG_?;>KCiux_tjoyAzidOVpdN5%75NT*DGa@d?$Xl zYur1ALA8s!24(aI(>xsU$vC#ok_gh#%PCUQLC$c7fsgtV6HerOc5B(eJuem&ZCyG2 z;qcf^-J?6dkQ$tn*fFqTdhx}MU0qxjUO8u<7R~F#3PI3HCk0BqBjq%w6Jn)5v9Gq~ z?c4Fkbnke&TwH!w{P)*a9y_~V<9}?(WZf%4QpeF}ygpSWh)?>gTK;I;_^e{Dsp;cq zC;gb7c;;Kr1;vZ|tl3rYm!&DEj(d5Z2>k2qZ<6o)QoJ?n!mCif4BSmnBsf@PG5~*S zO*=Yy2|CEV6_M(>&K*ZhE8OD~FnZI`Gc_jn3;UDH`j!nUuQq-jncJz2pGRL=|IcT; z`Suole^|fq4S7gA$^P%MUb|DXxki*KI#7CN;?+{mo$vj8am}v&_V;}YcXskB+u}gq zJN@X~Pde9G-F~R<9JeI;^2&tl`>4Z0TJ5y&z|xg3X*~D2UrIkxvDPE#Nh33XI_4%%E0fx8k{j*_tDC|sV*N0aMznGnwh4h2Sk5c_xZ=t-GoM3RjUp;>li1S92~!YiE_o8C%XOh)=v)z za^ZyyZ7Z(tksN2%mQ}h}94bFhc(L-#K(|9lV;3GGdgVngI#R8m>0UqIo2ux2uhOe; z&ZDxC2?0e!rgTPVfuG~4iaSgFXYanS|Hz$PiSd)%3WjWtyBgwrY0!w+O7c@kmJb8ZzLHc!6HfG}eGYy5 z=mb5N>c0Nhgu&yBmYpAXC}??DS^N1OU*3&Fbp8s}Mo0N|y+axOwYyr|V-a}K-4RboFgoyWP`nqBqnA8VGHNd8 zKRn_j=XMSlJkzcAKI*gPx^dO%HoMa9@7?iok5Cr*FJAI(dyV_VE?QmUozlHK?+1+C zxUF5;3T8%kl{cvOa>Shk{}n?M^`oZEJz7(^=_jrZJOU-d^wL1Zdy2=ktM({j3w*wq zmy%jikWu>4nlNwqz6sTtnTe=lw$t(3Zl}ECH_ZApcG=81cO&%GdwSnlJEQ(DHyze= z?Dpk<_4qE5qIQ`Y)<+N(f#LBMKWivECI_>DY#OIrm z2U|azzj^Gz?1zbCjMvU?rfW0zwjF^w?sSxu4m$XKOu(hFUwGW?xpP^J%bHo&j@_ht zoejxG2dB-fbUNM7VZvVNwSql!_8pK!sH?wxu&(#Wr1Y^v#Ii1-@_A>k7F5K#Bs%Mz zKRprIY5by|k~T;6`Q^9BlRo6azwiCA)7kv3g%ggZkDRbCsQXK~;sMvgoO*psafQWUcK7x=%nf?)i|iL?>>wCjXLYsP`7Ofk8;^UI{v5j3 aq316hQ$$~8eBhq5d-;y-wH3we*Z%@q+CHHG literal 0 HcmV?d00001 diff --git a/app/tests/test_api.py b/app/tests/test_api.py index 36763a0f..43b07b7a 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -6,10 +6,11 @@ from rest_framework.test import APIClient from rest_framework import status import datetime -from app.models import Project, Task +from app.models import Project, Task, ImageUpload from nodeodm.models import ProcessingNode from django.contrib.auth.models import User + class TestApi(BootTestCase): def setUp(self): pass @@ -191,16 +192,86 @@ class TestApi(BootTestCase): self.assertTrue(task.last_error is None) self.assertTrue(task.pending_action == pending_actions.REMOVE) - # TODO test: - # - tiles.json requests - # - task creation via file upload # - scheduler processing steps # - tiles API urls (permissions, 404s) # - assets download (aliases) # - assets raw downloads # - project deletion + def test_task(self): + client = APIClient() + + user = User.objects.get(username="testuser") + other_user = User.objects.get(username="testuser2") + project = Project.objects.create( + owner=user, + name="test project" + ) + other_project = Project.objects.create( + owner=User.objects.get(username="testuser2"), + name="another test project" + ) + + # task creation via file upload + image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb') + image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb') + + # Not authenticated? + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN); + + client.login(username="testuser", password="test1234") + + # Cannot create a task for a project that does not exist + res = client.post("/api/projects/0/tasks/", { + 'images': [image1, image2] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot create a task for a project for which we have no access to + res = client.post("/api/projects/{}/tasks/".format(other_project.id), { + 'images': [image1, image2] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot create a task without images + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) + + # Cannot create a task with just 1 image + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': image1 + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) + + # Normal case with just images[] parameter + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_201_CREATED) + + # Should have returned the id of the newly created task + task = Task.objects.latest('created_at') + self.assertTrue('id' in res.data) + self.assertTrue(task.id == res.data['id']) + + # Two images should have been uploaded + self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2) + + # No processing node is set + self.assertTrue(task.processing_node is None) + + image1.close() + image2.close() + + # TODO: test tiles.json + # - tiles.json requests + def test_processingnodes(self): client = APIClient() From db1b4147a526485094d11a80f84f25aed9de0bf0 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 27 Jan 2017 14:44:10 -0500 Subject: [PATCH 02/15] Tiles.json, tiles request testing --- app/api/processingnodes.py | 2 -- app/api/tasks.py | 4 +++ app/tests/test_api.py | 55 ++++++++++++++++++++++++++++-- nodeodm/external/node-OpenDroneMap | 2 +- nodeodm/models.py | 2 +- 5 files changed, 58 insertions(+), 7 deletions(-) diff --git a/app/api/processingnodes.py b/app/api/processingnodes.py index 8ec57978..006e0bb1 100644 --- a/app/api/processingnodes.py +++ b/app/api/processingnodes.py @@ -1,8 +1,6 @@ import django_filters from django_filters.rest_framework import FilterSet from rest_framework import serializers, viewsets -from rest_framework.filters import DjangoFilterBackend -from rest_framework.permissions import DjangoModelPermissions from nodeodm.models import ProcessingNode diff --git a/app/api/tasks.py b/app/api/tasks.py index f5c23842..4fdd4d15 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -185,6 +185,10 @@ class TaskTilesJson(TaskNestedView): task = self.get_and_check_task(request, pk, project_pk, annotate={ 'orthophoto_area': Envelope(Cast("orthophoto", GeometryField())) }) + + if task.orthophoto_area is None: + raise exceptions.ValidationError("An orthophoto has not been processed for this task. Tiles are not available yet.") + json = get_tile_json(task.name, [ '/api/projects/{}/tasks/{}/tiles/{{z}}/{{x}}/{{y}}.png'.format(task.project.id, task.id) ], task.orthophoto_area.extent) diff --git a/app/tests/test_api.py b/app/tests/test_api.py index 43b07b7a..b0035872 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -1,10 +1,13 @@ +import datetime +import subprocess + from guardian.shortcuts import assign_perm from app import pending_actions from .classes import BootTestCase from rest_framework.test import APIClient from rest_framework import status -import datetime +import time, os from app.models import Project, Task, ImageUpload from nodeodm.models import ProcessingNode @@ -212,6 +215,7 @@ class TestApi(BootTestCase): owner=User.objects.get(username="testuser2"), name="another test project" ) + other_task = Task.objects.create(project=other_project) # task creation via file upload image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb') @@ -269,8 +273,53 @@ class TestApi(BootTestCase): image1.close() image2.close() - # TODO: test tiles.json - # - tiles.json requests + # tiles.json should not be accessible at this point + res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) + + # Neither should an individual tile + # Z/X/Y coords are choosen based on node-odm test dataset for orthophoto_tiles/ + res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot access a tiles.json we have no access to + res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(other_project.id, other_task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot access an individual tile we have no access to + res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(other_project.id, other_task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Start processing node + current_dir = os.path.dirname(os.path.realpath(__file__)) + node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False, + cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap")) + time.sleep(5) # Wait for the server to launch + + # Create processing node + pnode = ProcessingNode.objects.create(hostname="localhost", port=11223) + + # Verify that it's working + self.assertTrue(pnode.api_version is not None) + + # Cannot assign processing node to a task we have no access to + res = client.patch("/api/projects/{}/tasks/{}/".format(other_project.id, other_task.id), { + 'processing_node': pnode.id + }) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Assign processing node to task via API + res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), { + 'processing_node': pnode.id + }) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # After a processing node has been assigned, the task processing should start + # TODO: check + # TODO: what happens when nodes go offline, or an offline node is assigned to a task + + # Teardown processing node + node_odm.terminate() def test_processingnodes(self): client = APIClient() diff --git a/nodeodm/external/node-OpenDroneMap b/nodeodm/external/node-OpenDroneMap index baef4f81..254ce04f 160000 --- a/nodeodm/external/node-OpenDroneMap +++ b/nodeodm/external/node-OpenDroneMap @@ -1 +1 @@ -Subproject commit baef4f817e1928090b522a2e871de0568377d043 +Subproject commit 254ce04f55db521acbae1b38294d2ec65e3c8a09 diff --git a/nodeodm/models.py b/nodeodm/models.py index 02313fa5..fb33d65e 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -58,7 +58,7 @@ class ProcessingNode(models.Model): self.last_refreshed = timezone.now() self.save() return True - except: + except (ConnectionError, json.decoder.JSONDecodeError, simplejson.JSONDecodeError): return False def api_client(self): From cec2ed03901c25fae32011244b2d71f46abc6d25 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 27 Jan 2017 20:33:10 -0500 Subject: [PATCH 03/15] Tiles, assets download, raw assets download before processing testing, view urls testing, permissions testing --- app/scheduler.py | 2 -- app/tests/test_api.py | 59 ++++++++++++++++++++++++++++++++++++++----- app/tests/test_app.py | 38 ++++++++++++++++++++++++++-- app/views.py | 4 +-- 4 files changed, 91 insertions(+), 12 deletions(-) diff --git a/app/scheduler.py b/app/scheduler.py index 5f6a3eb5..12607acf 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -10,7 +10,6 @@ from app.models import Task, Project from django.db.models import Q, Count from django import db from nodeodm import status_codes -import random logger = logging.getLogger('app.logger') scheduler = BackgroundScheduler({ @@ -54,7 +53,6 @@ def update_nodes_info(): for processing_node in processing_nodes: processing_node.update_node_info() - tasks_mutex = Lock() @background diff --git a/app/tests/test_api.py b/app/tests/test_api.py index b0035872..49e87a04 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -4,6 +4,7 @@ import subprocess from guardian.shortcuts import assign_perm from app import pending_actions +from nodeodm import status_codes from .classes import BootTestCase from rest_framework.test import APIClient from rest_framework import status @@ -27,12 +28,14 @@ class TestApi(BootTestCase): user = User.objects.get(username="testuser") self.assertFalse(user.is_superuser) + other_user = User.objects.get(username="testuser2") + project = Project.objects.create( owner=user, name="test project" ) other_project = Project.objects.create( - owner=User.objects.get(username="testuser2"), + owner=other_user, name="another test project" ) @@ -195,14 +198,32 @@ class TestApi(BootTestCase): self.assertTrue(task.last_error is None) self.assertTrue(task.pending_action == pending_actions.REMOVE) + # Can delete project that we we own + temp_project = Project.objects.create(owner=user) + res = client.delete('/api/projects/{}/'.format(temp_project.id)) + self.assertTrue(res.status_code == status.HTTP_204_NO_CONTENT) + self.assertTrue(Project.objects.filter(id=temp_project.id).count() == 0) # Really deleted + + # Cannot delete a project we don't own + other_temp_project = Project.objects.create(owner=other_user) + res = client.delete('/api/projects/{}/'.format(other_temp_project.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Can't delete a project for which we just have view permissions + assign_perm('view_project', user, other_temp_project) + res = client.delete('/api/projects/{}/'.format(other_temp_project.id)) + self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN) + + # Can delete a project for which we have delete permissions + assign_perm('delete_project', user, other_temp_project) + res = client.delete('/api/projects/{}/'.format(other_temp_project.id)) + self.assertTrue(res.status_code == status.HTTP_204_NO_CONTENT) + # TODO test: # - scheduler processing steps - # - tiles API urls (permissions, 404s) - # - assets download (aliases) - # - assets raw downloads - # - project deletion def test_task(self): + DELAY = 1 # time to sleep for during process launch, background processing, etc. client = APIClient() user = User.objects.get(username="testuser") @@ -290,11 +311,22 @@ class TestApi(BootTestCase): res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(other_project.id, other_task.id)) self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + # Cannot download assets (they don't exist yet) + assets = ["all", "geotiff", "las", "csv", "ply"] + + for asset in assets: + res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot access raw assets (they don't exist yet) + res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + # Start processing node current_dir = os.path.dirname(os.path.realpath(__file__)) node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False, cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap")) - time.sleep(5) # Wait for the server to launch + time.sleep(DELAY) # Wait for the server to launch # Create processing node pnode = ProcessingNode.objects.create(hostname="localhost", port=11223) @@ -315,8 +347,23 @@ class TestApi(BootTestCase): self.assertTrue(res.status_code == status.HTTP_200_OK) # After a processing node has been assigned, the task processing should start + #time.sleep(DELAY) + + # Processing should have completed + #task.refresh_from_db() + #self.assertTrue(task.status == status_codes.COMPLETED) + + # TODO: background tasks do not properly talk to the database + # Task table is always empty when read from a separate Thread. Why? + # from app import scheduler + # scheduler.process_pending_tasks(background=True) + + #time.sleep(3) + # TODO: check # TODO: what happens when nodes go offline, or an offline node is assigned to a task + # TODO: check raw/non-raw assets once task is finished processing + # TODO: recheck tiles, tiles.json urls, etc. # Teardown processing node node_odm.terminate() diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 997bf0e0..1b03732b 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -1,5 +1,6 @@ from django.contrib.auth.models import User, Group from django.test import Client +from rest_framework import status from app.models import Project, Task from .classes import BootTestCase @@ -57,6 +58,12 @@ class TestApp(BootTestCase): res = c.get('/processingnode/1/', follow=True) self.assertRedirects(res, '/login/?next=/processingnode/1/') + res = c.get('/map/project/1/', follow=True) + self.assertRedirects(res, '/login/?next=/map/project/1/') + + res = c.get('/3d/project/1/task/1/', follow=True) + self.assertRedirects(res, '/login/?next=/3d/project/1/task/1/') + # Login c.post('/login/', data=self.credentials, follow=True) @@ -84,8 +91,35 @@ class TestApp(BootTestCase): res = c.get('/processingnode/abc/') self.assertTrue(res.status_code == 404) - # TODO: - # - test /map/ urls + # /map/ and /3d/ views + user = User.objects.get(username="testuser") + other_user = User.objects.get(username="testuser2") + + project = Project.objects.create(owner=user) + task = Task.objects.create(project=project) + other_project = Project.objects.create(owner=other_user) + other_task = Task.objects.create(project=other_project) + + # Cannot access a project that we have no access to, or that does not exist + for project_id in [other_project.id, 99999]: + res = c.get('/map/project/{}/'.format(project_id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # We can access a project that we have access to + res = c.get('/map/project/{}/'.format(project.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # 3D views need project and task parameters + res = c.get('/3d/project/{}/'.format(project.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot access a 3d view for a task we have no access to + res = c.get('/3d/project/{}/task/{}/'.format(other_project.id, other_task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Can access 3d view for task we have access to + res = c.get('/3d/project/{}/task/{}/'.format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) def test_default_group(self): # It exists diff --git a/app/views.py b/app/views.py index 3caa4fc4..978ce2a6 100644 --- a/app/views.py +++ b/app/views.py @@ -34,7 +34,7 @@ def map(request, project_pk=None, task_pk=None): if project_pk is not None: project = get_object_or_404(Project, pk=project_pk) - if not request.user.has_perm('projects.view_project', project): + if not request.user.has_perm('app.view_project', project): raise Http404() if task_pk is not None: @@ -59,7 +59,7 @@ def model_display(request, project_pk=None, task_pk=None): if project_pk is not None: project = get_object_or_404(Project, pk=project_pk) - if not request.user.has_perm('projects.view_project', project): + if not request.user.has_perm('app.view_project', project): raise Http404() if task_pk is not None: From 8939ba9b000a1d4a247dea8b6dad1cae8b7d1c1b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 29 Jan 2017 15:29:25 -0500 Subject: [PATCH 04/15] Removed django-common-helpers, upgraded django --- app/tests/test_api.py | 2 +- nodeodm/models.py | 3 ++- requirements.txt | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/tests/test_api.py b/app/tests/test_api.py index 49e87a04..7bb0bd62 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -223,7 +223,7 @@ class TestApi(BootTestCase): # - scheduler processing steps def test_task(self): - DELAY = 1 # time to sleep for during process launch, background processing, etc. + DELAY = 5 # time to sleep for during process launch, background processing, etc. client = APIClient() user = User.objects.get(username="testuser") diff --git a/nodeodm/models.py b/nodeodm/models.py index fb33d65e..ce1aee3c 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -196,5 +196,6 @@ def auto_update_node_info(sender, instance, created, **kwargs): class ProcessingNodeUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(ProcessingNode) + class ProcessingNodeGroupObjectPermission(GroupObjectPermissionBase): - content_object = models.ForeignKey(ProcessingNode) + content_object = models.ForeignKey(ProcessingNode) diff --git a/requirements.txt b/requirements.txt index a149868c..d5b73920 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ anyjson==0.3.3 +appdirs==1.4.0 APScheduler==3.2.0 coreapi==2.0.9 -Django==1.10 -django-common-helpers==0.8.0 +Django==1.10.5 django-debug-toolbar==1.6 django-filter==0.15.3 django-guardian==1.4.6 @@ -15,9 +15,11 @@ futures==3.0.5 itypes==1.1.0 Markdown==2.6.7 openapi-codec==1.1.7 +packaging==16.8 Pillow==3.3.1 pip-autoremove==0.9.0 psycopg2==2.6.2 +pyparsing==2.1.10 pytz==2016.6.1 requests==2.11.1 rfc3987==1.3.7 From fa843be5fc9c2119cea0b82a06400568ab5573a0 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 29 Jan 2017 16:17:05 -0500 Subject: [PATCH 05/15] Added timeout to api_client.py --- nodeodm/api_client.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py index 5f58e72a..13a060a2 100644 --- a/nodeodm/api_client.py +++ b/nodeodm/api_client.py @@ -8,6 +8,7 @@ import json import os from urllib.parse import urlunparse +TIMEOUT = 10 class ApiClient: def __init__(self, host, port): @@ -21,28 +22,28 @@ class ApiClient: return urlunparse(('http', netloc, url, '', '', '')) def info(self): - return requests.get(self.url('/info')).json() + return requests.get(self.url('/info'), timeout=TIMEOUT).json() def options(self): - return requests.get(self.url('/options')).json() + return requests.get(self.url('/options'), timeout=TIMEOUT).json() def task_info(self, uuid): - return requests.get(self.url('/task/{}/info').format(uuid)).json() + return requests.get(self.url('/task/{}/info').format(uuid), timeout=TIMEOUT).json() def task_output(self, uuid, line = 0): - return requests.get(self.url('/task/{}/output?line={}').format(uuid, line)).json() + return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=TIMEOUT).json() def task_cancel(self, uuid): - return requests.post(self.url('/task/cancel'), data={'uuid': uuid}).json() + return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=TIMEOUT).json() def task_remove(self, uuid): - return requests.post(self.url('/task/remove'), data={'uuid': uuid}).json() + return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=TIMEOUT).json() def task_restart(self, uuid): - return requests.post(self.url('/task/restart'), data={'uuid': uuid}).json() + return requests.post(self.url('/task/restart'), data={'uuid': uuid}, timeout=TIMEOUT).json() def task_download(self, uuid, asset): - res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True) + res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True, timeout=TIMEOUT) if "Content-Type" in res.headers and "application/json" in res.headers['Content-Type']: return res.json() else: @@ -61,4 +62,5 @@ class ApiClient: ) for image in images] return requests.post(self.url("/task/new"), files=files, - data={'name': name, 'options': json.dumps(options)}).json() + data={'name': name, 'options': json.dumps(options)}, + timeout=TIMEOUT).json() From 792eee94e5e112440317d2af817149972539aa05 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 1 Feb 2017 18:11:39 -0500 Subject: [PATCH 06/15] More tasks testing (currently failing) --- app/api/tasks.py | 6 + app/tests/classes.py | 78 +++++++----- app/tests/test_api.py | 145 ---------------------- app/tests/test_api_task.py | 193 +++++++++++++++++++++++++++++ nodeodm/external/node-OpenDroneMap | 2 +- nodeodm/models.py | 6 +- 6 files changed, 252 insertions(+), 178 deletions(-) create mode 100644 app/tests/test_api_task.py diff --git a/app/api/tasks.py b/app/api/tasks.py index 4fdd4d15..1649c342 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -126,6 +126,12 @@ class TaskViewSet(viewsets.ViewSet): task = models.Task.create_from_images(files, project) if task is not None: + + # Update other parameters such as processing node, task name, etc. + serializer = TaskSerializer(task, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response({"id": task.id}, status=status.HTTP_201_CREATED) else: raise exceptions.ValidationError(detail="Cannot create task, input provided is not valid.") diff --git a/app/tests/classes.py b/app/tests/classes.py index 96fc6061..fb5b139b 100644 --- a/app/tests/classes.py +++ b/app/tests/classes.py @@ -1,10 +1,41 @@ +from django import db from django.contrib.auth.models import User from django.test import TestCase +from django.test import TransactionTestCase from app.boot import boot from app.models import Project +def setupUsers(): + User.objects.create_superuser(username='testsuperuser', + email='superuser@test.com', + password='test1234') + User.objects.create_user(username='testuser', + email='user@test.com', + password='test1234') + User.objects.create_user(username='testuser2', + email='user2@test.com', + password='test1234') + + +def setupProjects(): + Project.objects.create( + owner=User.objects.get(username="testsuperuser"), + name="Super User Test Project", + description="This is a test project" + ) + Project.objects.create( + owner=User.objects.get(username="testuser"), + name="User Test Project", + description="This is a test project" + ) + Project.objects.create( + owner=User.objects.get(username="testuser2"), + name="User 2 Test Project", + description="This is a test project" + ) + class BootTestCase(TestCase): ''' This class provides optional default mock data as well as @@ -15,36 +46,8 @@ class BootTestCase(TestCase): for some models, which doesn't play well with them. ''' @classmethod - def setUpClass(cls): - def setupUsers(): - User.objects.create_superuser(username='testsuperuser', - email='superuser@test.com', - password='test1234') - User.objects.create_user(username='testuser', - email='user@test.com', - password='test1234') - User.objects.create_user(username='testuser2', - email='user2@test.com', - password='test1234') - - def setupProjects(): - Project.objects.create( - owner=User.objects.get(username="testsuperuser"), - name="Super User Test Project", - description="This is a test project" - ) - Project.objects.create( - owner=User.objects.get(username="testuser"), - name="User Test Project", - description="This is a test project" - ) - Project.objects.create( - owner=User.objects.get(username="testuser2"), - name="User 2 Test Project", - description="This is a test project" - ) - - super(BootTestCase, cls).setUpClass() + def setUpTestData(cls): + super(BootTestCase, cls).setUpTestData() boot() setupUsers() setupProjects() @@ -52,3 +55,18 @@ class BootTestCase(TestCase): @classmethod def tearDownClass(cls): super(BootTestCase, cls).tearDownClass() + +class BootTransactionTestCase(TransactionTestCase): + ''' + Same as above, but inherits from TransactionTestCase + ''' + @classmethod + def setUpClass(cls): + super(BootTransactionTestCase, cls).setUpClass() + boot() + setupUsers() + setupProjects() + + @classmethod + def tearDownClass(cls): + super(BootTransactionTestCase, cls).tearDownClass() diff --git a/app/tests/test_api.py b/app/tests/test_api.py index 7bb0bd62..6a7e8fdf 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -222,151 +222,6 @@ class TestApi(BootTestCase): # TODO test: # - scheduler processing steps - def test_task(self): - DELAY = 5 # time to sleep for during process launch, background processing, etc. - client = APIClient() - - user = User.objects.get(username="testuser") - other_user = User.objects.get(username="testuser2") - project = Project.objects.create( - owner=user, - name="test project" - ) - other_project = Project.objects.create( - owner=User.objects.get(username="testuser2"), - name="another test project" - ) - other_task = Task.objects.create(project=other_project) - - # task creation via file upload - image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb') - image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb') - - # Not authenticated? - res = client.post("/api/projects/{}/tasks/".format(project.id), { - 'images': [image1, image2] - }, format="multipart") - self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN); - - client.login(username="testuser", password="test1234") - - # Cannot create a task for a project that does not exist - res = client.post("/api/projects/0/tasks/", { - 'images': [image1, image2] - }, format="multipart") - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Cannot create a task for a project for which we have no access to - res = client.post("/api/projects/{}/tasks/".format(other_project.id), { - 'images': [image1, image2] - }, format="multipart") - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Cannot create a task without images - res = client.post("/api/projects/{}/tasks/".format(project.id), { - 'images': [] - }, format="multipart") - self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) - - # Cannot create a task with just 1 image - res = client.post("/api/projects/{}/tasks/".format(project.id), { - 'images': image1 - }, format="multipart") - self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) - - # Normal case with just images[] parameter - res = client.post("/api/projects/{}/tasks/".format(project.id), { - 'images': [image1, image2] - }, format="multipart") - self.assertTrue(res.status_code == status.HTTP_201_CREATED) - - # Should have returned the id of the newly created task - task = Task.objects.latest('created_at') - self.assertTrue('id' in res.data) - self.assertTrue(task.id == res.data['id']) - - # Two images should have been uploaded - self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2) - - # No processing node is set - self.assertTrue(task.processing_node is None) - - image1.close() - image2.close() - - # tiles.json should not be accessible at this point - res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id)) - self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) - - # Neither should an individual tile - # Z/X/Y coords are choosen based on node-odm test dataset for orthophoto_tiles/ - res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Cannot access a tiles.json we have no access to - res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(other_project.id, other_task.id)) - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Cannot access an individual tile we have no access to - res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(other_project.id, other_task.id)) - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Cannot download assets (they don't exist yet) - assets = ["all", "geotiff", "las", "csv", "ply"] - - for asset in assets: - res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset)) - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Cannot access raw assets (they don't exist yet) - res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id)) - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Start processing node - current_dir = os.path.dirname(os.path.realpath(__file__)) - node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False, - cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap")) - time.sleep(DELAY) # Wait for the server to launch - - # Create processing node - pnode = ProcessingNode.objects.create(hostname="localhost", port=11223) - - # Verify that it's working - self.assertTrue(pnode.api_version is not None) - - # Cannot assign processing node to a task we have no access to - res = client.patch("/api/projects/{}/tasks/{}/".format(other_project.id, other_task.id), { - 'processing_node': pnode.id - }) - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Assign processing node to task via API - res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), { - 'processing_node': pnode.id - }) - self.assertTrue(res.status_code == status.HTTP_200_OK) - - # After a processing node has been assigned, the task processing should start - #time.sleep(DELAY) - - # Processing should have completed - #task.refresh_from_db() - #self.assertTrue(task.status == status_codes.COMPLETED) - - # TODO: background tasks do not properly talk to the database - # Task table is always empty when read from a separate Thread. Why? - # from app import scheduler - # scheduler.process_pending_tasks(background=True) - - #time.sleep(3) - - # TODO: check - # TODO: what happens when nodes go offline, or an offline node is assigned to a task - # TODO: check raw/non-raw assets once task is finished processing - # TODO: recheck tiles, tiles.json urls, etc. - - # Teardown processing node - node_odm.terminate() def test_processingnodes(self): client = APIClient() diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py new file mode 100644 index 00000000..b4726acd --- /dev/null +++ b/app/tests/test_api_task.py @@ -0,0 +1,193 @@ +import os +import subprocess + +import time + +from django import db +from django.contrib.auth.models import User +from rest_framework import status +from rest_framework.test import APIClient + +from app import scheduler +from app.models import Project, Task, ImageUpload +from app.tests.classes import BootTransactionTestCase +from nodeodm import status_codes +from nodeodm.models import ProcessingNode + +# We need to test the task API in a TransactionTestCase because +# processing happens on a separate thread. This is required by Django. +class TestApi(BootTransactionTestCase): + def test_task(self): + DELAY = 1 # time to sleep for during process launch, background processing, etc. + client = APIClient() + + user = User.objects.get(username="testuser") + self.assertFalse(user.is_superuser) + + other_user = User.objects.get(username="testuser2") + + project = Project.objects.create( + owner=user, + name="test project" + ) + other_project = Project.objects.create( + owner=other_user, + name="another test project" + ) + other_task = Task.objects.create(project=other_project) + + # Start processing node + current_dir = os.path.dirname(os.path.realpath(__file__)) + node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False, + cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap")) + time.sleep(DELAY) # Wait for the server to launch + + # Create processing node + pnode = ProcessingNode.objects.create(hostname="localhost", port=11223) + + # Verify that it's working + self.assertTrue(pnode.api_version is not None) + + # task creation via file upload + image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb') + image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb') + + # Not authenticated? + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN); + + client.login(username="testuser", password="test1234") + + # Cannot create a task for a project that does not exist + res = client.post("/api/projects/0/tasks/", { + 'images': [image1, image2] + }, format="multipart") + print(res.status_code) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot create a task for a project for which we have no access to + res = client.post("/api/projects/{}/tasks/".format(other_project.id), { + 'images': [image1, image2] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot create a task without images + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) + + # Cannot create a task with just 1 image + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': image1 + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) + + # Normal case with images[], name and processing node parameter + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2], + 'name': 'test_task', + 'processing_node': pnode.id + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_201_CREATED) + multiple_param_task = Task.objects.latest('created_at') + self.assertTrue(multiple_param_task.name == 'test_task') + self.assertTrue(multiple_param_task.processing_node.id == pnode.id) + + # Cannot create a task with images[], name, but invalid processing node parameter + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2], + 'name': 'test_task', + 'processing_node': 9999 + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) + + # Normal case with just images[] parameter + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2] + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_201_CREATED) + + # Should have returned the id of the newly created task + task = Task.objects.latest('created_at') + self.assertTrue('id' in res.data) + self.assertTrue(task.id == res.data['id']) + + # Two images should have been uploaded + self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2) + + # No processing node is set + self.assertTrue(task.processing_node is None) + + image1.close() + image2.close() + + # tiles.json should not be accessible at this point + res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) + + # Neither should an individual tile + # Z/X/Y coords are choosen based on node-odm test dataset for orthophoto_tiles/ + res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot access a tiles.json we have no access to + res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(other_project.id, other_task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot access an individual tile we have no access to + res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(other_project.id, other_task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot download assets (they don't exist yet) + assets = ["all", "geotiff", "las", "csv", "ply"] + + for asset in assets: + res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot access raw assets (they don't exist yet) + res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Cannot assign processing node to a task we have no access to + res = client.patch("/api/projects/{}/tasks/{}/".format(other_project.id, other_task.id), { + 'processing_node': pnode.id + }) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + + # Assign processing node to task via API + res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), { + 'processing_node': pnode.id + }) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # On update scheduler.processing_pending_tasks should have been called in the background + time.sleep(DELAY) + + # Processing should have completed + task.refresh_from_db() + self.assertTrue(task.status == status_codes.RUNNING) + + + # TODO: need a way to prevent multithreaded code from executing + # and a way to notify our test case that multithreaded code should have + # executed + + # TODO: at this point we might not even need a TransactionTestCase? + + #from app import scheduler + #scheduler.process_pending_tasks(background=True) + + # time.sleep(3) + + # TODO: check + # TODO: what happens when nodes go offline, or an offline node is assigned to a task + # TODO: check raw/non-raw assets once task is finished processing + # TODO: recheck tiles, tiles.json urls, etc. + + # Teardown processing node + node_odm.terminate() + time.sleep(20) \ No newline at end of file diff --git a/nodeodm/external/node-OpenDroneMap b/nodeodm/external/node-OpenDroneMap index 254ce04f..a25e688b 160000 --- a/nodeodm/external/node-OpenDroneMap +++ b/nodeodm/external/node-OpenDroneMap @@ -1 +1 @@ -Subproject commit 254ce04f55db521acbae1b38294d2ec65e3c8a09 +Subproject commit a25e688bcb31972671f3735efe17c0b1c32e6da4 diff --git a/nodeodm/models.py b/nodeodm/models.py index ce1aee3c..0e859dcd 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -91,10 +91,12 @@ class ProcessingNode(models.Model): except requests.exceptions.ConnectionError as e: raise ProcessingException(e) - if result['uuid']: + if 'uuid' in result: return result['uuid'] - elif result['error']: + elif 'error' in result: raise ProcessingException(result['error']) + else: + raise ProcessingException("Unexpected answer from server: {}".format(result)) @api def get_task_info(self, uuid): From 26339e0c326b435b9866472814cb7d918c97d80e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 5 Feb 2017 22:23:02 -0500 Subject: [PATCH 07/15] Moved @background decorator in separate module, added proof of concept intercept class --- app/background.py | 55 ++++++++++++++++++++++++++++++++++++++ app/scheduler.py | 46 +++++++------------------------ app/tests/classes.py | 1 + app/tests/test_api_task.py | 12 ++++++--- nodeodm/api_client.py | 5 ++-- 5 files changed, 77 insertions(+), 42 deletions(-) create mode 100644 app/background.py diff --git a/app/background.py b/app/background.py new file mode 100644 index 00000000..0d28f424 --- /dev/null +++ b/app/background.py @@ -0,0 +1,55 @@ +from threading import Thread +from django import db +from webodm import settings + + +# TODO: design class such that: +# 1. test cases can choose which functions to intercept (prevent from executing) +# 2. test cases can see how many times a function has been called (and with which parameters) +# 3. test cases can pause until a function has been called +class TestWatch: + stats = {} + + def called(self, func, *args, **kwargs): + list = TestWatch.stats[func] if func in TestWatch.stats else [] + list.append({'f': func, 'args': args, 'kwargs': kwargs}) + print(list) + + def clear(self): + TestWatch.stats = {} + +testWatch = TestWatch() + +def background(func): + """ + Adds background={True|False} param to any function + so that we can call update_nodes_info(background=True) from the outside + """ + def wrapper(*args,**kwargs): + background = kwargs.get('background', False) + if 'background' in kwargs: del kwargs['background'] + + if background: + if settings.TESTING: + # During testing, intercept all background requests and execute them on the same thread + testWatch.called(func.__name__, *args, **kwargs) + + # Create a function that closes all + # db connections at the end of the thread + # This is necessary to make sure we don't leave + # open connections lying around. + def execute_and_close_db(): + ret = None + try: + ret = func(*args, **kwargs) + finally: + db.connections.close_all() + return ret + + t = Thread(target=execute_and_close_db) + t.daemon = True + t.start() + return t + else: + return func(*args, **kwargs) + return wrapper \ No newline at end of file diff --git a/app/scheduler.py b/app/scheduler.py index 12607acf..616a6731 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -1,15 +1,17 @@ import logging import traceback +from multiprocessing.dummy import Pool as ThreadPool +from threading import Lock -from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRunningError -from threading import Thread, Lock -from multiprocessing.dummy import Pool as ThreadPool -from nodeodm.models import ProcessingNode -from app.models import Task, Project -from django.db.models import Q, Count +from apscheduler.schedulers.background import BackgroundScheduler from django import db +from django.db.models import Q, Count + +from app.models import Task, Project from nodeodm import status_codes +from nodeodm.models import ProcessingNode +from app.background import background logger = logging.getLogger('app.logger') scheduler = BackgroundScheduler({ @@ -17,36 +19,6 @@ scheduler = BackgroundScheduler({ 'apscheduler.job_defaults.max_instances': '3', }) -def background(func): - """ - Adds background={True|False} param to any function - so that we can call update_nodes_info(background=True) from the outside - """ - def wrapper(*args,**kwargs): - background = kwargs.get('background', False) - if 'background' in kwargs: del kwargs['background'] - - if background: - # Create a function that closes all - # db connections at the end of the thread - # This is necessary to make sure we don't leave - # open connections lying around. - def execute_and_close_db(): - ret = None - try: - ret = func(*args, **kwargs) - finally: - db.connections.close_all() - return ret - - t = Thread(target=execute_and_close_db) - t.start() - return t - else: - return func(*args, **kwargs) - return wrapper - - @background def update_nodes_info(): processing_nodes = ProcessingNode.objects.all() @@ -86,6 +58,8 @@ def process_pending_tasks(): task.save() except Exception as e: logger.error("Uncaught error: {} {}".format(e, traceback.format_exc())) + finally: + db.connections.close_all() if tasks.count() > 0: pool = ThreadPool(tasks.count()) diff --git a/app/tests/classes.py b/app/tests/classes.py index fb5b139b..32435cf9 100644 --- a/app/tests/classes.py +++ b/app/tests/classes.py @@ -56,6 +56,7 @@ class BootTestCase(TestCase): def tearDownClass(cls): super(BootTestCase, cls).tearDownClass() + class BootTransactionTestCase(TransactionTestCase): ''' Same as above, but inherits from TransactionTestCase diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index b4726acd..c5b9f59d 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -15,8 +15,10 @@ from nodeodm import status_codes from nodeodm.models import ProcessingNode # We need to test the task API in a TransactionTestCase because -# processing happens on a separate thread. This is required by Django. -class TestApi(BootTransactionTestCase): +# task processing happens on a separate thread, and normal TestCases +# do not commit changes to the DB, so spawning a new thread will show no +# data in it. +class TestApiTask(BootTransactionTestCase): def test_task(self): DELAY = 1 # time to sleep for during process launch, background processing, etc. client = APIClient() @@ -167,6 +169,10 @@ class TestApi(BootTransactionTestCase): # On update scheduler.processing_pending_tasks should have been called in the background time.sleep(DELAY) + print("HERE") + from app.background import testWatch + print(testWatch.stats) + # Processing should have completed task.refresh_from_db() self.assertTrue(task.status == status_codes.RUNNING) @@ -190,4 +196,4 @@ class TestApi(BootTransactionTestCase): # Teardown processing node node_odm.terminate() - time.sleep(20) \ No newline at end of file + #time.sleep(20) diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py index 13a060a2..046e9e01 100644 --- a/nodeodm/api_client.py +++ b/nodeodm/api_client.py @@ -43,7 +43,7 @@ class ApiClient: return requests.post(self.url('/task/restart'), data={'uuid': uuid}, timeout=TIMEOUT).json() def task_download(self, uuid, asset): - res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True, timeout=TIMEOUT) + res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True) if "Content-Type" in res.headers and "application/json" in res.headers['Content-Type']: return res.json() else: @@ -62,5 +62,4 @@ class ApiClient: ) for image in images] return requests.post(self.url("/task/new"), files=files, - data={'name': name, 'options': json.dumps(options)}, - timeout=TIMEOUT).json() + data={'name': name, 'options': json.dumps(options)}).json() From 209a1b5603972bf54b0c86a7c4cc9f37a95aea9c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 7 Feb 2017 11:43:17 -0500 Subject: [PATCH 08/15] TestWatch class to monitor and intercept on demand multithreaded calls, changed location of assets during testing --- app/.gitignore | 1 + app/background.py | 26 +++---------- app/tests/test_api_task.py | 26 +++---------- app/tests/test_testwatch.py | 40 +++++++++++++++++++ app/testwatch.py | 77 +++++++++++++++++++++++++++++++++++++ webodm/settings.py | 2 + 6 files changed, 131 insertions(+), 41 deletions(-) create mode 100644 app/.gitignore create mode 100644 app/tests/test_testwatch.py create mode 100644 app/testwatch.py diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 00000000..b2e60fb7 --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +media_test/ diff --git a/app/background.py b/app/background.py index 0d28f424..57b31cb8 100644 --- a/app/background.py +++ b/app/background.py @@ -1,24 +1,11 @@ from threading import Thread + +import logging from django import db from webodm import settings +from app.testwatch import testWatch - -# TODO: design class such that: -# 1. test cases can choose which functions to intercept (prevent from executing) -# 2. test cases can see how many times a function has been called (and with which parameters) -# 3. test cases can pause until a function has been called -class TestWatch: - stats = {} - - def called(self, func, *args, **kwargs): - list = TestWatch.stats[func] if func in TestWatch.stats else [] - list.append({'f': func, 'args': args, 'kwargs': kwargs}) - print(list) - - def clear(self): - TestWatch.stats = {} - -testWatch = TestWatch() +logger = logging.getLogger('app.logger') def background(func): """ @@ -30,9 +17,7 @@ def background(func): if 'background' in kwargs: del kwargs['background'] if background: - if settings.TESTING: - # During testing, intercept all background requests and execute them on the same thread - testWatch.called(func.__name__, *args, **kwargs) + if testWatch.hook_pre(func, *args, **kwargs): return # Create a function that closes all # db connections at the end of the thread @@ -44,6 +29,7 @@ def background(func): ret = func(*args, **kwargs) finally: db.connections.close_all() + testWatch.hook_post(func, *args, **kwargs) return ret t = Thread(target=execute_and_close_db) diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index c5b9f59d..355677c3 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -1,18 +1,16 @@ import os import subprocess - import time -from django import db from django.contrib.auth.models import User from rest_framework import status from rest_framework.test import APIClient -from app import scheduler from app.models import Project, Task, ImageUpload from app.tests.classes import BootTransactionTestCase from nodeodm import status_codes from nodeodm.models import ProcessingNode +from app.testwatch import testWatch # We need to test the task API in a TransactionTestCase because # task processing happens on a separate thread, and normal TestCases @@ -133,6 +131,7 @@ class TestApiTask(BootTransactionTestCase): # Neither should an individual tile # Z/X/Y coords are choosen based on node-odm test dataset for orthophoto_tiles/ res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) + print(res.status_code) self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) # Cannot access a tiles.json we have no access to @@ -160,6 +159,8 @@ class TestApiTask(BootTransactionTestCase): }) self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) + testWatch.clear() + # Assign processing node to task via API res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), { 'processing_node': pnode.id @@ -167,28 +168,12 @@ class TestApiTask(BootTransactionTestCase): self.assertTrue(res.status_code == status.HTTP_200_OK) # On update scheduler.processing_pending_tasks should have been called in the background - time.sleep(DELAY) - - print("HERE") - from app.background import testWatch - print(testWatch.stats) + testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5) # Processing should have completed task.refresh_from_db() self.assertTrue(task.status == status_codes.RUNNING) - - # TODO: need a way to prevent multithreaded code from executing - # and a way to notify our test case that multithreaded code should have - # executed - - # TODO: at this point we might not even need a TransactionTestCase? - - #from app import scheduler - #scheduler.process_pending_tasks(background=True) - - # time.sleep(3) - # TODO: check # TODO: what happens when nodes go offline, or an offline node is assigned to a task # TODO: check raw/non-raw assets once task is finished processing @@ -196,4 +181,3 @@ class TestApiTask(BootTransactionTestCase): # Teardown processing node node_odm.terminate() - #time.sleep(20) diff --git a/app/tests/test_testwatch.py b/app/tests/test_testwatch.py new file mode 100644 index 00000000..6e9558c3 --- /dev/null +++ b/app/tests/test_testwatch.py @@ -0,0 +1,40 @@ +from django.test import TestCase + +from app.testwatch import TestWatch + + +def test(a, b): + return a + b + +class TestTestWatch(TestCase): + def test_methods(self): + tw = TestWatch() + + self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.test") == 0) + self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.nonexistant") == 0) + + # Test watch count + tw.hook_pre(test, 1, 2) + test(1, 2) + tw.hook_post(test, 1, 2) + + self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.test") == 1) + + tw.hook_pre(test, 1, 2) + test(1, 2) + tw.hook_post(test, 1, 2) + + self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.test") == 2) + + @TestWatch.watch(testWatch=tw) + def test2(d): + d['flag'] = not d['flag'] + + # Test intercept + tw.intercept("app.tests.test_testwatch.test2") + d = {'flag': True} + test2(d) + self.assertTrue(d['flag']) + + + diff --git a/app/testwatch.py b/app/testwatch.py new file mode 100644 index 00000000..f56c4c58 --- /dev/null +++ b/app/testwatch.py @@ -0,0 +1,77 @@ +import time + +import logging + +from webodm import settings + +logger = logging.getLogger('app.logger') + +class TestWatch: + def __init__(self): + self.clear() + + def clear(self): + self._calls = {} + self._intercept_list = {} + + def func_to_name(f): + return "{}.{}".format(f.__module__, f.__name__) + + def intercept(self, fname): + self._intercept_list[fname] = True + + def should_prevent_execution(self, func): + return TestWatch.func_to_name(func) in self._intercept_list + + def get_calls(self, fname): + return self._calls[fname] if fname in self._calls else [] + + def get_calls_count(self, fname): + return len(self.get_calls(fname)) + + def wait_until_call(self, fname, count = 1, timeout = 30): + SLEEP_INTERVAL = 0.125 + TIMEOUT_LIMIT = timeout / SLEEP_INTERVAL + c = 0 + while self.get_calls_count(fname) < count and c < TIMEOUT_LIMIT: + time.sleep(SLEEP_INTERVAL) + c += 1 + + if c >= TIMEOUT_LIMIT: + raise TimeoutError("wait_until_call has timed out waiting for {}".format(fname)) + + return self.get_calls(fname) + + def log_call(self, func, *args, **kwargs): + fname = TestWatch.func_to_name(func) + logger.info("{} called".format(fname)) + list = self._calls[fname] if fname in self._calls else [] + list.append({'f': fname, 'args': args, 'kwargs': kwargs}) + self._calls[fname] = list + + def hook_pre(self, func, *args, **kwargs): + if settings.TESTING and self.should_prevent_execution(func): + logger.info(func.__name__ + " intercepted") + self.log_call(func, *args, **kwargs) + return True # Intercept + return False # Do not intercept + + def hook_post(self, func, *args, **kwargs): + if settings.TESTING: + self.log_call(func, *args, **kwargs) + + def watch(**kwargs): + """ + Decorator that adds pre/post hook calls + """ + tw = kwargs.get('testWatch', testWatch) + def outer(func): + def wrapper(*args, **kwargs): + if tw.hook_pre(func, *args, **kwargs): return + ret = func(*args, **kwargs) + tw.hook_post(func, *args, **kwargs) + return ret + return wrapper + return outer + +testWatch = TestWatch() \ No newline at end of file diff --git a/webodm/settings.py b/webodm/settings.py index 304f57be..a91cb304 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -224,6 +224,8 @@ REST_FRAMEWORK = { } TESTING = sys.argv[1:2] == ['test'] +if TESTING: + MEDIA_ROOT = os.path.join(BASE_DIR, 'app', 'media_test') try: from .local_settings import * From 6d42de9cfd8feeead644cb85e2a4cf6650a80f4c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 7 Feb 2017 14:28:50 -0500 Subject: [PATCH 09/15] Happy path testing for task creation/update/deletion --- app/models.py | 3 +- app/tests/test_api_task.py | 89 +++++++++++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/app/models.py b/app/models.py index 95ab5bf9..9908f8b0 100644 --- a/app/models.py +++ b/app/models.py @@ -217,6 +217,7 @@ class Task(models.Model): if self.processing_node and self.uuid: self.processing_node.cancel_task(self.uuid) self.pending_action = None + self.status = None self.save() else: raise ProcessingException("Cannot cancel a task that has no processing node or UUID") @@ -373,7 +374,7 @@ class Task(models.Model): try: shutil.rmtree(directory_to_delete) except FileNotFoundError as e: - logger.warn(e) + logger.warning(e) def set_failure(self, error_message): diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 355677c3..56440ed7 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -2,11 +2,15 @@ import os import subprocess import time +import shutil + +import logging from django.contrib.auth.models import User from rest_framework import status from rest_framework.test import APIClient -from app.models import Project, Task, ImageUpload +from app import scheduler +from app.models import Project, Task, ImageUpload, task_directory_path from app.tests.classes import BootTransactionTestCase from nodeodm import status_codes from nodeodm.models import ProcessingNode @@ -16,7 +20,20 @@ from app.testwatch import testWatch # task processing happens on a separate thread, and normal TestCases # do not commit changes to the DB, so spawning a new thread will show no # data in it. +from webodm import settings +logger = logging.getLogger('app.logger') + class TestApiTask(BootTransactionTestCase): + def setUp(self): + # We need to clear previous media_root content + # This points to the test directory, but just in case + # we double check that the directory is indeed a test directory + if "_test" in settings.MEDIA_ROOT: + logger.info("Cleaning up {}".format(settings.MEDIA_ROOT)) + shutil.rmtree(settings.MEDIA_ROOT) + else: + logger.warning("We did not remove MEDIA_ROOT because we couldn't find a _test suffix in its path.") + def test_task(self): DELAY = 1 # time to sleep for during process launch, background processing, etc. client = APIClient() @@ -64,7 +81,6 @@ class TestApiTask(BootTransactionTestCase): res = client.post("/api/projects/0/tasks/", { 'images': [image1, image2] }, format="multipart") - print(res.status_code) self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) # Cannot create a task for a project for which we have no access to @@ -131,7 +147,6 @@ class TestApiTask(BootTransactionTestCase): # Neither should an individual tile # Z/X/Y coords are choosen based on node-odm test dataset for orthophoto_tiles/ res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) - print(res.status_code) self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) # Cannot access a tiles.json we have no access to @@ -161,6 +176,9 @@ class TestApiTask(BootTransactionTestCase): testWatch.clear() + # No UUID at this point + self.assertTrue(len(task.uuid) == 0) + # Assign processing node to task via API res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), { 'processing_node': pnode.id @@ -170,14 +188,71 @@ class TestApiTask(BootTransactionTestCase): # On update scheduler.processing_pending_tasks should have been called in the background testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5) - # Processing should have completed + # Processing should have started and a UUID is assigned task.refresh_from_db() self.assertTrue(task.status == status_codes.RUNNING) + self.assertTrue(len(task.uuid) > 0) + + # Calling process pending tasks should finish the process + scheduler.process_pending_tasks() + task.refresh_from_db() + self.assertTrue(task.status == status_codes.COMPLETED) + + # Can download assets + for asset in assets: + res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # Can download raw assets + res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # Can access tiles.json and individual tiles + res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # Restart a task + testWatch.clear() + res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5) + task.refresh_from_db() + + self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED]) + + # Cancel a task + testWatch.clear() + res = client.post("/api/projects/{}/tasks/{}/cancel/".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5) + + # Should have been canceled + task.refresh_from_db() + self.assertTrue(task.status == status_codes.CANCELED) + + # Remove a task + res = client.post("/api/projects/{}/tasks/{}/remove/".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + testWatch.wait_until_call("app.scheduler.process_pending_tasks", 2, timeout=5) + + # Has been removed along with assets + self.assertFalse(Task.objects.filter(pk=task.id).exists()) + self.assertFalse(ImageUpload.objects.filter(task=task).exists()) + + task_assets_path = os.path.join(settings.MEDIA_ROOT, + task_directory_path(task.id, task.project.id)) + self.assertFalse(os.path.exists(task_assets_path)) + + testWatch.clear() + testWatch.intercept("app.scheduler.process_pending_tasks") + + - # TODO: check # TODO: what happens when nodes go offline, or an offline node is assigned to a task - # TODO: check raw/non-raw assets once task is finished processing - # TODO: recheck tiles, tiles.json urls, etc. + # TODO: timeout issues # Teardown processing node node_odm.terminate() From 84c0d449beea1e1acb5b17ea78f3d1c84cc2be93 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 7 Feb 2017 14:42:17 -0500 Subject: [PATCH 10/15] Fixed issue with removing non-existent test media directory --- app/tests/test_api_task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 56440ed7..43f1840c 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -29,8 +29,9 @@ class TestApiTask(BootTransactionTestCase): # This points to the test directory, but just in case # we double check that the directory is indeed a test directory if "_test" in settings.MEDIA_ROOT: - logger.info("Cleaning up {}".format(settings.MEDIA_ROOT)) - shutil.rmtree(settings.MEDIA_ROOT) + if os.path.exists(settings.MEDIA_ROOT): + logger.info("Cleaning up {}".format(settings.MEDIA_ROOT)) + shutil.rmtree(settings.MEDIA_ROOT) else: logger.warning("We did not remove MEDIA_ROOT because we couldn't find a _test suffix in its path.") @@ -242,8 +243,7 @@ class TestApiTask(BootTransactionTestCase): self.assertFalse(Task.objects.filter(pk=task.id).exists()) self.assertFalse(ImageUpload.objects.filter(task=task).exists()) - task_assets_path = os.path.join(settings.MEDIA_ROOT, - task_directory_path(task.id, task.project.id)) + task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id)) self.assertFalse(os.path.exists(task_assets_path)) testWatch.clear() From c5b78675a4d7eb6ae3365f0deb39bebb9f99e503 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 7 Feb 2017 17:09:30 -0500 Subject: [PATCH 11/15] More task processing unit testing --- app/models.py | 23 ++++++------ app/tests/test_api_task.py | 76 ++++++++++++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/app/models.py b/app/models.py index 9908f8b0..b1406042 100644 --- a/app/models.py +++ b/app/models.py @@ -189,8 +189,8 @@ class Task(models.Model): try: if self.processing_node: - # Need to process some images (UUID not yet set and task not marked for deletion)? - if not self.uuid and self.pending_action != pending_actions.REMOVE: + # Need to process some images (UUID not yet set and task doesn't have pending actions)? + if not self.uuid and self.pending_action is None: logger.info("Processing... {}".format(self)) images = [image.path() for image in self.imageupload_set.all()] @@ -223,24 +223,25 @@ class Task(models.Model): raise ProcessingException("Cannot cancel a task that has no processing node or UUID") elif self.pending_action == pending_actions.RESTART: - logger.info("Restarting task {}".format(self)) - if self.processing_node and self.uuid: + logger.info("Restarting {}".format(self)) + if self.processing_node: # Check if the UUID is still valid, as processing nodes purge # results after a set amount of time, the UUID might have eliminated. - try: - info = self.processing_node.get_task_info(self.uuid) - uuid_still_exists = info['uuid'] == self.uuid - except ProcessingException: - uuid_still_exists = False + uuid_still_exists = False + + if self.uuid: + try: + info = self.processing_node.get_task_info(self.uuid) + uuid_still_exists = info['uuid'] == self.uuid + except ProcessingException: + pass if uuid_still_exists: # Good to go self.processing_node.restart_task(self.uuid) else: # Task has been purged (or processing node is offline) - # TODO: what if processing node went offline? - # Process this as a new task # Removing its UUID will cause the scheduler # to process this the next tick diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 43f1840c..b691b951 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -9,6 +9,7 @@ from django.contrib.auth.models import User from rest_framework import status from rest_framework.test import APIClient +from app import pending_actions from app import scheduler from app.models import Project, Task, ImageUpload, task_directory_path from app.tests.classes import BootTransactionTestCase @@ -23,6 +24,15 @@ from app.testwatch import testWatch from webodm import settings logger = logging.getLogger('app.logger') +DELAY = 1 # time to sleep for during process launch, background processing, etc. + +def start_processing_node(): + current_dir = os.path.dirname(os.path.realpath(__file__)) + node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False, + cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap")) + time.sleep(DELAY) # Wait for the server to launch + return node_odm + class TestApiTask(BootTransactionTestCase): def setUp(self): # We need to clear previous media_root content @@ -36,9 +46,10 @@ class TestApiTask(BootTransactionTestCase): logger.warning("We did not remove MEDIA_ROOT because we couldn't find a _test suffix in its path.") def test_task(self): - DELAY = 1 # time to sleep for during process launch, background processing, etc. client = APIClient() + node_odm = start_processing_node() + user = User.objects.get(username="testuser") self.assertFalse(user.is_superuser) @@ -55,10 +66,6 @@ class TestApiTask(BootTransactionTestCase): other_task = Task.objects.create(project=other_project) # Start processing node - current_dir = os.path.dirname(os.path.realpath(__file__)) - node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False, - cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap")) - time.sleep(DELAY) # Wait for the server to launch # Create processing node pnode = ProcessingNode.objects.create(hostname="localhost", port=11223) @@ -138,9 +145,6 @@ class TestApiTask(BootTransactionTestCase): # No processing node is set self.assertTrue(task.processing_node is None) - image1.close() - image2.close() - # tiles.json should not be accessible at this point res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id)) self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST) @@ -194,6 +198,8 @@ class TestApiTask(BootTransactionTestCase): self.assertTrue(task.status == status_codes.RUNNING) self.assertTrue(len(task.uuid) > 0) + time.sleep(DELAY) + # Calling process pending tasks should finish the process scheduler.process_pending_tasks() task.refresh_from_db() @@ -249,10 +255,58 @@ class TestApiTask(BootTransactionTestCase): testWatch.clear() testWatch.intercept("app.scheduler.process_pending_tasks") + # Create a task, then kill the processing node + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2], + 'name': 'test_task_offline', + 'processing_node': pnode.id + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_201_CREATED) + task = Task.objects.get(pk=res.data['id']) + + # Stop processing node + node_odm.terminate() + + task.refresh_from_db() + self.assertTrue(task.last_error is None) + scheduler.process_pending_tasks() + + # Processing should fail and set an error + task.refresh_from_db() + self.assertTrue(task.last_error is not None) + self.assertTrue(task.status == status_codes.FAILED) + + # Now bring it back online + node_odm = start_processing_node() + + # Restart + res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + task.refresh_from_db() + self.assertTrue(task.pending_action == pending_actions.RESTART) + + # After processing, the task should have restarted, and have no UUID or status + scheduler.process_pending_tasks() + task.refresh_from_db() + self.assertTrue(task.status is None) + self.assertTrue(len(task.uuid) == 0) + + # Another step and it should have acquired a UUID + scheduler.process_pending_tasks() + task.refresh_from_db() + self.assertTrue(task.status is status_codes.RUNNING) + self.assertTrue(len(task.uuid) > 0) + + # Another step and it should be completed + time.sleep(DELAY) + scheduler.process_pending_tasks() + task.refresh_from_db() + self.assertTrue(task.status == status_codes.COMPLETED) - # TODO: what happens when nodes go offline, or an offline node is assigned to a task # TODO: timeout issues - # Teardown processing node - node_odm.terminate() + image1.close() + image2.close() + + From a4554f2c5fc2647df4c53d6d12429814e9ca1e50 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 8 Feb 2017 09:53:01 -0500 Subject: [PATCH 12/15] Fixed textured model orientation alignment issue --- app/static/app/js/ModelView.jsx | 5 +++-- app/static/app/js/css/ModelView.scss | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/static/app/js/ModelView.jsx b/app/static/app/js/ModelView.jsx index 027b6ef3..133ad6e9 100644 --- a/app/static/app/js/ModelView.jsx +++ b/app/static/app/js/ModelView.jsx @@ -46,10 +46,11 @@ class ModelView extends React.Component { } objFilePath(){ - return this.texturedModelDirectoryPath() + 'odm_textured_model.obj'; + return this.texturedModelDirectoryPath() + 'odm_textured_model_geo.obj'; } mtlFilename(){ + // For some reason, loading odm_textured_model_geo.mtl does not load textures properly return 'odm_textured_model.mtl'; } @@ -508,7 +509,7 @@ class ModelView extends React.Component { // ODM models are Y-up object.rotateX(THREE.Math.degToRad(-90)); - // Bring the model close to center + // // Bring the model close to center if (object.children.length > 0){ const geom = object.children[0].geometry; diff --git a/app/static/app/js/css/ModelView.scss b/app/static/app/js/css/ModelView.scss index 726c6fb9..8e11dab6 100644 --- a/app/static/app/js/css/ModelView.scss +++ b/app/static/app/js/css/ModelView.scss @@ -4,6 +4,11 @@ .model-view{ height: 80%; + canvas{ + width: 100% !important; + height: 100% !important; + } + .container{ background: rgb(79,79,79); background: -moz-radial-gradient(center, ellipse cover, rgba(79,79,79,1) 0%, rgba(22,22,22,1) 100%); From 7d5ab4f4658b051023de7170c53ba83bfa9ef557 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 8 Feb 2017 11:27:23 -0500 Subject: [PATCH 13/15] Timeout simulation, watchtest function intercept, fixed #44 --- app/models.py | 16 +++++++++------- app/scheduler.py | 10 ++++++---- app/tests/test_api_task.py | 16 +++++++++++++++- app/tests/test_testwatch.py | 17 +++++++++++++++++ app/testwatch.py | 12 +++++++++--- nodeodm/api_client.py | 2 ++ nodeodm/models.py | 4 ++-- 7 files changed, 60 insertions(+), 17 deletions(-) diff --git a/app/models.py b/app/models.py index b1406042..b1cdb4a9 100644 --- a/app/models.py +++ b/app/models.py @@ -2,6 +2,7 @@ import logging import os import shutil import zipfile +import requests from django.contrib.auth.models import User from django.contrib.gis.gdal import GDALRaster @@ -146,7 +147,9 @@ class Task(models.Model): pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.") def __str__(self): - return 'Task ID: {}'.format(self.id) + name = self.name if self.name is not None else "unnamed" + + return 'Task {} ({})'.format(name, self.id) def save(self, *args, **kwargs): # Autovalidate on save @@ -343,11 +346,10 @@ class Task(models.Model): self.save() except ProcessingException as e: self.set_failure(str(e)) - except ConnectionRefusedError as e: - logger.warning("Task {} cannot communicate with processing node: {}".format(self, str(e))) - - # In the future we might want to retry instead of just failing - #self.set_failure(str(e)) + except (ConnectionRefusedError, ConnectionError) as e: + logger.warning("{} cannot communicate with processing node: {}".format(self, str(e))) + except requests.exceptions.ConnectTimeout as e: + logger.warning("{} timed out with error: {}. We'll try reprocessing at the next tick.".format(self, str(e))) def get_tile_path(self, z, x, y): @@ -379,7 +381,7 @@ class Task(models.Model): def set_failure(self, error_message): - logger.error("{} ERROR: {}".format(self, error_message)) + logger.error("FAILURE FOR {}: {}".format(self, error_message)) self.last_error = error_message self.status = status_codes.FAILED self.save() diff --git a/app/scheduler.py b/app/scheduler.py index 616a6731..ebf1f962 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -7,6 +7,7 @@ from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRun from apscheduler.schedulers.background import BackgroundScheduler from django import db from django.db.models import Q, Count +from webodm import settings from app.models import Task, Project from nodeodm import status_codes @@ -51,14 +52,15 @@ def process_pending_tasks(): def process(task): try: task.process() - + except Exception as e: + logger.error("Uncaught error! This is potentially bad. Please report it to http://github.com/OpenDroneMap/WebODM/issues: {} {}".format(e, traceback.format_exc())) + if settings.TESTING: raise e + finally: # Might have been deleted if task.pk is not None: task.processing_lock = False task.save() - except Exception as e: - logger.error("Uncaught error: {} {}".format(e, traceback.format_exc())) - finally: + db.connections.close_all() if tasks.count() > 0: diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index b691b951..d6495cc2 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -5,6 +5,8 @@ import time import shutil import logging + +import requests from django.contrib.auth.models import User from rest_framework import status from rest_framework.test import APIClient @@ -304,9 +306,21 @@ class TestApiTask(BootTransactionTestCase): self.assertTrue(task.status == status_codes.COMPLETED) - # TODO: timeout issues + # Test connection, timeout errors + res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id)) + def connTimeout(*args, **kwargs): + raise requests.exceptions.ConnectTimeout("Simulated timeout") + + testWatch.intercept("nodeodm.api_client.task_output", connTimeout) + scheduler.process_pending_tasks() + + # Timeout errors should be handled by retrying again at a later time + # and not fail + task.refresh_from_db() + self.assertTrue(task.last_error is None) image1.close() image2.close() + node_odm.terminate() diff --git a/app/tests/test_testwatch.py b/app/tests/test_testwatch.py index 6e9558c3..7986cdfe 100644 --- a/app/tests/test_testwatch.py +++ b/app/tests/test_testwatch.py @@ -36,5 +36,22 @@ class TestTestWatch(TestCase): test2(d) self.assertTrue(d['flag']) + # Test function replacement intercept + d = { + 'a': False, + 'b': False + } + @TestWatch.watch(testWatch=tw) + def test3(d): + d['a'] = True + + def replacement(d): + d['b'] = True + + tw.intercept("app.tests.test_testwatch.test3", replacement) + test3(d) + self.assertFalse(d['a']) + self.assertTrue(d['b']) + diff --git a/app/testwatch.py b/app/testwatch.py index f56c4c58..4631eef1 100644 --- a/app/testwatch.py +++ b/app/testwatch.py @@ -17,8 +17,12 @@ class TestWatch: def func_to_name(f): return "{}.{}".format(f.__module__, f.__name__) - def intercept(self, fname): - self._intercept_list[fname] = True + def intercept(self, fname, f = None): + self._intercept_list[fname] = f if f is not None else True + + def execute_intercept_function_replacement(self, fname, *args, **kwargs): + if fname in self._intercept_list and callable(self._intercept_list[fname]): + (self._intercept_list[fname])(*args, **kwargs) def should_prevent_execution(self, func): return TestWatch.func_to_name(func) in self._intercept_list @@ -51,7 +55,9 @@ class TestWatch: def hook_pre(self, func, *args, **kwargs): if settings.TESTING and self.should_prevent_execution(func): - logger.info(func.__name__ + " intercepted") + fname = TestWatch.func_to_name(func) + logger.info(fname + " intercepted") + self.execute_intercept_function_replacement(fname, *args, **kwargs) self.log_call(func, *args, **kwargs) return True # Intercept return False # Do not intercept diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py index 046e9e01..4e224e60 100644 --- a/nodeodm/api_client.py +++ b/nodeodm/api_client.py @@ -7,6 +7,7 @@ import mimetypes import json import os from urllib.parse import urlunparse +from app.testwatch import TestWatch TIMEOUT = 10 @@ -30,6 +31,7 @@ class ApiClient: def task_info(self, uuid): return requests.get(self.url('/task/{}/info').format(uuid), timeout=TIMEOUT).json() + @TestWatch.watch() def task_output(self, uuid, line = 0): return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=TIMEOUT).json() diff --git a/nodeodm/models.py b/nodeodm/models.py index 0e859dcd..174e1c70 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -91,9 +91,9 @@ class ProcessingNode(models.Model): except requests.exceptions.ConnectionError as e: raise ProcessingException(e) - if 'uuid' in result: + if isinstance(result, dict) and 'uuid' in result: return result['uuid'] - elif 'error' in result: + elif isinstance(result, dict) and 'error' in result: raise ProcessingException(result['error']) else: raise ProcessingException("Unexpected answer from server: {}".format(result)) From f50868371e9418fdcf8dff7950eb4804b65ce16f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 8 Feb 2017 11:38:16 -0500 Subject: [PATCH 14/15] Cleanup --- app/static/app/js/ModelView.jsx | 2 +- app/tests/test_api.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/static/app/js/ModelView.jsx b/app/static/app/js/ModelView.jsx index 133ad6e9..8f2736e4 100644 --- a/app/static/app/js/ModelView.jsx +++ b/app/static/app/js/ModelView.jsx @@ -509,7 +509,7 @@ class ModelView extends React.Component { // ODM models are Y-up object.rotateX(THREE.Math.degToRad(-90)); - // // Bring the model close to center + // Bring the model close to center if (object.children.length > 0){ const geom = object.children[0].geometry; diff --git a/app/tests/test_api.py b/app/tests/test_api.py index 6a7e8fdf..5dd83120 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -219,9 +219,6 @@ class TestApi(BootTestCase): res = client.delete('/api/projects/{}/'.format(other_temp_project.id)) self.assertTrue(res.status_code == status.HTTP_204_NO_CONTENT) - # TODO test: - # - scheduler processing steps - def test_processingnodes(self): client = APIClient() From eae691cf73b4dd7c2babf5e3ad499487b6c6e557 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 8 Feb 2017 11:50:31 -0500 Subject: [PATCH 15/15] Increased api timeout to 30 seconds --- nodeodm/api_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py index 4e224e60..afecb0f6 100644 --- a/nodeodm/api_client.py +++ b/nodeodm/api_client.py @@ -9,7 +9,7 @@ import os from urllib.parse import urlunparse from app.testwatch import TestWatch -TIMEOUT = 10 +TIMEOUT = 30 class ApiClient: def __init__(self, host, port):