From 253b1795e50b2bc46118dc56074d348a912dfae4 Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Thu, 23 Feb 2023 23:41:27 -0800 Subject: [PATCH 01/28] Deployed 091041b with MkDocs version: 1.4.2 --- .nojekyll | 0 404.html | 492 ++ _images/model_bundle.png | Bin 0 -> 41350 bytes _images/model_endpoint.png | Bin 0 -> 65535 bytes _images/request_lifecycle.png | Bin 0 -> 111799 bytes _static/favicon-32x32.png | Bin 0 -> 1717 bytes _static/favicon.ico | Bin 0 -> 1150 bytes _static/file.png | Bin 0 -> 286 bytes _static/launch-logo.svg | 19 + _static/minus.png | Bin 0 -> 90 bytes _static/plus.png | Bin 0 -> 90 bytes api/client/index.html | 2659 +++++++ api/model_bundles/index.html | 924 +++ api/model_endpoints/index.html | 830 ++ assets/_mkdocstrings.css | 36 + assets/images/favicon.png | Bin 0 -> 1870 bytes assets/javascripts/bundle.2a6f1dda.min.js | 29 + assets/javascripts/bundle.2a6f1dda.min.js.map | 8 + assets/javascripts/lunr/min/lunr.ar.min.js | 1 + assets/javascripts/lunr/min/lunr.da.min.js | 18 + assets/javascripts/lunr/min/lunr.de.min.js | 18 + assets/javascripts/lunr/min/lunr.du.min.js | 18 + assets/javascripts/lunr/min/lunr.es.min.js | 18 + assets/javascripts/lunr/min/lunr.fi.min.js | 18 + assets/javascripts/lunr/min/lunr.fr.min.js | 18 + assets/javascripts/lunr/min/lunr.hi.min.js | 1 + assets/javascripts/lunr/min/lunr.hu.min.js | 18 + assets/javascripts/lunr/min/lunr.it.min.js | 18 + assets/javascripts/lunr/min/lunr.ja.min.js | 1 + assets/javascripts/lunr/min/lunr.jp.min.js | 1 + assets/javascripts/lunr/min/lunr.ko.min.js | 1 + assets/javascripts/lunr/min/lunr.multi.min.js | 1 + assets/javascripts/lunr/min/lunr.nl.min.js | 18 + assets/javascripts/lunr/min/lunr.no.min.js | 18 + assets/javascripts/lunr/min/lunr.pt.min.js | 18 + assets/javascripts/lunr/min/lunr.ro.min.js | 18 + assets/javascripts/lunr/min/lunr.ru.min.js | 18 + .../lunr/min/lunr.stemmer.support.min.js | 1 + assets/javascripts/lunr/min/lunr.sv.min.js | 18 + assets/javascripts/lunr/min/lunr.ta.min.js | 1 + assets/javascripts/lunr/min/lunr.th.min.js | 1 + assets/javascripts/lunr/min/lunr.tr.min.js | 18 + assets/javascripts/lunr/min/lunr.vi.min.js | 1 + assets/javascripts/lunr/min/lunr.zh.min.js | 1 + assets/javascripts/lunr/tinyseg.js | 206 + assets/javascripts/lunr/wordcut.js | 6708 +++++++++++++++++ .../workers/search.208ed371.min.js | 42 + .../workers/search.208ed371.min.js.map | 8 + assets/stylesheets/main.113286f1.min.css | 1 + assets/stylesheets/main.113286f1.min.css.map | 1 + assets/stylesheets/palette.a0c5b2b5.min.css | 1 + .../stylesheets/palette.a0c5b2b5.min.css.map | 1 + cli/index.html | 590 ++ concepts/batch_jobs/index.html | 598 ++ concepts/endpoint_predictions/index.html | 1008 +++ concepts/model_bundles/index.html | 846 +++ concepts/model_endpoints/index.html | 685 ++ index.html | 695 ++ objects.inv | 6 + search/search_index.json | 1 + sitemap.xml | 48 + sitemap.xml.gz | Bin 0 -> 277 bytes 62 files changed, 16723 insertions(+) create mode 100644 .nojekyll create mode 100644 404.html create mode 100644 _images/model_bundle.png create mode 100644 _images/model_endpoint.png create mode 100644 _images/request_lifecycle.png create mode 100644 _static/favicon-32x32.png create mode 100644 _static/favicon.ico create mode 100644 _static/file.png create mode 100644 _static/launch-logo.svg create mode 100644 _static/minus.png create mode 100644 _static/plus.png create mode 100644 api/client/index.html create mode 100644 api/model_bundles/index.html create mode 100644 api/model_endpoints/index.html create mode 100644 assets/_mkdocstrings.css create mode 100644 assets/images/favicon.png create mode 100644 assets/javascripts/bundle.2a6f1dda.min.js create mode 100644 assets/javascripts/bundle.2a6f1dda.min.js.map create mode 100644 assets/javascripts/lunr/min/lunr.ar.min.js create mode 100644 assets/javascripts/lunr/min/lunr.da.min.js create mode 100644 assets/javascripts/lunr/min/lunr.de.min.js create mode 100644 assets/javascripts/lunr/min/lunr.du.min.js create mode 100644 assets/javascripts/lunr/min/lunr.es.min.js create mode 100644 assets/javascripts/lunr/min/lunr.fi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.fr.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hu.min.js create mode 100644 assets/javascripts/lunr/min/lunr.it.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ja.min.js create mode 100644 assets/javascripts/lunr/min/lunr.jp.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ko.min.js create mode 100644 assets/javascripts/lunr/min/lunr.multi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.nl.min.js create mode 100644 assets/javascripts/lunr/min/lunr.no.min.js create mode 100644 assets/javascripts/lunr/min/lunr.pt.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ro.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ru.min.js create mode 100644 assets/javascripts/lunr/min/lunr.stemmer.support.min.js create mode 100644 assets/javascripts/lunr/min/lunr.sv.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ta.min.js create mode 100644 assets/javascripts/lunr/min/lunr.th.min.js create mode 100644 assets/javascripts/lunr/min/lunr.tr.min.js create mode 100644 assets/javascripts/lunr/min/lunr.vi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.zh.min.js create mode 100644 assets/javascripts/lunr/tinyseg.js create mode 100644 assets/javascripts/lunr/wordcut.js create mode 100644 assets/javascripts/workers/search.208ed371.min.js create mode 100644 assets/javascripts/workers/search.208ed371.min.js.map create mode 100644 assets/stylesheets/main.113286f1.min.css create mode 100644 assets/stylesheets/main.113286f1.min.css.map create mode 100644 assets/stylesheets/palette.a0c5b2b5.min.css create mode 100644 assets/stylesheets/palette.a0c5b2b5.min.css.map create mode 100644 cli/index.html create mode 100644 concepts/batch_jobs/index.html create mode 100644 concepts/endpoint_predictions/index.html create mode 100644 concepts/model_bundles/index.html create mode 100644 concepts/model_endpoints/index.html create mode 100644 index.html create mode 100644 objects.inv create mode 100644 search/search_index.json create mode 100644 sitemap.xml create mode 100644 sitemap.xml.gz diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 00000000..e69de29b diff --git a/404.html b/404.html new file mode 100644 index 00000000..89dd6b76 --- /dev/null +++ b/404.html @@ -0,0 +1,492 @@ + + + + + + + + + + + + + + + + + + + + Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ +

404 - Not found

+ +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/_images/model_bundle.png b/_images/model_bundle.png new file mode 100644 index 0000000000000000000000000000000000000000..75667ae057bff39ef6679fad8d15d194aeb086d8 GIT binary patch literal 41350 zcmeFYby!s08aGS}B2og#9awVJb2f0+ zpLUlnaGc?4z1NH!XvLvR-7Q^>(1pLQxJ)DQ$y{^vQ8_0D_*ke$djY*URAjh zRXlE?+j_8!xWgltI(|NSgY3bsC+{Fs!G)-4^gZje5)KH5ub`}~;+B=#f&wP1 zn8p1cEA*Lfyh_|!OTSG@da2+kw%^GxF6N!Th>$RQ_3+1&km83@-7j$L#dcVG4 zKd?TjFh_IDI`Wjh*1_q-{DJ#S!M}(~lZ2wQf}=YQYc1T7AdLS7v+r}$PfzoUuwN&4 zl7~>Py)niO(U%VrMw*`5Xfry0GxXeYZweF2&m`VH%F>M;syoom6g$fwF9Lqn6`{7l zMs*rZK%2L=)P3RX!hCV4B=2wZl~bkZd5RFT3NbbeNeY=49aV|yJsYwT+O*i%Pwh$6 z{f{)i0x`3wFwq0!6u9_lEMH1JD#iGWbBz&$&=tJ)?$L*bVu)h_#3Rh5DpZh{RWD!4 zJoZF$p$I{$KQIA3R}@}hBNl!~GA!BzFp0CB$psdlHx(3`Sst@$^hi8g{hup_mz6=dgG`5JPAIK8GpnBzy4h zn7>W~PL-{F$YAs*i2SBKaf+B91&v&uwbs6+pS2BceWHgX^eKw;0O^+M>078e6(Q{# zt>?Hp`skDOPn1x(+Jw~}DYQK-BXmG8Y<*oux`X^coZJS5p-tZz!w8YDRiw^auvKvg zYd#2sA})uH^@S!lg!okiKQ%Z68!zPTvtc>m*sv@b{g9_IG+Z$ka!eHurvn`%3!+V< zGrQzhkVb-M#VMYpb&;;%hken~BH|Hu9Ky}R?vcnGl5#^zey1XiIgGl&Sw&VXb@sJ- zL;eKM8`)9T^sC6z*0C38R*?c`pr@qa?IUGMN@4vS3T2!{c!g~wix60FwYl+>Y--4; z3;iC7t}|`=+v5YKa|d507~(?27Bx7+z%L$D56O3pC?31e^Owi|A_k< zFA`^f@?J zyZvF<=NpOlS>~DeIo~JsKUW&Be;mLUl&Q_C+!Vg_y?lI0?ERiyeNrbeGo{$Dpi#9} zyVg)uc2JT--znMT-TaH@*JvGOX3kIRKdgQT`jGNL^^*RQ)em}$cSCYhcuDyvm>7i= z{gJ|>Q_|Vz=N4VukGT3?7}@GspSPm(xjrW`=N`AI945{zF>CmK{r)gG{*e&>PRR?q zotI=z!rwg`UBX?WLi6R{hW@ZMmcn<%> zYQXkh02zH>bW#?P4nzx&`HnWehP`4;wSc1~Ek92a5x<*)5TGqxVjoTC^>5S`m z6n9U6CM34qC)SHZr>1^pO|e$wS9+P*zw9k%t17N(c9N~ctVnsKYwIzWZnJNmVcybx z8-~2KxJl)f=UlfFBIqp;M#d*lgT?UZjGUdqMBq9Be>`+mYf)?Wq$}wc*{;!w(fYOB z+wqVpTo&0>(!=rgQMyUX_q`&z(&TK#7yH``rEV6XQ93L`*hQE-Eqlr zJaL^2x^w|@7|QzbBcRtmru5A@vPrVTG`2_OY`blRM!h@UMHmW|m^ZDTFCFqROsRb0 z{q;>MGqfH*N_iypN17XBT6B6WjNyihFTt>Tw|mjTp>JcY+-uvD zmzCIBK_8JmqJCke;5+eeoMwq3Y(J^I)5w#W>8>PI90p2KZ)ZQw-gw!0ZFyDi@fdmL)-{vRM7anKA zJx!@zEd=#XikZFnL@oWlBgLUYh=lc@mX|f?W$GFD+Q-~|WeY!S|Jq)xIGb2xdQ|<^ z)U|vpyQ=8qSYv)9T(tgH>sIrwqR+Cpw>iPdMBM0^i3`MXsAsOQN|&I-yTbcq>nBei z51svdaaz@>M3U@ug8N7JBlq3a;Z^u{@kFWKomY_a_lU6Qup80{KKAKzGyH(j@&2|H z`+ERM{*HTbne4gobZUY+$Be*PstZL2EX))S4FjCUKCJu*s; zN(z3OH(7h}7_gP@%?kPN$NPP`+OkiztJ$^Q;SfQ1+=YF8sCdc2hb=?y09t|Mt(C!l zMaXW*D%i#R+^n8+|JHDNId)l-nHkB;cg{z4F@7;=9I+Nb(0LNcCaNL;TlBmO{e~~1 z9Gg=Tr_YeVt!fAJu-nPs&h1PEi{QJjT~8lW<_?q(r14Wa!w%mpyvro=W1lcSYS5Zl zns!~`8(fI2@~)mRIfWSow4W(R&UDo*>jssiHxYX8?zj)Y75QF)tLBs%ysjqB4R>oV zYE5Umn~yKSZF4Xm$UgpFqVSNn+Ud}_+3Nmj%f8c@o#-j!iF)ty+s?D}gY+g}W}nM? z^ER%Zl@;q&gM}t_%{Dua_M476>qG@j6iiOt3GVW5);n}xmp;BDxYkuXo7#(<9GSdv zN`UeBHg2bNHm;@DrN7@logw@}Nve?g#V3Ut9^s7FCQ^kvr3`fe#pImHF)NIwuMXU)S51O{bmp+Hm zia6!4Cb9Pt;YrIuXd8mN2c;czjAcV3Q+?h6t<=Wuet6k^zzjU2BcLFXAfN(Ih`m5`3?d@}@?YEET1L7$QT{fuY;{D&Z}DJd@x+%?TzEiJ(i8z;91pXVXK3k+vD zec(<=cYh(uYtSA5?N8Wh>AC4CD+!uAIkKBsIK8rD_jGi=ZwEoxQxJG`v~)A0@^o|n zLj*lVsQ)S<2t41v=AfqftB9Mu2(_NF8kMAzt0fgL`*Zf^)S!n{R8+#Q7FL29QZoOl z4*U|KwsCWF7UbaY@bF;w;9_@jwdQysARxf;oRfo-lMN`r2Jr&BnR&8-AvAwC^6z$} zEFtEuw$5(0PGGA0cFkTnxx0x_Q{Q*=pU>ZPT6)_4(-Ro-uVDcL=bs*btEu--O@6NDzt{ZjpZ~6@4Y72U zbaDi`bOZe}UH_{5`=9@+D9mv`^4~b|H=F-@3-B5AP?+OCa|U{7NDe6grjg86N>vNE z148!Krx^HV1g`r#aPi_9SaMY%An?q~ONncFBJRv#G%2m#wH|WFPqSRXf}`0gKBi?! zFdHh+#_n9f_AQ7iv?Sh|zOLX{%}r@8l22eI(1;0OyjxH5Uw0cw;(`FV@Ln_5*>RIe ztylUwD`ep6!h5dWu`7y`76 zi0T02O0f@vIuHQ~o$6m-Q0$C5P+EzzYHK-#Yxt9L?)!AfLpUeUZp+EXZV3SGLlhqI#?JtLrJAWg`!H-W9f>)0wZBqZ9f7ThjC``|aK#Og3Dx}wDkwDPI73w7Yv ztG$UF7AgX?NEzWo96cl1ik8zAmK83`9no#8Vz~c|65Sfwm&DZz85CiFZ;i(pRNJ+0 zYev*U2j5{5S!8j&#k|1Y0J%}{m^=f!kou8Ri1jLj7F|X+UMGq8PXq(t?MV7qnr*?&yxfoR$}Yv z^3q9k3h}W>QNq+0n1pHpQF3h(?`Yk(#tCG(wBiD)~7g+Mv^2T|~q?sdU^eCL10XmqknVTgYPMH2HFOB+a zA)tlZcHT&^Q1%tKIb=&s=AU+`ZZV6Dt2H_vld_xoU1U$T<;cZIpgnp@l_uno;ImiX zX@4q~8ZAq4;txd3>bV?}32( zQPw0hQT+xks@=bQ5yE?qlv|ilg`=qCM|L zh+M?k&B>^0FSh9QOCdLr{ef8R!l&8&ITDEf3=euh`0055k?Jwu4Q*F)GPf-?rT_Ua zV;?G4UE~bMnHu@spViWXw|2^KQ-A%OJPd_6Iz_8u`l#P*1`>wYVk^vVq)dfeAYdkw3w64VDzvBU$p6D{l{vg1vu8OZ+o$2++4I z2fgtuBP;Dhs}OkEkL0kze20m`P<8jaj894)bO-tkZuw)Srlxzd_2v}zfH|<}PUg|r z3uWjNt)?_0KBX;_>00PS3?k_ideg^~~T6 z_1O;}L6T_tFtea1UMK3AVYNo+w4p&FMNw?#e)1XAw=%@0>-i!%%uEBy;JpZ)TX)IK*mDB?nMpQHWiG zLjxVGwx7(Bi=lSCJh45#anDG{V%}iVsw>l%Mf+!dfu{q8p+X*s7W5!nD_^6g-aEP4fho?a?|Ed@pS-8CK@_RC_2S7LjPFhCcPs?VKk0a@?Ing< z8B-5*&@|wN?w$5snt5kr(!RbPmTof-wZvZIKBzMC(LW9ic)}rolF!uiK(q^|*B+#C z+s0PePqJ^!Nedhh3%xFK+n(t6{r-_jvnO%E_uhOy-|qUiH~hPqVce%YZXu7hMO$az zQAq6qQXLEUv`Y49rH_AkAO#8@DLCLE`p!H(1{}Py^&yo?f6hQiNh@G{vMQ2!trzy@ z)=EsgRtYXPGNbxjsgf8`L0BZ*(SU-aTybOyQaGM3Vg65jCa4qa)`0mVD^o*bL=5Qe z`uMYq7pCCBbgfgyM=dA2EL!>4E_QdS3(P0a3p}NNQz}H1S6ZKCb5%GUm!u#Wg7nko z#PEKjMH+m!Ma^2Uv3~ihclC_}&EDOuX5afbiT@COsvK1RyBiOte2SXuyoR}_wl|k> zjZ7J=%#Aaiyw>%DU*9#dL%L(>j<4uhrT*yuE6c%c-&xl8l;Zw5 zts*`38pj!{gd58@4}DYpe7v#9sG3=6SkHZE5}y5G-iC4S&82Vwntr46ZBu;;$Cw^6 z;Q0A^VVsB$KnyCW(bpHpDXKmv2jlMtQ)3>CX;R|-IqjrKnhYw@CTW5xPT469oAgdG z*@4IzfCl$SDO!JxoNMe|jG>{K5$*lmSt3B6;ty$VJsF5sWWTwttRK9Od5%_ltrO-r z5m7ZT1-paAwvkd((aN0!490`$ zCa+4ZLY@BETa58go85_GtEB%DRYN4%vCsCFz$G&Rj7^g=fa!BdxpdRh9DuV$ZN{uZVJ?r;hx zPp_eBXQ8t5Sm)lIU*BHs#{xcVvNn?Yz;?E-oYkmqXx6jT8|mZzulN1`Et>2o-mmvq z_f`8QTWE`Q_< zHp3dZ@dE9BxDPf7qsSGUuh-(wF6*9jW$+XlH95CI(7n}aXk;VnC?XeHI|a(xb}o*y zVOz(Nf4YmMpw`iN=I#MsF~gCb{LyEC=M*u74Rqd){j%m+@b|1}YU&ZJ-PQJ?-# zU-2VbEQOHGV*YAB<^92&$AWyiuvcG2%pvLd2Cr+OPBG_1{Bg?-JrGiH|1XuE)+^}5 zV^<5Iij&2L3_SJ|CxWysP*oF+8uSkb^&`GDH=L-~9*Z>D z=?><7ISRyml@1oyXdu*wPokGQqkJaVt~WA|7LXr&*!?fdv}CjxzB`kxv)Z%fqLePI z|Fg!?J|W`6LMuZd$+KU8dZ)W9CbF@lyx!xkStj_yr-eYEeg*FgvPZJ}m!E=s>rS@D zds#_|0gAWA@>cE&GqalR zI>++8cdbPXgJbzLn2Bs$|HC7~^ItpqOdq*&y`BO=QJj1neUc^3GS@-9%W^QSYWl3+ zAXh+CdguGMe=>*&TG(<;EijV9gPrSjyvAugKn5_#7Y)=9ngZUkq7`vrf<}F5WBFYv ze>^}WueDH)>k$6f!3@?fdkNL&-x99JY1&@+#2{+GX-ZFHT$RtCKn6SBWq zVQ4fyknPngb|BLHW@N?6fckA=<)GiYew&H_RqZn7eEjrxhQA>I>g#-f|86R&R#4D= zCH7)nT7Ty+P7zSQ>{Zbp5#loju|<5>#1F{7#pwvpm#FBzZwSo&{-9m}5h)|O(Sq#x z??j=n0QHNWZvR1j7Y5Gx+gZBhz(0u61M0tQ*Q5NMdI=SPB`fU24?g@pHEj=Z&QVx8 zF8`qZ{|ifT=8?81iV{dC?+ygJt2RgF+k3Q|wt|YMj>mOf*T$Gi`(v}^*i}W3i*)^b zOZj~_q8+B|+JyU31t8s!K~bj!a~24EOMANF*TS+~Du21KVQmTE&}o0ZFB77c7)S?^ zpZM?9uTywE-<3RCl*E?b6ZWE$4tpZLPxH*_IS_LU^I^z%=bO=Ve`1PLKWUop-v;~8 zJOsui_6SJ)e;`md0tWXJzFaZ*lV5iDWQqLIO1B@|<2;x}+Z)%VbC*eXj=Yp6wMGGd zj}fWO>GC))PV=+7PyUS|(Uh6_#Sxe0s zUe)mex`-6scp@Ng8Lcf?Qw>UNdNiTc8~^4&t;q4@#0@bs%pIb3<;`wpqsaYuiBN0iusC`5%YFV z(i(HUMH(q%sQxt>6B2qICeC?Xqgpruk_+u(({u$zhhhYY7g(>@y=~<^HVG}wct)wz z;<6(B(PiUBZ-A1>VR^bV?E1oTuEsI;nYoNTk5_o$@d7n83iUlJkn3jw(ko}e`K!45FDt(V zt{4?9psEr_6)r>QZ%YG@t$KHr25*nTft0tyF~)xGKDbpINOj$NPid4$8V?vMse{r( zk%M7tZo3T};)#I3-IZi+d-~C}zVdPPbJ@QLey@rK@Vf!vcb=(uBqj7YKEUsBt(Fm? z_nhe>0kAQrS^h~ab zl7q@pgq}vvOawImMpjgnN$cRWH3&N_J*6Z0)z_#&Cp&JTIij^1JJ9Rx31z_7H?B7q z?c?i%o*-rH8rF8*X9p_MQ3o~tosSKTeVL6kCF}0;E}4yCZt}5(^*JZjtj3ooCK*9_ zWXIZYuf6c_5+IQVDmw zj|nn>nE~1A0ruDB86c^4liA@rQx(#u`*ZdbDr`KZy+-+Ybi*hzz$kHN~x za^-k*AM7$pzu=swo}U^rZEg%^B5${|=z6CmrptMpR&81}!!Rb5S^co`UY6u>YSMp5 zbWGt*(OzZIiB_3wNU@|D%~vM9032lIhVAP!o8edK2q()?Ydznh*198$4f|JRWCHw2 zVMk$An?uj?p4H!-FJoW09WtcffM>4v4>xbNwvEf}w7oL>4b$trPrL;0*gD)6E9Lt! zt8C_d^_v}%bn&J9ln9u06LgdLG$p4dkI6DhXzp`{VO*+%&`63I>Oip%n1D_h)H-oY zyILf|j2oxtxBFIm5k=yi?bjyqWwGVs1>+y>jZW4Kb|Gc(u;|J)CKNZl!JSbddxtTa zD__ZHfadnycew+L_)G9i`g89pbB?|C`V3#{^}!5GJ0#8B7d{e37gLb6Sis8m6%;QShl;y6xH(wtP*8(aFdD z&{@~7O;lRQzHJ2HBiL% z__5st(X`JKb_1=)9ZADkVW~s-E9!&cIP^(vX!5aq&TfS7lX;ztIo$oGWpk$Sd-t~j=v)kG zIbdjhd=2C}u+2#}R8D>&$(+b!-iSNu=ax`pLcSS;wK1;PVdy_Rn9P)1M2ei5+L6^gHzNSQdsHH)-#KhQjrH5{Pd=X>9^fJQzWV^7mg8&DEa!(Z3$_hhBK602-@jK;HA{0TJ%w9G<)|ebFp8i~(dcTO$nt=d~5+w&xYOx|req>6Tm!CkE z6>d%oZLftF3Lg*Kj%HVm>thBKM$TYccfZ1X+HkVRhCEg_0sd~-hze@3)$0!BNl>;eX|oIE=V zj{TdI^i?ZCB13IoN?Q_O1zrh;3)s!grLD4j9LzcN&hGScozg9`D&s$T2G}jt7pzhAWV`q_2oK*MN*X`v(v#BUkG14s% zX?R`UkcqhJ{VcGG8(?^GSr?#{(P+^ftFPJ?=Tzv=|4GQFM%S-%Z!X(qG$)+f#4uKF z)%9suUzr#PC_}UW@2x?tbDzFGIUqRSfZ;T#zU5I)6X_l8UNErkZxqR&hWXTPm?k`c z@3QPaq68byxjP4GGuL@tyd!ehCScW)(?m>5y;|yKN!5@g(c7;3+XY~xX zBbxlZv#$#E;$Z3``A90u^y7MjGr;8g_$kGRkcaP7;wM6*D261@gj`AGevJ%@DvrCZ zHl-sJ|7?q^muTa7;aG+qKMBQ5QnfGm>IJbXR+&JvBND0jRG&}08yE_Zo2Nx za$ByQg|un^$%#!6M}F(rglED@PvADQ?5+O4V+!>e4sUl`|8=eDL=wX|Y%PEeksPtXi7=`>8DeZ&zT7-_dpa99=xn2~*)b$5!(ddp+UU;Lv5h@7!sPrcEVaS5> zi4%l$sX2uN?%1)?U2{AZ}0iz{>>nD230VY*&*n#kcQ8 zUy~!M_7ESFH{&IKXH-h~2{tzFg0K1CD>Bggl!Ce48!)&mNViCoA|(2`wLhmKmuHUG z>CAJKY6Ojk$S;nq`|Yk)O+6^~i*MZ?(jK@pCQTwUZ$Jr<j>z<5e5882n`o{>WMJ=9DO{Vcw6}OIV%U^W z#+4N5XFv1M+4Ixw?P?eaY*rvKuGAg5cv88$qww?Cr1P>1!MSe^(v_d6U*6@_i*a|3D%r6%4;ymoIb?whp+E9?65Ca|;aCYR}YgP7E@JK=%^iKwf7C zq~f$i+x^qjIQyv7n7*@XqXOQiMHc1UH^XUnaE>1nf!{8|J*l8km7#i{Te=Km=7OVP zp1aAwx{1-r+iYUZtg;dcEqtoxFBAzrS*yX5#J~>8Yb6DdqmW&ZS6|DHTM^+WJL(W! zV7UAl{7Da%Xflz%q^Ak_!0Sf}&q(;iyam90%Ro4Qzx|lx#kWr4slqgU0uR73>kQ`xUu2 zKvan#6Yh>HwpfBGMR;xmBm%BNGMW(HUl*^gUgvAqSM2I}Il9jz7*?JD>Zi!ZWsh)}#SB{MJjc2dESGBFx2KOA*CIslT{P=x|+-+=Y5vHF80r zj^)c-kn{HHY|lMiqN)e_cmTJU&&(VJ2VDqQ04ZzCO2t9?!iJ;2m^r9|?b2w16=YAl z-VcLtv(&18zNfYmFMf?D^F3DlmjoEm0@_bPrPKd>Bbx40}k@iMY)vZ3u% z^SGLF3&=G85T(?%;1|$*cq_5d(lWjE&#IbR>>KTa;c8u{ZFVG*@qVx6 zGl?LyBj{Ehxsq#2W@jCud&&&$RMKW=1O*+k&nYK%H}QF0zhu@`wU)CmE|&++%bZ#} z9ZPa$4lN`~KktE7sYwA@jBu63W#4bOM9{&)0;Z{>Y(;o}(k2m)aAS+V`dS$CGH;-S z7IH>s_>-8-_t=IW9Q}sX74boA!l$0LwwK=FFZu4*vg8);t>86s>n07(0t&$Dl7h%# z0WD{9Wf}YzwYiLqQDldGYEH(Y(F~vg(v`fELzl4GSyYQgVwK%fH}^DM zm#aNjDQU#0xG~w@qQP<{@1?B!DEZcL-Yw1LU7U<7h{rW@i_if-y({^~xW(U?vv7E$ zD8HH{-TUM%#zcY5x+D|e`EWfVJ=X}&uZz$!>-J$+*9JFP$uHJ=Q>SZGfK=dkoVM7B zCt&V1*}z(8&AZ5Nx^{ZIcB?(pc7R)|!#Z%0=$HNotm4evGTm>8co+t@u7fcv0{}Hp zpc-oJbOc9-NWRZ9n69W1`>s1$E{2prteGoU`+C>0bN}_RwPBlvGpXKZNT~^@^VD+) z+xo$*ciR`uTwj$UTW}ud`e0hzn@D-NG*#b|%Ynrn-RCxQwLGH!eKz{=P0mVY-E?8j znHLJWc_<(-zRwnZUkQ4%bpnIH*Li5^L&|4v$3irl!~5xOsTt)T z{)+1pfw-Q7_+C#`P>`^E{WVpsD?|T|f6@%iCJ8aE**Pl_jyb zVYslg$A{A2rVw)lZK_5b2h|!P-fUa;U4Ps0Bmy$|de`FgWV6ABew|F!8wwdhH|CS& zdGDF8PLl>tnnjwvmr44o@dtiuN0wfTZ4dk?T1=cb*E8s%dPkqu4k}SiioBX+&j%Bu zOSZh9S=X9uq_c*oG)Ed9Yu7sd1qvwwlt>5b2neLsk)UXxOR>3RR(;i0n_H`5t|P#s z)@oTFaN3n5 zAuqC0UAjUTz2QCwW}HhG-lTE?_TXY(zxGgxE?6)RC855`&$ZtyNpKYTBQut{6iG{^9}VvH~gojwi{C~Mqm)4$@crO>AQQOt~|10u7|nxz`%h_ zv;SN*Bv*J9@9C!4a%dPqaiRASSk=$%pi-vTv`7l06b`p>cDEO{O!+*n*A2u-y(AOG zEcUu3+&afiTe-Si;}M6*kqp+}Uv6DD^1-p9$y}W^$~rtRcA)qSw6-HzHJ4M5ePL5A z93uj@Zknw$1ztagCEr`#B=eM*2=dR9;64k*XQ)XVOd8G2mfM>JB3x(ErxG5}8!5PB z^tAApTX4F>wc}&N(7r z?rLzHDbltw;bUXYN#EnI9cI~3pH~pp(RfJ6%qn{0sm`-H!?Y|L!CC8W<%%FV2jN!7icf>vIi$epjUqJ|C5F_X^y-Xb*Z+KZVB3yv=ljY~7m zMc_(oOke1Ul;fGJlV7?;v6@zl!67A+lT1KTHb3uZ<6*o|C5;IW5<~UMa>h0OXl}Q$ zO}@3zGdo==V!0jQRtA&9vRitjRlFU2?*M#(0PQ^X3lO|6NdbJ5rTlCtN@7f@5Bc;G zDWepdBAOx0N7QS0A75^SZ;EtG^RvV|%qt_m(+`^?o5j4K#4CenlL(UB6c^%dAhAnS z^|dZK*`Aow(no1y{MCyvX}?w4=^ z0}zoY??9~52v9vjblwB~M!RfoH(O>d*KeM-qp+aCOl{*B?<(k#V{d4=WVv+;#?cS~&n|DTOtuY%oceNUG-4~9GQGZZ#+69!;* zumSPOfI6EP)=xsZAj9ULMr;kh{KTXO6^0*~?r=H_l6&_2+3sLd>m1sjUAf-@do= z+oSfdq$_t)G2o+9GniPX|L795_N~@zBJ8%X)N)!f(!w`k~0KYB#%@X@ea3IdXz> zUFlr8TUT+0=>dmyg97!A8ADs(1sEId-A(DvLanZGRL06=p)Q@`1nZIWEz;2e4Qlqm z-DU0%4kM)J!ug{`?lu)EBXJp^v{m^|C5maKiaXp&S?UqE;ZnvjHZ6X&(;Uh<1yC9uXmF(*!1t=9<#(4Qx&>N=?9$;xEsain^OI>g%jgdXied0^9RZ$zkL4~*eUHg64p*yzFudlKNJT0@oHH#eM zQ-2EqIv^djLGn6Ujk}e6uh4u6r&Iqbg|U0G+Q0DfXRLQ6+}QHYY26|2FsU02PgVd zR^o|FV;9Csvj@fsv{^Psvcp#fGZy}}F}9EDcEDvbL^a(K&@x%BQ>-qz#%{t2fXz=0 zWTa0}&%vRvXFdQ8=S(<{7uYa350YNEhn;MV>ATU|)6TgS+m7aNzHCM4j-h$4-hU6` z!Kpw3z+z|DqPwk7@n0RIohreqg%%5vKr?NxzGF~(iqQ8Ha3`PND; zM2i61TpQ?bw}5$j=nGn?QMLon*8w;1#EWib!5?6C-UAs}roPVr&?Z-bXT%$TGIcR?d?0nkgsNSE<)1?b%QI!O?UFdZs!qvq+yfdi zL>Cy3!T@;(jzq-J$Y#B24|Cldp*t3~pDbiJPGsllwRnTS*ht6EF(HEhJ-|oL-cLZm zJvY*pYzE~gm!}E1_gq&2h+~zSdNOa$y`RYGGsXV>cW2-&!il=*XEZCs)&wn>1$A}a@ zo(J~M^S8fzbqDgQvUy=oa3O~ROe0E2J33YLXB3HFjdVKzhF~FpqkX(=Os0iGL7>ZP|fZDfy zi=#{Nx)z6TP}g$7062OC*6gRn2YGgnqNReOU*DW9XnYn!6fjF_^*Syw*4R4i#19cW zQHk46*tjU?0oa%;=6eUm*lt}vFE>B({PpcC#~YO-PRm#T;GhV=7Hj|HHGpDh1|>r=0f-?3C$_1kG^ zmFjEWR=zIQr#B~BY;4~Xzh8`U#y>Z~VozXDrGTjWUVt%XTCFEeY)9(*XXwv?J@+j4 z?TLTkXIn*)8tP}hNWK?-ceU53>aV8S31{gx4BRJl!neD}UhqNB&*$)JpsH7V3m>tq z2DnRuZne1}YkmJT(CiJ51!OvzY-T+Au3a$!z)0Q_EH0YV>EUI<c3mX(xIc+&eYXCr(Q@x* z-YfkoHAz5oAxVg9KPhRUR9}U-{L{Un00)@}nm5NAYd<{YVRqH+9kDNMUioshk6mHv zKCZpG93@(!qyC8lmkLR$)gGlIf|TI_h38AH4wthzmd0I}>`G2HBH{oWmfnyIN(cgO zmqcr4J!(_7-}Z!hxev=cr}xQ57r8<5ILYC(*>S(+-tpjrRRC=dx4gZ+Ff%@gQY!xoq;$gjq6oM) zV!#F@D3;gprxb;sD!+KJv4`y^dfwffW&2&7s-yhfT1kAE!JEJ8 zi{Z;4nCC=!@c7x|4Y0wg(R(FPLIMZT&-rWXBA0pGJ3Fg2Kpun%5%ibe!B#4yjgp)3*KXON!-(BF|vzGa7U{svuy8;t6-V%H(@lej^ zxKsc+DgZfP7xGSd->oXm3WIxc5GffLC=J`)znzrtu;Tw!YUE#IrUk|CvGqlvdwyuG zUGQfax;Z)?6zvVTyE4_cjMuhkC(!*!muNJytWjxW3se!vj;~I>1-j$u?P>}|0GfhP z{d=V6BjE~ak=E&6GK3r^sdD-Hl~xtN+HVh@B5RDNZpecX84BJ?K#_Z0d)cggR*M%B zP5qfXW?f{na6AARNmz~)DjDV{(EP=a;j1wZVA~n()(6ro??q>JwW8_vavO@bh=EVv z#ZuZt{gmHft+wT+c&BbTvSQN2r<=WThuWFQPAn@&DG4vSKw_C%xJwr}e`pCG*G=S^ z(3A8(PFZ*orxX{qe9v9v|0{R1KV0nAucx~LLSzW}#ABb}(70PO;Zc$fkX2!n_!WNh zlumtyfdsiADv(q^m;~}!YqqZ;Kp4P$0sB>Wu#6`FWeHYFYZrk zG3BOG4}z{7)NS_xM!Qa1Cdzk4lBX@3GvosY^#*<&qh8!3GWzifY_z*79N zfktdeley^sC~TF zA6{@s0~~~$YxGc!P+W?87%>{IfJ6D{A%975T8Tz-x`628r^vZN_{!xeu1Iho6N!+P%msemZgog7w zcXs0ch3BcAhelx+g8{d3nqo4d?o!le8NjkZLCe|g&3>LRqoI>FeKH!#>6CfYQ;_#n zoBxPB0}l#&>nUKk*T$re`l<*HnW&^i-+<0$A)$i#LLhCXS<1WfPCfmx^5WQp<-E+u zu6Hx0EYSUMMK9k0^5VEBznF-a8!5F8mse=p}j(y|*A*)EGT_PZ&Lj z8bq&yAbO(L=$!;X5G6zzT|%^|38QzT6OyyXoA>>Fr+nv}%YV3L_OthX?!E4{)_w1# zE+T1lxT0SemEwY1ob+!W2k}XxXO&rr$_C1bo^+GFuAgQOBYA_q@_Js0K3M@X$tmzR zu6V=7@v6<#+vpgSJ1#B{hlM^5%vPsMc)s0k-l}RaNMP`WF>^-OCcF0n164i|imIV{N_d;IgKk(DWp&|J7`I zwuVrpk@v5&$6uAyvpd5EblVqN2 zZcSDetEKUE%#*HluHfRZW#tB(7`klGUzn@+TNM-AzRt#;lavj}!?w$?PEGLInZZ1H z&>hpt_dRwvUBF6#4)^s6W^beGQV}Q(#0@eax_^R+_{KzGEIOqqmpgWEJYr9+X^E`F zceA{7;Qn_LGKtVN@wF_LP^??g=l35+#F?)Eo^b767=$9{?}E1vAy& z>9zjKl!T}bt7t}1#|Dt}rUfe6WaVq8Em|Qbg6@{`#dBPmc)|9=0+;&|nsz?^gpe?7 zAWDmdF;7L4HEGraH_=j-o97lfLfy;!({0WaW;n83#{I8E?`tJPr9>w029(5ly!~h|CjH(aT|1YTpE|wm8v}(XShw_m-+0y$h89%ib#b8oPc+);uIdmQ^E0i5sdm;LOS+kH}pLqbv2 z=Pzjhj-U~*UrLFjEO`czZ5XFZ{p6AsdRF#JWYUd7joT^ZMwH;zr)~=4sgTtHSy(b_ zlfv({9RI5Oe@fOjmc*P{=qFB==0?f?&{YV5fYc3+~u`bj!e164^;mh6u-p7Xe> zED#3RzN`&3qfWz;r=R#}?2CLlFdmtPqe~0vU)vuJ@B7Q{xx&5;p*yol@h51OM-7AO zoaf!Nh^p)L?(cv2;O)+ugvGzheIu^MK_-%T3%Xf@NpkV;ec~sfSah8ulrZ!NRXhTIrH0J1Jp{MvG zqDP3ZDv7!Tb%u6>Z90J>{SNMHUp0yh9T4bO@)pJ~wURNNG!F4^vO=7j*Tg>VGXQ*e zgT&=8=v#}8^Oy8z5PW@>x4La26v?dB6&3DN zk85{Ec%apcB*R`=_?`jCZb~u>`M8$d=DRvoE{l#VV5d{G5#>&W$%&5*eNL%0r9t8d z2rp$=sO7hhJGv)d)qgh=`ruqQ>ott3u8#6hlZ90uV!IJ6l2{n84+AK~df+rb500ye zAM-UU040d5w85LnP~cGRKa@~?dut&>5ms$gaF5vHBkzGcl?&5P0*2k3M$%kJzy<4Q zd%;9Er@&BD<5ckIYf=NJi6ulqZqI|?l#>(TQ5u5O7%a;|R8;D85>pA)8E(W3G`^1z ztxnUT1H<9PuNS5xiCnOw`cJ!@t=fJ)wr>e}{nq5Q(uu2EElHfiuO`j(`# z85ID4|3RjzqnMYXzk{KJt(VusmRl3m)2-q}v!FC3j2_}+(> zZ5rbZ@5*e7{Wbo z3nL2C-a}hPHM&`xQy(4z;4RXBJw-;0CD(W+ub=7P_j+8%`wKIqtfc9eOFJ4i^jVxK zkvp2-yytgC3)4sfx=?gibC+#m3=o?ztEPM`q0leTqfUw(8=+W4< zHl>HYcNODaF7nL%A{wZhN-B$O^%QUwwMvfC@D3 z+5Ei3W7nvEWSN)|sgo9USHRWPZQ%Fw6&8p{L7wnDJ83HjPo(&UJPYzmaxh2IMYF*3 z6@J{((jC69#?M=Cci;CU2!29tnRnXD%Epe|`1v`AnSgcTw&hUvU1Eu`;(eepqx!%`a~X07;I30r?y3wbYW6;U@tA*4q*M10H;Y9+~h` zlRo5)EJh*2&!Yp}EDv8t&rMY4se|&0*xKF70bx{LIX}(|dN7pE1^u}qA=uP>2|N0x zl<;f9z&r)C0a0>>qO#RZALtNK<6B9QgrEPWiF10TnklB<^Y`7l3i{%=YX|ufa%Y$l zle8e>;5U4iy%&$Ri@OD!W`uQ4pIJU%07Z-++W}TtTnH9WyZP!enHWb0gU2nc^X8}E7U!X6&P6P@rPrdHvUm;eTTf;RF!+CTxf&P}1^4s3>G z+uVRj3Uzw_ZGmZv2)c^BHwC3}OO0_m?M>ai`W6`=V(ec757g<5ocMb>ugxN5uQW`B z&WzsEOt_Mo@43EG7Uh$KluX%eL&zQS8XsM4SV;$Pbj*F50+~PF03e~vqKaRC8lq#! zOw{|J9@gx%!qXcYm%%<@-(W5Kq8OuL;!km!nd-}iowKQGWraE)nS?6dpP@vMDbnT* zzjLuz%4Qn9sQoH)yi8{;%P-gY+O5CmMY(OLLzC_;jj=jMKKL4#c{LuY<3eL%Y)3xM z{7-tKd~l_Jow(xAUy5PW$H$OVtbs5qAO6O7mZ={`6ouhBnfB@MURDE}Gt0SnQd{bz zqGY+O`!D&^SGoeSbl-pt%a?$CFurFQcAk=W#;B25)G1ur8WEMyTU+1Vv|D_))u_fv8^`W@*&*irMd?^Piz z7R|2Wo0A`(o8J+zR3O(N!O;my?%T2KaFwKO8GD^r%bw}nY`+jzm_9W_a4YubB31G` z--CV*MUq&k%VC)@XkVj%>kt_xzM5`jxhqF!?A%n$sq?eOZbfWU#A_QWzMdCqdA@K~ zNUFxiYg89`PPrv^(8sa@Hg23^FH+%%WNa$7nX8st)uZ{&c)ycmb5x{iaJlfa$sw-S zeniVlt{4mg-Vo+Rt3^Zh>);o5otD%0I;j%|)(@?YZa&)k4OYMXhJ?;|5bH&lB%4hp z%TVRVA8%fKvF^3)i)YfgurPb8$38l6P)%oxLhgl}Hd1)=k9AKSNv(ywVU7dhg*TH& zbYtmqK?4=G&4`A^^VLi>UAJwFaHn_Dp13dk9v1&V>R|M~oy9H~N1e-;TJKuk+$K@b zTaQ(%?`jF;diwdAULX#hS;cv%O4DqCHoRu zcy9yCDJWJiy`A5G{!2{!vA=h3>gTPO)5{XXchylSp!zSn#Cs)KPot!@x!K=Lo+^)uiUe)lkTcuY&0ch>QP22a*A`bdm~AH+MA zG;A80lg7`b=GBHyIK#xAJ-s0?(l~}GFjA5s<=?Q-rjx%~nj4rn?=`DI5ArI7N5!@( z5+3XE`w>dzd*CIUOS$;<^AF*(r!H}opua{!3zsqMLBx)gzgX_A?|5;)-+K+PdsB5icFwgIm~&}Z>yAd%@O!%1QlSN|xI$ivBrl?)A{YY22Hd~g|DGIh zGR@)E6(Wx5E#)dLBrLg8msC+Zz@7i0UwWeVq0R+~^Fgt+-qJ!r*&>z42NrmUo62d_aFbX0a*+29v~cJizANm74GK-C+d3Q zN>fm_;5Q$M7a>7K{ECa?AdffaIWI2*yo>PF9U^DDTKQM}G`YtSj`wkcuL_+Tyo`Q5g z2TB&UMX8La3`brA9^dgwYN--+UkW*x3oYm6(o~M}-7f%s0j*^CO)ub(a=W@cGCL4c z4S8p?KY!cc=X{9z`7mJj)1l@wAD%g1k#0x9lLUX{uZyU&`M*<2c+`^|jLs2&kXL-H=uA)`zhg5h6m{o=#$0j{Jkq z%V$`xw|DvJ-C42X&!w%dIPqt~gwp1g-${`flHSgQy+>WSPEy+TCgwhiVQ|{gVP96r zty3}6f0!qP;Hm)*QdS_jY;#=t5w)w#0|P{8wLRs+L_+-B@ptoFz1~qbd>BKjo7KD=A zHlGETf|Y@I`Q6V;-(Ng(7^Jgg(9#68XZf3RN$^q&-8zz-kzvoy@s=1To^LA*2@6|jDJDY-xuS!bk;=U4K$aJsTWk=dmE&cPfM{{^97JS+@OYzp#GUI<_nx$Ub zF}BJ@5>d=GV20!vK>Q)Wm<_ms=5KtS-!-5{0?I(EqIc8eK$fZwb5vPqg&opINxJ9Q zdBbGu34VmzyI&^u6Vc}&DJ5WlJopv?NYGwro5r*(VvL_FoVEAFBb)!Y?J14)(P^D1 zA=+$XjtE4QkE4f=10*J|OO5FTe7f1?tT?pdKH38yy?-2 z%dz~!cP%?4CgRs)KRSzES9ni)U5i7|Sr-F!lH)x5_caZ{nc=C41L_OCcK&jZX`ip{c+MAn+pVTu&c!|S4`5+N8Z~?dHpn+0F z!;C|)k9DM~CRx@UbR|{RD-(b&IgcSkT)Cb{SdSqFYD77yoYn6^rX^Q z$>m62>@~fVgxf3+F&WXfjbns7UFiXwrQKpZe4ALk*skhIP*81#~4*@7Y-Ys zPViB<+_-R!;s)=K83=X%xT*k7HbT>rgA z8%l);76hWm@}pI5%9KxAFUvX1coBsdMUgYNh&&1w(T8-^cyF(7c4BAi1}1AO`77>h z;~hm*)Qzf+mT1tR$T_uC>`gQ&WzHbL>XR`F!;TvV*>-!DJR}3rcK5`cwN!1P|+jR_X_&c#i@(oIJ<%hLSCv0yn%&P~P1%{B!cNtVs3 zqOwAjj@YK0Dra-RRx6rTrOFdY2}Aie_70|N8XqOz3!N9)ctj-=k*8f_YZ00WW~HhWMAUlSB2NU zq7~O2uh~;Df21U%SgwT=DAvCy2OF;`ml1EOYCi&C1(ns=Qrs%qD#>&vePVsWAypue z!#l;*(N8p=chx!1$vO1cf$r%0%`6GeP4cr~s#kQ9B8;0gGHhOXt@&kAb5i;@VV2@~ zt@{i_UsO1GM{=L?tQTUZuJ+aEM{PcJy^qzAG zU9(&?iA*T|Mio|w!;oZ(y{C4}c$P+xQR@Dk?{4JOUrUq}Ng8Q)J4;f29u4af1s^Op zm)}uJNR!EZ<=czCNRF)0M|Z#+mxCmvW;E>Tvwk&HzP0;KZs#Sv^dKbg@+@nRF4vMg z{VmIkxT1R2V2E46=S>NW{7vR}m#dPL^|vO&x!Mn3i~is&Q*Rb~rk8>>gXjGWpJV{z zHVNO~km7=#MqxvBoif})34#AIe&a1NAS4&DzlpiyE|9vSZysueFV%XPA8@ig{q-e0 zl@gI? zL7_I3u(|xqv<=9gK*8j^2`DHLgpQN+cU-+atQ1Db8Fg+$r2KE_R2J-@KVy_vhoic$_AsZrrW-cidWzS6X=I{7p$I}_=| z8>JN|osyoHg6fkOM}KLm3Jq0MS`sO)ZPY8ETJ1>}F7kQPN@^A<7O@Q8%?6jnPLsCo z`sZ-$DFjY#Gk=x$YuZ?g;H-2prca5E)cE)j_{iW*+y#Q}C7WMGbS?5ep5AN&L_5KS z{@O3>Q*Z5pMn#Gip7pt^6ArDMEOfN-FQ0r5 z-A4S$!trz2T@`3)7GLBvI?yELmHm2Jt~kI9VGI%TE8CNJ>mS+#Bv`&+A{X7$Y~8~S z{J)Ab#|QiK#B1rX1*HH_M=I4{nQ*ALa!gSiUHs~q$sfFHQWmB~Ov7XL;gyp`WHwV~ z&4zv`q>d-_{g0_zc!MuFyh0s&8W^#iH<_DxeHPM+IlhP@Lj9HPv7Yvna149h6s$|vL=O6^2s?tY-red#4yNH;RVw(Z_T`5qh2ui z8_87r7?7#kLxTUJJpI7~I+P@v7mjis{L8cY8E|qSb($qe@k!Wnpu>={Dg_1(gjR+i z8wqwNk#^j^cQ2hcYv=uG5>kh&=Gwdl> zT6Ix-ILA*{mUuNGr)}bu+1mt%4G9uSBN?oe=lMZRRJ8p!G`>)w0}=r6Em z)Z^kH6`6i)5MmyL<&dOKBS*^r=lsiwN5yGeR*dT8+veJTVQ-(-%DIiy%HsV??Q-jvcSNKaLiY#yn?%5U0iub7Fpo zsxzf*wwPHBp}a}$5$d-763r?5Ii+eG08P$7K8j00x7NMzJ*p7gnYyDZky4O5>P?3K z{MYb!S8UuUKKVP_H&#F(j$O?3g57_$_czwPtnA~sp7F0J566EqehOw>ASz5(q0DyY zX~Z&nK!+dTJte!x&=*G~D1L+V?m1!FTi1JVbxPkUz$A-qop+f<29MkjY6H_9a>wLW zEd85fe$Qg?M~~`dA+aj!l594M;Z*vI)EunHc;}IZi(hsAGJdr4vA?YSE-k%b+V`aB zrZf%;){M(q>7$5#zpKKM)AN8i?G%-+To1N1R-xX6tJT%+IyMVG#(>fGzSD4XJa5rV z_mk$PM0U*M2cX@QB=@6msCM}_7A&^JgRYBK8~ovl(QNe_qZIF$GYh2T?CIPo7aRp@ z^#hkT@R2BC60clUt~Ogm&I#VTV#neoFw|gIL9;_o4c@nn;7bZizUK7;D0Ks7y@nlH zQQZa>e3l1)w=-p6oCVqOXKErHL+q!|;@=rKPY@Q=%D51&bLF}akDshZ^?VkTa+K}P0y&rZCy{2sJ!AMTU9F01YMLf z?EG|d-uU4w>f254EMzlgK04D`&l{)MAFY2&$o$V6g%VoxJ$_F`XBbbK`8(;v975T2 zOU9t-FhVvXeF?8^Dtr;xnTt3;lvaV~v^MACojUBASyYO8qtyzgCX}A%kb^Pa&A=Cq z^%MG|z%85i0M%akwaAPzfXj+^Bs|K>=EFG~nIQ+?lo%A9q1Ayms`7POXb%=biNe2r z8b(=n#$7X^Vv4XL(EVHBvzuh{SEgy7rPo%|JZ7JulTfSI>E@|ja5>!f@j5epsw9er zUenfADrIyGIx*kUb2qe`-4sX?(my6sSbx3n#R z6YkeZ&AMu1@b-sch`?u6Nqu9d`#QRWg^m*m(&B)LVEO30G5hiTD=Goz16{aMiB>Ls z(+3`?R@@ySc09{xrR&AD0QQT;ElGrq?Ent<$ZtgR8i2mN2B2OhK=HgN*4|TEnL&1Z z6RNL@yvqAtLTuH-fze&kd`H~vDNh`~mU3ICvs!A$$|e7U93ZSCoNKx%YWP=+&nu3Q zge@k1^wqy=mqJa&4GJl&qWrSl2Dob0yLZo~0K{V!zo(BA(?u&if<+u`7Afn~O6Mer z+b}P67)w+%v9ZyfQ`}e=FUZW}4MWKBIqno$Ih@?87f*bb;+o|RJ4!9ezCOm4P{;UV zf491oJu;;1sgNRpJVKXgb=o#pgNO7vkk6r{BG&QWP24qj)Zd^lE1SQZi5v!0EbVM_65@n`$7aa*6G76HY|cr%At25~4|`aZsm zgkd%yT#nC8p1igV8b*hw=xN1g=d`Ry^nUhBcLnb*P(I#ph7N`WEQhwxG_)wS5i>+? zdfgX6!`OsA=JBh6uK}Q~{DeSv)8U1X^8G$#@O6~wez1|WaUJPWo*x0H^9twDY>@nTMQgO(AIsoroUoS%zr3%P zkgxK}MfgkiRNL3t$l;f`WTc+eynar;Gae0jpTKw)HslmgYB>g$2oo82<7aD>7mfgP zJ@Z84Y}!j_oGt;{h|;=rBJlYd87bJmSg-^%K%Q*rt?G!n^Wgv5qcn#lqy{V#m4{iJ z?c1xavLKnjPtmaaQMp!>{v(r9+^Ka!@!hQlixRAF+VAqhv=A zQt<-``cvVu8#cC|sruCesPYV;t`0@4OE)`x3V52a5f-kre=&e{h|dZP>-6o>L238h z)HBAQ+1(s`fsPva4@7w|#%IlFviw^&5NTQwgkE7oJ}#7MFm3kr4}xPu-hXZ(7B#+Y zigf3QAkZzLPF8{N7mz|>;Caraf+TNF&f?wPJA+!}|8tO@kT4?4w_}N7lWza&3cxgd z@ZRa$D@^0bFy}~W5z27r(R5q%-G0lcYC~_Bivio~c*ZzwjN0GtYj!U3V8pJDLQoI+ z(@4XJksYvwGh7yb*aQM27mBO4t?UIvX;S1U6(ThIfxGLgT3J_4uRo4hMctFBAfdFH zJ(kTZ#q~u^PKC8x6SX|7deA13s>t{E`PyuBc9)K%aZoLnMK|A_(b1qVTADhFJd7Nh z*~GJ=xQ!}?>Ngc)TWMx~D&uJc6TLwKU*6m&^W5NN(iMaa+b-hBR4~Kw!?6m)XY1_A zB+ieh(9CBv1I|-}uC~lcbOFM3V+;{JC`-h&>V0)b=qVt@JH!gupBg$ht=x3&AU@rS zLFz1*-J4opaiIu(t0>s5`0Raj!`raWC?-S}c2o|`K|&(oh%lqG{_@fsZq?ja8F-!1 z*#PJ|`4m`A3fiBMkidDF+A z-t*PKN%MvLxUyi&O(=k#l|1R)MCG9@{MIHK{ zT)0?X#`F8s_ar&i>f|bN^>Qid}k&0rdi1 ze@z^qLZ_F|>X<-$of1Jq-w++_&k)M%K(?*;}6v#2=d_=&`0a)b_MiczmIY z>DfNZb7>#&niG6Sa3QJuKwap3^a{M^V+E;Se!}A6mk5-gw;#x>{}Y{Fk&hl zs7py%+)+n_N|eo)g&cHP4(p2m#)%(iYwO^pOlU=e9YFg%#2);1BY7bMVYN%^Qy_vy zupUq=!`#cCQ(-9}03>+bZO?<0Wpe#QbZXEEGs!8lg_MU7W9Fn#L}|%WB9Vn}P|nW6 z9aWd!(tN$NP8&i-R!Ff-y`cT|Xc;miP3i8E=9b9A4vsNf%^PQ!yuRDy1YX#0WfK@T zLl+j`@l$m2tpeB0R}0b3)YRS5*vn-=c2csH$9^~5GY067S@wziq^3?frGoEWjbm9z znc8|B2aa6vtw~J3J5PG&Ddr}0l=?{bj;bw;7rttxcI;--6P>{17)M9DV#A$24yiLZ z&usvr*n2dw_9C>ev4{u_zw8w32C8>7aI4H7cIX0d01Jk8&ozD)259OTgYXvKhTE#M z15b@P-IVwTF4bFO@34>pm@m_HPmlc42P~PqeoWB1!byB*RkYrvMrkFrE` zxQBK$X=Q~KbKSbxC|XPdV3YspZZqXh4)cQA>|$UDvowjSL+9<{7G^ht(GY(eAVw=c zEpMwGcu3cb^qUDiq=QE)2PYDxiY7(}GKog&es zImB~1+N_d|KgXIqpP@SZdu&hXxRhD}JH>~SF_>ZyB8o$SP7QzeXhAyML(*fNQFX@H zMGM0>9m`vqoGZz|Jm7Jvp-sku8S(DX%hTkceV3)!D5G)g^RmIUj(M>;j!vm)P#h!O z$nrGCfLFTSB@LXwVyRM_FIrgn8;!%7G*kr0>R+Ti);ClXnRAgh1jPrTzakBxa8llU zbZ-+EA`)vQLsx8d-p$YxjLF8hqCRI2%6B6b3BQ4@ZUtDvZ}lw2K`xZ_f!(U(D#NK^ z`XRaRmu>X0)wJ7YBzX&FBo|D(REs%}!F2}3z*^GL%6fD+xstfBh%%bFp#Xy(!`pA) z{Dw`#9~q{NNj>Oaf%%Ae_>Bo^TV1vm92=j;MCAMH(T~iV`1!jl zDf7Mk{dZpk^1j-7xnVVJSaQG|A9t24>RqCVajEccRlv6k@~Akhhqx7OJUMFGDeP%4 z?>ViJAv+A5V8>&(icIO;`k0#247`9N|NV`Ndj2Z1k~r(<(bgdzqOkdUr5&2LScMbM zlF3$PR9h0SzIs3Xhw{7{%5i?ba_H|THM&ji93QT}aXG;t>SdGc4O5M9YSKF>-hlz_ z^oom}Z-jsE1JafDR535ujPSc!4dkPz9``It3HzgX_<0)Y{obtgrt;YCnK0WY!|&$A zWlujJ(z*XL6~HFzL7-9W1^eN*#j8W9in(S9O#0_Cfsn80A04L!c~lE?N6_;5c(H6B zAy@D&++DubCkw8p=zdz5Y0T_6H+Un%AkaHXhP? zMm=VH&al=(W0icfYr#>Z&4&4^IMs+-tdwA$>U=;?xoyQ+^1dCmZKW&r4Xjf$m)|ta z&UgZCduI=T`Qr8H-Isw84f<(K=&$nB(68)3(EuMt$#^v+Fhz~m@wT~G5LboeoeWR( zn+|B)Vm3yag^_WuXhdTrAuAY7J)>2q2c`aR%}rNWA$G*Yw~Zgh(%#{e8n1m_(Wnp| zQ?9yHTT#ibO4Y=>6JFY0XQ!Y4 zNUd`;Hab!-jD39mQ)ZezQX8V@gg{>@7ySuy*H$-S&wb$QR?(+w@{DonMh;_%_8euy z>*>9`cFI4J=^&xQOTLfz^Hc~2yEsz&Iu9EY)CHRkA8$^{ef$ev4DXB2k4BK5$|)W;t7@0_scq>_0WL~)v`yWU+3ON zq(3}9Hoxcgmd+S?4tJP22#r)SI0SlucvQ6lgxC--8`CCdLSl0gU_z)NW9s88*(`;| z0>b?5ph#{nlHO=BZ2}yU>~n${D9L;1x%!;Cs6rHXl*D;r><@#29mdrU=AQ#GC(tdQ zA4bhpp6G}k=NIHjY#=JW1fnEnOy`UF$#B;)7g zkOW&{K8DRCn^9*nT5nMRp3j(ie}FMTY|iCBmUHqP4g1~96=p4Zxnqv<=neODg0|a>cP<}go-L0pJ2m81WzVOK7yPuoir6O{xQG@2G{3~+ zem5xBpcun?r8kQjz1OdrqZQ!r{--F4iwudzRl|EWP8X4-%aghub+#D*N?i;oJI@@R z3o(+ecH+kr3BT?I)iO5KeZosmcP@%}f4Dd*&96mLJl*^JCooTOTHVqw4gO1 zoc94~nZoQM)sx7m-wL^-^&_EEPtfvr(+s<`-UxH)*!rY9p)o(^wMyDdtyjwmmv-qC z;|Jo6%m-g6gXi#FTgi|ZP~p~11!A;$Un~fsfE*Hek~ITr$0&aXU1)I$Rqd$O94X`_ zfJFMAniy+lh^mmMWpIQk6+kx$!B0xySJle*Z zewFA(flt|^Ls{$8fi7LlCXv?Qj-WwyIKN4J3vTB`_A)%Ll#Il}Df;r_GUF*0bc(?1 zLheBqN2JEgSEY0;ul>&bMKVeG60mw)*;td=cx-2#Jetcr*l4q+jBLuxc!kEB34l{C z*2Fu$u(a)MK(BcDV7()2=YN(^0)6LQQpoxYYY)r#O*laaJ z1k~=R8>gatR;99lLc>dfr9@#GD->Lo0lz)PXVS>8AFEn7{{ju2^tfFU3PFD`)D|N# z^_$^X8Ia!Va&wMFwOTxJ6WOR;H>#}uY zGdV-I3LrdXY7Fm15r@DHB3Ti*N=Ay-)zB@pBj#YkyjcuJdhe;yy@;)s8AQ+j5%+{K z|1**c5;Ls#exu|S)3>Jg9ysl7FZPjoUDv{FxUs-V*;errBs1bXeXbunz5et7b>TwI zC8&2|D%Sm60d}xsvZ$O9E4|%dDoREKq&-rt;;B(}_V=@nN~V!=Y%|W_7AagVF4%}m=qK1lbX0X>4UB9pZEL(N2RbodXE!1c5F zE{zWSp=tB&yP+qz-Vg-(O-Fwo6X5a$F{Dtd#Y%;0~WyF(nJB4JQ7r7~wCW=YvC+^C1v z8QT5n&W{NvmYWgues;{Qa^98d2P^nU*K!Gj*#t++v**>PtuP^p1yVmf+XC}fwW zHQ-pJLSZ(e<&B>qwUy|Wp%%O}e#wCq@@H@ZUNq+g%22m+J+6xY-Y^5d(B@sQCJy!@ z^MU^Ul6!@ZI&TeyF}x%ucW^9tEB*Uy2`bpQWs$URz52cdIWf(ua#0{Y)LybL3|qSp zjd7!EQI5XmxU3fT6{9dX=-e%;(uxJVl1#pnAze4TEok1H zHOC+GSN%Q$W*WGOjvak*ma)gB{Xar6%fGI2;Yg!0)>qAryp=}v(U<rXd-Bd>Cm2vW*kv%3fuP}UfC{2Hs8I+ z`okhMTp>`H9!Od8y2fcJT_9hN9GOs>mOU*=5-qxM0j@(#!`!xcHFlO!RZ zU!lz*x%SF$e!mySwl$?5dCfX^;fxC^EzmL4?-NiK6iaM+4RDnz*fa0Q6*mWfHNaiO zh(9Qz$-LPIEN2gi5*@I`@wQ%-zCK&{5l@eG0vM^{68kokl23+Ifd$I;e?qrusEjJ! z2t1j#C7b#IBnF*o)jGPF(QO8Y2!3w)avoL)@aNDQ-(`OgrB(jBMm+*}kV;d=0Q(3Xx{OB+^rW}4tp2X z3xJ21(Ax2398K&VAeU?UlP z7J@nb*k%a*;*Wk~x5PB=RmW?;!0U|2ezh-(Ql>nQSOKTY72Zcq9RcUK82l!OTDZO# zFwQ8giliqb4t#j{Ix-GP1J>L!_s6`YG$X8BY)okEJ8MLQXy^FwJ#Z|lyV;QE#hEYQ zZ_d{7l&!M+UCR9RaTm#_bjEL6=>z1fF;am(mm7Fdu4CEaetCujxQt$(nIH8P`K%@B z`>c>C&@kgVs+I#(7k>tqh-jwD6^awG&?&Rii{S$q+$(`%Z+N=uj2|5W|93! zcoe*jeDaZm0bWm63V`a^&{HyJPoxxZ{xc!ZG2em+CKXIH1GFKC@)LVM6F|8}HKJn? z@P|Gw+HsWB8j(&JUf?X94*Dhy0W`V7BT@#^T+2xHd{<={X^@oP^;xNlVv5NS?w}MM zLn&Ma5`B)Uq|)JN0`e*u{v)1UcC8%RFah31`B47VJ@B##g{~4p&K49JDtkIe1^S){sz=s3aKV5?2+j#BuOaGyxnI{@I&|&*r`>Dd95V;1&p6 zr_$3FepRq;E$WPpHc~*_WPbKXW;!nhPib~YY<*Zn9wCp=fW-wRE_2BwWxXNIW7);OD5SB$lx!Amal{eZ zA1u=f?%Qnwbicr9T+k(pt!m_0SA-Ym57=skEveG&P=%j|G)QgD{Sw^xa! zE88p{kmiQ(2G8Fz_V3yQ1p>od-^QEH57&UDpJR=#;x;h~A3-9gY z)mxFgQY`$ka3GbOK=FpXGZWEpEF4a7hjft8{Me~C@kq?x%iTf?K^7|FJ;?i!d?JwP zN8|Uf4vsJ20A)0q4TDBlH%NUb!b}7^j9k?+8dnREBoaN6iwY3&3NPRh@LpBe$m41n zOVS&&h~XpoEO3)BWO`PZVH+nMdC*iNefy297-~%MrKpHIX-PT@>^+f`_ET zm7aYvuM`OoV+uFas>_YZivr;QgN}e&xNj!6cJl><5I5W~9DGM96|^&4!>_iEx`U3o zy+a--AOSUNUm3jjSr~ALBfd{tPDw4_O3PBXR1mMY-%oLnT&HxKr-NtHU*0>olcb#= z^mq`s2|2$t7<$)6gF|*6<;H@ViJ);Zzevcl-Z9NCe5i&yWrb1);JF8s7y?*9DOEDKg-;dGY1SaQiU%cDgVA@Zpe~P- z8=v`tG6n@)0pemSMWee}pv-{Lq4Xh$#kq_`Lf=q~{&<10S-|mQz!+LT1x*+zJxi7K zjqKTk?3VQv)Y#S2866VtIxKQKoN`8fJeheG!~CRnLf;rPF#J-mU)RX=;&Z+PHJ55x zc$vR>0qdtIW*%9|f*=%UIo^6_8uVB|T^sBiMV-(6%_};-KLitBVg4Rjy%ksVlNbp? z$|J?2_rRryeN~hWaefq`iH$Ns&_$af8S6opSz3M@W#H>d`Y-jdA2m%&0VpLe2j!_b zAbI}3@8^^j0d5_ws_^AUZZhY^hDJ*~gigxk5#`^Tc#A}RjN1SH zW2CR^k)pcz?vifuy($wbv;GnqWk4B>s@@oHZLXz|!_AEcyir?)Z?cdp14pUbRZ0Z=-~g(|8N|G@cd2BeYJl`aWUAgNJgAGW*BBb!kT zXRceLl)tO$8@LYM)F{7<1826h2$+am(hscH|GD{h;NQuS=iqABr}ZnTF-8~d=tRi^ zRy_*NM%U+%LLM4gq=Gppq+yN=C6rWmZd;1Ib}zd)TWB$&jE9Y=CM&vWaBPj*=uy!_ zFjrWWW0l6D86r+^{M7_lNx*;Q;WF(5u|y5r=3PLY>v6FmxPLIn@_%bU13(h!VBWhMr)(VzoO+qFwY*bpOcib^#~ek^F6c;hrpNS-5?`ev4~FuxreYA8=p zwL!Wc;kLGHSiUEET9{vIKt3_h%Mg_HO&E3!#q64K)YJK8{f;+pqrI?l7TbN8r0~LB zjg;)-dZrvvT*5dnQrzcfz)G7+Qizuwpg-~m$h4?Yi+zJ6lI4c$94eeFi`#F?kJkVC z3L11^fxb7wPh@FOkIIM-YUom%*&v}8=FFjdZLAJ)(;sXHfiiWc$({){O5$L1Vlkji zS%V$`B(yQ(gy#eADo8=0j|>6RNKgw`@_i?%@F)Q&6Q*81a_JJy^sJy)EoffrRTg8%8bu7*caxPX*~8M5cp*`YHAXUiGJ$ekO>7Dw*KW}l zk;CqT0Q&ImfaK=Q#*=R2hxQD$os?h3PXhTo)`$Q6KK)!fuB)bK=6}4+Rk+(G>1Hr? zQ}@KbczBIWBl#Hgi&zHlLWNCDx+rRn+VL^+TCn{nUoPF(o#MJKeWGcKq{Qmj(W4`(d5!@J*xvII;|wr6(ES1#_OYtia~EJSXc#pC3aRLgnM*X;_xxU<;Sw+>9t!JNLG~ zX$Rg$@rT`s^kTt|%h*meM~c>K3(B6vi4K9OljYG9eVDpnc{ zXCAmaBTE1j2m65gg-qpF-iYH(wxs*@lG<7?#vDq(15dF_EWAK35NlJzzR(T0dVmVZ zQN8pr0|bM9Ta_FDq`PWU(xT;GhtzTIn+$hNF;UkJgTqmUzdE7>bsOy2+q*02WSRu6 zXFMbzkf~y@CV_u%H^y)XaGrjs;z#JcX7Jd7R+}Lcd?BjhXPGGIOrHGxuxoFBis&41 zQ$YQZ8^HbD?mO<8XPC18=jlS)Q7xDBO4M<^sy#bH4?db%VBWcpbe^{8 z&2}i3xNoTWmG}O%BablHOm}d`yXHc~YqFbv#`&x8qbf=fI6gSC>`2+00KYgUk6f5G zV-)Cg^+BYcw<58`ui!<(5!FNBe&RIXP`LxW$^ zUmt`rt&G)1nIdkRhPJA?R&n?c?FqG3hkW`hV~6|{=tHoq(Xx3IYpH|Sw~~JcNlCwT zPSqU#1YZN`Q5t9;B+M$}o45^13R%2)q5LYh+LlQ(#GGkywRwf922qfZ`{rj;r~Bf^ zLqf&xB}QZDeBW#`wKo;}O$ph8&sL3*;T`7v^?$T=?eR>nVH}a6q*QXyCSu4Xom@(D zi4@1Ak`mIXxg-=(ZikT$=?J-m)|N0*BxMm(=cE|9S0b0>lFPzaGfU31rCOb{|MuBu z@8|bEzxRE9-{<>&pYKn@)OauQEJ~;!D%%9Tk$0r${p&Gby$mjdT-Y-b2r)~1n7V;@ zvr8R;IqNicZTD=fOS8GAoYk^(Qd^Cp$`ng-3xtpSb9(V{9z^>{B0;F28G5kaxb8d4TroycopoCVL^!7u-+3HidnKlg1Cxv)oiP~ywK`Gztvl)Cckk;qRyf}-ZTSoBT@P*|nbc$XL~N1Bhl zC`kKlx9qvWhRJuh@wtx@giu}&7w_7+`HOoo>XDDGb(PoF54YZ^$kpJ)XJCl$quoxa zU@oiJB!R8j4m*7=_8Cie|AvbeX3re~@28~0Vh`a&t1Zl?9`?*(!9a^%L2kYp+sHN7 z!^b?Rj+6N_sHXou#W%#=A<@6Md9@HiI-*m*>V0NInO}U(!Cs^U*j&m4g=)x zjay}d8A5;MT~F7gA9-EuMQN)2pFuVtGo(eya2@|{{>g;7Jmsm08ot``(h z9cP)NGC5^~j$XC?R)QX>?;oIb7H}eeJl08FF+5s6mhb(*HBI#4wsz*JjUv)SI3qIE zYj<4~1ZeEWwD#IxcVmq?R~Nv*k8|0FfNbqHTBPmkDSvFD_pu+;np@rNbS6z{my8%O zr^mZdZ6wef>pBo9Uah6K{huy@{mfmBU+*YE=g*eoqbA7PS`3xd7>hc|;(xQ38dS{VR(?aK}> zLDdIIrYr3Z{sQya&3y?vP{ei_^pwFivP)=j-rwf2O+0pyFD6g}yG|>dsW}CE@<@bD zYWgO38rPMsrbx>AM(nauV)^Oeb-tC%{mJ8co!&BNJ!fE!l#Ksp(`zA&SYstE?`*@9 z;2;+x-dzQmyEjr|-I?pDJEW1v49-lqI-3^C8@@}5kPs@; zbV55*x*b2y4W!wf9Ag;4)8(%f+51=YMXi9IRmYRfJN4&ZkuNy`&jf0zpF=Dqs3C-` zn5T~A^^5e%W=JY*sp0A)Q|$MPRbB{Ay*PtnRn5_T{oY0|W!!&z2Jl9m$nirL&VSkY z{CE8PBX{B->Rrh($DJcP!CO(e#Jy26dl$LdpQ)`jX_^ZL70GYljI{XECBtFmvayi@ zx%-<}StoFDCf_(F!Xu0EtnM&}z#dQ4gVRst>fdHnn(=)e6U;% zmy6E!`RI`K-e;}ho(z58!jM*;#+)xaU}Ea}7GC7D_QG^1fWGD8AVIzSaGZJwDs)-) z=9hc?`l5v9hK%PE;XH%}0umXGp2COYT1w9@rQycfbp= zZOvKsfSCn3=QuW(=iT)^aGxxCQ$!@PSr|@@Tv6p}jbq^HH^B8j2OXKp;9)Dv*Ri$- z+B1Erh6ZG^FYc`~nfdav7l69Y08EbT#_(E0_*Pn;lJFzJ^O9>&ubcu8j(N!4e2`c! zz6m^(hYt)E=&Vx0gZ}6~Rh!~&4|m28y``E><(@LR#14V^n`#JJOp3F% z$vSX^x+#UEB9-;ivd)ti^3`_Pu=hhcnsre@wXxwEA$<~b0g7`RmW-2ke(Gf842jXc zIx~A^rlhQ{x!aJSWE!!w>yMD(@=>i8?=)xJ_Vr?sKPLq|LvaH1r?mRvOEjgu$W%)p zt^L zEM5l*UuwgPD+wGC;o7~7m3I2v1wR#g=I40p7hdOQ-lwp^PX`Aq7Brln4l=71Ief)*NjG~@gXWsKedD%1%P0-F2! zrn*C?Y}i2Di2N}-4m4A#MGo@EeCE-D=xu<04s9goHh3@y-7Wc0KDxNdQ8dpd#`d

mgdwbss$l=}3<7Ek5r!?m6yG zy-MTXPj@igYC!QdVOOVXm#hM{5iZNm(?3lxR2xXhG6f+$fT1(`eqC(8U-;#V8OXTw zsvg>eS@coIQPmFac+nJst4}kJ?{M zhet%^hPSM<_S_^dUgEc6ezUrL3fpHjbjbi)pF^l~eU}?t-R`2=295u-oD{MpC7OJws zN^=xJeE;*(T>Gh={ln8;1zB&MPn=3nj%2}yN~Cy=$Py?#7-$MKfvhMBx5mJO<1a?g zwqxjj`C@0$U}N}3%W?718^4mcU4of`e}NeZY71C;cl!et7-`UlwEtm=5{=`l%2%(X zaNTa%Qw1T{?&yhd$O|p7k_x>e>l1}K8q9xnb2L!ItJ)bA(0R zle*0yb*Clx=nDZ=(D{7d4%zKw@~yY$xJ$d;X*(ZaQf>-g#LSo`6YoEay2M{JgW;aG zyr^nBX}tRdXW`(x_ztKIH_`r<4Qh%V-b&y*bg|wvmClzxiaulQqi;1C?@SG-j793U zJ!Ic!_Z|~K06}TybuKGX-TwUv#|j^Xyon~8PWdP%!_KEbCJX1jt&Mj0YAIRY-C%9 zE!e+~e2x@NJt!I(50LouJEG;WV+3^UX`5^O87IvHnu#=!1rx%_caSe>?)m#Yqq%$k z&2tWdmzo$OwL}VNTwjG$Zp(eeD!pq3(r$JsCEq|n7o#*oW%{aVhpB_a+bsOWU7%SW zhBN(E1XWB11Lq@sKoIHcP(C`FAiO(4{ zaJdtP-_zVMDCVYo{|>2y_9x!S_j%u~=5Eia&N0q;za}gTJ@FqEr;EHT8no24}iZ zDl`T9b`spEutjtHtdhAp1|&zvkjxpbs+9LJ<>O>7^o6>5jmq&0>P$A(^a8zNZ!jEs zMt>OooH8FJR*_OURxSRS>$Bv~uAgnqpCsY1Cd5f~=CWqK=u_&W>UZZ}A?qQdz-jzR z5sDS66bf% z`<3w)+kK50@TY-UhM9ypEhCy46}rnYJ%j>MHCYv_LT8?5xM!s9fo#u4UdCr87is6$ zE7!cJ(N>o36=&D9P11We{kXy9*3VJ{yL+`C7C*fGko-aUjPdM|m*3?bI2pX)jQVx} zDJnU}ZMoY=5 zhY62N*eX&WNn_SFR#fI}yfY@#7SpB%mXSu!jLK%_{1|@pw{i|Rgc?^E*Tg0cnH40y z9D30Dj5R|pqRT`(yc3(Dv1l;*UD0um+ahPr&Q?gO%@n6Id4h1hhpMAc0(3bd%VpC^9 zXZgaye<-MufJHime0S)_0Kr?t3 zeC`-RJRXxrSsr0WItybPr$0I;@aNINa;`T!Nd{gVy51EO`H<5x;BABHZv$w3U&G3wF<)GW(SmANv&E0x&OBlEM z_7eK`ZSS6PQ{#%Ga=mw7{AI+${5@ZtmzkC&KP`g=!J=Tp9I2&Qr8ow7Ev(-vz7@38 z(+km?NIa3eLzhXXE;T)uVCR1CbVA|kL!fl2z zhW6;BXzu7%CUpiM8B9gZ*nSb0@1vTA?Ac`5A*$;GGUn~(f&=b9--T)m78^pBPv&-c znMRcocz%78$PBI}3|H(=`JU?ZAT=T_s*wrK#T%#HzS%x&WYx8@R0dghV?X_R!F{15 z=KCsk7GeTk%I_l4MA9s%5O^o{jng1Lh;^LCJ(V)Cf#FVT#|nuu$OiXOW2iW+;jAV;G)k_c(uD=V$j$kfpCw2Zv^#QJ{s$EP1f@{{p} z`g>LW`VM7-*_DL{`>NCZ??r1bpI@q7m3J8zbvDG=>WS&l>e)LQ!#bu4D%FXL-OJq% z*2cNJxEU;`i&86(o+e6<#W}|~?>TQS_AMT+7Y&zaTtVL2wSvGS>=hqTtFdwCa+n^#siEywH5>2B7~ zyQUZK73Yr|LQNsuTC62olY-vcv0fj~*DO;ksy9tH*o8s3ZFm>E@@MS4cv6&BA`6b6 zmr4(Rg-WkT%UMK@uU1Vtf2%t>8$7Gd%nakjJ|9XcJ+4P6Q)ZaoNN6;0)fBBDbfkCzX%du5tIx%)3kyFScRb zy}IWUb7Kw*yuCAFmF`u;dPj{qK0l7-#3$OS=hfd9r$O(!Z*DmE9Ln>)wyB&_sDqpj zpJ;E^oYv@1wm0mb*?gU9^l;oJ+=>^1xvLz(P7D^ekD9h^k1a%x9vnRDocC`%PTNU? zdOq|xt2O+}HC|D^Y|>kx_ocyXwY7dJ?MqtV_Wty-*Mn8=jVXBF@_el_b9&mbB3$82=kmv;e9Oq^ zlH>FJvvUR=X4hGD&|94DF4Wm>Hjr5!2==Qslv4?EmjYh`;!O#j6*X9d*n^><&wi^CU8yY|MjF@EGuM9Jxi7J@x$R?#49Y|DW$(YdY*d89 z^Bq6t6U=XB{RtG$Ur4A7;y`h$Xh``xesXa1+no%xBudwsd!(mdOy>%e_ z2r<@>HBnRqJp|rkfKZXhKxn`_B;X~2ME;-mQb-R#DA%7OgFu1iAk>>{lz_j8p9tWE zxaap@l&Am@I`FSMz{@QI`Onp885t;l-Xnbnz5|J=iOb3Yf7J{fjE!v^&1{_0XyG#x>py9|gIlC0|e9bo(s^XD2)8j1=6hPKvh21d58joI9+?GWRDgxmyxx7Nl^ z1~hKgRyK|TZo+ieR|o*_5g)VD(Oh5RWGPIip{PP5ZtGx7!^6hG#z7~7MMFa)Cp!Ulc2`$dHdih-TL)A2$Nc>K>>QlzoSdw{6|9aB8z%!dRvSn9 z--G-aN5a_A(81i!$=ue41~IO|Yg=b0VLCd*gZ}gLd!EK_=Ku2~8^@bv0SjbDe8c{j zjf4F^V*|GeAwCsQF?Tbz(v&c_24n_2L*xlJr_lBF|LdFodE!6r)cBt}A3u4*@y}cT z@zuX?ec@>AAZ}|7Jkv?!fA;I<=6`;9bE6PDV(I@7#cw%Ze+md%1WSnhKbt0km8+t5 z3hX0=xrFj_;4h$M*FST>?+3sO@fUd6at~^87lA-vkgSB*b2p@oDNN||{j1j9@Vauf z;rru6HI?~=K1EL@H19}`d6(bSoTz-*#1mXD=^cm9y1>+L5*)+BDi`&@68jnE=}d3U z%rPIA>ynGfQdfiN60~RjYgfvC>UPS$P_E-LbolKzdsrHvY!qNO1=A1K=QxM4DlI_@6x^hYh-+x_aG!EEsa&8|7&`{ z!VUkgk-`7pEWC|Squ!{>kZbdS@J~w$b`<5y91}5aVAP4@Se~n+1)z z&Un#Z3hp$cERGt6$4*Y_EH`C_iwIP{LWO;tI<(^Zt&YElM=Q;v71Bj?r#-fXPiNoa zDa>9>*fzZ*@n@C(hv5Axu{LUEyj80#Uhpr*sAY-0z13f8AkKd>`*z?gQEW8@B=ErF z&Ed^P`&D9^eGzhAd!EQ+orIZ6!m1UsUE#Fwv}%&@evRDpgLm^`@%gPUVpX%{+8fU| zSk7010-(~AV=8+YQK!?LxBs~wjNIm>bl&Z=_wCl&#>tn5M$?(NZkpAv6Ca&F+Gc+w zdZiG@rvLh~j#KNiR45ta2b;mqkxwj#6`o}E{0s&OkRLys_+xvYKJK6F&M(HQ>qU;} zd9XfRfQjSAx-$*OM=KRF9V!dS<}baBE|-oCHEY5C=M=!r?0&YxUi6)D?6Lg&B2oP5 zU9q+4o{z|RY=TxUFZO%m-?KMrut9ge6=G##X3)EEvESGWO*EP}{7^lS!{80{Y^{LP zy#KzbTIH>YY?A88Te+#Uhik*}a>Nh$JKf&=1r0>+no0D%X{mct>9>2lmG#K3K3(i} zviDe2T8;Ii_Ko375rPHlW0`IcBO(u(oYyAi>s$2BAOd;`akJE;aZ7go-EmX z6qD|3iQqelb&x{df1;34ZF3l<;6f;K^x`{v!^I}sCfcT}%lfg`ti+>vYD}yB+3#Z> zB_=L{Ff%GHW{vlMFRIN1iIO(s>ptwGxMDlb)bQlI!{U<9)T8(s8_O-3)b5GiYTADQ z9ry_I0ua&O9E+jx$>(3?4nuMkeq<6e_N|otwT`BJO*>s#z&=GE-CYm4k%83gcGFc{ zE47bKo%1>)InPFZvdinuI*0AeW0}soMJiAKp_oXhoO<9A4@Skb2gi7%H2WmpudA0~ z;&`!0D7V_UM|9{tHp-->tmu)Pq&65|-nqe3dQwX0;GI)uQJxf z3BiGjD@KC9C()*Ygp+*8dp=d~8n2kllbDP>J+#VJo7>s21PQRYTM+t}#@uHp^*VPU zjo>A7-wFDMU>&%gvM40>*M{NaL*`an7ppyN2`Nc+tF1H>`RqTX zxXlIoeWdlp?J9mHd1F1zSVBjmh7RjRH64JSW~siDns1N zyqWeF&npn-AqpO6sj!iF#RN zd_bRRJmYm7+83qyFAw~37Qj3%2L7Lodv6k*ms4FkQ1FCkBy~xWK;v7qf0QLd2JP}> z9?yKDy66KrZx3n@{jJc5BWppTRe{}Ke~k#v!TbQ(GOu0i(Kec>w(ieSB!?W0nKCdj z<+ky>1=Y2!oW8o5V=1zBjZN%QcXDzvPMpx^EU56Vb)K3aA9v@5`(DsrTW|EH`2sCn zrXsG}Xumf(TeHH1)BD10%(e+0Vl;%A@!mfb?~iw8DYc)MGVjj{ex&n71y^*#F7fE> zNxN`k{{Ipv_zrTXs5Yab){{tT?+Y=Dkz!W0T%}lXT>ksPX{sLBNKO{sQ1TmdQpHF3 zCxmh|p3Gr0yE^eF@<5Ve*wty@Q9)S#wNu0zRqnA*y0JetX@FV@k_QBG$w?u@H? zkG&WPs#8fg#s9NUcSlPN-VT=N>%T0~V~u5b(Sx0yE1yW;DZx9>{r9L~EadejU3SGD z%SY)xyS2~rpUWTu|Z+3%A0Yf)ei87JjF^9sg840biwd5Tt%v-h}?`8^acSg$$1D{w7 z%9!+~SxX7|a5z^}=pLCVZ)X$yYnBekmQ8vrxli~t8?SzO@WT%VjK?d@RX<4tF9^ub z>Y~ChU-kWog;WVo)=QeQ0LgGd_KcVzDzc|Lsd)Qq5iewm)TaNoewta?SF7a={j({z z%1@WO9WGCoJN+>Uvoy+Hr%fkY(Ic-IExJ+POy>S0yVJ$^PGHi;+YQHYiC3x2&g+Bn zU+|{?0ow=-i-0yyDvQKPE%QWePSkWF&=fE#qrFs!8xv9``u(JvfEK)lEFOp(%cff= zqofoDz3g9iFo`zFbCtvUOZ!2{nAlKm=a7_{CC|PlRj-wgfeUfkHnLyvN^EI-j~+@2 z-t4=K)~mixlBzW}x~KcSqQ;}m1`hJA+R{X_~674}#q;oZ$;mzY*#<&qYIpJ^?J3nTZFf%cRBy>0#FgFkV>A1_+jTuYM9 zV%xyQustmlYy9imyYlVEbH*ukDDNF`YqWvLjQ~&RGI%Ldo|s3)h~_{aoxJ*p(2K?c z`9JRXj~MeXSGqAQ(^SQWi}abS$15KnZ^#z4UTib^_Sk--xLNFu?5l&7K8MY!k&ZNv zEoR4@-bk_`%d4mpc3nW2-qvb6mYb^y+$$Z0aciFMO~d(SIOT6s_@m zOE6EP?32|*wPFD7y@xf^u=d`q)qiPsGn3yhdg>T9-TH(k4c!L!Dhl_Nk0OVUo`N>X z87OZYxUcqXc_&*miN5DECH~dtknN)5$@gwyf5iWLg7_SMTsPJU!mXx0GO4H?; z{It)Nm!slxtHmAg`9~S*n_Zg4Sc#9d7|L%CA!b>qPPT~_gQE-nJAR_U*YOkF>#!3_ zP!F3wnGcU=kdI3clguk-MRLl21Es#PbUM@kT*`;Kbxx7}(lK$pB*jr)M`KK{`TvA( zf85p^=2GL%yLnpd>W%O%ZBibq1gEV@{zYO2U!}wd-2Zs1i?j6-hl4)pP5`zSPy2d1 zvq`GT-n&Ws8RjN_0{M}&hQ_*#R7E4=Um(O$rFt#_N<|V(eewHJ8a32gcvO#Ery;q| zJBt|Ki|C{<7j(P-OIlx0EmF1A;vQ+!i3i+{1Y-87s{6aSsIcJVUEDtaN?U5!ip#(D zIddU5-8(A&C-go^6_ihk#pIt68nB(Hy00cr;up&c*rPd(JKEIsy%>Nhg=`5Yl2gAE zl;B2OlgQ!1M+r_Fqt70y=RfLPVvQF*8f|GvSpJuplhTzsE-U4!7L*ri)=Q;%?#b3J zC0R!4IRa70s@osWt$_r0P%V2cxu3k-7ycm*aG0b2OCyXjn$@mIS+D92^#NHCYc_h; z0wL3*ydoN3rI1kG|ETG!y@mGvPm&@mb2pBK?P9Hd~TH|ICE|X|$>M zHKKUPZ~J*=Sx;2&O*-~6R1u@XWXpE{#80KjP4J^+gi)0PahYbt0SmLobItl7>6@+j zlPTT4d*aY-k(Z*5?T!FrYM+Z|M>9TG{xtz00RzteAhL~LZkl_?WW?NHv~sj-E_7Y* zyW_t{BH`QY8Zhda3^(dZc--IUb^gBx0Wzfau0*cINZRnW!x1MZ_BsB4d8h#F5AF_G zlMbz{{a@@2f&5MDo3=P8m4npwPZG~k>J4{i)~+tBU7bDaZHiT8kpJJNY~=`FvZ>{Wlh9^D0A~6msr^Sg@$c~ik8&DN?4L{7H~uz?Nzm!$_cypiP$IKV zRK~Qc>BT?Gl#*=?3x+SUAUU~Epje^(p1C9)tj^hzl-DLji09&QZ4SIjvK-g*!M~Q} zj+RET@9(rR`3&&4^oFfriYqM`8H9769^YK}1r3*f0e|d^eWoYRHW&ZVXvcrq^*esC z=_L{Qhxu}eoW@2|by@><~=z92jUtbxb^QxrbC@Ng^ zQo~}nP;+5DDwHLos!7j%l>uS~6L*+*<~5#YELQ}c4vxfLNw2Sl78-wdcdD;Qu575( z{SJMj_oV4_ZQ<#fd~W+{ZVFp?xZ5{~Zh|ruIsEhFz&h1_WF|EgD~X<6q%F0m`JO~= z%}7H|laKBp`>N-FHRhq0_xir^&{{8{dwBhB45r#bLS|>JDRIZXQ_@)(67Q6EbeEfJ z-RfFmYg#0bjUQ)fpIz(P17yEIDV+F;?FQZPDodtbk@Lc3ZXi%lkQ7H~ke>qKsAI2i zcRCuY9#q4@!GY8I5T&W{MgH^1UePO^yWM*Chw{_#PY#XI06F+65NUid^Bc5Q;e9^x^{6*wE+$Ka5|&a7^`{DW!+4cXS|dF!9kV z_JdC~2Ag{&*vsGogR_>6ra29ZaMU}o9s=(PmYUZVJIu@wvB8BBZWK@9`>E`Lh!l!1Qf_C z5|w7}0aVChQ>k4l+cVZPFX6TvmZ^1^mQyAG=WZ$ejxv#mOOYyz^=xd4RYF>tC=ZvC zZRPftx+Sj$4@1?bcS;Q1ucMJ5{2&r=y8E8_-d2n&{kT+208%pf0Z3p<2Nkw@H^&E94~=TZ7+SZ{S)n;6P~I{Jz+~YYUkOIW zd^5*&GDNeFw(%efP}fc%8}?VFxZE5Fq;i)La9+F5tLG6d^yGy0N+5^nT7NN+TtT{J zo|32bE#c)rd@*2hn5(}ub+DBXbhJeRr&^(JxOKo&u{NB?w7=BL-RnvMww>eFa*+)e*xcZr8eQNGK$r;oyt*;XNz-QACavRiZ^6=5jXeaed~CN z=l%u9kkVm53{d%sRur|3n%}p0sop6pb_7%lJ<~}n1E!erD9E?GT{lK)1zoqE+!C;B zR_*y}awLoPg0Oh>P43oe}}Fz znzI2tylsSrR5w6RDSRjm+6)esx<@Cv5X4Q^uaPh2x(A0UG3J;0x*~_Q$u^(-dACyx5i^n?0T$f&l^3Aj-;taHJ8dXqbnAM z*KH$>S>iU282z#D_QaFhH1%P2Sxci+*!iYR9Z-=s4v!WZh67E4o~bk8)rJ6_AWWt9P4fDI!-8D?dVmP8FGK2=wq#2!*#AC z{&wV2ka@;*MUmXYb)As6QgLw<=6ezwJl|#7^OxYr3$dp9!Jo@?OT>gVOU=9`1kb;9>SK&n|BkrtGwy5~;K__k>nMxT6fGOjMqZC7WSnn;lpnQ?uN7WkJ_=+$x*B6oYka4LxTJ>{j$+aNg{i!cHOWHdjMHYdbq-c z>8;tVmmdwKk7TA%m_M~MHTl3qCZS9!$}l5<3J7@*_NumidCFwdw)XNR#Qfs@>C+d- zIp!cw0HwDymmm*w_Rw)%2pY?&%x8ECV?t7_B-=4}XX}YKby!33)k31^!WnG6f zg~YR*RmU|;wd_99(9}z7knYjva)+M8?(QG~Q-eW5;iGAz{!lN#6gmd3v+gbarzR-^wFZcc~>q6Ct-u{6UvYE2NlWX8J+s}#7Mone7Rw-2=r?4GeO|*>!glbHwd9B6d2ragKSr}d3O1bEt$a1$Z z1}GLUP7I1bvU$%IVrOBp(xUN#btyUyiwsN7PM{)DCgsoi-I(dzvqRuY@6@w$`0*wr zGtsoOt&cvnpIQsc=+}lMBB@QfJn(T`^1h#&3Q5}6I{&g1VyO^}-hedqxi2&2ctWZV z>41gsr)+Y;_~?(NW%+vf37|?sVbbv!D)h_Pl(OpCL2pLJEm+U3Md8|tfz+Q1>$bxk z(Lks^Q+WPU6Kb_p9z@WXmsE#hwk*DvrsMvM+%;nciEd2gm>=EJ?%DSlfC_$)N? z&}DCabmtCuqIWPD4=l}{OXNK7r=djdb4AyngN7-A4qF$! zdb!n3t|`c8>tCdV;(-?+G zUlko^YYh?7)eRn+IqA_eNo=`D^R7;LsW%tlXjhxB4*n|AD+*9>w|}kYwdSBiO;)*v zVddJ6>I!PGp5SRO@_-VyO{b|(%SxjzFD&qRnZ0sfIyDu@)SEKM;@RW!9mcP`c)Xa} zJ3U3nD(mbtmYHg@MF0~}Q%v*O<5$C*;5nP?WY5|nWfI82!*Q3ffo?I@^BE6lARQd= z=5&(ga@&cBr#p49)bGhmKYqpJioexb7AXUoVlN?Tr>vPKEt}b18JDD@s4dv|P&}WW zd{tgxK~;-sfQpVtj{PdJyL!)+o;5x5;{WAOxG!M0TFmfMKr433p z-^Exo5avc#S&qm-BV8{&u^O(c2H&wVmc94NQu-H&-e!JXdV_aY0$e+jX-w>twR0j{ z^7D2^jqMDZ^h>r)|BNjazE1;C5`npcJxey;3hvL-2Ztv6le$~trr*yO7nrb@+GRtP zIJf(*YDGJBxisYiJoz?Tg7Etr+jU1Ymf>HPm~t7b(vd3RLM7q*Gs*1Sds5Tm%mTi8 z7r*9$cE+mf=f+u#Hdxpb;*Td?!Vl_8njl+&=1w5WlCHpjt6U!8AxD(Cf={H0x@ zU(?~F9BwBxwM^ST-wLG8Tp*KPeRDgWpi4`iTC;+f{(R2WO9`%*Rtfhus;sn~pq=yD zB3Uemb*$F4-5FWC!0w&{YM7$=C`LGU?PhUPU8WGH(};}G@r99MCd$eiZ*w8L z70~K*ZoTFYd9A9rGUae$(J-7%m?a3l`yo7AX_IjOvH9j&1Kw0;;uVmZ6J&N{W>Au) zA=Ka<-OeGee)2Z0aYsy~^ZGF4OyQq^>8Anc5c;*KfY_$#0&rI4_M43SKvPp~JA&%k zr~&dcRz{UUIJDFbtY`LFZHCHgahgrHdw<2wy}Q3e!glAbE$V?!E|hk;hJOa?>MEAY zlRuJ?rY1E%1(bgCn9>;$8Hmu)U}jD@62zEmtE+&T($*m!&mJAN6-7%~(nwokBx!_+ z@9NVcjpJ%@iQ8VFKFM5SQ>Ns73kGy}Vk(l-7B5GU)8mEL%$W6{Uj}{(Bi9Y!q#C#R z()Ms3NB^$&Y}OHE4zqiEekU|3qs{^F^(fstoIhVjbf=1!vHhL3E83~c^9JhHfTK-r z-(mgY6?V~R6eBIVno7>oyiOhzR7pD;(G z2P&F)${aR$v_j>vVIVMCwu|TZwsp{be(D<6c2VchU|Xjz!b={$7af1`woiBIX^R0) zuyY6rSoC5PW7YGx7<9Y@jqJx;3Qb@$a|jH+f?S9?K-Q~s*!DlT<|fw&i=IIoqwC$g zg!WEa`d5pUOv-RFss>vwvya~&$008tZw+hLP1Ra16sIE{WRWnn*P5oQ&zh1{!T^OE zd3+|ki)>cpoEnnn9VX~o>|jN#$zuySXca8kOjldKt*!SX+)(`7k(x_QKlx#3k6B?w zcZ}LzI;=6nHw82vvWv)#h#Gw1wGBWcH(2DDlK|}T>=O5fOpaLg=e>El^fg z$dR&#S+=k9^PszS=-t(*f!6a>+*X#iUDt=qclmU5 zmnnsv>51*wMhLGTiOkj-GPkUs_D=2M|{&FIZ`r_LPN!Foc{Lf9gmmv7O zvS>$SYh2iQ+MBxv?ezq&N1)_%Yea(}f$2~>=hS&LSYDFF%oa%C>}Pb{q;QeeVJ$_v zE1^uP6=<`y>?0h4lYxDvBnB#qQa~)j4t0Ya|B$@|s(=d9i!pnPsEb5IBBPcjxI zes>ZXX$E#$gFSrG2AZy|}-Q9xQ0O^H-Q zQsNQUVV}gFSRoN6$~FSoJPh{Go}02O$!N?U?sk^*IN?VYNCXMIP@7V61r-@KJt^rUaJuu&nsle_DaBj3 z-Pk4bd{6z2X&^?j^YvxQOvyT9pZof(w+zj;-&Vey4X>Ppz@ws;!1Q@2%`eUu!YF?@Kg9 zRgk70VA51r`jm5D8zD1>8)gRck4Op*Bqs-(>*kZp^Nb9T*11FY_)t=Bwb0Mc2AK9ndgY3A;7fZfAKoau z=iFek=*~a4L!WXAK!Nov+p(Q$L$vYUm-bU+Bi(SRWp0NzGwNwxl3LYf#%EWDBl_8L zn;lB#VMp<<5v^42kLF53s%@qo7ERQ|F{{lu4YyN*d*KU){MqfyJ>@lwPzFUZZfCu9 zLL52U+WEVX*In}_-iH?4MqkmfQ+URRz~=^JlweEiWyuPX&d1VM``G9|9OE7OMz;D{ zfrMaO-{2N3lbV!ubHW7{ZhRfV%VYv%B{KU}vg^F30u=}a>RxQAQecy?#oS?#GaCMR zcW?X-n1qv$IuL~Xk$6-W)UdZttXicvgpsbno5J$IH;k?5V>X}r^pslu;>&2Dblg4( z?Z0^-7m*Rw3Le{rvNDaSHp#55ferZctmkiU4({uBI4uw-jLrIwn% z^DtP@cX))?sqrgj{9)B_l}&$X9)-gWZ4S@=Y+yj!Idx<1`>undteR~5x4O>nodbaI z=O~xgOTnoB*)mDWNI%rLP5kZRR+?+_a{Hj~FmH|pLLBMDo$4_KB~Y!kbPau%i$?J@ zuT_4?>JW)@Wa1#yW4f^>nz~i04rS$c^}F6CD{sQ}yt_s|8!L99i?MsdWaqkUHCq9Q zDkLMB*EUWznz22J+fqT9%e3c7xPU$iW-8FVVE`I3PkPCDtjwr7g8GIRJKo`{K4&MaowE7a+L`41K;7r za1nkvo5pz)FRj_Es+?9)QIQ!%WM`*6;t^fic}A$t?76dl70xFhW=ZF{Fp}yV*Mgxr zJ6(Alck0$`V!2rhHldhJl#ECpADsYIB?~eZUp6!C0}>Kj=w1L3yPnV|&3wel&0LEt z>gRxdX>@pw^yRk}w)yMs2#qrm^2+_&+*ZRd2vB`jYDl)~5*D|4Z?|*dyB$yq%>y5#BMtYXkCam^$FfsHWI+w>YKAok3fgQTSTOqZc z3$REBg)zeg=+;r4 zP)=pJ>eWl+kTI#p8>h3%FR&6=Pfc)nF1+o}wj`f3HZ* zrT=&c7Z@Yx7f=?rI?h6>4CqT0aDHu6NgjY`TY+gY?~{SoEeI|rjU#bYvy6%TLte}5 z?`aHKp0!vGvwr9~Fml^#cEJ{t2Y^{2Uo>EX@(^uQE^6dTeK(^OBeJWZg ztHFd!Nmt&Nr=|l+EtYU0oKK0@AvBgUO(G~sm16n`52=!uN$XlwX;hvg1L4poEou>j z48SxVfcL^4jVHT3<1%5b_N$KZn-aWq^2+c)T980<;>(NnfNs z&xK+l?rg3Ab0Sr`C^3gk5fXy8r-SZZ8#xA1Hh>Y&*7nB`+u(wY4{jZ8(t16gUr@}M z8SQLe>#(S>8tR1B+gFhB)fH$WEf$nCa=t%s z7DgS`W_dW2k%b0!&lK=PPBKnCS-U?P z^P6-f{2Gj93pxD7g$SvvNMryHCfGYtpx!&^w>4f>aIifiI#GLPTm*T=|BKm&>oyhm zBQR|e0e)%(==lRIUEsRb52S{2HfGi)4P_aP!?QIHm~=^sc#Dkropb|m$W3gXYs9R7 zH$`+N#~wz=fr=ObW%DrtR$pU)`MVb-25@F^zv;cAL^#$BA{yVz8Sf!-obkvWfQ7;P z%Q3*47IuW z?%#6B(wzlxBK{Ox^l`cxP&bb9@Eof2VWb9TgF%Cj3&;ryFFZ_huB=efveayurcojj zT2~}+?spz^{_O;Jl$*mrWrE3$Q?>Qb*TK>%o$fCnP18GSMudkRE z9T(D8#?@o&Wvzc+FLeIsOjH@rj|g%>$d?!Z+O8*=F^2(8Au+b!1ADMgb0D3#YgP{< zgUfBXZ0+~NufqyJPq2_b;%0Dn4fXX3uxgaj7~1UQ8c1!9S9QCiL%vsQid_Pp7sDvR zS0Bh69ZI0dqeoj)+76D6?^G%1P+_D8cz!SbaG?Oz1i;sm2*VmwL~D~N6)Iu77avSp zjMIDg)1S+vt1V$?vehuY7;)JQ+QsdRDx$zJ&*45;%pA~!g1^eX?m{6n92f>L5OLI} zq+rXHLi(VQLQQ%x=&y_sL+N$!dMGB~;cTVsEbn#yu8lx_0cTwN?y|f>ufg5q9Y7W! zSyQF>;Hz}`Wg+$N$O3wfXpejkUN=L!8sE{L{U-$Cg&2LG1{5sHhTrn&ecMW4jfttH8v(_24ayo{CP(GNb z+n_fWK`$sn>n8wmC&PGdB&Q9-u-d<8Q;qHa(091B`sq8YoUMvs#o=cyV;qVpe9yG1 zER4S$us2o426I#7-fL#kj=77Mc)D&XuLc3BPu#N+V5$52`7CIj zKkH@>L4Nnqo)Us5B2CBm&c?Ppq{Or_5&c`g1=N00@H*MMS|W5HT5r9-{fF7|d-pkY zX^4=Vs>8jj(P=X?icrG=xq%&!SAi`l+F+Mxc^UE+Bv5M@<@y_1udNOEmB@x40&)k! z1&5&*>askD+domF0i-ZxdBDIoZd%7e7(`Ma%3)OsR2UU5_z&mm8h=2@nh$^>>JS_; zfLi>An1by(FPeM+aJf6IYo-_h&>+E?{tfj|Y+$S+OTwRs(Wb-2wf^RmjNM0!_B>DZ zlQG}HlILP)d~Xgs-c}oE8`Bd%e9pm3KUrz6*0|Vpt@1(aDnNmzSZq|z9)K^=EsZ}^ z(BE2PQ~@4FhgGvVbSH6l2US>2C%Q~LF}hkSC^7o^f%Fpt|2Lko2zslD?jt&1Ums*a zmkoTJ%mMydd3%5lC4P3W8r>7lve&p-d)nLWLws6S`WqepM#P3;053sGr<~i z_M?tK^Q9H#t;zo^4WU1Az_|?5_OK7rw>>^mSI_EFeuHlIwITt2kA@5N8_?uTOUBIL znolrd{Y!%I8U^eq?tf1 zZ}=@1(}~E=qZ^35Q7klUBWIAXVZW9Ah z=Fq1BtMNcB2LM`_mkU|IBkV=>byxiwm=V8rfKw`)lFTR*{`{Ubz`UJqQGF@oM8*iJ zCz3win##7FY1I7qy0;r3Vw4GvVf?}L>-(F5T&B`Azsmy1Nif5@1YTBkYr)KAz-nl`y;BT2X^y-TLZ5gd=#QfuEF`Po6(fG*JS)sB_u_ zC<}d-d7Li-CyS3bv3UID(f%eYB=@G>l zqiS$<%v`BxJm7%O-HbN>(k?#{WmDe4z-;NRxYPmSgI*J*qXqj0bUE*WI05jbM4h2FcrI?pR zTCxCKD3XS8SXFqCTem%X{_7m*jels;Wnz7cXk3lQHwyP8^C}lNUAiZV`ZNwK15al7 z5kl;!w}1a0vg2}3tjp#&M2{3+>j2@jutrU4kPn2R_V&R63R^vNhI0B5Q_ly`cCqNhl^CN48fKrY}m z>6}$lMFf*18zTB-(FacSK!7%S7J?(&$U61~`6%V%#1X(e?(ja}Bx&n*il18IZ{?C~zK>3^dDI2MTZ?%A7p;P^!paGEKxYG_AO+$?7n z>pswoEINT7&tyaPmp*L|<=M~wjp=4#-v(bmoIsk=g)`C1z)SVWv{0r({dS`LMBC_Sagxn-(XaGws1|T}55wSPvZ4$a`6o1hDYO z)U5S3ZGq@A2elgr3R&AYZf4-WV{vo-J8el@_Xco?kk*qYTIi>pprDs~&1&VWH%_9UZYO zi@9mNBK=U%llC|Wu=n+DQwlt< zb6S0EvhF@yq&@Osc4VI4y}zFxetsODwh5&I$c;LV-P{P0kZGFNr}aW(`MIgv9GWZ> zK4RhRw_xs#4(RYYU7$(k(-=5UfGB@^s387eYf7*{T5O`P7ZG5MO~0aY zSk-4CcR&^P9VYKE4}J$62@_%q37a0WQS9^c;Rmt$MscETCs)yL-+JzX0sH%qNdy5haOS1Li|9#a`qk;NiFjBBI_#=& zw-~Ix?y{Z^w@lTiR5Z^8oMd}NuZX4I+RAcC)8)8-rORMmn94A|%G+gfTWDY90Ktws zmz<#aP7ZBSn7~k9?iGC7NAj(PpyY_OD-hR*dKIeWhGCLYLly|QKA@{DJX$?(H#`rJb4La;_ljiIvQ-o}TXRSbzr*5=Ux}o|O(96Zqa{r&~<#xS3_J z@Qy_9fk22(je3iO;6p9eO8;*_eZev**TPt|BBRv^*^D*tqgZVi*08KC_%kn^lXBj; zrCmYOq}@dRIVMxeD;s}4`#FjE<;yN7R9K!(;HASepZ3quTl(;KA>-!eIIN_ZO)(Ws zL4Cmg{a^>ren-XYp7v!iPP@zF&_^IdD;$S1xTMwLpmD%`9cTYCWa)Ee*d5knE~B=H zpji+2|6}j1qoQ8h{b3jf1(XsIX%G=4l@O#;kPf9oK)SnAN@)?021)7e5D@8(p;5va z970<9cMtA;-tB&$=e+0abN+hQZ*A7vE@S5YUia14=lXn)_UxCVC3#P?cb??LE|$WL zAqYoGqTSysTBrgwCwk?Gg>ZCHW;)5~$)FG`nf(h8V^_Exp`}F?g<97I7TMjkX1UUx zKK;m!*FC}$p0DU-ZRiUa<5g*AXR-Rq%AMXi?gZ1J=WSD^$62qODAx%FB~}Fggpb$; zZwu+0?^0%D&&}`QuvFwok6i3WXxDW8wU2^iGpb&=TDT!5%1y4{_$4u;u*N;CG zpLj#9sVPq1WTu@h9h>aC<$sm+ezwe-l&w{!Uq#%L(wLx=>Pm@x)#^ci%1kj?EsjS= z0`+>8&ig=RZ=atXw9r0IvFOIox-7N;4{nNRuE|{3yH%R`a^2lbxF}n5wx#d~pYOgQ z-)v?E6Ki2r+PcsaGJEmhYG=24DtLdAiS*OS!Qts8cS8}rsl8Dculpg0oi(A!k`!MH zGJV#wxJO5?-*jdKLxbUz4JJ#QKr}SgJroxJZzHet&O5`1G8-pKE*>V8oG`-K=Di86 zkwOiQ3Vu*jDW%aq4zr!NNUKG1BfFG~(ARL&toWMjlIE`s$h&9qc*Jx73=kN~6MU`k ziVS*PeR;3xVBX@pF_Qg&*EA$QjR?8m8__dit4xnG;`FqTN;Ri_?u3okcV2EMq@!L= zv;GdRx}%K?`@TCzAK2EOo;u&MHvW*!=*ZcGbAF1Ot}=(~A*K*o4f5t;Jp*wC_rAPL zd0zJg9mY?mHtf8Su~0vLJ~$-JfB5 zk9@j5*RnjMXKnfl`vBp55Uk_5)iDsx{7Zs5gG&*71NVsD({5MYt&2Ee1DFjDjzz&5 z_%$nhFa&eM#YtVy>Kvkr*M4K;Gr1zMWqNLfU~{W%cK%O92BuSCq_POPPSD6SG9Swc zuC{d_FNYq{Pae^K(Y%uQx#s5pC!Lt|{JROl?!D2IMLCdiaC7B3)y?SY<+>BLJ*^RgiAYvjxX~SCbrw;M znitLFW}Ys|fg+(7jMBi0Qs-NmSKgZv#5#90kEG5=qdO(IRzqx9S~F-{g^m!JAuY`e z`A)A7>Lvg_AQgY-_iA$O2Br~OQDz{Q=fnsUBmCX9BUV`SEbsfI5wd8Ho(q9le7RNd zBG6i?9DSF>B$+^|f?JvXj0xvEXxQtM)`BW>Go9;IW-h9xN@|oCJlNK>$;ss&9|iN^ z2Dsz_A)9aYvDS=23h%SThgPJg6SXC$rLSk|EJPg9ni>op0D9&Ehrx>Q;K2=` zG-LvUH=n}lKowPnu|;JCI4C*D{6Z%+)IcaB+cVkzg|kyxgpx@=tie}sXpCF_Mz`Y z(gBNBHS^bghe-du>%vGW9SWnwKCmwhNgjHxB4dbHW{qA9Zko6@G2P1BQ6OP`^V_M-Y^~i0Sg+Y( zp91~UyJDAASct!NI!`){b)xo3P`iF^AnT>zKCt&HhI=(lE$Vu#uP~RxucZ($`A!_7 zx&nt987qcsI9yK69wR<@^|(het!?ktx@-WdL}(N&YAfBMO4>D1Hv!9GSZ=@fjsV^z z1I%}9rtsC$*wjluhP|GEi==^W)Cd``k%4QfwH}%MjT;*|4iv9}j!*Xx-j%(dlt(LC z0`~9&xM4DAMa`(rz|q9ejMtL5Nvm~g)^%YEnEOjfkC*%`BDSDGu>OtkTw^VGT&StT zCr%cvsHx5-IWPMur}}s|p2xvq38?k&@^Gbe{!ELI-2_Ft#v1_FnOAs-QVN5@;;mu5 z$Bh6h{hk{(hBhZ4asIwNm7`J)tUR207riR>vY2|awbUsYFb7X|h%hxva#YwDA=C7~ zdvrH;_3Y%uQ4Q!BbR)JXf9p6=J79R})FbME1Cl1m1R5M)XAOIQ#8C zI}=?6T6d5j!7IHybTIExa*dp4XcF0v8GMc8x{FYuUW+bhrx?W{`hMJIec?^x|hadQj zHCJ&OihzTee*Fu6@MnV^GeC@b7xW5@aWXk;N1j~L!Et`cW2RW_S0}GYrP^+qa{>YN zpat4GOoUd1!kN1RK?r)J4ugocnqbsG2oy*Kt0YEp- z4zP95*O}@(ZD#z!e3#dhI*`_M-}$ualSuMB zr(Exq$$R5y(@n+Gxz_RXrUL>^+dWM0>6q8DxT(i~jN>uQm|WDXq;K^+)~j+XaE9Vp zb-sJb=I~`cSW%h+*@40~{OTUFzag2Zo!eMM22gwSd9Ki#S@*(K7h+tmE-{Dc(>|uxu$}8dl{&G zdSuv|mZ5F++Lz3=melkb#G1fn(7Wf#@Xkb3!RqP7Wi|a2wTQ^E=sJ`xBH#646DC{^ zdWrICz&xjiS!!l);|ZOVj^@sHDZUh?2+YsOXqrlW7(z*xlpnj3fx-U-goV>Fu)vxN z2W|4l0ofm>8(ylM;?Kd{y@v_fWa5|~7%|SfF5$p;_MSMRoi2v5^%%wLe5+J)5t!dy zNSmG+p^$&sNWX!m3C}I3Fn^CMaEfvoxlSI*za66TO~Rra={{M`IeoWYyAQN{SgAaJ z__4C)@v|qmEOi-aB{Y>m zgQTJ`FsCsOcOzkWE*5eB%255f`;G{ONJGOD$=vU9qhKL0Ip-C0Na%DohSO$#_5;Sc zq~^Ay85hW@KbUd}K8piT=vl;{p!kl9+A2;`sDq%!^A1upJySKQiyl+o-fdUGv}> zObn1>x>2MQAzLdwfCAolhSv1F6~ggmtoTDiPT@cWGI#K*W+&);b%RD?RRT$sr44>w z$sG6K==OEZyEGG7dv;v+E34w?tT=CsEy+rrf@a73hD1()FN>J%b~k-5XbRpc<3H*^ zHT+2z_f|*g-J{*+z}|I*w&dH#3LD$pydz^1{1Mjh;yp4QQIVPN2*oD-G>@!;20R>% z&3a_WKvwC`U;uoLrgjIX~F@Ti`(ZO1|3MiI|! zb@6W9|9B#52f7S)-&P4(y5F~m4-@n|gLX~xL<{KD;Rc?mUbz;zH zoAtREZV64=K`KYZ3TWraJ;J~#1^=hl998v6$S}HpLMc+G-+q}8?K?siXTD0-FK7T@ z-&r^t!QS^A^i_S6W~h2^1fpw9-N`RgE&qPedm+AY7-)E)71G8yK}$%@5!#?HyVLs$ zq!xTF%OOO$kpIu-X&}Y71Qs7f!t`G6P@M@_rIz_#P>16wR0s)0XekSKQygp+ za#_7-eTC~W%8Pb`_5e@nzcDY)4U>z1^mzG}&JyLNKtRU?x|vhz?(J`Zgs`>6H+8*Fo1PQA{{oN_c7aSUmvpkl$PWYJlSjN{Y ziD#m+851Ut?G*Y#Cjijt8oJAB_#Ye9FUt%Bivk6>ubf_uR!EZJ1_OTR#iwVN5*zEz zjN+n0{pmltF}NNFaEVe|f>FxEF8(FmAiFQ|^4`f}?6B@TJIZaw$9zzKTJX_4$NpR*)P)cc zp}&0ei+}Oa_oDj$_-HQba=j2t?={LL0>3^2%p;6=wI>5&y4j||!atxqco;TmrH?TK z8@*dnuj0W2Wws&Sitx9UK9OXtKP+K;% zU|epAP6%yd#NXYCI!#^T zq6JhcU!FM_& zra&7g1>B|51ii4eb7l2RQKpvAOt0YsuKq$C2K#McXwwC^*6}eT(BK(g)~-%K{YtT+ zr~EHk;I92rSm}g3Z`M1i)n=jfIrmWBQJ56!I974F9l%e5vv-5$NE~i|lBG8ilxFY05~tZ!^#T0RS#ZtM97o zOs9c7r*)dG5C<|veR1B;|Kz;bw7$4nppr^j8x;S;LjR%-0I2YK7}xI$rY$A*K=X4s ze9Qvjp+T{Ij|f=UTQpuj+9_R2j}EzXi^K~6N0tZ>ETS}azOHnhDsqqtn`-+k?I^!c5n~G+PXUHfh6da z|3xdf;9vFQ#LVLG#kKg~lOweby5l>Lm(0EJxqDq$ivR%903T(#o~_%Qyodf{wDo6U zzy(U8T;M*2g}<;mhQIe8F7OG)5I_!ITRF1w|Lr@(}DlPU9rQJsRD#QzuN>-l4unBBV^8!)0>d6|JLGp)+v^2J3C)Sw(p z`}a@IDENX1!4D4KoZZLEx>Z7iBtO6TI@Yld{+ft@nc5Tcdf3;&H|uhv9r zd{w{nwBWS^@)8m=W!m)i`kf1tFXb&m?bQh@LIFtBYBd~eJM=WV?cAZKr)NmgxP#P3 z-5W8M5Y=m8)vG9H$kg0)pB50F{nz~e1uJIMW%L#Z0^t0f0E)9dQc&%y=c|>Z237SJ zvaSPjMf6SE6tX}PewlyJea286Qq9B7oT5{@(<+g_w*tz0H~Gp`Pu1cX;8vU z_Gy9?T1wq<6qh7*-i6M6H~t6GppHeW2Y|DY3KNCy{(hpSt>^Oc@}v{@9ez0=4i~EO zZ5^jTxQ)T}K@|E!Z?+J|R-xHX|i!#-IrHR4<2_m~bTUi9=i<;kZjHU6v)&L_WTIR_Cc$r7{!r_A|M9if0=gD7dF4F zXn(`7`kLSSiyTgk_@7)oSm6a1g)@60*8XT0P#Sro#JZxlH0gk{cSlRAZ@2MZfM6>! z&sRtW`HKHr+C4k{N80^o!tv>9`n{%T-s%hYR8J=WZaxd8?YChM&zMjrq*w@{10Dqk zLGj=kGEmqR`X4C*N;w<;uAG3QK-%W#B`Z|mI$lb!YU*<}$(gm=NlATJpuF{>iqu}?;toq_`v+~q?nj1XNk%HQ(Xv07jy+>>m_Mm=%SD;Dub<1D9l zsGlE*C=Rmea(Dc4Dmyh4a2g+h_6ah0M(O(2t|c2hW4AUa8}?_Izwj=_fA=nqXmYDK z=TYl}pVBXj?_VWojr2L@k4}Bl^krm_=ddznxrJjP3NR?qZ$~#*Rq5?xrF+rJ)zUWL z`MY~>)r%wtH%e#cQvCWVwE;2xsnOLr`2n%jz5f0Amw)DNsEaB4`^9Xa)q{E*#f+)} zSdbhy=XG}P2FP|_1zto1CuEKIHUzPJV8^)WsAo&z{8PXDdqeDD{S08I^*ro|FE==p zc1s_;RQ_U1C^P&@|cf3ABJvwWeAV|2U<3_Cug+~;Y(8d zDK>HFkl@BHw0_&Zb!B4%-Va{B7X;^}ag3=T?fuEE=fr)l629Lni0+sX4L_#7kX3h0 zo1L*OV% z5e)6SkqWYN>S)gVx;AvjVe&VxbZu8Paj7^(!BXyMp>3nmXuV=faM7`Q3RXdj&Os5m z^Cx}fy#VcdY8l$1MtsdW=S7}-E1HjTPKF7%a@XU%Af^>eua|wCsi2^;Ckot&Qv0x5 zxy1jL5>~p~p}TfH>#H(&NKwZ;NK2c&Iw*QQk54>nSQ6tMN; zCm!aZN1uv{qv>#V8nY8?mxTN16xaTgHVJGwwrW}f5Uv3WukC@iW+P?oINM8jLXZ1k z#H5-+>__^Kkh8{zaZ@Ck`%BFp9=)~jPKjNmW&<}fZ*Pay3ApZ~?;kVe93Z9zsy`!9 zWf^0eTUWtT_h~yy=|~H8_Z9vdaRMpx>&yIabr&o*1^y~&0*k-d>2n!dQ`oP)eS_}q z?kDZ^fhE68)bn`0_a?nV>)x)RVb%^#O~OW|q%dlqd!z(@%I^4aXwKVn1{VM)6(aD9 zjyF=h4_Am+ZC9VMuNSRuOc4z3T@Io0y5HGH>l=}NZ;Tjq(AWBZG$(v!KpO?dVcBFy zi9JK9Ft~63>*zRt?NTiN-Ca(iX}-(@Y{4q-?UIHU8G_^(^LNJKtHr2=?5T5?2+TlO)v0|;2o%i&5ogjxsQKaHp^3uC?F)d{KFmv0&80a!V4y9X*b_@^_~@Z<>gjy2z5L5>0DxU-JDiVE?6T}*-m*4x0JmvP zcDy?2g+I0v3R|!F=Z1*i#ejay@!*$d{-MGd9IHpH?H-&|r>DuNROpZZ<}E zvoED2jFf9IacD_OBSXxJRtu|AoNoI*yykbVhbl)P_MilOFQt3`8!feY_C@6do%ba6ReN5|13!>4xd778~kActBv1k&)3h ze8H#h$SdN@J0iJaJWre2!uypEmhkjd4OXX>f=57a?PlEhLEX>i+l!OKOXb^s zoiSSEtq)RUn_?c; zqnU%Nb16ejgA)q+tBHzVTFBC%WQ1yM4RH~m^X65q+_X%@fpyP&yOn;=Fj#94vWlO^ zQGxR#`(M4nnkwIlU7Q(=4c&~R^$O0;%DVHw%xlwYYJd7ILYpUOso>JF(Ue#3F0y-Y z6)s6^O)ab!L=VG5$?4YrlAQ2&TeKBd3m^&mDe=1~0C zuDzF=IPpY54~;R0{-+w&@RgKbhye++j7sxxhm(sfCCy~%A5Q?O9R&jJub$BY7`=%o zvWdy(rZ`;YTwD5Vl?JjK>$rWq=Fx6Mmk~|^sf<;xcOJ`dbo=}*ocD&-=L!-wCkC?O zGS|H(+*QHO{OH@le0|zh)kU@x+J@K{*N2b!+PC->=8#A>OU?6^sq-R!Ev@*I=E{KC z;bEnz;Yu0Jd8-6F^1w_k!L<`%)uImxTW(*Czv3W7wNJe+sZH@KpO#i`SV!0ArpBZ$ z2`GDe)a{;~jXrap?Mn3pvWfNmAIk=LdorVy<}myG?KRogUBtm`(kpMIH9R*3%JU3{ zm+eTi9U|;JJ+l-<@d@QH|-Iu={qx=6C#~ASeFIzl6zm}7p^5@I&)>5!$iAK*mTi!^E z5{N2knhrOcMoVS&mxNnN1Rlv?7fr{!XMPfR$K#2#vgoNa_PU4C9V=;N@iv7)pXT7r zckqV?f+iad@c9G>A^2W?-9cYMMdRQsco|;8j^`j2i7GXe1wx-}>o0xfg=TFOCR7*= zD*W*|WLx%RA)^H{)suXv@O1BJb+T1>0xuSVjvpfbWtSXdtUG-=9>F!}dD)qP1JCo4 z-+cmK3TzddjAH8Ncm2|5Xh`4)^DKH{--#9_qw=DwYCYk4A`j6d+R*|2)|72@M4%G!ku1@y8LA3*zTUi|@#HXNg=V{05{lm{0xg zOuenZi|{OJGzjVkpAj510F5HwTFgj<`Uccr@asq*10hEv<|ti|BQ-ubq*SbR_39WD zIua{9v>p}a;K-$nIYhbfBEb^bwim_q#vcvS7yJn4VRms4KpgYxabhw26hU_WB0z57hx#dW zIRO)$_vu_vkH0VjCe*7f`>N7iWBhcdSExq|29IWY1mQyWbENz2(KP;xkA^u^|0xO= z>Mw@o%)sO2mk5UV*D>oU-q=MHKgLF%Zcky3vC^rfEPt$gA#HSgOd@o*yQ03h&uTKt zsqwKPB5Ag(qHkZ3KuvWFq^>|(Y0&s?BsUAo2~+D}LFaJ0c_h_n1JTyM%ExtAvufjM z(B8X{5E^E{YhQjs8sXsl8AT0zObowtg<40No=?{xm8=i4F8O`I2O1!+mwuNzsu=2T z9sD9Wvmjd3ci)!2_k|X5ad2VujSC5(K}98mQ(xhyi;Y|5F=Gb!3lGt2*W()*Tt?T7 zzcu0~Y>W937bBNVkq}=>4I)w_#y&7SE=42kQ5%EOB$(KQ6!yjC146Ic)Pv%fLmA&6 zKD`MQ6`@8Dvg(AeKX{2&By(+5n3@OrJlKDfI1dXmi|Idf5Oc^isQbRuKC3dsNy8?1O%@q~9Gz7zjfjsCA1{a-cuw~qFod83>Pv?lWobIo$w z2Ub(tMC(V+^y{T(>(gbS5tIU?`OV_Mncwu|ciUEsz8X4&G11aC(E^sRmDk;?J}Ae& zE@SPTs6%-lGI~M|Aq>+=nz?!5`ul{w6t7zj-WUYOmAt2;Vrz9}IJ0hxrJ+XSbM&Nt z<#pjis0QzwhoM~d<9}BAx-&sR;8%9lr$6&c5?=jMb6oS36-PqnAQR55w zonUN_uIzmu3a^=$0;G(uglOZLXoMSt?ZjJgA3gtKs0qHgr~X)iT6#CMltg7A!%QV- zn0_MEH7vM%<4V^3cwycdnliRu5u@eOzY0AMbfAbm3vR_t>n5|Zsy0c{KjNBk-03J737k2Ij~#j%DggWa9ipQL$jC+N_G1zXMHw?yT*1+wbZHN$rG-3~xis_$Y zq@RgUKd+~_e*FGd?x5E5uX2aGSVPDA8=1XKdLo{U%|b4(P zR}Uv0E0#k}6qe~CI5XAe-8C9^9d7~|R@xBWf%-9HeZ=jc2RdcsW>~u@>667BBk3BU# zrSXyMRbYef@rZqdxy+xz@+Im~qSLIoa^26D9$e!5Lz)1JZATB92biV=_s3CR_WkVV z_3jsB5J%4w+duT?(XTJlsp%oAbl&5whi&WdSuRUC@M5!|``sW!7#%883AuAVE7bNA z)Gbpic(}?4p22L6QVI%51&>6qdITi8?dkCqf*Sgp z-7guEpdQN~O$Sipn{|}O?(Xgk4zE*ea%R39!p6tM_apusnR&dqDjGjtEJEeU&fS~% zm+y&U0phz-v3!N^E#2)qfN+rDem@Z3z{EQK>EZ{K+!3(Fn|HktY##%np8~`yyUTxg zf@}28U!$GwCd4Y{p0mSXLT4D{iQ0yvv96uuxBuV?Ocmp?tsOoX2jTyeg+v&kfXTmY(G+2Nu_E z=QHnU-uFpeVK;IbbHuoMGb>akzU(aat}(&b55V?6JQ^ufoDa*^;%2n{O!B=#jZ(xa zZ48MR&Yaj>yLJS?{ALG(e5-F&s?*L=fShDFy?>=oHuJMJFFL)Q+nc9WF#ZVl@5QTeFHF@xYrs!@_=aZS2eaO z_3;L}`%w}HA|h#8<zm%5?~aa;V_VfT3bvyD z^sZCaza`aJ7sWoDdgbhSTx74%x>9_!EyJi(o|W&fM%i8L!oR}RbYy`#S512N2?h#fLdX}`|K1z2x!Q@*ecqd~&Nkhz_dIN;Y_!pf9{5O(#K8uwd zLbnQ^ETJk{_g6fv((ed9$zv&}W4s>%oU*x8a);8pbOVi#gLG%{VeO6I_-qGwO6jeVLZgba=urTgsH@1 zu7944`TB>0&ZtjEAD|=}Zr*IpE!-!3GuBJsL5efyoehs_m^! z#XOE6e-emI^d69V7tG3N`3xKaiWEh#S84hwgaC_`V9M{0qWA~ayW7Hg8kvGcn$550 z6Hh61JajTY`bH+(>e!7x%-gf@+3D{I54GO8EJGYzm?P;c}(~<4gl?M&CFHZliMyxMJ1RtoQfwNM&d>gLB zChlYg!m^zA65m^*jKV=f{^bD%#1e4Q8>ctb)&M=(2e6d(eHy&jSPAT1j+vO8v;|)M z05DK?E0LjpCUu2SS%P$?*nH8M=IO^&Y(JnmAC;!BLxyw#F)taEO{@t<8k*-Ty$C=MzBG~S3Fe+GV ztWS<25*iMmx)-7rxbxi>_t^V1Rvx^G^-*l;QpU?GtCgb)^g0pXtzg6{fN&-MWKF)- zyK4HA_RGZv{+nYf6MDE4B^smKy-`T?Xhh1%Y6VGKhZVKPbiWa0YC2r=UQ|Ra2 z>CfTgZq}L0k1WR+h+uQ1#NLMCr_;`eC_C@3ZO#J4=0nu&?$P?W@yMS%o1+RGnpI|o z6(%>muHg0j^gMNM@Q%YXJ)TwvoE;S`OP{rD`ky|mF*(U1UPq=FU0LCPC2ht#V5bo7 zaCf(;)d{^c*Kb$ z;mx00k`4XL5BYWozShEg(GwQw!jq|Zw;vpz2+TDUJd9;{)-%BU=IYf&W=0u7&*YPV zusr4f8h9qHu!5IOoI4i2MzJTH_}Ufka{-mZf%>fNodd~;NpeS@`0i@5SF8$05;aX9 z)~IUAJB)$v3B6t_=F<`qcdQ)&ns4q8*iIpQy9b-MmO!mB3*NyO#$m>PXJ5jl{3`l% z;248N;MJGQPd>?8NIb3l0qBt?V|)SA3A-lyr(7Dpg`)1X6(;_rtQdLe_KD1H5NK9~ zg)fu8x@mbmHf@Y;jgU39tL1P*R;zZV(NOZP-SD=LWtzuUI5+!-*BA7?xqH|ZQGHC- zf!#Z7Ey-m|AN{WB5(rogSb8wGFQwWvpU)30+)FM`=U;kRM`BTJ-T#FMCjWIzV+V9e65p7*ugukP&;9jDztJU=S_n64;Q%C z+YkH%@0JMCvvI7f8(0iigK+Y#ZlCNt+TbuFtI-+Z&@&z^=wHyQz5g=M^-*VMArw*M zu|K)0am(}g^byI-{<=y*aSGFlhn9~Q-O819hV}7Nox44hS&URA>o}S|uRuz4L(BJ@ zcMyeUDmvc{=C2xSlxssRJ_JEq_(sq9m#qh5PEYoZ8s08A3`LCK&egF&5tzyaY9#}u zCx{Q|XO0$himHC2_jf^#Vx*pQ!#I{4J{o}5V6}B*C`-s`dKc5zmj?RjGlmuQoxSLz zRcxJZN>}4&sVYD9VFV+o(T1r5cUa{U`KJUs{mYZk)^C*Rx$ce2!W;zDd7@9ItS)zM z-6=9y>0wMoZqI#fPbfAlds=LVJ;dxU!yh^my}o?Z>!urc2VOxdEaN>KA!tm}+n z>YPU7)~yua^6jpjy#CQ>mBqTNV9R;U0=p|$)fgWWdc9b#JR{51DP~I<63-W3&yV6* z#@#j|sm_1Wj=GslKlP5Q{(i=~azH+j3ER&px0&q;qIk{nvNM&`IS-kFPgm@HK&+C$*8)@*AZ}Qzb6f&F6=Eqc|?J zcER;IZ@FHTK7@=kU0c0I_pz9}Fg#>Tz>jDs$eBG6145l{Nsh;>{0Q;tS@pFLNBVbL z@?j_S1CPEgvRhj$=stiys~BKUdT3L;sNuQn%8j0=#>O70-j;=#5f3ZK-PUL}i<#>* zq;HP;AbU?wwc!yE$E7x-vRh26$&$MIEM6QxqV=5<*D!~eyLiu^oDCQ*OT*W&hob%Y zYqpA6bu0%)jEjk3FV$pZl6QX2_Oea!8(CU{^+JJHoIe$ zA2D)yZo#NGlsO<5BCU;Q)?i=aws`j1`uW$y68)bwT1gl* zS#0QbO*BHFJG<(^mcBy$oEN>!L4?RJ%e$ViPh7#aB1tdnNBhRjEIDM$RL&;}-7RK2 zwwA-3*A9f~8c)AD4vQ-m7ld^4yM5hkbl2E6<|edWcQaVyU<_CBjdS0L4pgr3c~dg_ zOrjA|IR?`Z(#2`CJKn$M+(9GC5q~d?r|6R;w&s-J6r5|^$wg=nH8u+_M&O zCN=NDZLL5Gdtwlt&#LxH#kvW>uNVvkK8umu29;4O?&@@acutI6#6}}AWLGHb?)eUV z;hMDJORnK<(x0TwaeU9gn<;hKB;Ca@Ly+O@5&iMk%t#Z_QMq#hW|uE*^vieR-{|&6 ziR#~D-T&}9@`krJkNepEIDg-Cfj@G^(WMLyPkWY}gw_vvuoF^kAGc#bqxz1b@}Z`; z!L0i6agET)7e?deqX{j>aLl2#eUa^_6Iuf(0HvB?D-dLz-alBR%lc?=}dwXR{50*L_S z`*RiCi6~=r(mVbVjy6Bsx&T~vybs0>hy&R^xfkU_lGcsc5DLlIuot=Q+-EvV5fjy2tEMW(@oXSQ20n7`O?zHmM) zj;vTQe0Q@*U4)yZnMbcmXd#fQfcfv*E^16u%(>nPf+@xoPBM8j79*n6d4YnxdqKWKfeZZ{N!7>V%r)`c^aweFU))WIa%lV z^*U+#XtyV)j%u3j8sRZvmC;Pz07O=&Jo|(D$3`RBe>nt%?PMCrEzpYVVZkpHZhG zGL3pn>2A-eE7z|hOJcJ4HPHK$pwh|YEOral4KvuLG9XnYZGLEvvFqpa8Pb<`@X@ar zR+@yrI;zf;p0dF|d7eEs?za4FyD{U=`BFGj3fzBNV>o@muohx1Tsh~9>D|}f5`qUb(YjNqtJ(!&DhIP1PKxbl+)_SEq&eb-1?~kRy z<8JV@Fg_ilF0MwbG|UEh^!lQzR~5@0tPFRW$c1#9X9drD#y;%3wLnex1DW@});|%* z;?LE>ebiYjy^IQ4$!S&BwO2aA)T83qm%CPEG|Hf+@@!rMvr}@fmuWh`DhTSsI|MW7 z1f?X+3N|*`qaV@`xq-+Ob@Unu0m)A35E1aR1`e|OUYmcB)YNJ8Gq*o{%Pl;*5nb1Q znYGZ3WS$}n1fYmDQuh%})~}Q8`uSm zfz?{iZW&b@hm*cY!KYi=X>eWd?qZ>|-eR==+kEi$^V$;E*zz&`F2l%a_)kw&2xrZc zZG(mJMthy@MLF7)z>iLV3U$j~kU{c83y`hGep`usV>LI!>76}aiV;HZ+BE&(v-!rG z&Ru#w1zVGhlNlB3CbvI}`9w$y5P-T9PNydzdx$#I}d8k$F2St zp`i=G|_`3$;M}KfN+NCYGcal)`^*>d0hMJl*Tc_1|Z=Sk% z=WHi$$o?diUvjoWrI%uD)^c}jzS+4ox$m1BYmE27E3T(&*nYF1V=Lw;Nx!L-w)E(n z&3U75Y{C^JipA0O$TQZZ!}Tkxq$TTnp&pHgZMYpb?PnuRn6ZTIF>ok!x>ECIKK^A8#Vp-UQn&L&K%k?@}Xj0pqQCJwJ7|0Bp>k0lu z$$FDWi`0qFzWHbNI8<)$dlzd%KDR#(FE1go6H3HGv_?kQu9Ts&^IEEul^-U?tp;VE zO6JQuSeZ!i26|VOC5l%Jg9lQc`P6M;ODB&3R3{<~Bw!1Uw}fQ)lC2z$nOFTs6qNCR zgKNVBNoB#km9rJvHAQprboG6uqt5<)o~1h}CucA111S6{7CkuZJwvt>?^}jX;>;ax zU?mULGZeKgP2m0juZp2w8B9x?DA7|0)NyuJfJW&|Y#VbN@vKrc%3aNiYUX|C?XU1+&k zIa38!-D{1O^o=9RijgTEx8*K^=|($RuQD^*6nkkwMxA1EA~q#|W^QFA2Lb1`TUK-B zffI%5BDrZwzKm)gq)pNsX&~TbJnmX5D7R%+#j<33|`@Zhyey;2H>-i(~3g`Fp{fy&tjQ8<=ug&zprOYTi z$IRah?Z}GIo^e(qODq5JcWD-VxfM)}g0qC4WAOs?2ec1WyVXA)4Ng|^~y zPzWWyBT{t)^{N|^Hdtq?Vj0M+OJbbFt{*6yqt!n$42$$8Bh`YkgR$RxDkdw zaU(;*P`I#ukBRRurl&-2)Bk$;3D44d4bRV6w9E0FYqbS7eAPGi>OGI{I`$iE+W8eG&2;C2 z<4yI(DvBA33P;Zcm(?gKixMo1g_e1p-f|lA!)Meskz&BaKSle)biX2V#xZ)(oTyDd zM2D+#MytgA-XGW<=RGLs^&N-UaNPAeX2H$*90HZX#P1c$*yM=}SOoIWJsK$Em5USV z57?6AZ+`Db-46Hgb7GG=A1XuFv-HNxtVm^^b-0zcOnc@zJvwYn`GNT@*BV zBtc{)TmJES!Glmu!WcCGXyu)9UxxL*%?t}LBj~y5hFv1upLKv;{N1cydxBd61Zmt) z)*hv+q#SnGUi|Q!QSz|xTvXArfYnU6Cf?_agdWb7C-dc0$m4}kY)s1sU+6llQV6cJ zh^;KVJMxeM&|t*mWKfB4563}IV617=52MbVLMELm?(Zj^8Js|%QjY}MB%y{lVPT_z zvKyHf*G6B}8^@OMMeT^>>4FO_36&-7yr|Pz3g=xwQz8%~$IUtwroUvi4{(D==*o{m z${=WeczBg1AqyJi!o2_R{C{}35#9s{K`Dl^sCnXHbwIrxF|A?l-+A?$XbMn#e^^%d zyJqR?)gDGnz%Cz z6>8XaA#<>>jdc$oBm z|M-x_gXTz@x*(m9{q;=RF}2#;hsu^VK$s#?ML|$4^BIS7vedCC(kZ??)9903)NMAu z|0vSpH2N6GP|w0Q+jzT+di$cAlfaBv8K{k!ak=1}bN_T0Bl$^T@XF!5CEn&PWdos6+)r?nn^RUl6jp7To{uNO4{u^F{uVoJ7n zq|qDj`0BS3wM+t%OA(-~wbX!^~Y z1)mE$Irf&>=4qau`5+fGC17i54gANGEeT?^MhHQ5dd5Ai4f!r=ua#mF+}@&_cq9n2 z8*!sh1QqbY_pX5}qw#`dhuKB;)50lP1eg2Y@AmgZEx=wA4-DI_U{gDZCa(IO*f0}D zpK0bbA4{cZv0eI>0HS3opdE=);UQ-Mtv3@ppa~=&vl9!){`UNy!{$wB0UuuP8$Dvn z+1}8r0kpFp$}vR(#hG{C`z9dXBlve-bq4M;$A(7eR9mHr?nSI~w_TT$%{y`{<6#1Bm;_=udBx?x{D&i_T_7S@@hSYrm9hG+Q zXuOX5(g_QhS{}OjH2mD4Byz2tZd-k3>ioMW^@6b?44$3A?Q z&wp6I?{@AyOyB$nNk^DlQWWS;fP_yCcx-<3SYkpNu7m-qT8ErES(-x^53Dgzaf z0oaT8#uXFRk_*JXFcHCYSv<*ezwINwCuo7|Bne~-TiagBq(Ksg+hNl}cvG0$7O3@I zfm&Py7#Uvz>_yR^hF4T&onH0%(_tXaJcm1Y1Pt)i-?k5T(G%5f`0eo`ZmFijHPYMy zs~svk6MHS+K)?1ji^{7*KpE;2{@EG#!2VM^O#Z)~#)vl1S^bm|XB#>Xs!IFL;EKb< zlGfWt$2+sMqiigHfa@X+#5n}o)XGZ3_lgDrt^Q>h{+DLRUB~&pQ!~f}(y#3a*#9t{ z0>L)A9ok+2L?Og(BEch7D(F;Yvm=!-(M?)lcxMThm++l&mYk|~=2i$l+6dj-_xcANQBaVMKJ{IGzUC84L zAN9b^1^m;`K~Gc?z&7him45Z+fD7v=GP~Ol!h8f05iej1>&bp4YYn_tIv^z;6NHao zh&+ojkoUfjHoA9sC)9f+k4@#EIL7gf_)|1}Aout>R#4|JKu@9yP*j)!%pO zsefEfY9eTM5*fx7!u^3*>qNb7R+c1asXyia7)!tXa}nJC;btf?|AHlK0gB%o#dO zkCXi|@iD3k$Kk&zkzg~ocS2x4{E%n)aR`-m8o3AQ{(29##^fEm`Y(&g!cbt)n&`Na zmum|UAvHq4F3=>vlNE};pMRTsf@EuOZ#PA&_)%}Z7ZMqPukfXa=-NIo{MgTdq$GYp z-aaip0JdQC{@b^O{k;P)(RAYg19tho9pXp9N45dWv^W@&<)i?=X7je>YaJC1PgoS= zadHq@!K)aa!^viYs-cwHE z*Ox2C`Am_ikxk34nePcjN3jrQg|#H7_A||J7+ZXgbEU#O z5gF0TM&r-;Age)hB@jfbpR3PG-i6ctiJpEWYwn!@Mdk}WQD=EPhv$T-VW#~ssNx?_ ze55jNB+rT0Y8jN@KL)wI0Kk|1v8~qP$rsGoI9VX5_FxTaOljbq7`i#@-d~XMgKH*( z>I8TxvQcl762R1rVK8PI@cP~_WPW4PBXB;mT0$5t!*4y83<12a-O9wUWaTTPSP-b8 z*V)lZmLe@kyUX{Q&12dxzr4TI6m#C+*|W6d$g(O<;BUO|Mcfj#4E2?@uO=#B*Cs)m zMp{O<&av-fF!LbJ41S~r77P$#XV0Yg3e>DV3+ng2>VoYh`}vYYiodhQr?3jqSAzJg zW~31O>E|e(j3m04$T?deZVlroUs2v1_Mn@nqtZstge^P-nKb7gc{=B3yyl=fePgj2$~LIArYi#L+us`*(~QO;a~Oz z72N^7hb$9GvKRv9{gm3upY3q}-R%bBnoQIJxx*wRlYto5jiv9;x%B0QH_2?TVr~sr z!cMP^TCtoTh(Q$JJdY0V@YyWfI?V^v{*gdK683hz1MIRvbFAPLC$Y_FJwyc_g(^zs zkkfsli$-iupa`5nOXLimLC#9kTDUq%orn*0nd3%bb5=-7%A8_I|3ZVW0eRV z{TqBZO6bi^{~UasYt2Ggyg5ZpS=z2gYda5h8G>O{%}~#D*8nm;2Mf@8YyT9%HCtvo zS`Taab#Rg43cB~Lk&H|jM`wBqNZ(0}$5_WsHNqxu1G$@IOYLs5^5rJ}4hZ7ZX5rRP z3%hswp&Q*Q1#+Z9FbV7X5m;n7;P6bQGKz*>x!T>eP4*cbt7m>Dc@FoJh#2}(;q zX*v-y@u~BWrIY?)@{!}LGhMCYgwQ9IR&*L8dP22D5?CeW_uYWW#=rOE!4;_>ZqW*Y ztYd4UgwF&>ZzgHz*wzK?ip55vHS-M;&?j|pci`OiI*n!8+TIF&S`CU&T!%g%0ecz> zx$v@n=52mZ-a3RphdIg?MEPmyT*c`5q1@=&Kb}KxCU5jQ=kS8&uLzdt(XwE2CjcI0Y`|6SMIDuO({@n&4Ys|d#&5Ash&A5`W(k6U3zTkdTjTR zevjfO9J}wO;VRf_JYMYPeC?;((NqS`=Zr4k0?pd=;W_#3Gd`)Zg7eO4jSO`; zwR=5p^gzt}Wrv1z-oe|Zh>u|(s#Gmn z>wP$z3YNwF3Y`iHZLl7DbSq)&ymfQk;C0v#22Q+;)|m5N`?U|DpnvpC9LVEsU$UFj ztO4Az1%x>eE4Gj{7^okOwpr!cy2-Bo`dBWj2N=`SP0_(mdLZ;MfU+*=VH#={8lU_Q zyS$iUK{3*k%l;ZbheY}IM!BIW=B+sxI2^>I+}&PQSlv3ORg~kvvE-)WT(L10H3{{j z0NKZV>%i0#gur^hh^uWZpRvxZf&USidB_KT87Rncgg$pDS#Ep@sB8(sL1Tw1qpu~F z8A}SlC4vaI~J{Br_6QxnUT&|g=#S7lFLO$EYIywOx#ePPuBTWH3 zLf_?BL3s9)>|~YbyRyjlYLu45wJTp|99DQV|S!1Gb*e5siZXhWttL-*lZCxJHIhy@>VOo#9()sg$$Z*hy0*e$>vEq5 znC&^Dg*Fv;Q&UHqT-vwZZiOM-DjoiEwCV?o6%q75*kl$LRJ zQ)#l}F{l+#WR>OVGj$_-&2`%QM~y3=&umjwBCpBC4=#9Slq%3}Wv(AA8)y4Yf^vb; zp&>8u^N5hoSZ)o7T&C12>-4ZQl)h$)yZ*rj&tMY2Y0K@1Q2fKJ;xj52dTuu#UnC}R z%nec-lDl^U`(o(mi->cBMCiFzM^Hl;4+#A&&Ij(e5 z;Ft0e+?fEu5KV_hV{MyJ5wliIZHaUYOzk{ijjlDHI$oMS%vefgM&wEZ6FYcLCm9QwG$KruJPwYTb0Bs)Yh1Nl2qlY&VoMr^ePAiFBHnA0H4(!I@=Ck&C+A z@XYk2Enb$&b4QTf>X6pI*32PLByi*V=Zv>sq5Q0&Z5jHwzN%|km=MWYLiy&SM!{o# z{?{|UckDeSogG01{CN&%51)jSe z;eKB}4@6m3FG{s3L|yhn6rgoGgX-)l4J6$Ck6JR~8R&d($Vg+5PJ(O09rr;U&y7KxwEw!G1J6|A$8(B9BBHIWP`2DVRP!3kHq#)&cJSw3R(+`T~ z{DLVNN=eUlVua5#>$rS&(ePS+V0PH+*2{|xBfZaG*Qld2XR6aDiR6%+^522Do&-|VWxw%8ye>DxAHpCphcy}J7d0C`wdVHSsn2-=%Ba4nB2AME^8H9m56O| zmlZaqAz%7Nh0oTJTXnn^{=Tw&)uOC7h=<{j494u)G_{QLSZH*aQta0;yQa_M-FlKx z0iPYyKaJ(zk-Q@gb3bsNSl^AI|(fi<5}(0xXg#{ zZB_26#HpOaE{}!FQ?_o`tWUOkfX1_!+8_iny;(Y{;?8^Ih+m{2p@O81R7B|XT(MX7 zQ+6*o|2}I8IioYO&Xf$U8XQY3YP-w!?hBY9VOvhna?B0KsCI0sgeTqdHt zU1Q~2eTWxSVmwb{CB4m&3_k^@juyvfi@w~u_}VtZPpxQ7yg%OIz=y4K>K-6!W-OLm zra7Zdy_vUOo1|vX@j>6X-B^(ZK2T;bAD+QmhO(fcYs@hgT9?bxi$NBgCAQz$erL67 zY8hgijvZF2oOoSm-QS3u{J8qPvx~-tOH?Oq_UOe?z0kD4v%1i=&I#1Wnc`OWV+hvR z9_1~txol-a!dyB$tjItzcgtQVNGSeg*=u#IF6geVy^jI$VPz-0D%ORVopZKNXV3mj zyv&CD#;(KQ_7+Fxz-UhCA*eRq*3$!z{K2yv+tu4EH?BVSV!6T0N~PKb${?X-WZmCAvIrv1 zsw-Q)PJ*A=lq9cr+TGAEV9`;xkckaqeMeqBaPE{0ul+r1b_j-EZc@$ijF(-eKyhz5 z`S(?Y^p`G_*NJiu29!++`TlVrpa0g>l)znID~k3B<1AkZrrU872oc9_*Dnzn42WMm zE3)13tiIgGlk3y@7dShMALvvdqSsb}=Az{XHz<#c5;kpK!J9x>3?NJDsP#h2di@tJmQWUaL@P#lyOHwQ$XpXvlC;iS1!_xuZ!nC9y>mpr{e5iuoRQ6&)Br? zr*eI~6Ph11=Q71}#@}=s`{?TL(jLFgHoiP#Cw95$8+QqcyEI_9bDPE(dIs!Z=XL7u z254v%nhfeVfDk;D-?0ldLSdam23FHGmceSPYft@;%!oxgNp3HlF&QZH=c?X4>+l*m zdk=r(?A^cVObghq)rnTI;=Um@pWBf`TV{+;1)HJDeOBY(Ot727L4F=f8NILF3YUDE zqj*ft=&W8b9ja1Qm&|xXs(-}{_7RWz-PR8FT2^c&qyWtJPS+suE~r>)bheVl_gE5_ zRxO;cv7CZjQKz0fF5egFUIP^O;1RKBhm>rKjXJ0@jJ@tqWE|ohmvDaZ(578{oX9f8 z=ec-J*khYZygt=451#g)EymN){v=dWr{|M}v-9Pq38_hbJwp(*paf0VIx(ELoKpSz zp(r|pv9ddy-@ON{WSK%-DeF# zZ4&Or5kR#8UZm7Xek;|wIlm?;#?cdfk2pzZ7^goD-uMRPnp``P zDvmq%Mx}(tzh%JAbg1T3x$=Vno8gzl=lh?j!ezTiCAf_v_`vt#LIAa6RA`+F4P#Q(3>=<|3gClVJhm ztB(o^%Z?8NCY&RT&blv8|H$GQCmL@G*85v&O@Oa97Mr@ks>PdA>Iq(!dVz$*mEFSo0>?DM^IG$7!A~3+YZAk%VoqL@NJ#dBM)nK zXrR61SM{BGbI1*c5)4bG&^LFG-4bq@IpyL|S@C1UCoS`n z@XNeo>J|1IibW&f5M;zY!i{XS@csaZ!7_sI;O;744llV(#inGC8vRCW;SYiDoj_xem6`5PN!!*7u`}jbXFT-HBY2B0S zWZbRy+oxnMtI9sdnVo6;DDX1Y{+pbvM9z5S%q{Daw+Y(Dk<5AiwJ9K5qgYV_ou-g| z?8)ZeV^5G|o)&5ai98GY9{cqAdTTQrWxL)IAS4od?o3t2eaJ1!*++veHB>1t)?L~x zwXqUWEO?-{`qZKG@kzI1i1YeV)cHQ$0MMyBi@qBIS;ctg&8EG@Mq(Xa&fRp7j+cM3Tq2(HhS+ag8Kr;|JjRcl^vBAsWtLeCkq6x|do4$> zQOJ=MWAl`2ug-6~^Bv@ZI~|yx?)(@Hh}gV2@*#jO`!dV7dBEhJ+NFK=DFMo)Lo#qgZE7sT~3>k7;* zOA5^GI??t`y|^Npq9YL1o=XS8TTP|LkpT!r4TwM3$L^5CLEOtCxHo>3If<|(+IZ<( zz4Y~O%{0d4tLh`4ac;c+=67Jwep&A3%=hI!Yc^FGDR-RNzWBgpz3O}@eyYBbJw8Iw zhRCOq4ZoC4;ruS16KsF?LJXYbHWxe_LP#O`Zhl^FFrb|7p<00| z%Y#AYrm&;i@wag`wIWSv`wBlrOFX1(WaQyqIfdPW`(~;lv}EzVZ&_q91ol&P%#{X# zVa#lbfbvq`@EU&-p}mr`0}u^&EqPFaN(N`IXV482VEgfx(QniwrfxXTb!F2*nOAn> zz)2zdB02{bfdg0L!MS{(#W1VseZTy3)+Aqte=G2sJDai2rq}G>CK&V zR%~ANFh$+SIFIIWnr7BJ5gxDaeJ_H}&FFx3_5)4 z4ON=~8il&%pD#(@yHBdgod2+^NrxdjM5xp7XMS5r_cwB7v3Lg0gstba9(3m<(Cs)2 z9FQJ@Wm2xW9hm-smDLuH8($~pa|VaoQP^I57ld`LakyyQZi>7l>eE%DTNUxGLA+OO`m&dwxdiS@iZKy;NeVi=o& zUBe1Dh*P)XBU4009!?)5u)b=$FEf?m4!^NcdmJlZ8=d0GbKvMfw(v^8T1PZBQl&KC zNns`6ZI-OygjA|g_O?}Q1|YsQKTppPk8jN6JRTzl$4%BxtDa`|guk`%$cMomqdLed zHPOvtIG;2*YXmGG41RMXE+%dCz&dNC`{S&@u=W}AQh*A6Smn89`tpklkX<@^XB^C z=>*-(H)wUg9M!+PK%{MDmiuJg`1B+5<7MY=fmcJOss8#?_w0KJ)eRupZm>K=3O0+nKLuBdSQZt)H83MYO^nnxj>{(3v>&z#gA`W<)VZw+`mE|R(j@gGIY5^ z9ctium|@IlS+Q~D-Y$QP?N~&*e3Rq1O3xv+6)lt94_W-(gS_j!rP0G-bhn95%}pUG zTcO|BhM4%BqNqUqHh}A!iEmQKYn}!gQ2S(7ixvf%0%vd&TeFmKT^W1=`nbO11)uD?o|+?xOUYmgOfAUt>GTZ9D) zNl7w@J;T7~o)jbkEa`&F{^y$0r3dQ0KwP*RiJ~ZAZgu)OX2t*UnE7z6>Z+;9i~(CQ z9dCDX7Nz?3XW^{>4oCUt2c-SOEnYGaiQK=a?aq zaR8pZYIw%#|J(wX2*^C8Fu0`FmULr;EMHVEBLk1k0UoU6OZzp|7$AoIJJ>qp0D!J^ z)L1n393zef%{j3ER{$lDN&SRkpnSeGNy98_QXeOC#rBdTAnC6`|LAP|G0PWG@)|TV z=&>$`U&Cw=VBbZuvi;jsIB0PwpcZ1`XQ_)9)+|=dD)c=Yr8l+}k2UhWRK-{2+bc<^A%V zRro)a?|?)%pp?+|;Ipbf>R!2bBwShjt1O6FP$>F9yq{a_#=Y@I!?e5IcoS_@&tD3b zuC=@}seD-@0V#K)2!POHX8Us88YtPNpK_j+r7p}4@Dg_q1k&X2Q0wUDUN@jyb0+Y> z2_aYK0BD&>eE0OEchf1vZUT*D=aibk z?)dVZi&8uYncCQ_a|nIlc_?_dDgo(|z8e>PpcD#7Z6Zu2kweD3BrmGt!W2lPK?6x< ze28WF0|gY%!M<4!euLxRrxXXit!gmHU|=^59RJvO3{`Jchl|@A zeiNSr*X22M8l4n&JdV*TK-o!ECanBmR8SnOc1yM1!(F8EWQF5-hs?%i2y6Fg^I^Wu z0XwBZk@CB*Im83Hgu4kTf}C zPZ5US^JtTH7J{NErf`=3wuv=j5UP%Ab3Y;FuKs0uI2+pc#(_O{L2i-nqp)S zkUD@M+cFCW9>nR~-*D*z>E}&H2nkgHbUS1I1EYhok5et!ezLTaV>PP@099WTKNp-? zYR!9x0E>@7Ej=PBJ!0S}Vk_d3xB_*%Hl9Hz1$|1kW0;!Mg9PDXx8|(}mC~SofPiJJ z_R6@Xy0tW;jgkGhBH)kobU|0^g<)uUt#yM9fRW~Cv1RGI`>^ff@g2+j4Xif;o(X{AfoNePg< zXCYaXmMS{YDEor-Fi?p_U z-G*E5@!KcU4{F?xFyljr8E*_wLTil~bRnKP`M-7 zTAw$u=J07kY6yTFs@I|TbZQl9?4eShUD-XLM0>M%>dJbb`Rqdoc0$g`CHCa$XILa% z^-F{VS8rGH(gE~DZHvM)WInwnS-*ji!f2s@BU83`S5KqeTaPu9r0rmf7!)0;1Rfkj zF)*Gl(iMJ~)GV2LKc>-OE^)o!L7YE|+s)-HNnwOK%3?wp&o4X4hhItRKaF#8&#;`@=7%>xc zYv|9FjIx@kVwu1h zFi%zfL2yN5i|zWFE?BZ)b;dR|guwIX=zwW{^GR$*9xAfB=CEx^dEha0K0){c4`0f7 zs3YjuLG3^lU)Q2lV&!|7E=w47uAM!Ae=R!IG}4%6N=BfEmFdo1VEYDy!PB?K{SQnW zKki19Ka1hFhNEIH2AT0hXmF7zr$Je$a>pwCnDst@${**b#)Eypz<3Litv_yFeju?t z)|7IPgt`p~UuPCYKb9Qq#B;%7h8$#!4SJsoy}7(ok!2_rdC~lhRXL3QJ~l_Jw~!EA zhZkV)J;mwUDB_Q^%qYGnq$G28zKk0(8`w=X9&49d6@BaUC(ls&v*R|GxJp|C&qH0S zWbr-q_lF&frK7jjY1kGXU$3y|%?)g1h{O?u=Dq?mhLqd5QT_Jf&!BD63xEw-_7@v@ zDT}g;$4*T3u4v=-i4gH0oWU9%-)dO5rc)H>IMf-bH=)w7XbnTXXl&v=kv2i z%Zdh!heH&Elx9u&J&L-~+ZJGr`7S?+2Iq=E7?&JxqMzSdr9XXj!ed7mAw-rY*Regl zeex-#GuE4+ujeg$`9|+U3;6`|i4Zlqj3i^VpVX`6GQe5?<_to;itu`_mOnj8A1_mN zkN@6loE=DRD+4xGjV=#UuHPK{C|lP14AvUDI^O#pPioLCe*E(BC<9Ly;?=_AhC}GhIKYtqqt#3*(WBE~+jULPKIqlj%d0O}o zCH|giC=59y7jj3LpmP@&EqgS+_~r|Tis%y^C0AD!!?xyABwv@)gIY4WjC8_i$LhV{ z>QQK&^B8kn5&{;^R+WR3Y*?S3(VfM@7FLBFyJhhBiYXq0~=3obnP2F;p%K-$YSi74B zhy+Hv#X73l2a8ud0c4U_>4Re=d7aLtzyb;bUK%z5)I1#R(&NB7D@W2d{y8c|%OmH= z^gE9u8I3oF#!SD5YA7Sr4Eo#SG0wVKSHOJD6Kes-5QYN)^noxry z>5Vg^I7TgkK9!B)9!K#%k2qc7;>_VCC> zMLfh{k6`OA?)N%F;3vBgK*Ou-)j-n&@Rd#IKbu{a$BoR^zuhyL?zW>6#;F^7ue}y< za*BX~@h^?d-q{$Bo3*k3NQ*11WrlTb(uo^-0H$Mz{3GPQk#DYi-A%G+yT*){RdbQv zTfe5xZ}^%V3H`pIXig6DWdQ4_Jz*bI zmcQ(IWvD}|^817QN4sX-eA>csW?B|w0v1~qWiba(j$~3){I!u919=MS%h=u5pqbg-N<+R^JQXPH(9fpKCVL7jMRufWNMR;{xk&534eO5=25tOW)5!AupM2m#Y2ah z1+*;pR_@@l1Xf~)SR(ofsdx-#rz|b*wqIW9kDWhT**Q?*hOhKNJ|?dZhu8GHA_Hb2 z2xsq7PhPWcf~3cxmvL-SrgOIAZBcozu5%hUe9&e=PdK$!AKqz1#bN1Bz;-scH?oX- zG6JZnnMm#NT}BC$qfTxv4h2|P<>q;gyMq@nx`@@fBg%+H%CA>_k0nG_eE_BMV(t$& zpKIW1qHl3L)xQ0$F|4_u3i^A)Vr|EcTQ(KnNL=28FtY`U{bqfu2u8NwWaIN_m9eC8 z>Fi~GKdmFUy;p;CjzEyaCcZxVmFK3dm~#`QkQU<8pipIoa{w_m+qR~>U9aqozgG9 z`thW;6hD1ahljGbZx3|WwnNm=;vn9e(9p@!9yKrjDzBOMYs9BzeIP^s2wZW?RXi9| zGS7Um$@8?mx#sxV7KRr_@QV1YW=uysH+i{H+}T89oz*nT!>(?4(&KT*0OH5a(!p8r zqh_sh8;HosiQ8zzv183p&!9q3>B9Gk$eHEydFbpnzK3~fXv+jnGhm5>YH8~zK0tSd z6fpI@?a!=LAwo~sqz<2eEq)$^DcauaQBRI2hoQ5kyvX8T-Zm){h@gF9!{o5M%EQ6} z{*>zcR7P60B}tmr2GN73+?Cgb8hg(k2lK|buv6jiKsZ?JMQPFML-lr#`R11anSMF8 zm1KvhLW?}@1Xk$#`Zt*Z3t8?0q>rdCtd0UDa0RNB2C)O9d;+yJC=?UNgCICP@^&Vip zUIH{UtyxDMo5$+s!%=cJBhjCKZz3t6S13X64Xq6J8IA)@P4ZQy1`2%iM6wk%lGl*J zb%DxOMwn$IC0kNMXnJz|!CW8N6?YlwsHLf>6%b2oa}wTsqQ$Uy1>uNFb^5IKqT8IS zpd?LCH0lW4SMoqEVsCo6931kxITN73U9P@?kwUvzpLNt`FhG3>OkBI?8 zW}dpvQI?hYxaHgc(zQ#$YD4|Z8+=v~)h;_*>S;^s;ype%nzE>#RT1|IEnv6!I>KIBt+V)_K)u!Qd;k`-KAFeeWuO+xj zz>lC9mt!HY*0NAVxCtd8xc8cS$JQX}F-aqAAT#XXg_>Kxi{ z2xql4Bp(K!$6{*p$Z5c-GkIVE88V3g!On%eQp_033{gq-P~MI;wM`{L1axeepuLds zJQF^w9HJ!P&M5H=d?i{!bkAWq+lLqX?Zz`Zke;wh4_~M^iURWIz2k^NMBO*2n^h#YVQ@@e299>#eYs75Id!hikR_{UYFg|C zz6{ez7XqdUWumolaO)saoY|F_1zY(>!FgS8Xy^Ddl4DoP=jLN}4v|NLi;O-XVMQM( zjuL@!S3}#-Q_411Y!2)YybNJQYK)X5O60BByQ@O72RukrpLENGpBKq^vOib{1k#iH zqtQkC&Ziu!K`V0CBf#kW+=kTmM$RrI{3rX&n+vLKDB=OgYG$`kjo-NnP$Txwv4UGg zYArJ(VwcZ;+nnvm`yd~37B`Z00qBK$S&L^64vWk`w_Lc8oH}HcUdl2%xy3y5mTLEa2Bgq1yElpy;hY z<~?&SyJC~2-)=f{&3bnTJXWP34KC=okoevX?P_<-f~ViPt|Md}o19O5sd@ZuF6Fxe z5#`c&Q}iHYN<1YNpnR%4P-gp*fQDQO8t~yzFy}A^0^>ZEp0I3Z?JLlxvzEM>Q(I{* zrGs1s#u3$Z8WI`?kNmR;LXXBH?5&7^(j6E zW1k{AUf+Ih{o41*3pyTNZM}!7$Bo{ysyR@yfU+4jD zD{=s8L43pRer1$50IQ~SJTiAJ$G1_OmZl}gI1aI`1OhL&e;;j zO2~j@9SJ0d(Vr)L)SV+z{=7WrG2(2MTH2 zNa|%E+)kAvh;vx$F)&I3_&RyED2Hll;KNAP?6P1=4z1KCy?gh%4X=OG%ME)6{$rZ` z#?l(^ZqD5ghXDy1+a+@lkkcxVQnxUR!EA`d?Qj{p6kXkHyWOUQ!w?MJrk-+&928DBkD>w0`ZaE*hi{ z>Q0@v5_J7}^_zWb2$cywz5UU372t^m;LWCQb2VLYL;4B6C-DK<0wJfu#c%#&Rg2O#4CP^;?7h)CZSj4p2?Vw;z6-uA2KeehPn>S+U=x z!5Q=qL}E;BAGn7ev--$-AsPQaG^rkdGK|*MOVeG+&9h_~nqjJ8XOU z*~)+Dj495xkl&?EiQRe9M#9q75dC4HVfiOWbKCYFR8nn&xEdvYl z;M@5Fod!TKFm+hl8;3nJ#5ZN@@h>N}b z2Ot?vLFS4Rl!Zi9MK_V=s;5{8KqeuiS*c6sOskbRjcR`ckC-Gt z7F(It=vVz!flX*v1CPkTj_CX4dvkk*);w6XP+OW0PVT8K zfd+3%D~2YBDx5hzO^7pnrddB}9=)F>YiyVq8R7p(F6;HE0mseD{w6L3X_Ju|J{#(1 zjHp@eKVZqa6VK6Rd~2He)w7{1xE$m839IjC^2x<|0o&6zI(CIQ*CilO^V#s&o*y88gK9?f58`6O;4C zIYrw8EK1gpe5RU`QbJuZ>@@5gfC<1$S#V0I>@2)ntw0!k33347QMB}1gGX=hQ@d~ap z!ykd94Sm6rqzNHE7CmprlM`HF8Zy)^q9IBw!E`Z5en0EsyoFbExY+5?1TIRQY9W8R zC1=aSu-T(@6~c6+4b764maN6{OcD;N5U(P_`z{m%dZD%TvnmZ!ya{#gQykO)r|I_w z3Os!LC>P2Ywl4urD|ZX=cjQJs%$`g*KtMl~gRSe7_Gq*d70D8dJx{cP>#|8`dNMtZ z?jlVeqeMsPY;*vBpab&$H%NK7yBRskme{H+SfnWj$wMx0*&Z)IZ=sKbfG-d`l13*3G-0J zT63$CwfHwE9mb``y~}c#kqnEtxq{F5i;Jc_5p{`%3^MJ-mR)QoOaN*vi{sorus zBl_c`^f(&A#8^*yHeX*nr87^^)Og`|jxi?5Lmvc8?@!*oKgaSt2} zS8)AC*QAl+FI;c}cM($b|F*mQq=$6zQabrx6|%6fAo-MKLMMGR7S(&)yThfbkN{p^ zTvD>p#=?aPe(RQm3y2XYBcQ{;?gTrhtu1vX`ul&G&iGb>LnnBp@P!7lfn#eLkj<8w zmDjNRe!0&P?b&+(}<85WY|Smo>#!$)UWidD%S~w_2U+zxn>o5*P^ic|t;iln`ZxUr+5O zI%e)EarJ6?S(#Qgr;|4Mdxd<#rw)IacN|)XZS;`!4EzGg^z7Os%`DmmEzydjeA zNO4q&h;16pub+K{ha^X#)}XTfwnSrVw?g6eZS6<@<+IR4M7u91Znu`$&8E{jtSDvP zEX=~yEY6cAYq7S=@%Xi`N!S!`+(;v3ylMc9b`QS3$~-)`*}dIU^;uc)FROUV&}AgA zuEfN~clXI$rMHhyJ5c$6OYn8*3{yOHbojwv_Pt6y{`G6w+}zxT`g)~8j{baRuA7`5u@k25!-r{rqXk!1R=zDOmU!pk z|9_9qg^qqp`VSvIuQz4uAS8Z35}3Ks+EV>4f7p-YL1)`)fLG7jf6QZ52p&hMR$KRn_YdYJd+zi2 z&8q{Ei^Z0lAXB{HM&YxHpXTY7lFQ19#c5{dr~n`6+Lxyu>`t9$VWm~s-mnk|AqmDd3ZEs(R+e6VLW~O+3xDQ(NDrvTCMoK|9(N? zqP0#dg4<6LwVRII&D$y)_w&F}13+U;&AWHnog^M#8E|LMpTzrPlWC*W)YRYtQIVGR zhQB@=TNn(AaHV|i2npCx-`(Avn>sQ3*K1myw8(7q8Sm%!Bl&eKLN_0Mv27MyM?5H~ z`WbO6xPoKVkbNNZm(%FXj|SXeB5=$^%?AgQ(F-Oi*$Bn9@p8H*~Y(_Q@litpCCqh1~?KR_-E8SXD;}QC?2tC8JRyIE|n1~c1 zqd(4|3K}`093{C1t*0m`DZg~|j4>N=Up9Lk{GRDty24=O=%YVf+5M-*GMQc(S6t<$ zrhcA#No<@mI(t&nz#ug^B;=CWh&Z^&ug7fe_R{0_hhO!n_%#bv>qPdDN7SrzEQh_R zs-vT0?s$Ftd>x5Tb_FZnmolyrzs87Z<%Ue0`i0aOM5azTRHoaXD+V>fe~&R#~t&O@kBI zV+2ut{S_U@w4Qr3O zX0N(x_Ws_lX+1}ofqPW*_U$fxy==k+f!~M3ES;EcDKY!bKuWCO(3=t;)5gHSSmNp8 z7y?WAt672NdDNV5C=N!ltD9p&!?qI+!fMD#A1Q%8$YK<_uug2bElSah=z$ziVp3V4 zYjmLnHS-`$*b&vhvSszrOnxLwkbkD_F=|$75)zVQ*EFu` zlaL%{Cn2Gbr#uYa=@3v-BOy8J>a3!2{hEpj_w_pv2WK~X5)zH5B$FdI4ZbjDm_Ln= z?<9MzL4}~|E=;5|BPY$j8xv1XuNh5s`BUs;>JrPVZ{HO1JT9S#S5;m3`etC;_OiXb zrNFtzU(@0vO18glv~RSf?`6nfVRsyt5QichgiUz9s8*7EW|+~M zKVkZivf->??zGm)0HG$hJ~MZd>q%*?DF2I|59y_72hKE?pGJ~JuqL%8Igq~c;=-g* z)CqjfTxGFkx@3Lku(G#w6IA|m^6|Gk>)!(dpPi~x9(_Yg2!c>yNK7IEEDUidikQ0=nYQLh?lDL%98j?X6_cym82q)6xc zI&azcfGtl|Fy@ zQ_saO$D>TzKb?O%BAuk*ar$%2^;tpg_4rd~@}?)gY>j?>(HMd-A3a-LKB#+I$@` zsKTp+cMLZPRb0pnt7u}hi!AdVxVAvn_XtJG=SNo9Vpe8HEnfl8pe-u73|T$2`a_ws zOKHc;Nbyus;(NndA9#py>=aGT)I~|o;CHv-hKXmtIlde3zufTSQDA#xiiePg2gy04 z#Ml0Q^X3LAavO`CoQpw6lu7Er*J_#&>k_V!=KFg1x{hNoJCbjWRNDIMGcB4#Sqrbz z4b}JgMw}luoV-aU_acsMg>09b`9AapH{+SRh9W17O)2}|pVX!h`yhY)sOAUiGDbHN z^9H{%wuM7SE^|1M^M5e)q_QBDYEY;Nl5NoHJT`Vuk^Hg-<+0bi4F+04f&^!vv^G(F(>efC2VP&e@8tS;igg$Zylfc`PvV%?uR3nIZvm5X8XbL__d)S zlf-5B&J%gZzh24gyy`=i`at(GO&7(iXa#%m)r~imv)9(>gATc?Tfb3YZs@&q*dbQN zMvha=RA~*3olW`xra(>e-t#L)@;3BHLiTZXqWoA(D)coL3Yi_Id@7l_jO%75xv-?R0ny747XWi=2k7#9Z@ z50~#Q&MvlR>)9YT=-5XIxYAv&T@!n8g&bmWqE#?MoGCi%NZv zYM&Y@F6`>(oKWdjF<@p>8RyhfX<2DkiLFR>S?)rQx(6o9n4}s0f=vQmr10T|)RB zZZ5O&WPTq#6@4!{Em{xHhZhKi?$Q%j359sBqYqig*(i@{9$jY}5!t-`S^TKjw@Vhz zHyt+{D5dU*u-b{EoXWdcGD~dWjXRO650j3{NiURKa#^^=4v}vRsP%f{6&IC%?S53# zZN!)J&x8yfI+rFpIE0JfUI;h{mA?#39d!3_<8&61LOYDP zj5*r4CfvSp`z?9`diGOyi)hcC82fVj>ZBBuQ(=lR%Gkg7OP^xjm3iO3nIffufuXDc zfp_`kfi|wUTHgv*6jvc2>g6==(ritf{YNvLmhE76^!8!>~wlG#y$78H3~^)b0Ev(BcVG%RgX~^(}j!G=>xI zCdjENQk7ATA9ka0I)3|b&JmAeZpTHBPaHNntUz0Jbo$8h(XjS+j`rp2@2now+}BWv zy&rOG`>o^Kv@35rA9X(M>=Q{Z%PKo&bG%XTWBJFz##&x^UWcpVs`NaWJSJ*m6E8di zUS81MIJbPGTrEBJjoR6(?vH}l7wH6JJeXr*IVFN{v)h;YE=|0E-=u4evgFw2ZkP1X z94WPXWH-@l7#+yC%&H`|YeA-9qyI7E{zla4s8)vMvx8^9JWG8h@vMd4G2s)k^BTPspsZj7Qc+&cKbFv{WfquKzQrJj>L}c<%nBJ+`*vTnM?-q=Cx|DeO9o0L0h~=gMny31iaH7%$N`bs{v+1ewxK8m!=Lu)Q9*!L0 zl-)X!x|JE0&7)({(<#$pGi@Q_Pb6H2-ER9eFHFlk_h$Sy_k!(`|Aze6x|f0Vr{13| zz7Qm(WFOW@_Kd=tN#2zCZ5jM#=1sE@*Mz+{f=^aIz4=tEHIiIp{j>7E^_{o9*%d`= zSpBi?Cra;k4R;Op-hHz#ZbdwYSY5U_ZRO={-}!a4u)^e2Nzl8XHO!F2H;J>ZW5wwe z>sM0L2cP>U`u_A?oa&mw%@_BT-rNhk=h+zZIQ}t#Ek;UcaMOk%ya(0xAbj zpU#C&^Na2$hFvbYUudmjsAh5tui_1Cc@GN>D;aXGDEFC3ww$nBtey0SmHaFz7_zH# z43sbvEL|Fr3tLVKeZ7rwed$uQ=(r%PKtlWZ_|B>pURdh58i$+W5AVF`GTiqV^;vaI zmxQ6Y%2D5s@O6AIzA7^_R#IwHihT>U)o&Ry9doK>EmlxTR|YvAuov}_K|$wfPRTP< zewetP3)0_ZA%8x%<)w!LgYWdt;7WOJ``h+(X)aIX>Rt4MOr}twKFgnQ!{Ld+J3pj4 z(6JRkm3>z0NQ>}K8=5M^pQ|QK?v-TJF$OIz__pJ;q-;GZMz!IA+kKnni`84z)+1jK zSiHxFQDm_9GQ(1`d}q-0_0CP3spa+hWypq$()ziz8?BS~TQ)LQGU`Gu1moY^eGnTe ze>daMQD{|za9TLJT=%o3MoHF6(`w!K)LuSerrE@=ly>jbj)~UBz*1~~cRvC094Qe} zJD=WCJDpLJ5xI;V+Xy{3C$TU}=$e^)uXi4nv7tlI##`)u+SO|8&nw;7#^SfnTAcSE zHzB!qtnC~5c$*N3Q$9)O_xCZPx`B-BbXNnzIF!egY^DOelAKf!ri;ONF4&f+OOf(GD)Yn zYT#ib$`ctZRSz$8rlT09?{ZxYR3r$sa0S15)Np8LV2>sr7T2jp@^c<|ACK*CUUSgV zA-MovQ<9LAvXW4MSES%uk(BM<*J`BaNDlpapNxbg(wT((_cglUH}NYTd=tm)|338e zA;}T&7d`k6ct!Tl)rZ+%9s1`rg*^C-RRvym8XWO zHwg*jS>iY8HT^Ry;Qj;7hBtj~>S)W_LEMFGZbNMCg#z3?iT5Fq50C{f-R*sBxC7kX zJiKKC6nK7JAq!p;-xlWK{&k6us{+qWo$K5xkURF=l0qUvB0P%J+}zyqcWyh#>R(m+ zJskX{z~kiO<0&gF?C=}55;w3S05&2&O{@X`?9{JbMn|}?JkQD!W=wBcG--jA` z+uu=vxPyoKDE>LH--G}D@b^G@VdC2V+Kc^-{`D@{X+>&z;eSs}kvjC!n+b4`?9Ny9 z48dw9T!+Iq|D3k&Tb8DV${BlN;IH-Dv$8HoF@U$3VQUE(nX199!<4S)<}y|+G$jZMoganFJ$1B9D&PQ zeCoqaMy^SJvoEOk%Av@LSL@HK>$0;tR0WBf%m__NN^+Npz4u?vs38jsymq`4kc9~S z&E}xQ2Aix!esJ&S=gOj~!lI^}9{){FgMa|vZJqPEPXhf${`@WHS2AD8oILUB%HD?Wt=79g^CXq1!1K;Q?;TgHk;UX?_PJO!lh^mX z;QxCgnB$ejJj*yGULCj2C#Te=yjr9#DB)SwAh{YCd^w^af;{7AVy>Y&yH}Ihr|Cus zYQ4WnyR4Y3t>MGoUS)9|ty+X~963*4CONGFD>qy-LOGm*hi`M&WlrYrJ)CBull}&A z)FUJRO8@9)9KCd+#6v@s2<05I6AU3aC*i7p(J%A?H6=Ua*ZFaB$~6scTLsPW&P>ub zh2Sp}Zri)drOz`M{$iC|VWiN;9l!N6%C7{Vk8tOpDyDWC66BP(FVJ23;Hdf9{O?uY z%maJ&x{g-&VA8}bJlN?2@;G3`1L^TcL>|cL1Lbm{#t&?df9%WygXzFDJ+P+_JePmn zvjcbQ|5L|ywzPq3Z?grngY75{+r`36X43twe79Pykweb8=>=&cKlo6wa!Ui~k83Y~ zLkFF_q)?SOMFGpR%C}yVotVEt7Wp773%Ix>&^>G5Fgxl+G~6Kb!i!s2-0hHLk5`q?I!i z4xQ5~bhW~!YAybZ;?KC59`f^xtNYUXnG96uaxl8GAFC)cm-VbbIOID`+8E)nN!>)j zhsqM)VoX&v!o3pPE(gIgs||6VdHV7Iy`!hfJSZf z1O$UASyaEoROXT8rO)my9S1T76<|VL5fd zWNxT;Iru_68T5$a3bYuZw9;^>kg$QS$N87ny?^)a9qY}P-t965X}Mx9?XvSl`5WG0 zyy;XvA1IiLd{^hK_kxlbcapaP`pPqm5Z{kMZn}yJ8rJU-maT&omY*fp{Wyy{9my`b zyyUt)iYxIi?taU;_UcG6V&+R$gH0rdvBD%v$IW0dieRqIh9iAex%^67U-Swr{YQN# z5RlM$^Fo`IrZg)$?oL-g7Ntsx>HNyK*eHGhG{c*uvL^v5)5q}W#d!bxlMSjL7vkIL z0zRv7zUxDrRAl#TAayPE4!ZN@Y_j3pGCqAdWac^CchS|lHVB)Bwoq8GIX>L&r8%}W zn;BOUwo!{NPO8}$3)T~4EFp-N^RrRVnjfR+mReo*$(5@#u0z>XbNYOMP_sK632Wlz zyn)z2EEkvfWM6DU_9$1Kls@|7o-rSkg`AvLjC&A(UYCg|6czj7OBE-tMJQJ&H<5E$ zmakK}8INCz3;foUEpq1=dv^-+Tv|s9XYj(qvpKEa7K<(>g+oZ_*vP6FY)ji0kX7wg zRez;2eB{d}Z|G!QU{vBNdS)#ov*9!j+iRd#?BSg@{0%kb{Br1~=mxtlpsA(C}d9$|? zsZ?b3aWW0P;{eDNQGhdBI{#97GFmJM+eWAwV!~N{fvUiZ`Yu|nKGx~U_6iuCxAGs| zen81CsRM^MiXMtw8!YvIct!ReCb@pc9@higt-#0aZN^1u>BN{kZ{PLu>#fl+qGIP$ zsN)g24ZyK@rO-+SZV@56cbrDnd%iYt_-XTtzEIaa#=iJN07~QbhT@e1`lmmN3kT70 zuY{AK2}}Ma7%du|otUipXR@A~^)aG>#%L#>0$6@&@M2}ZCcz!oZq-Ng=_1$mK#MUZ z!UiLa!ws5X9yGVx9d*CAy7yeWV_u9sNM$8TFT@t(%~_g@OxV<9Wk?GR0em#FQsX;b zHmLFtg5F8XQ~8{|k&onRz3&>z*H*P=Q%_h=llEXegtPzXAnO-uH<`3iH=|&%oohFO z@2k$3V^$fOje&$r-Zu(89-03$d}oqN&+VDOe6X~(`^+&S^WRb^K*L~T4BTa*M1k1u znYTyNa}UVNCybQW5}7O$CmCKCjDMsNE^3#@e!Gy4^8AsYZcG%s=5dL`EVStDv5kVX zv9%n7P@RpF($g>O3Vfb_EQ({$m3|H19$i+f4E(9#s1dLOQ~u&8opnJ^W-1Z0UR_Vv zJ0;&(&Qv!}ftt74fcsYaJZyxEv+dfZfUCBjE&%zi%h4%{h5%_!|{ zbrdQQdX&Lr*pj55_5IxR3aKgD0t|n zY%~jH9ic=ac{Vh&cDJK0Wp~Q&p$c}mBR_Uc=O>L&%`Ktzs`9g(U|Ai$MNj=*Vw0b? zdumjoHf#raTTHCq6yl;^dGR|+yFK=CpPWVq&^k&N`Idy;=?EV4c3CQda4I6djKky_ zZB|rBNT3p=^EX^iW&mi}lZdlAO84%eQ;q;}!YD}h^qesKTnm5MH7H(`5#>?4D2cU_ zN&g;Sk{cG|YGGHfJg5Djey6qmWl;EYDIw(M1jei%-q(a1>@lUEfrW?pzHhbkn4rri z;JYsrE@z`H2^bUWZNi`<%o3G6V5(qu#vNPJqEG@y6?;^!Z&&dvZ4Y)7OvWjN;U4zQ zrS)Y7lVyoXcc(C!)O0IL6 zu0H2qzxhenSgCbwhp14SSgf2qwoCW$rI`A??fOFT<=l>tWJ|w%kAOxt<8YiwxKVaT zP)5sKF3c(vkfbR`?%Xbu3G!!>NlW*lhktFQqR`~gNT26c1$JgA^~ni_|6wQqi3=CsJ#|>A0(b3Gi5*2xS;KpjE}K*?AH2c7=_R ze8^1So^$n5Fd&cX!gPALa%5unbEO(4#H^}8OEFUzf2_H%laBi{NsWzQ>CN$bxnb2^ zEF^Qoo%v|C_qncfO3Q&A=0N3ziFU=6FyU;+HL+|H^Oeu=E70R1uW%Ws=Gb=T7yH+D z)d*&{5p0!mc*|i0b_|_u0n$hBp($5K=#n@8dOeVF)jzwFkv#zu?X|+dozV3`$Ek`QUGvefj9_WEjhSKg`uAWBXs4jB z-an*vzq9*~co__%E{|{W%_~C2^!N$sl3do?L;ZZQN~{o~hBu*=9*^h??u`y#cMi*E zT1<nIw(+qeI`ctxjeu)~{EVTE1WF)-quXeAp;pCySeh>`p@*;JF=eifsx~ zvSFTE3NGf%96(=^5#uh5K{NUcA*7l5RKV7G0(54*AcsF4+3>k|kEi$rN2Orj8?u56 zF?+%I9++a~i$sI*c*gi%N`0%36u61KBhYlp)2i0yJTPt zFec{U;?*7=>+**&R=E`88^{d=L#XNUr!I9aM7~|zkWPtng6Ga#4rt?>d5+~}wT@x3 zNc`^ELaA*ImMmCD_%2pT_S?|%QzP%on;gplEg1Wh<~S)c60702t67SNvb}Ev>;%jO zx@-*C#ksTR6QApSXKPhU98fcH)h$ZuOUwc zIW+yuKu^JUVslPE5WTm%VY0i^u@^Eym}X`Tw?tRpK1*?uO+nxNc-YE4M@0u}(c>~d z;-eO;=X1K|M{9!Xa;C8-$*q6^$d)wgPfjUty&X!n zcKxGa73d(Pos!H~Wf(8VbI(yN)}VkBqK?0r*K z>-?n>^-z}ND8f_p9#~A#Va7=qS`Dfqd+}DtaGPQ{VcxplMvAB~Aak3UyG2aW0za{B zGM>__O$@mfJ#RvX?FOML_-~Tl;Tc_yFWXnlmJ<|b+U_6Aw37_?P#^D%)1;WzLAtkD0u-u*Fk@$K#1rX{hbAraZP7D4L>M>+ zo_Uy7@96qdKkF997;sX7CZ&4FXj)XVq#0u4Ly@~g>D+E#HL|i`54My)IuFq@*#LB3 zXg>Dyd11e4>AiawM!jDl-*5C*v5gk;Sf(M~>mlgOP>$t>U*=@{-|HqV&BvD@tisvC zD`OlE;ijtkT#EL@5Cu$?g+3gcYkBo&<=+M(C{0JV+WoCUm-Tjk=oR4KDKpetgZ+&0 zt#hi4Y^+A&iczj(dxJ2-VAB}xN~mvj?Wp?%V*1In!r|`9?VcG+7($=W3e?fiG_XQ~ zLbocVcG1s#!XF~mhg@-CT$A;MhvEv5J2!k=xC8+2vaLG-$CV?`Rod(B!@S;gj- zYqm5dWn$*1d1Fw9!|ssa;R`G|frW~hIqwID=6{{!m12ah-q3XKn!l-am9uT_8c)og zPkq8PFChkH^AUs4?Y2l-f<6U@((I01#hBE?hamNYR7kK+M8jPQ3`G@25E9sGc5$v= zn3%O8auphn%Tsj_rk_&9$V!ruh0;ILNZkIBSxSCKvGH{v;wN57--Kw62rsI*$~B(G z*{m%~lm@n8d{>5r^z_joff#)W=X&7Sg=EtP=D27iBj7+>m-{W;V4Le!caOt2cNz@m zY?EbaDHd;R&V=vIr)T{Q(jr9q13=?Jgu!+A8T9*rz7}ob> zl#Np~bh0)OcF)Isdrc$x0-I7_kfU^kB_p5vb|x1guW}J5G>VHjBLB*F8~<>OfuIqp z8M?cctE5mF%39jC=gwaGdhuwaYB;i0zVWpcxjEvtDcsEzYYo>n#@Fg5U)F+k)U?iS zJ+a=IIN#I7cEV9Q8gGO?8JbP|Qq9tIdutIA9_+G{6WWc|z$Pp$clsHd!gJMz&t_r9 zvwgRHc8W8;$2a5is=7LSs2apgsuWT;46bcwnRTN}ftA&Is*JPJ0#bZ}JHAvITmL?} z%OKg%?BP2}N;>8`AoZ03d`J-VpJhCqYNK@s?x^go5+>;^trr!eB7SvtIH|&1##RiZ zDGnlvlNZd&W;+qb*srG4lO3EIck*O(7%VJD(fy=k^g2Z7%~Ji|Zl<*kfn6yq2zg_? zY&IxYwtux5$RW6RSAe^-O<1PtmyWRYa)#mDQ1$*P*jZ1FCpB%YV_6-iV@iP;BH!z1 zrCmw*a6HV@w%Vr{Kh5J+Fb9l>)Pa(%i7Damt=_i=?(RTmD532S(@w$&4lrP*ae08! zidbrP{ee!#+p)6!eWm!}y-e#;LJ06Z6pa>-1lODALNWG(;`Jk=aT^;lM*o z8GvM_@kuTA)386kv7EP8zB_Wx(@9S52e9tVH#&(J3g5SG+@gA%DOqIId}lpj5yHh` z+X$PRo^I=SjU?}BQk=Wl=57T{16{YC^$Dr^EuHx*USgg%n`efT8%_%b^(RxfOj?AN zy4!@G)Ge;<+1pkX6L;ATb{!L!4P82K;X-q^Wms$D{I%hjxMxhrl?Rh)MHawoH~4UK zD0lJO!zhk|xsTBTi%Fauj~$Ynkrosxa~GWeH$LTm(5qR(k5Z%CGA8 zL8IMRCIx=wSL(!!jJN=wqy~(RL>i4MPJ#Vhe17S1#mYxGct66!IncfScV8(}AG13*QoQU@DT!E9oEG4u^&_=L1T=9q(^ zWvLW3dpGin&yJWTnhp}2r8+QuWf6E>$VweYD)u1{@iD)-w90#u8R8v5?|agh#NVo7 zH}dPtIqThD*RV(nOkx+c#^al{e_Ae^V^2UEFIsPXjan@mdx8L1gzYz}dxgSbt4|o} z^(u4HiaLI3OcrnXcUN_yC3E{WJYP7yaZlromA17_i?I^#xTWd)4Zl&=rG=Nuclc0TcVU^C-Y@QGmZ-*XH1tZ}CB2cI<0g#s*1v7+#wKVw+j@pw#F1!;+Rl zD_lW-ST2OQ1HYLYj?axE^jsM;=A2{e!yU6S=_Z@p__f*jag)T&D;mA&Orxu|0ZG~7mOP_;ON*H4wmO(X0=-ML zF#R;baN){=KW;D?qNJ{x%dL~8nYlIsIE3=Yxs}9R`Rsyvmg+tU?041&{N%tkIw~6p<8|Hit#z~DKa)!3SI0J=$adyoR4Xw* zp6Qif$~lbpQ2i^VNZ_3tg*t`1WJ~I(c8Lp{X6J*{8K~J z4Rwynsy6Qo>`M%KcM;uZ)xY78_jG&HE;rpTb@M2#+(KD_RVOFXK`7=Js zy?dap-M7R5b^4m`r;Aoz6aI5pZP>g(9R8gq3RU7)8Ku)RFbts9{xS3B8FOp;q5?<= zqR)ZV(YekY99!@k*&p#Le6$8=w#*@~77qR~^JUdg>qOMv!@PUh`8t^L+XdoW(H$_N zr&@rIeir3E-%vUfIFrsl<{JFARjAewV8~-br9-=8NFp{%1aO8V=dR&{>$_q+#8<6` zpG?mTywsW(J6nK9ax&EB?1ehDA7WsOMmU*C#la-q(+uVtUKe!KnaOp_p|{#G%eza< zMC1`SG4&9O-fYHAWf$Nb#wQ_0?%~4>YylvDtcv<(+>o*E5g=Y#$lCU-ZO_l&@a3fm z)<|Shn{NQAT-w|`HXH=r@MqZFlF zU(Dc$u0&OiF|g&iisXT{1cSf)QsnN|!b00xRDJ68R9gwSi4YMd;#bV&2aY*|x~9~3 z4=cVL+FHJ_fIT9s{T=+@^0Fac0;CTNEqN z`f|wc-p^@tr-L<@_qR)F=RZ{eckBl6DHur?LAKc?qKp;FH~hA8gOZd?h6ZROnc!JA zXpV9_v{9AGQv7X%gxPJEK9P=jU_USwN@?gMXR;7n$GB{te|lS0l}1yO83vhhq_$3t zt8z_(RK+X;%iRc4z8hoJK`)&(|1mB&TQ+QO8(qp#<-RIC>J+%sbqW{&AEK)hGQ#Jy ztXY+=&R}E6B7Mr|r&s%H>l=cOp&aeC*)xAh)HVfA4L@9p0E6xq1xpW?oi0fS2ZXav zPhek~T1U_hQCe9)!)rfY?Use*zH$u_XlIkoA~eU9b96|tee`8?Jw!$>d6G`r?ekY7DOE~W$k@q^)qkWf<>a1~axXP!<@7o}~@eU5RNj64fwQ@HqwZ%Fb zlAKX37?HD>Mw9J4T-=+w?hcTfQqv5uCp*CTO%vT2Xi=>4+w^H}tho$79EJs(@)Dpe z3@sWhZ0YHp@_j+2af_iioWbXmV)^l*M|hHT|%QTRnmsET)qj$@VEbXQ&f=Q^kg){1WU5-cnDZ2<74 zkYujs7Vi;s+pUI5Pj}$F60c+wy*}y5VHK;m`;HKNkSC|VK}zI zrVH0TDFKvZY?@+Pf+{{mNH2+VZXq>iHBxcyH3}WG2P*+6^{t?t&ABaydiPO_&pIv4 zp-A9-$K}NC0T<4%+`5@pOIGJB9 zgs`UTP{FoGX80K2sy(@bi_xE1vNMm57rNLJQsHib9y(iq0vV6Tc$kHNpu;@RS}+;8D819}vayu9o%2-LwNN0sgwnRZt1vKHph&E5%^k69id`#_V3dC&2+HBh+JM1_Yf9 zXv~~HlyAL|@^mt<(ULlzflKhXnzmFdg74pH4x*+?Z_iv#HQe_qcY(qegqO z%bQ&_piM-`JIcK;IAO_L6bT6$JIelib!+{!rxTlja8TD1J}2NHz~%Y=sfVXGrNtRSOAN)emV zQNei;cX>lg2?p5T@lyEhN2L=%b)B;1O)^-lb zt(}A=!AoO%Yw|JJ5_PznA*)aHI;bR*wRSk7}3pujSyNYlo*QiwSngKfaR6_!>PMy+l8)V;&p;{3cxkaA3 zS1X0Su>ew^p}RD{ASb+%hoN$W@^PZTp2$CvgbY{r6~-s^S*NS z?g?wS^HNdVGd;yXmrF_1j1 zTDKHBt!3=n*`z*mX-gl3rGtAtS{qQX3tZf~)ZaOU=&rQh&Hgk{QcfpSGk!N;bxAxn z)v>(7vYAMm9h$OoPMIx4FsBc`zb7vbMT&LEd*gP`wOvZsT}p+i!h6lVYpcq0o0XEQ zKH+_$?T^qp%a6_J`ln*{906#*x}>Y17tZER0>s;{HO=9}@rR0+1*KYxq2hp!7gC{y^@5!457M!~Z-@lRw zN-wz-4shDi;1_E#gH>z^IfNdmCkC&&)`=9<5(JJ*3WctC;SXZaRT7m?6(g&(`m?r` z#2eF8Xk~6=20*_1)p@U;G@=eRzxUiQ+)TuJxl6u>$1?RcqJ?MPva~H`3~XAbpWRH? zfTohV)Kd>v$n%ENi^)rGFNJ;U^6F;`KHsGE`m#671VqB!M_<6Ve20Vuhf0_d>ws;V zA-kSe>@XhBYOyv``whroc16nxH$?b(X$24oH|{1VSZ#h}D%q~?S(i#ytWKER?7#`> zRNNS-Txd8Q+U{J!yu2|RNL@L^IVb8qC-A7tZ&In6UOEKCV$)?Er~@RaIrO>GE<-SS zvd@2^26WN_{pPN~RKNHubotp|ewKxljDiXmSsJdU44+Vwv(3`JLHF` zV%v$WP7?}z%1B8>uS@Cxo@F639&yMR>?|xF8MBN{Dh;t84f6ngKbq0Qs#d8ojj>4| z-TiS8Se?~Rh3xKta9YU%aL0-XZsxb-mTD%=szb-H8P#{1SsaV9R69<(h5QQG3viWI zZ^@_QJTdvXB}7LsGY}F{0ipJ{BF0AQ7TqCf10e}GU2*ZrCvD`kbMbKjO}u7025rQ+ zW1qWkU&HIg>2Grb&e2(|KeM!gvc*`DYuUoZ#AuEahunD*&F>n+ybs9RIwTRg;S77b z%X>)Kv0qkX-&3J3RMp}vq6(-J03dH3+Os|9_RuG#2qZHU z;oR=7Dyovx=4VW^7-lMFx0Z&^s&JWEHt2abe~;uE3vy_5+6KTxK9}<@Cx`2fqb!ZU z0~&xLv}?+GUnGQbzwu)d9GtAblWB*7^ubD<=5#bTbF9532pa9y$tkqv3qoNbr%*V zLf;&qR5=>>s3xn0j#{jYl$%IO zxhCkT!F3I;>pz?=rbeSlJg4P_STZF z@j|3nq5(MipvkBC(gJkEk-tOZ=1juJk3!q4 z=x`7%$&fAQzTm0hn>H#`)~UmyCPYwRK^|b9%DZ0QmXn1f6tK zjVr*NcJ_;5wqgz`@F>4hhCZS~DyyA6B@i!^Bx;t$LXZbBEh!_fVZw>-yURRWa}LFG z-HK_ZK6=TN`0*o3HQ%r72EckG#?<*TVOWdf@}qVBh}L`2v2ys7C9LPpYD=8I-RR6* z^nFvtNlw58C0VU=(VWE0k6euAq73}#}#jR`_xT$sG^4595egfH+E+63!H>m zbv@LOL@ZIf=`B2Cw5sLIg@xfYho=fsk#oLqpAI!rWpi`>m!4N<(5i5$&me231|E#1 zmj!O0{}65M`WTNQdU}w>*#gp%>F(j;?GY@@l3m|&@Ifm z^84Aqz$g)#k8;dkf58R|mT;;AolEvCswy^PV|2Xb?&yF3I8-Tnr3_0j6oUmw zd3_^T<#p7PX(L3Qk>%=RlcEyu^m)VDg?C!SNUSjt=r3d&89)45A~y`xk15raTdXKR z9u6`Je-cm`mv23XKE$~afdMLiD`)#AI`WO|RH8va(COL&CRAae<)RBJ#=|EQ11iPr zaq*GzS@33wl1skdfYK~4SZ2CT0G3}d=0P0>AP+`H;hFE@3--ctmF}U}UKb$0X&MPf z8~Z975rg5S9$rt(4Sfg1Di=4|o)q}k#UK^+CM83SofmR0B{ZuLVir)V@fP#to&`Y5 zgrRgm$ojoz-Rzqu#&K!QsvH7_I0c1*QNQ70I5E(z0D`4Om|*pau)Z|sZ!^QiNxN3v zj`6!{JtKzbdzl4HpG@}Ji9xydO6aBM)sQD#fOHDIRYQ3fQ7~r@9f^@?$?AO0-IYj_ ze#DY*b?Mtlb9gL9wdoACTRttxFnVY8omQ4q+ z?4$PXAW7-Se#sV>+CxQp?#^H$w`@33|5e@<(ZCKI37O7?D-p^G``!~z)Qc&B$khpQ z@VMb^i}5l5@fmMO%>0$Set zJ&sH20&td(5JC6jH+>TRnPr6X4YD+aR-3wj4~;d=eHk?DRnv|TP(68Kzj{(E%n;n9 znK-PO9j^9gbsEsXjyudPbn?-z-!?!38!Qb0tPH(O_dxE0s z&htiTK%y3Q-W?RQY1nrlMliL{6<;~@F7PUI^Afot^QZ5Ff9Ke6i@a)5Aks^KI7c@y ztXyXBF$whN{?ex6E}o+2j)xM#^UH&2OCL|Dv)>?WwkVCQh57wMr*z!$&l)bZup5Me zEW_bGAu3LXnUm7rJnXf=S{e7(s*q!-9-+)fTq|ETq<{^?a~BIrL#8gK4Yf4?VVH+! zxC3T)D=273)vCCaZ!zyvolUf$5j{7Z0I)rdB+Hv%$s*qmEc{EdBFfNThzJqZyg)(6 z`3!LPuXotU@?B@MJxErZVD!gPP`oCPaXU0lLj!n z4g(TQhWnPNS^t+^m_Pi7z%EmS1)?oa3jjZ9 z6Xj*Jvi~2(J9>rt!Ljo`0~nP~k@{B+HeqtX^xW5<@6WpEQWh(i^@>(S@;|)*P7zm- z-6{BoDLzrVwG1A+`>^=xCPC?I{K&B`TG5ax?f}VCYhw&>hpO4XY1|r zxl+qxE5P+)L_T-C3fKC>3H^Y9YA<(|)x??q7h{1eGA!l<;;izkTp~s3<@==lbDBU+ zkV|^|-Fs!T%2nE6v)XJ1Sd(ji6T_m9AwY@o3~@}y7a!Q4F%WVG8RB#5;ta;iE><=zvoTldqyg{&j$O_ZsukuDCnjp zCg{@e8q)qbWdVvlIq(#^4KFJ8iG@n-2xY7NiPbeni~(B9BZ}2?+3)_G?rq*qKd_e5 zyOSrRo5BwV0bqf1e~7$dFaoR$r14Q9J@u#loDC%tvl8(kyIw`Uz`{}^nox(BesST^ zkk0ckkT60CWJhbh_Qk*W?!1UB7!;455%&DNEY`(KMsB-L1D9Zx##D67FTm84L~N)2 zQ2Wh-hJQH5f>7Q2P@?ZV$k#sejubp@U%(x`?Rowg3G^mW=2Y1h|6TCSt)~3{P#*zc zMHeJN^CuhwL2>}L64SuB?`frwQ#KJr;M-HKL^$%_m0SYM&+K0eWY&Jl`00ADSL)#W z&JoY=bKaRhSCUiu0E|L5$_?a`EAOxM5R1iEpvR2?lf@O^1WNf|Rs6sD27mS+9EySR zc;~g-08Ky4v!84Hd0e0Gm%e8Rarb_nmtP`z(;}6vV|* zh+c^LdvWPtak@NgAWMVp7vnT4A5I4L!XqM2+gY&G$II>c$g3YH z$F2R~{7<$cK>!tp5}67btqIce|DCdOZbZZDXP*!6TQ|TE5Q+bgtqn+vXmt!cuKc6W z!J|RcUnRO0(AwvHMHi2j?gV5=eUivVw+;U&I0gJ;-FZO5`&)NF!oNxAfP{Yw ziUTS9kDxe^!oTgn11bDl*BuxVzjgkBA@SSxIWQ!C+oA`C#DO94H@tY@&i(OC4h)F{ zL*f9DI6x#K4-kp}9B@AX)ebdPwM@(%E?$cp(4v8eE+2G3=sbv$9kG94=EXe*iwjIdd<6N z79^7@5Odms)v{{Ip9^`;WhWcAaQIu`y5hnrKrd3o3Rdw7--)39+FxA)3_;h#U|dD6 zLNT%XPG42Z1~t9(#)x+zZ=C$Q$~WqYKNC>LJUs99q1*7�Hp{wH`g#(67E`#6F8U z2pE=)SS$!Syb)X~pMbi26&z^xRRN!tD1F~61PhpmLVe+f)%8th1TUG6f;@jqV)sp@ zo$tp{pqR(X5;W7ma@vj(h}|lU^guO8;qAW27Gj~}s+3ox+{o*owM?$j$dZnc7Vs}u zl2LM?GS$W2-rg71YB4QpgkP3=js13C@=SaZ$k8f{Q*orEquk#P^04t#KIL1|z^dBu z^f7|FL7xc0oqS*K-e=Iw3j@6jO`N^fC!PpfSw5tJg0^zjpL2L%vP+YCrvwojByX8 z>THU?RM71b`=T!5>h$z2^O33w-F1NvE?U^!h(z~Ia)Pc3D6x#y=t{es@x?=+sl(7b zeMSh_^J36QsWeI-N$hzh1@DUC8^hiufrf!>D*UR<+*#0vaX+b0VAmT(%;+|*KkeUi zLcQo(G*Y2@{UJYj5O+Ldi`CXw6JyZ0Zcp}YLycAcvNvqo5VSEGE&xqb@$>HQ);4;i zl6t~D5!m$Q5&hwQ(r9hU1oBz)MiXb(ia5(S_u*@yMwZ#{Qn_WGQS7#;6pI zZ=0+q4A+l-^Pm}&OixKbZiOtoZSe@$>75~#hZLigAxoPn9U(bz&>M9$WO&}}RbW0S zUc@3T?%IQ%rJ*&s89^?-X~QE9B^@&99Nx39BdeA!dOF$9h4iq1L^kuW1{PX#zL8$i zU)31rD>xJ~KqH6}t1;KLA@J{Z;2_=YuY~A!Pv+(UeQ*LFh*rJlmsDYmUwtr_LT2P3 zvTn@$0igLKLy$Y($6)s4i>z&l3WashoDOp%F38XqwE4AL3>uxoFaGK|3kk!fp@>aS zFf`ViAA3vwv@y*mW875xtBvXWe{5qa1ruVbMybOSyH{JdtSv#?t6|LR=kMwpY|ejk zjiF@kJ12iyq#E>I^^lbVdC4G+6eMuIF1wrs{tpvqSN38Yo7h>XNG;BURg=illFw@W zF#^iNi~NS&t1qNeBzSz>0p&Z8iZ4SQ6&f3jT)Sn&rWl~h#`H5{Wp1HVh(}e5(2Z4M z!!qnJW=_-zbZ6r)Xpx`3WRUy}4!YTxWd~oe0Nf4X4C^e_qGj)nY3fkiUdS((hAXc4 zq$HjkE8d`!F5)j=-ENa{N!ailz$nN|mYhMOX-|7W7=xRz?%sFI~hb(B`ZMub0C{;e;-i-VD-JmmaFAgC`S~Zi(aAeG8=N|Ki0o%CrCdQM^q91M_ z{mN*U^2(N8^P1_?8@6A#69@QDp1!(3ZLh_X|3yRKlC~+&nR|~OJ@S2Ymh!T8U3fSI z6^dV94Bkd=1aE|H`-Q_XGajR%?OSW_XTEF%Zv&!$bH|^EGjr)}B|$&%$fgBZx|g}_ zB9^bzqHJ&!UsYlJcR5dZ27hG8C^OcT60i0GtcB_I@^TyY^45cYIX?g4bDlc6AF;$* zCw&#PIL>e}e=)#E@$o&r6hakwNATg7!_{?!14O*)|mL{LfuEFeulKzbJlogh^KDN=$|3DtxaAe2Dho{hhAJpX6i zFP{6oZ(OjJ3A1O<%vI*PK40JfJorIzwY-S0TC-F3Sb+adt3NMymj@3ye1Nd9XD&f#_{41^zDey9r2cqL5S<#gO3MEdgr zXttfINyurXieZjDXPoI$B@#>_cK z%asSEEv?g_A&g+`%3*Qpm3Oiv0B9p`WS9GmC}rqe^GCp%Qm3aA8JpDsW?WOY8qj`B zMgXLNgO~=4K+TLWrmMDx8vnh?)1Vbi&1SEM&20X2Y-cwG8VPdiRGe-R9-> zuO1$C*ddIx=2z7r;u*^YP@;v$4w^V9qFvy zg>ajOvE>({c`?No#}|liJ#T~N1k0!So&cd!HP&`k;0dU{Jew+aPs%p#7aQ}mB~C&V zus66~z1B3l!~tI@kQ3J@Hx>4Y-pH3{evl-F+L86J13uXZN(6QCjC%it46UJ_g>L!N zi4u;8u*4h7!e#1F#z!165OYcuA+kjM^v1GeJjJz>R%B$hJvE?fmbs+hPYD;E6e;NU zzaW$QvWVyHoivfY6QQu>E{TF)WD=nt)BePfmC#+z?YmFitSp3$i(ZDrDUT3>Yd239je=bU?TuQxzt+y%$gct*z6j8+QO zgHs+glWZr-H$T2ri$7jd1KN9xH#hSK1)mx8pI4O{4Np zlte^sB%mzI%dYn!cB7eE<9LM;;gX6$<7yQFg0bNT$0_ikc*va=oHE|LN;<~K2P$hNP^R#HLa6L z%$+uUQL5ma7qgV^CUR1?t)~(}o0i|PM@?yY6B@g+GAv+5-toP(q*C7GonzHQzavBFMa4sQO;@w4Cj>nxvl zh-=0b84g}k(>{_Mwz3#Zyb0_>vEmj-JHD`0eSOSliF05eWUD?c+!FgCMEIzA!7@q2 zF+VJJyVg%-3qO~-GrjQT>I`dSJKLC@XB*;=O6#F$sY>)J?y55eGwI?{!dh6U7XW17{z7>|ea z#d->TvNO|#jwN;ef{wl>OD-3Pqzj5`@cGUDLO-}$q7|F=5fl<)l%>I)pg_9k91C>7uoN1b z^x~>0CS8^Mk$(E)sRGg3B>Liv+j*zyChvK*-ba)ZF_sMvZNrmXaw@QZ_i8+zx%*Lt zo*qbdI2HqB9Ye8RGf)Z9i}E11pc&EgZmd+z8V_uB!;5>iz5{B>{ctg z5CtLLnDXV_E`TQ{mP6q(Q*o!}lj6rpdHpxXuVu9Nl)aVDS8r8-KiKPpq?yhnP!=DeG= zy3hXdU&D~bOKn-=7o7~&iKCs8tk@Vb1mbg_gT?8@yM&9z`#qFJg~KV@D8KYMj|0a% zlcdi|hp4D)CQ%bk>=DFUBa3Gg5%(&@P%37^iyW=?Yt+GF`n`-6n!OSy0bbWQ!#$Gd z7{4Aa840$2e#=6hj7UEE`xN?1)V=;yrP_o=@31n*obB-BE+dOe19zF1=prZKoR7;}GfTv)Z2326lEb~KEEO|m@W zytepsD@ymFUPB7s<~n)F-~XDJ2wv^Ew?#aK2Xr>!tMj9K8&slt)_KSTD%`CiFXUy+ zA>Wc`&ci)p&SWOU`xqzdD#@w0$vaaW+OiN}ttet^wRIRA_7#5ccAzn_Fsa>vc-j07 z-}yr~^=>0h@nyXiVgvzVMAK1AzD4RR z?gDIVh9t{nYF=2=r~y&tlXb(x0aIb(4KvdB7`13kHlB+Yj2taWt;{|Z8Yzi~_w{XA zT-d~+E>Z>i0D@1ywy;cO^+#Mpv4&5t9*5oNMEA!jX^PduT&y$On^6Y>CLp`Q#rmQb zT?&3v2zYr=rnoYU{ev9ub+%rYzQC)KnBt!c?v6BHm}6x9q=33mdHy+I8*Pic^c1m7 zbCaY9j`pohs|6XNAL7+5ntD;gLS>z#yLRe0kJ=HSY?3^fw(HgB z(X%RgI2BJ7yiiexP!m%)@Ao>t?x&McY4Qd-&W^@5J)2)UL6gcasluemJ*^#sq3kez;cyBS+)hjiOG# z)ed{0T#Yg_g5>i)cca*xn`pF&1?ptMC03~MojW4ZuIQq%%APj=`D(_49RxU=tb<}h zirvDgTvU)exghX*#L{g9vQ8FtPPWDS$fW)2AbGB1Ev+pUBZ9@R^Ro;3C-G?S$e-j? zW3GlAIC7R4)aY(9zMvx3R2Bi3a-V|b1%EFUK3rp^TB$87v-m8>4C#IQJgsSe*QD2Q zsY(Bh^1KgqI!^))r!(4&;w+Pg9T69QxgIt z1#x9I!Ewj;hvU1?-R6#u3GrmJY1fh0t7=mt)p zt=^AN;8AObU2wTI8;V&qi?TaHZ`ti(9^*0;1;u@Fo`s}T6>IuwHx5NF?t7X?29@^O z#CMP@`pt@4_!(RQT~~XMyyg*g?lBhhAh5+P>dKMRQTG(Rm^zw#2%KrP+R<6H&MRNJ znNE9m2VNdKl?*g7T-VH)v-k=?0#@DE?1%!=e>2!W*0WG%J?2&u7DN_vBpFDH3VFk` za#x~`XTof>z`H+J%_Q)KoMF`6U>v_|a*PLw{xkqmaj%Egc>%?s@+)t-nmZ+zqzHy4 zT@?k$C)1_(njz9|vE5qj@aUpDe#yR-819^PlD#m7naoQXj&SJNXhc^<_r7MB9=gFkcQwm$&7?bEp1c0VJr@VAfI>O*rsvl9$3W3G@>eVd{<#2PV_w6d3)zh`^X@3Z0M!3@T!k;ThO(z(R2S?I)tjwlbY zrqT=c#yc)mW=m`cG6?naiU%Q)odqg&&xSq$eoc@cFPOvN*$N_i@V#Y3(7Qs!DiEz$Lg*iMJ3CHq=WS*{J(@eu@X=Gn|PWGE%n! zGX?)0gBp$Qrp{0v%Xa1YM~JNn6LfJi1BNu!GiUHVc8`jFkg3abibvt z0X>K%jQX(9VHMs|lhGksYADQBVAdRi1Zboc@vB}`!3OhHLoYnp?Z?09jYSJOM|d|p zlq_&erlS^2|G{Z&+>lbYc*l#onKRRLoSRm&zwXnWhKbhz;z%MOr9z(<uH`?>KO{LHWtz_Q!P8EO-6xh^f5tx%D8Mf|mOJG~5L&hC9HxmkICnOCS1i zK=x3}FtGP(AQb0d{d7b?_q<;|0%YCQD1CDRQjSeQ$%Xs*jgQsL_87eTwgBJlM@RPM z2pgmSQEX+5H|n_Pw>c49$Ob}WJ0?@No;l*?s;4gy zffMeT!69*egqru*3=)B~n(=fkqEt8{zucp?=x{Rc8>f&aquOww81>zo%DGjZxEgDL zt}DC53Mp5Gx;AaKcyRXtj}B=gWd5M@M>Mc6pn+L2Z<3{YCAi7 zs$i1AS&yVAk8%qSLvseQjLS-4XGiUI$(=fQ16c0|KaU2^!~IOPg@wVuzJ{bCj^#x> z_3Xo$@CXBS!Ac5d!WF9EX)|prz#*u=1Exq>{Q=M6wym^Ginlap!(gKNfqBM;GkWiN zGGw!l6k)w~j?P6m5P#jp)&Tb6#$~nYA3=?^ONgaV3CGJtBATy;RI4=f(wLfU%x#Q0_hf8|$cN3voENSQCRKZl#0>|=do(46x!ET=d30x#4 z$k@KHd%pte5oMlG>)T}k?~w2;-May&B_ zR1a=fL>WGY%6)^dsH*Snw@*VlM#^@)eTP)RWo&l_kL&X1qv~-A^(A%oA`{*8XJ0l4 zGL5Q95t_>G90}j+_b+C@`>aYCVIPq3$$a7$$=SI01xTL^_q{`HFsh0*8L`9P#`@*> z9aW*UOF${oQWIf_R+AktFPojE>$i_bOT9o9 z4D{O`;*9vb%T>58!csh&gLGym(a1;pD1}H6{fp%+_p+SB}pm)zUYZgGC(?i@(T_Q zYLvQ_Ok*G&KpOArw+lE2kSq`E{w@h)E|~Zdn(d0`9Lm&sB5@uiO`JXb2&`8}B8SJ(Z^Q8!6`>aY zSs!x(&}DIE0enRKx+AiVMZ2Yc`%drpGe|GO7Cq-`?_mvbsuCpl7w5AbiO&nu)SRsp zh8aCv`tW;g`gJb2B7SaGPA6MnRxN;eVOjkIa_(e?FCofz#n#GYZ2)?ejrPmgpOSsP zx`vto$8<1(LIk_%W9;_pu-^B>0-m2wWVD6;l0~`L#Yb3-pkgg{K@NMj18YvblF^(w zxj78v=ImZ~TMvQkUWhAnk|x{;eJ~N}x0aTXs^l6cs%lCX`p$c02J*i8DkA}lv1PR- z0lEofq2ue>Jr>(F#Mvz-645#fnCm6(rnWv-j1sef;asfojP3;}pPE;0+RjKWR&0fV zMVfb~C;Mz-{|09$)7?OlAQWOJ7j>vCZ^gZ4{h zciw19HI|MW>FX5JmlkCG+KA$sOobP506Tq0?Bw)4JMM!N_P>o5Di`Ytx*nbcK<>@q ze)q6|6GDfJT5lzNlTkWfUZ;4z%C8QZMdGOYCcKvd-S2Yo!zv`7#y3Oqn`9ZUaEHvd zG?peZHpiG4d$Uu_;&H7zSat#Fq!F^F17&Wi1I>f#<%Sa&^Y|zrC~?RRN_(Hf|DX>+ zdUfU6pOuvbIt~M%0ONhxv1dx=lR&qE{cu@pvZ!>6>1mpekN}G4v_^6RS48AgCj^`? zE3aL!r`r}Nsiiy2w`mR{aIxu&99v(MdaYOQawiIv^eNI&t0HfrPhZKy>!wNDGtxLj}Sl=NK%j9E+*~j&)R6J;469TU)akpW+)>SqoB*S$hux-4usoe;;BipC~!(F%;!N2 zVkRu%@$zO*3{b?F6^H47$%boTDV9sZ_77eT*fp~oeidqA*|S53G~ zc+2V%Yy7|vRm`2UtsXnzRx>^aEqd%sEd48|*tKQAkDECDJB2VjUi$t(fIU6fFXhEv zVYqBL;|Im(<_*`pIf-{Mt0xu4?=5Y$P!3E;-fG`?Wc8=V zqo-)>L#AdT`uw4gDPHf%liu_odE;k0P<2Gb5IfbG*bC<;uK(j+?x&L~95Oyr9(!~Q zq?4EzbRMHA<1-#*9}N~ENk~qd(Z9mjl&@}A{Muci{+gfj7;3?~;X)?|TBM+>5q-yW za`B}h`b@-XOW{*d+cUlV&4spB(HtHoI^_nd<^y#Ezg|?{x=6I}Qf)Y}8;j2}Q0(AN zVl|9e^{%co|4dtQ`8T5jwcZgKv#S{$Iq#S}ZVIu<;%g0BQ)EVa^Md4~O7ieL6Gfg4 z7YgJl7esFqD5MgAfNtU(3pgIk^YR3=WOD_Yw_o4c1eI0vy6>x-*-n-$;^IAaEATl< zXY(2Rz~ie}i3k*PPH5D+B+*%eFyGpp00^0lbl^l3-ouPB{$Tl8e(bx=u`>oQEYbBRQPJ1Dym8Au10HDpLEP;`-H zHnEH^x3;9s&Kqs>4Tx%N0Hp}`oP2w7h0OGQO7~$@NtcXIzD-q5#_k#9eCPcGLeWI% z;+^P2gT5VExnYs3-s1|cg+&tG$+eay*?veEiz7_VzU}~t);jwgGbxcoblnU2yuCBva=j~emRVMI z?n9tbXmHmObsXyiz^Co_UOs4JK(`ZV`cyP zK=zWXLqbp^ukaGxCKCAWFkUHOjRwIfn3edfW=EF){*SgJ4ATP116{REsRQlHLBMjG z{>J1+p|6`3#uJ>Nq)ax2Wp~b%L!T7IrL{E2$~2;Gt)_yz2FaxwS7#`}PyvbBj4ZmB z^DO%BZ8Je-URwBb#A~xS0Vp&{J*mJjT^OYdf9c(fUWsXH&Ub+4p28HH1gBo{*ENX> zm`^>K!&JRkW#{}hVdP`QH+ztb2f7Y?`y~#Et2L{gJZ(gZFpvl3R@~wlG4GPz;BAAc zFhSV&%yqs@`fF`ACSfl8YpL(s(G^4e1$vM=Fn`;luD-f{(Rcl0w#>OCDU!WO?p$>b zcdBfZD>;%RU-9m10~TmI$T^oJj6Ts;-2tJ*bqU8c&2iE5VTk3A4=!^ic{WYjQ%DjBk-MC`e#Xjiwy$o6f zh`*`VXsW%{iPDZ_u)J+6#MtB`*b;RYm3nfJsl1|@51W6=(Rs|};8G8IlRfHg;C@*F zf+-dK&g1q=5sz#p{sM0 zA`t}hxcPKVL2!j^xSyvk`o?7F45CI3hzWvgL5N%Q;qv(jjY9;WiluXqDp<$wIlwJm*pC9_ zgC(Bk0EjOPY~4C^ywU$Kq;9GpYi_5nfH-#2SgzsWAO?n~I;ed^+yfiv zw)OkD$)^230U9#np9aaFYHGiAYzOEJp37Gi3TY`YxOj}I5F-+C&pbd-USy4 zXk1{(g7tgpZe9O?mD(9zIT;ibmUw9Y{26$fzrc@}%LL`jF0o6vhjOt>xZlzNmmosF zhdnf{cl{uRJmiQ4Pvz$S9)er}7(oI0yDKH6eP9msJUK40aBiI^ymDoWzZdOftwmr5 zAEMtM$I0SMo0~vHjUNod(dQz=o!>RUC-uPd7RC52pYjKez5!&E&xu14*kUqVjgj59vni=Aiw!( zcZiMx!EknbTEjlMNnZEjY-NyqM70)3nI`>CEc=kfq;IR{aMUY+tbhn=@$dAN?{NAb^pK(i?k^0!QW%K2>zZy zrz^-gR7miE>J{$&zh_YEmkPkIc^{ya`b;H9<==5WusqCI37$_3dcFbZ`TmpRqNUCq zhV=oLf&qH-`XS1?f8d09HT6o1E?FXa*p!@N;jT-psFof>uo)2h6Vr&>VBrI+_AnIf zGd$1)da3_BV=e9AoGEpG%Y7*wVfT6Pgb`cfpx9vd_caKq*S{19^-@FkMQk$~!vFak z{ouSQweCr1pfgGL#h!G2(*zP1>igTb8}0W|5tOfKfQeZ;A9(Fg>G)i2eZ&>%5muXwYyTh zomVG{Dx#&qN6UYI(KzBB9(-*G^hK{T9J9`O0AcSi$XS+e9gG8p=J&kSzoO(~A1VY_ z71ghS+U(gE`=5Sz7IB}4>dYNR@OIuN)MR5Xv5GX$od0Le0uCORntynjXAE53GroRW zqN-vqb<5d5zOjx z$9X2s%E1PH=^W!M$d!z=C4m=c{QCti$)v-<3sit!;GJGwNPGT&kN$xYg2-b*EzZ)r zGQ-q|lTjc}6m_M9>gD%$#afAg4S=Mx(Sd1@RQe$C?^2^Ya^cL^7k4D)=PvgYdEKH4 z{x&QQPGvu;eRrAEA6)w4KtKfa^la%G|AF+@G7N@Qvv8erukpDfHeZo)6CBy6e{XnM zQiLA(%^BM8lUzB<|Gw)F9u$%;9)Svt1Ms6Dfmq^p8SK;F&sH>qP+9;N_%x8irf9A6 z_TOid6QC6F+5ZsC4}3@zt^_`${o5eLBNX$&hXkPyHHYZL{QIH$dncZ(EglJdF>UGq z_W2J9aIWTF`n}KdWm18RT3!~k&r@}BRR0~`gK0{U{o$<$6b3b}ghzu^lj@=MMp} z`x2OgxiMml)7+S_Ccvb+{2rV*#wZ(ng9_v_oGqUDL+c&X3jmV5HG`mpUU?5y_TYIL z*m+GZ#@~hUH!1KpkMTEK^uNT_-vrm6Z1%Cgsh9tl%D<_XKc@2U!uY!| z{$}x^Zn6LOOZ~s8m%pi(zp0nMsh7X0m%kUn{~xa1AEeu^0k}@(Cg^H0+yS-iJp47w z4cO!Q6}O#1DR)`LFI7h)69C;a%SL7K23SJ(=2fI_fb3d9{TsHvhEYL&o( zMyIO)#K=Nly?`=y-fD-{P)VUUeHp@cvdJrg20(BMqWiZn}t|S1vm{R1%I*HA`cMTJ* zBZN0Yy)T-17ybGS#~lD&A;O$zAD;n$GYk-f24B54Lrg7B;TPEiK*lSeg|_CDE?>bk zU_34Kpc6*tAE$BkzEJB9Ow`|s_lT2)%DB6y*?T@$AwZKPf6{tsrYa?gDSrmS{xNiU zb-IS-noM7FJ-s!)M;?a=mt##bP-U=EC=!V7@#{VkkT6~3C1~EutaZ!p!$hF&-l`M}{fD!P7V@S~ z+7X})9t+8dr{s(h3q1JPiCM}FH2A6(;eulMS)=GcX#79K{uzHtl_}t8DY|ug)U75; z>f~+)ZUW3vddQ>CdGCc5Y=W?_}`sZQ@AW+t}M3C;23~!Jy&Rg`N9T zF#?8*Kdg6Y;>@jIki5F0N^^V;9sGVPq#{Cw~r1GpDe6ZX1$osgHA}7@5tvbIeP#U@r2}w ztyTBM(lrCt!d~b`ivT0rgBNd{!~oth3}SLxB=t;gfSyx`4mQ!`W9DWI4HgwL^rxf_ z3|5>X_mxvf*>%v0-4)lXGx6k)% zy2#$M+1M#hE&kCHQ0z2D088xJ?u_GZMhLVG@H*& zmIkn()8VOXV$Pt`WCmMhJoYXR1?)T|+K(JhmtOjkI0dJ;m|(E}FDo)jGU6EqZcaSjAk zG%yrgckW0%)BzDWL#wYkfLF^{Ue%v}xwAi*Y0+t}QI}Uq`%Uwn;8{n)XwiS)#MAnL z6K>KI%b=V0<7TEmDKqqivdSbxW^)Fk@Wqlz$o=MHv2W12A}KdRT{D_HmbgF$UyGhW z4WLtOZ9{oy+egnWH&#bX58;x*>#R-1iIthn?G{Um2H@s_I2>;>Ld>3 z4R6gR!$#&Gize7^fmM#iZcauDnAe-QZG%(N7=Z9L;)ci&;B<~U6nFGUY>tv9RC}0* z%LUbJy7_&76y(AClH}|Lt#&t)w>q%>LrTxaABmbqr9!JvlAfzZM$sCTPI*fVKn|X5p8* zzqfOrrm|`u=Z!_01L!v0f*XWgPJ#HvcOk=+&~8;{b$A;06SUP#8T`s4>9iUV6{TeJ@gc(hp+gm6bG!Gxw%^v7|HTX!W3U7lF;&{BW) zYhkvchtLIuN*$vI4xALft8hyvBf}`{2VD}V)aN#hn`WqP!}xk#lk}U}?-Y%Im7679 z*mW|UJco(3H6F=6<9}APg#0hGkSqwM2iT*27`dNxwcO%q<@~4iFbM6ZsM5aHo4i{J zaTJtRSz9>p)nmkha1Zx9{i5|LaE`+XnX+V?HHVHgeK^v_6{~+72CSEoo%YUpIhu{R zX^4Sv^vaE1Rby7B^1ChWpjwl08<>qI2j|W#yC?c}E72HCSM1S(~ zd-7H;hK{;;(Gp?%3;Ms!DZ%Vj4-=s?D$OP;=jiUlX@7$`%o4;xRnmT-Y%--ROVV_1 z<}oWCgkwyV39mq70D`^*;=me76K{7}u!U|O-USRrjiw<=nx!B1<)cb=o=PXDAY}1v z;&9Jk_oWyUEl{5w(kIBj5|?gbyLz_x8Xut#MP*zwU6AY zI+H_SDSdJ8sb)ppII`bQJ$JvDQPE^OmK_Jcro_2|M9Gl}>J#7Ie7cB{%E%)DdY-25 zululTsgBAC7tP$ahsXU&#kL`qqY3DF&FlAEv240MSwUbN@7+}6ty1g|sPms(SOIMp zTH8M=-lJNhJaw1{O@{Oy51YSPA^P9v`4U=g@}Y+1A-O8N@S%A0cj_4J-GhJ{D1KJu zeYwd=8{1?T>grGgwi)c>%}_D`23x5tX!}n>4#d!Xvz0exH}JB~^A+Kftx)H%-Sqn~ zVj~MIGLl(#zvE&-e^=sR{*Hx2`TL__Guus)bGZeWhLU_nD)KyizsR#|N%yTr9FN)y z_Q1xwm&WHP{z;!42#98ybCJq9xv0KV{&2?wAujzrqXV+PYKgf!nzJLWid)l9=(JmP zCr=Wju$L`4ALXB%>Xe50ZtlYap)<{hdBk+CrVBtqFl~xOvq>J{pdoWX@GDFbAjYBw zCIQ8I&1h?~n>1m(;=NS6k#t3tSF~D@HTUWZB7~PEMj5jm;>lc&x7PRVGUr)pGVrNC z#D1M%SS$P7gSk7|c=nsg>Pl;s-BAwT<&jUV@T*LtW(h>{6}nT}ofBre^_~uixg29) zODE^FDn%3~+3_S>HKm!CZKvCtvj2=3m`i#ucH|j>$z{i&puDItpQq%d-8AQSy1llO zKbyRm*d72Pag*u%UAcBIzXiKuNbjtyE z(AT6!uQ_)2(=q|-W#o<^rh>B9L%yPy`ojKeoZ104aVEwg58 z1b{)^HQLHu!`ZlRZs}cSO}zn~1(`z(ccZWgfeX7I`i_RW z;6B)A{{p5;iI$q6oOku2iK9g?9{3u}y@tl*SeHcP)oPLq5i>2MW=HUsXaS8?2C1@< zU?Tk2KwR}_;s0`Dg0FUY1^vdG_$ch42@VMtirJ$)lU_?NpPrL?14SDZx0LT zr=uoH@R6$Cld_6&ER{3ULcg>xBj0gsH;&)g2)CX@2xFgn|=WDDAzxj$|r+(Ex$b0V#Fg|w?=H-d4fL@KfNneI)ggM;bzQgI zrJl#Uvu@DAHDQ?k;X^!1uI;_o=w(gG|DMOZiXToEF-%9@#Lx0y^u~8pFMoWGtMu$a zn)Mk(AEFlf;MrlpelzTJQ~V+0;^rzE!C-`VbwkrHPW~0BkGE8B7V4?$8QHe%F7up~ z92YN!)p&lhGt-vB8+OHe?g8WLNFx|YT5n*gE4q>ybJYRxts~7Z(@ch})x8|{I)wynVdTJ-qbZ7UDrdhTNPG_YWjG0?_jxs| zX%ztOqX*Gt&*opEEk?anm#GYq(HiFzVZukhEZ;)y6|-u(Ya#u@%Q#PU<+*_G zLqyD`L$d@UvY@2c?cSA?H*0?H;6{l0 z9g0#IHV=ORTo#{jRS7~9UFLeEMIAudiY~&+&!Ho#8ROeRhPft72!4~Tv5iGGJWSz9 zL**u(**5x1aD(#~(?%Xr>t6g%t$6)+$}0BAYljl;RBzsYIeBQZb(6tIC6f+)Y!9Fl z%WMi)lwWE)KM%RTvokB}C}qX`A#q4HLp8^d2wPE3n8`<;LH8AQ9oM{k2A9cJYbGQ@ zFWIWoYf(b5i2GuI6h0Y`idC|y8Q&Dd)#7W;W4obP=+)b2*i#rka3}nB+oN8iycCJLfyvg4t8Et2(S+H_RzrIh(evhR zlZe(xb)TW$7LH8W$4zM}*-?H!s`UdhhUXt2Zj8~~Ry-ODa22ZMiOuIn*Oj>j;?AC? zoot#Dd5K-+ZY%ljIPJr~rC3^{DyjCvW=mkHjDZ?l$Q-GO-n37Eo zi}@80$tcCqNC1S+witRtPxqXEOL4!Z|0@%mL7fAc*@rfnw$~mWh*CTfo7DmE4#1oH zrEMQ4Ec>7~;B|POMAL(_t8@e9IGM@Wl9{wzc4Js<^)C!XF*->&p2K%{+foW+RM(jDe+Ff3>)G) zNnane-}a|p;1`VR5lZiVt;}9&FKqJYX=oeZ{A8V`Ek4toIt(?qh910lBr-$cHe4fg z7QzTHUQ3l3Y&kGc3TTGed1F3cS>Uc_|I?wl@9wypJkX`3?)h%;a;su8!tOm|7Q16C zQGy~#Ra>HU%OY%^TGHe>ubsK7F#+=niy3?w&N)jg9!KFe?IFr!{{sgO?Cr%4 z90>XcC^kQOVpCeXww3VQOdZC$>yBohHvd%xoiOUK zD=y-mS`P56x14SlFe7`@S_&yTshsO^T{2edH%r#zYahcN`(u*4x>k1^Yv-w5VFUW=q?CGyYA*J8i!f{%WBS;jVa{|ENC?PJ!w|%Nd6wF( zZ;t%=d&g0xPp2pXpV}os1ikF36lsrk4j`iR{%%i9+)ZN{qs;_e0Yh&Mi8f2d1|#a^hiGfR}+doIOn z*Rjt#O-9>yCL_TC2T`VKA;N4rDNEX86prbhft$I+Hvv7`d4*7mn`ZO3(+i{iLXF<1CwG; zHn2#vSlpOhC?BuWxVQ6*sc!kXmh|n`830&Uv|%&chJU*+0Wwp;1CVW6jeP4_8ludX z{?5uNoYydL1T1{{SMt-{0Ne1Ecxth*o z6X#}zMaU-Ze%gz@#5$6bTVUiJ$z*fl325+QvT>Gtp`58Jc9S~38QKTeUsu+wZy?! z9Qr-fpB}w6cBq`bx1Hmi;!?tO=a&VuGcW1c0LXp6Ih!8Tbvn(m)1VsUKj) zBw|&Yr$Ii`jVHfUYcJocsUA4dvKt_FanpEiC5<;ETS$9eTu~@iYGz%vEcqtl3dAAq z?SAhUpYikG#WJDiK7DL_z){Q%QEW#O#?~^v+g{tas_u2~S1l0(8z+N&9Ba)6Q0@74 zm2tN90E0ybEi8~o@6qwbED&>5N-$K}^3ZLI_SDBwEQbtVv4MqzW^e1fsx4EPuBN07 z4>g@|8ZfjvRd#VZRh>8n!3&#yF5TRA<}l)3Yw1zuFwMDtB{yfGqT#~Fa?d5hM^Q}b zq316CX9FFe=si&?%pRTiaj=~tqgGXPw3J@`O3S5B2Nn-3Ls`IXZ^NhV%UgL#fZ8T* z0eh{P=Q_u`B~hFL?f5K2Gg)nA2c%3B;?qNUSRr@qbQuwl=~?^W?1I<#jt7T}j(KYM zC31s_x6D5EBeXgFn|Z4Vxk)wyQ0$a!IEtB4a*aMUHFQ7zL6s(N2{3Pp>BQTFUMOQ% zs&-s}7pqzD^+h@SspEIv=ZIi_*Y8Z;jO`HM%j_KkFBwuoxL+M{>iSk&aZsRtT$Jn; z)fE!tmzwST>=}>r+rjg4vEr_jS+OAx)=7JpQvSODp{1B5(EZ#q>RGacb5G>m`6_(D zp{#DImp|qi0=qI1so7SLTENNv&xP9a#E(?LPedZJ+t=|TCz1`hS$Ba$b-GNuxD0zE z@@#Cf)1dF_H>7Te43uLvu~l||u3(d3Af?f3xz&n?axT!L!s)1g^k0;>y!sj79BNka zX(As{22E*?=@biI%{Lr?^{hSjALd{s9z+70-gGNc|K~D=qa8 zH#2$5Y_@m-r|?7aJ>#N$%0Wm zU$b|$xJ9Ot1q(lMhi^;`|6)Ja%kQU>Z>xl?nts$Pbt-O)yQ%Zykr{5*s3+&Fyg;7B z%r1ML6a)9q%36!kRj*fB6yP8Qx;E55Ei-RDKxatk1(N}cuN0z9F1)1xgmRgp}mO?U> zsDa;XAVA+e)db~}9qM9N?%wRg9tCvfcubT?Wb>i^%pDogMTV;>Jhvc0s2JR9ba&J* zKd&v-ukyWD6G6DWm2p?vs44A^7+3${@7XHWAHK)T15*@V@g^l;l4Ey*r#cOYf)nXzN86ZZkvaGUgo~HCWn%5{}b^z@I5(KVU}A4LN-0VTIUo zDF{o^0%`_H%*Nw5r*5zSN+N6sXLFrbLjty_j`rWzU+6P+doEYZ2Z@f^^hyxq(6SJ{ zKONCzwrLS4o3HJ>4J}UVp-+*3utwH(YgQ}f^Dgko`&a?i`yH{5gLJf6y}Mgcu`J+4AK$#y9CdQ%X*R`1Rm{vsGftfvI{;?o~-l5Q7qp_l)k-hRqBlKi1J*eIqxp&?O5ZjjpF4EIO$hraM-E3%(uo! zL5^=dFgVJ`=_rF^=e3CsA8*gGGsma4!DeU*S)1r~aB;m!l@tjlOrAT)hY- z04=WsEJM;LQ!{?N+_KI?vMu*^(yKP@W0j@vign0)-D(%ZWu?71;&5uJO2}bk8su83 z7x7^{1!sqJ|2G9jt>{t>*&1Ld%Vom&61d)j>9wABAu z0o{rLQFPSCz;0{g-6F)?Wi(~}$$fpti@@-HeUdT0pA+>S*gE;)>5T+2AXOJElPj5~ zFjsf_?PXRs_u6rh!_V;i@vi6!1iMN>HKO0!?>Z31(zVz3h26a%8}SMwEKA)?B-Z>o zYj9;kW8-*WH(A83S=~9C?(`aZ_(o6dJsdA|y#MmF8y=*$NLl(EXO$#g$|38el_|4o z)Vid+dL;(X@uHYYUqYm;&$+B?&vyV@Q=?j!$;@-2`}Bu(nyP~(?nYKmd%WSa)JNGp ze|DcQ0?wq}>T9Im{0_$gHasSbC6}++fAx;o2snG2uPyr?1Z!dQ@Pm}FfBeIJPqDt^ z@Xm*+juWxw!Q9EdE>~Z?YNZoWY!^dz`khwIoN6_?ZS@YHx20*d!oxmwv=I4uGq<41 zHtQ5bbSNc3X!;i@?QQNvt-FVDCy681{qtN05OYlsP)2mZbWxfF}s^{aER zG9fl-InYa7U0zwC&>8>i_nfJ-xJ=}VIpfmajLhk59RAzi?Na6g^b%Q`Hj#;{E&*x_dyA`iHU z(&VSl-$@*LiOPNJq`{mrzZFxrmafr5-iW%oe)K|roMWoeQ36B!g|53RF}yq66`Md+(q z11Rd#-cUpG5ic^3TBbP#tY>v{X6R%mJ|K9mz=-P8*LK~3KEcKqfK2I2eD&1Xvz9o* zcuy?L9=LI4*KN28AZGAfkeU5k^)6kme4h@o=ZE!7N>$zWng}D&tO4(4f4JHmJ+e^0 zp0ng$rN2O!d4iUw!@m7!c&%PTeQtS2K$}qZY_$sW*-Cc%N7CYY9mdRuC<3g>>)|O6 z_-SFp_k&8ZW+{rIeAfm<{R=+NMQN-3+WU~DVtIE1Bu;-;we8x;)O5dD67YWyZ@3k= z>pQ-c<|ie)@;Y)267os%yJGmA2^ipB{gfB~clpg9)>0G8Dm(sS)_;6ElH>zBs5?q` zOH^?nk4ECxGLRhem8JRcvjDBS>kU$cROYttf_xLPX)|buX5}5vd3QrzY<&ON z)8pFzRsHzp#*1!miAtRf4GmU#t0+7>RT#r_HA=pBDOdC1C9~KcGjI!e+ajz+GkLlT zx}o*!m>Mv4Tv&pM=8gGnF9hNT63<;ZHebtpPizVvzow}lJ2%E$oc{H!fo{Vsdo7Nh zQ!N6uYv0Gfr3~h68Ob86w(8m?dW&42j%b||3;O!1bN5&-P4(@$y_f*=9{iXbY@hIA06NiQM-Qj{)L5a~^d^hgm@ z>JkA3DUsf#cY-J&BAw7f=p~d$4G@x?=hgk~z4pxcac0h6|Ew9;@Fs7ba@Xs=E^Xhk zSE~QG-2;s`y_$lqsV!kclxiT{dQb$(Zg~{-6ns$>+$jhp<=6*^tEBm3!x2zerS};# zofMB4{unB-LThyD*Gi#|@_$l>-@e>(k3657ZD60e8Mnxfo^6d>V|$}|6}1E$c>TqpQ~xVwv(vIX!$|MY)N_I%H87RbWPy7SJl%I; zuKDxE1D)sT2Yyh~?g(|tJeKu^!6hQp*K-B@y(fpQ2f{-6b|3}Pnr`t43S3GJ*G}H( z+Z>AYg`0(8p>J=2ZiyQflt)<${WRNJj^>Ib(?P2$&YhU1r8Jx%s;`0av21Q~sZPWC zf9wDpo>$N+>#8VIDZTKgFVzD7>}ZqWH3NCGshk#qEr0hN&Cb=Z9XF>nsjmU}Nrdv%y(E={qg9R$vefnm5s;zK! z!D5`?iTz57Ji7no#%^#1tEfrdn*(h`j5baN^OIz?b_rHu)CIC(ER{c64e38MK@fqfg|eL@h{a?_ zKJ4p<0v5dqCl3L!op?FDJ|~8wnn>KKm!va7Ru5xMKq2^H=B=qk7 zvgfMlky@>VWEu1I6ApdIfxKgCSNU@2^9u}AOfri9iiyc6p7MQuwVCH6l@;{yN=hyo z<@S}%rSmP9Pzo)A)3q~XPKb8A0je?*&UZzc?_9xYlXkRqKiH$kqTiqdYbIyQ?Xyy6 zX(8e;6c9E&g#(R)iDKaQs&J*}RlH%HK>^F)eV>h8I3Vo5!4%iVVd{q-Y9#FCdX?jK zYn_OAqU{^5Vg&J!;PcSFjoRD}ZD{VB-9^4ZIREB{4^S-{snJappLA2kK?*u9Kn6F> z5*u07DPV0m7O0H^4di|@g+w>EF;5f1$XKAXsko*Q0zKt9s7b&<9GkTQufaP;OeQ;) z+#Yoq)*fnKyR#7V@nMHd!?W);hQ&K`@y0iQ+E9xpp`J9MYjS!04;GT1k2PIB(eT}+ zWP=_Y3_o3an?K*@{^V7EmkjFnJ-2E@0E0*5df#{>7WrE*{$FQ)KnGf-TBjy zOpz9K;_Rny;csF2!sj-580qtIG?<8E5GApl0ZNwBI4m(WrIs`nU1v!Y*u!mwrxNIk z4!d1IxsGU>smvL*8NR58JSrf5I?xcKS^&jv9#ZCA3k#60Zve4P&Uf%zDn}anRGdP; z;-Pxi29RK*sh$SH1*=0iX(@m$GWGsi2`8+ksY6j|AHGpUIv9{#Wy;$+HFtn;wuEH% z!@@+C2rz5EHGl9PypB^=1o#fLBp+9zlx{EhP>V)eQ`bQ38c)lnD#4-duQpY zg>|&pp+}&0IcGJrBTD4g*p&DhBU(Jk&F)JsW4FOE9k7wO$~nCvcL|guGEKb4Y^bo= zN8he+sI*>Q;7~-JR>%+|2bS3J*VDe!lvYqo+F;_^gyj7NdBL7{fc%^9mtIVB9VmL%XU0hJ)gD5{%Q~t6fWL~Z1Cio$ zopeAb*9kbi6+=gyeLsb@K$5(c9zf;G6$g+VGW*7<5?*Hykp?EkXar%`%R>?68JB%K zSW|rL8&umAWX`a*s*>5C4&8WhoZbxr4}PYpzr1jjWAoLtZR6{fT5=+tLIG}s2W|1R>BVboAp*C2J@i^1k7u<25sKv@m~vMoo_;+Z z_k%p2iw%&Z`QQKqrOC9j6@X?aPo-%I-T=BF<`)QBfq+;ZA_rgHs_|KS^+N09Z2*d+ zD%K|r=)W*c5W6(fc^>R|dyADmg#(S}<(Z85=bp?@k)IH4Ndm?e+ z5mKt*Uf{L`^HPGt&Rf{{D+RU{pY917xE-VGP-k;6J^UNy)U;9SOq??PN;=`T_t$bd zA|64SAao&)G!}BMvGLjG5H0mu6{QiNEP?_y@zd)cW1tQ`3F%@^SO5N{fw!u{zCwh= z)0OtVnI!PT&NOz3_J z$n=1;=SmWHkA?_a8=xlYM<2YscoY|#=XHE60KC0^Gx{{ec~Ycge4TS_iAwK!Ak^;8H6CGm(>l>TBcy`3tHkGKSuCH6&&Ga(W>u!s- z=Ou$aEFglz_ zXa?77QWs=oa9LhA_$?S|latgh3+S)CAo_X~05|A@e+0VNnKilp;LWWCF_6H%alC6g zBR!u9S+YAxbqT~i9`zsA^InF1@M zrbF)V!6uPc%w^m^?45TzErkpj?B{^!J*jQik199=Kd2+_Da^EP-M@H`BNtU9EdX(M zhCRQ|0!WMN*iEk7P|xrwy-6Y+zw2CcFC7KU03n_O)JPFK*0dnUEHDB1QI>q_?E2qj zpm+3S!qDM-3R&W(kKPi^Au%gDlz{%1Nl<+4SXt~NZl$1)+MSzrhn0h4AZ+eN)UBRh zhOS3msUW($#(6YN)lm>Rz6InAeCNflvEALezx!BVCL{H?q2?0y?KE0gK6N$KUau<7 zE${6I6rPuh4JuDHfCF4TR12~F7w|W$OV3e~r@i2`dw*>v!wkdV*NrUAC%uVmzY`rT zfI59?rZ0L`ay)&tUdt}<6weEb%70AD zl3w>%`fNs8thqF9wtrDg4xfiaW2Sc-(}vU}ivTQl1Hhy&6sp(1E^@gp?EnfL>bCX0 zm}_jF!Y8OD0YpthvUq5il+%~rx)oY zCU1CJy(rNd!d}skptvXLIVag$m;j`=xHNB{ycAde;f{MJ}6^mw>4fj|NNd9Fji7*ckl&MRi32%P%2T~fC``X|DSM^ZSG^OzQq9yqU z>ios;v4vG%%>^oTSfz^ZW&IhFn zy7X>)Qv*)$k0{k*HM}MusA)+$Wu*VukWRL< zUQR%HhTcN8aY$ATXwNY?Cyf}!8n@)ufE*uJe#vnm=sMvNttfp$DrHE-R98rNRb-rg z8F4K7ryo>IUI9(Y(3CN_g-|d zZ#o)%qS!HI?!PWJRBa`_YZ`cq_8r6eBeV$&^`Nvc#w;;lbw#ohHndmpT)fys8{Ibg zmhwOy|0Ab;#q}9bKB14kYD`}8?0E%|DBdq$$d}HA^c!P-$`uh;dRX4RD_~?+RXBjx zWGj70CT6kk+{s8UR&%N}kt;2{$O>_Fe6R?AugqI2+|_?#A{?Xdzb%J zXY>e%DJ&J*C?SDEQNceMK1|)@{!^sm-Ck0n@d&TG(*#a~LXPT7F-|2qT51Gt0d{m^hWG z*iS$b%S})jGaENF^?*8JEvn5k@ZP0~{Osjh2uhONDMZ>1lehVP{1 z8iG(ACstY2B}I1$&IPc;W0;hS+IMdsrmy%H&u`>q=g)~1{4D4_I@AW0-uDaGiUOx5 zv>sMOPSj@I9wTPYci(;H zskYU-f#H@vP_swPdV8fodesF*Ih=jF$KAa4N)?LJcW5bVtqj}d6NZ>2g6ita?Z<6E znF=$TPQ5-Nyd#xzSgf~Tf0kZKc?`jLSouU+Nycq?-Aon<-L&y{#h zelde9IXrl>A z1{y2!?l#i3kCz#S3%d4F7OA<|K!sg+H}~?d(0FJFs!~cyqj8_ACgie|vfoGE1SLf! z$~XiF=6A~Rmv?#xhAU-gPT1m3^CToC72EqLX@EE_LFXX4-=iijJ<5`&d|PtVH>+JC$~SY` zp`s^a1bH$!NSQ3#Np79Uqz-=)iz&>%s}#xE*FBLa@8EDfb)}@jOP;tHYp7L`HK%9F zq_BP5&p}n+t-7`tcDHQrtkeXeRx@=8-h5%a2hn=D<~yf8Czsl8(;+}FjAE05Ot!ru zb@_So^9ofV0Pz&J+~#XIF$-HNH;CvB+N)63P1fUD0OhK}DNv;w*L`?bm$3Xk9FvJF z$Z6a>OYciijt#n!-HCP8CESwVz~sWKyVN`G*>>~4E-Ktonkx+OEK<^l@8H)nUy~&n zX~nz!DxDu}-erAIB|4^B5Cg|#p`$<{@)HhBhBT2(7&X3|05O!EG0RZ8@b-&!d9JI^ zHL1M_@`}g==Zm;)6}8HMC>nx%SkTB;ua~}g^YWb&2-1plB8s(u#G1qZT@*P0JeI-j zbzN`mn1JUp)jEJcmzIG~Q|VanYYG520F6~CJ*P0x6p7*nHZs=m>qc?0@6U=9+C8A1 zoDP>a+NE04H#D;eVN{po;VH2J_dpQoHS{6fcY(k!u~jjE0Fxl^_Oitd&1R;#rI_r_ zi)p?s5#0g#T%@furq*Ug5S|cA{F2n{^s(OA)b?OY^9cVoeuF6Pe^*K<*YRf>{PopW-pl@ zDd9Y*HB&}*HffN8d*(qh+Z}3*^cP`X7I6WbZK|3t&pk@sxC%deX2;}x>_nT?io?Cs zV>(68H|v1tsv0NP`N;LbZGc;(Rl1eTq~Gn1UCt$UF2S zdoJxxB&I($mzc@8qv*~_lNGHlKJT^JJ1>})om`sPQSf5pEtkfd!h|&b{dx2~eXpI+ zhsmwEM$ctqQFtX=41VIEV=h{&L_IS0%=Rc zsSkQUz^Z=OA>BYzyy(*d#p!_b`te)``|{Xj|M^%isT^Vx)lvOROjdPRP%Z&GRfL^6 zUC>n&bi5Y`FTpp55H(S*XAfcY(Og6*ytlTUGTbrRC0%H{d4#@4*ek0v1UJY(R!~s+ zSn%eb8`Lg+wHU+>EVTRcQv;^gF0st#z;nakRc}z5ti75n`)?x9o1((kn@DS`=qU( zAOjS-A@KGCAJ2+uSePucWH>-K<*SS;XMX+JYgK~$O zJD{1hLbs5!vMS2)otu=)?|o$RF)WlMTXL(!(V?L+K(;_jV4WFc;8WHbU7+|Q^XDpT z#+-44a9wAqLC@^0e&qt|0qjzQIsfO}TRp{zpcsu*Q=G7BDkT5$XoP{jkaH2lOQ9t< z`z{P0AT)&PuA$q$tIGe91>38uxKCzPWo+ByP_i^Si>^_cz;K-PVxjyM+3wYo9JTHb z?#aaA`mSq63f4k5_~47JCgC69W`BK>&fBZxnjSlq=!FQtk)wf|A7LT)2 zek^r?p}W5GiK0}G@D6z}o)0><)TjunDe8!vKqcJ^(aAT9VoaKQ zBtZt9rh*+o~mN5 zf{GdyMz~zk$M?}ai<~G#-){hZfp{U4Pq&j2)-NJ3tl1kGd{j?O1w&+-^BWcL!;UU} z{85tPrlZr`^T|JVf1Q?cRY_u^gyvy*j_yFynWt6v7h93~JFa=zQC{W!l5Iq<6_J<~ZnL`V1U5qXUAkHS3 z@{U?45g07b<10F{q8rQVSEIKp>fd(D(>Iz!y@*Knk9k}GXaL1nkJP(=1m&?Bm!GJE zhO$3nPefjP_ZuI2NhKIu@*YUNJG4NAgPv#c@EZR3s0}QjUf>0${MLe*_hJRUlyoSZ zq{+Zor0Ts`Gw~!|GbCMX_o;YsxXI&}%<2ujIUn4|jQvA#!;c&Vle%BZ`=kx;rmt;$K(#(rM z-wmt=!&Gn1;4*F|8$f=4rKLI5&V&m!JlLIXS_?WZ)bP;um)|(|#jE9abnGSIB18zT z%hyo*-FFvhDBR2(uz_0vNjqDOC)+mGr}--TAcR6W==w4T@fR2#^*lA?eAgcJmh}U7 z>gdPNRFG-ncW@bz!(V7fq?NhVKpFDDKQH}}=<0lB4d8)|9X8^gv{+3gR# z0zp5w6KPM!IvkMY_@2C^W70xmy}47PzMhIS6XDM)dl~i2x^nG2h*a*kT*+*-ypx0M zCZ|x=ZF_>qta4G1Tz%64>R`UceI=d-7IsrspeBy`R#zM4|1va3v0tB)PeXi;Sm^)q zYaRh}zej5Dk)GLQ5eIZxFUH*LfvAvNEGS}o5m=wtDjv=X9ophCrv|Ld2r1YY@CblK zPYyEOAO=O+MERNHXsD2dJ)c(MkG7O-sT?hg8K2vspy6e`sOAYG(G&KMvCyFDjE?P_ z8^~RDtdD9w`=I&E>5H}jx$YK@^@6a?Qnx%0;ZWZI7-7@hfS_x5{J;eoFAq#Cv*=y* zi8d>9I*w>Ej`J--E%Ti~{j7C3MWd;Te8NeM(zvQ&IpPxUlD3u-*8cf=shiyZcSJ1! zBlt-T2bmO_J4%UZK4(fvbpOcDrk5mOFFj03S|(i;C79S1G&efSRI9%5(esbt5>z}E zyaI(8_wz+jBGyoDoRJ7JHH$*4BGJ=*oW<}L5#)*6rb@xtM>+kzvYztfq}anXfyJXa zV!#Y>ss6yj{Q~z)hJCEp%0F46UgAL)>B8+559~7m@CHbrC67Sb%Xh3-qF?4vkfag_Eci#z!aaP z)KymLk(xQ3oZS8VBCd+umv~dH^)Uz9LVo5(A@KrD;-Q)%I^6J6vd7PJAyRD%gUG#-|ZY7k8?*6__@6DNMZ3N5Vl-P={ zr_R7*P*-N)kU(ebKN>94j}C-m<6J^y9s0|*vF;BAN-+lj7e46^bc9vclY7sZ0^ySo zPKAQlW8}B~xVQIXGuEmAvZPQ2t)dyLTt1^tUnl8nN*(~O3qmCs^HFf zupKGf?9(^g7q9bK#nw=3yOd9okxzcdJT*+&&fF>WAVs)-_Oe8uwVfPrdJ*wjQu&po zvq9TSQcvqQYh6AEjN9k#map)KyWA;Z%?eM##jeNwDgzgGL?tFW){ zEsJ8Hu=olT7C-D&nYybR&r@r3l!UUiEV|APl00P?k1bB=yd&pe30T}|^r#VRe#sWo zG(+dgQ)La>b8H0$poZX5$(Ed~XtPpuH}xMd6hQQ{L;G1LhkZR~;v_Qa>Kz;L(1hul z&LtU7<%Py8L2<^Nw5!?R0DJAYek)vwsr)|1P##EFwZOs*jI78$v$ zZ=d3*R`gL%CK!I10OTM+GQC{cwrNo@`)CuQcX`%5e{khGo6=zO@>6Vu`E>weox@M) zwO(%E{-TrKwc&nKY_&0F9f8Q@1aaZJNGQS3qDPDD_^Hl8hpU@pr8D9urpvRls$$K~h+1Sg}`>fI9MAjt$V1 zz0t#%Tz)rr)>jB_86ltj7IU_KOTo}OjNE()^zU`fDg{4+jfd!5pmEt>rCXQrS2i?d zB4v~UEBuzL&9bKF<0;LhN+2lrDfq;)i$Mf?|8-^akD*{qpEzDHSi1w;wE7V{P4Ou%oE}(+d4P+D}}k1-+23N zr2BG?Yhku(41n<8YmS*MjHt=jw3vP^Gl6- za&$QAjGoti6L_yTS4}dkWWJBtfH>gExfu)JNYQ&v9gPI&S$*q#i<0zr^kAj;HdjS4 zRV5`ykaX_5`=uEE3U$eSuYu9hZ~w?&&*xBA|6HjH(mCl^{$fxN*`)kk5NVg!atekD zB7#ss#H{n2(=-+J5i3_mvJWsTN#t&a$~zF)*{R|~5R?Zi5vEJYO_GC@c6&%G4q;Zu z4ajgku(x5*ZFu3JbX<`71}ucUEW5IV3< zRR~y4qfW%ReJ4BG6k}*PkeKNyJ=a;2f}Hzwo$ZJpUeKJWuLa?Wr=>pyxqLXPg zpre72V|8sT(~Ia!xB-`1dHz#->SYf1Dd#T#NN&(W-i)~*b{A>{A0d*Gd9xc08R=~p z%%LPC`XtEjqD8N5rXB`!5Jd2xYg%kjysKIYTeMt`op~gkVwLR<)?$A_kwccxXn{(; zu&1OLq^%#S+X$<5i<_~6$x}bjC4BGeIyn5_4byS4>7|jA6Xm=bl{=ZZP91s4Z@$+4 zR)8Xz5`$(PKufW-YCSIjbDga+7V0-=AL2`q;#`-17c$91R5*82q%`t+v*O%zG8#rX zWEp%%G!v0!JSB6mCA}a!wRM^Z9ZS)7q9?0V29XDgaDV3eplb{z%3xZ@yY3^T?R?sL zrAbGASpVt4&DgnwB(a`MG(!$J%t0h2U2<`dx?Rd1H(L53$3>xJI1j30LFD;82|o2? zby#+&#GCmo*dn4HS5Jeq!1D`%q!aEh6P@f(W)uz!wdcw;9_O8gB~6T}KY|d?-=G6M zgLpOTRa|xAc!1z_18~aep+#l4WOw+)>4fBLivZJseSUBW&+GXlRZk=*oQ7|CF{}*+ zZ+V{ganv9FLc0RLfU7aV4x`sGjXpq%O<bKMrGZA zgTR~~=HK>WOkSBx7givNHhS$I{ZCuaD(v}^d*hKG($}o00xE2rq^v*#iYq$(%oVw3 zlkOb}F)zon?YpfsmVb~tgHDJY%`Nl>$Q=p=l+dL*H@{-J#}xQivvKfE(^i^aqT_5)aBEY?*SZ-{?6}AB8b7 zs5uGYdVR;t{Pc8&NXPX`y}TP>V?i6A6k=haa{-)E8aFD#T%H6#vlj;Jcv(uCt*z~j zg0N6;4w|Ed3EuC?=zHU(_aMLn^Fi~Ls^)^OjW0N;RIWFRotb@333`>f^;N5LEbQ{g zh37QUpaMJrUSgHk{mi5Jc`9YSaB`W7d)K4m-N0ZE@wAie)Zzo(<485h<;528alb*Y z-v(gXwiS58!1;;%#=e0k3KDu#Aj}gV%4v-8n*=Zp<&^IECKXM|yiKBOe9?|Mk{}~0 zcQEL0pHhIA2Pa&oDH(JIQSt)+ljP_3{m;W_8R_Rz$|#{WM>W*u@a8fp5ALb#BKZi*VH_w?0S|+{Q+W&UX9M;s z(&^Hwiq{Ls?W|}hCA4oZy!&u6CdDm}rnQi+-rWQFkQ>wsBlij`j*F!R_tXwGo!K$9 z(WlJbp36vi#?PyFYC0Aeps`Y7Aph3k{<_}7Z}POdt9;Y&l-K+pKCwpg*TVBH#ixAc zED`zjO5cQq_xwP~Ymd2J{n~Rj=FxocFrCbHATy^JAu&XZ<^`Wf=B3!T0Az8rMDxq8 z;FA2xpr^tJu6gzi|J~tj&^+Cf_%ODrq9wU3efn^U?^k|BqgQp=q6`8k&HL9UsV$#! z49qRq_~)j5D|`j(I!LI%U~W-HGe|9GOdTwxqe<*i0B-N^2sFuc&W;IS%IfjQ7LXGyS<9Vc_crDDL|W^;Vc2>Ctf=#ed(Vx2wn-Wm&!;<(hVpo zNC{ce6pH-l7S{zvP-P!je2>NFWld&*B4lAF7AtdTyUZb3XS}u4zqLTU8x4qJyp9+hJjfRRED9gV9cY3VDu!d-Y;|XgskDf9a~8QLEuyl zL#GfQsx1;pj}$Lz>SrsIl0KNH`(g4!9il-~I6T>P4DE_Eev{n(g`sbMBHVn{;e1t~ zK^I8sz@U|FUNra>a@$tyAmH)5Yb$ViuEB1c3ne-UlZO>oly2K_kf9ojjzmPc5-~Z) zoa4JbftWF~BD`arp_~rIQ*XCU6}vk56`Gpv$i#Oh(gHVP)aR-Ins1YA-xz{ld~IT|$3JhFhu)Kre9|TGwt= zmk}J3uaFqlj9T;*Wnd|0AnQ(rQBDN*6W*K2bv!@c>&R=T{=M6;)e>GMj6ia7C%PtwBfnxfXNg!13Z7@1 zdWe{k%xgAfrn?7yjLG451{0CSN4JTE8*^~?QX>}G5l>yObGs2zhRu0_7+VRHr>+soy+b8y63!^-R zKNMuYmU}3$IvXim@-aw6K1iL-)OR7F2>_72mZgEg(7BMT#3SyN8iW;X`_PO&oLaEK zxDTCAuYvP&axI7nzDHBN+2xlrl=gIeZFJLuTVR<;-{!o%6a;2@^loCDUHwhm2muYV znL$gcySbCm9c=)h_<6r0?i{!YPEaXDBOf!y;C5AG_|o(Fe^lxBP?iruZ~BCQfGLaT zGnn+QTU(n}M4kuJ1JUxGAvZV19ZS88`@pzAC1DZ}G_4yzGY4QgVvi&{ccItK*z%|B z?3q}*M!85+g>$F80h4G5qQuDL#6eo%tC%6age zC}nKb3gFt$hy~Gzo)AVM+U9yEbHAkSTkAt_) zaxKyL=?A#i(m^|@IAY5Sl~1TV4faa{fT3T>-0r~wb@S!N@}d60$HTjc(;m=FAPHTO zPH@73@^k41L{}N0ddu9pAAz_iQSmJks_TAiS0S8$N`h;#ub-KLP2^_O*!9sIde1B2 zP#*i_3z0}tU9M3^)3SF!jdYg;eFOI~{ctUW;~^~4DOov*i&I^&AGhM$>u>T)ODdl)_6On0Mmc}jAqgOkS?Zv<2jX>Wgv$@^hOKTJoVAq+d52HXf)Yk!%lpfD*M-!dz@K?U)*akhFaUtf)M!WYB$(l(|g8 z=`j#WYz+H%Vv(aSoIbmef6F|#5%=IR#c8{Hzwb939@^;wW^IE%L8T&r^d+clG(j1( zlgYUo8r0^^Cz)r-{u^)35Uh5VemjrV=2D*+FcW6dvz07l6a|0Ug9R}suu!+^7KH4z zH45~GLi20FpLd5%8l1-eQIxH)zhx@!+&(R)4Ygr%fE^bV{7F2MC#=v4Z)SD`c&3Iw zndvwx*Nz=L3l&tj2RsX)ELBt}4h74SuTm?^IIAtwZ{xAcf}Xt%BAvcGP?ovtJ9eAg z8er(oM@A-`QOIcfgIyKe&=I90qZl{?-e}NXRty>>NIu)gh8l}5MnQA=Jq4mTT+xrZc$<+P5GRuRy=1j#dWH zd=OaIngOQL4S>I`NY~*tQ+|&593b<^gZTUm0euFvvV$OUOFpONN&=(N@&16W6bp3% z;(YF{{fcwDxAsB@;J?X>ta+{S1KW~hewbTWkF4RaXBHy$jepe{NV^=pgs94O3QPMrf&5vkUSnx_?741RC> zm%aD8Qsh9J8`M;=Ck4Cj0MTsfD0N90Fx=o}o93Hi0`Q&EIZ7U96fRMM-R5O`c`yP2 zc-$d-h_KWwy)*lm*l|e@Fc!tpU;)yW<HJ4DNlXf`04c zK}`v-2+p8d6F@1mq(l1#dgE|PPuzyA~bgi^`yHWR<-B$h6Se zIQO}q;=i8*eFCKn3}1xRf|?AISaP6NE8N&(57Z4Kum1kZ&ys}D)$e}!n_NNN+ zU&HT4Ji(*jW!vMx%ZhG)3{$>Uvc?QP@V~;J%|mLaut516Q6Kl3?_X3PpFfj(l|17S zkNKwjz`7c&;?Lpurpqr{X8$B_0xmH4;>k;&6(Gk>zNw@C@7FcA{8-5LCYozgXXt&m z2P1uj`x9aCxBGvtRwUZhj*R@IzzbkA5XS`c&ireM45*|0!Pad;TQ?}nJ#!h@OLW~|tM&0siE5C_?@|9wH7iRw z`>#-(zNZ7um3CY(A5=O47Yh1MpI@*&_?@%R3Ps^QhJE!hhoB1U%zw~!uCPxwV2|R! zyQb@G=hOV>G4O2anfGn_^eO_33@XunV1b4_tOUV}ApUG89?vWY?ga(3ofOHk-pnzQ zvnQVe+<0^oJL|tVi{jq40Q}KnN$u>L4+gKlDF#PvKrj|bGD!*%+Mge#xbj!5~e))N(}sy_0MvnmQ5-d$ta@0 zh>OqMb!|Nv>WJ5)qbL7+(KABHT>(uW$`Em=pG!fZVGPQBW1)s%Wz7Cy2{!g59)Z*5 z0UeU1(Z&zVAZ~v8ZtY*6kWF6d)5bpu@Usiz<93;e)ozDR|8aXi6(oK$@?xHiC_tY{w#s_?&U|J$H-sPfE)Jup$^#Q z|K^u?#D62QkoBJEeY7kfGWdakNbkM(4tPBKKffyrNd*lLSL#6z z@p5%5{m(tQbRoD8Nq6ny8Uq7072aBb4qPs}Kl>~iBU=Sraw23&1-Y|9^Y%Xn^W=M4 zrc&^JmO+vy0NmzA#xyWQo)C%};Me9-V-~@Z20%-HP#N{`e@@5g8~aDYa_`|{uEx}()@YfRjwFG}7Y>+4X%?|$NO@Cv8 zzq$5bOYpaX0;)`ZvxC3c!QYtRZ+7t468y~${$>Y%yDCtF>~Bo)HzxQS6a0+{{#t^+ zF~Q%M;BQRuHzxRt&iM_T`rAAB+dKH19sK|F4%Db+07d!+7=C{SGzD@Y+73MFHw|+Z zw`?cGR-}eZ`0?M+KQl%E;?K_11bBbTcfXN>U-PcE+5&7~OB94oyr7?o0+?LIX)w#{ z0ZDR^(16HYe?RL;PYu#$9EGGkUE(#xSDuh7y?$>UD_czg6zm$?Wl0CU!c7ub7BGyX zfiNGjfHgH7VhT}!vd`o9m<^E09xDc83)?013srj46KJA00;W08s4BPhk4qb~A*7l4 z_BX5^fb_*WT}E>*{X(pxs6WdX>Pt@3ONX4K6w27Ay8J@vvqILX6O2!yc_KqX?sa~8 zIj!XORYyRi=rJ`@%IVXu9=<1UEhsx?9I6y|K8`&&>_vHJbLKuVkfZ0u;NZ63)J^PU z9l;CPD{%{}h4yc*!!q`M*uW>&G-iX@a*v!+Ho(I=;kdm+$#C*?o|eKFczvv%0>#O- z2bZpLW>0_aVCb3va>$=qgQF;eB<2Pa6%tzjanbxD#Fk!(SCz#)=q~HvkPbBe5ek6D z{N2z*(c*l(_S9}%GYOAC+@1+Ln+yg`%IW|!Xwxdr6~jtgNhjvtUVLx87!Ct$!Qu=7 z)5n+0rGIk6NH`dJa^2;*4;I zuZ{&bQ9qf_WMbRfGxlWxQrO;Hab+!7hZh&m-aEalaSs7TPwX~;h$eT`Q{NqCP)@CQt%N5(A0`BRTAC%ohyne(2});_w;R z?QQ|dk1*mvxAxIDZ2>xsFOPYnE!GsLFVWIhpyUCuxFd01o0xTb_SMy=7JEJlmdKE= z(D2e?I_V%i*EW)06KFr|uXupn8D%4*jOacnih6e}c$en)`w!zkNJ)eDuco|t&S!^0 z{oFnzH6Q4hsvR(U*5j){@$@fnT65~Uon3B8v=d}r7qU+s4%di}G;Mg{Jd%(5x=K90 zfZmICC2$Nqc=TlksCwBGf$Bhjy$=L}Pk}U9HDFj43h%3*OG1>RX3oU=+}Vzcka;5U z^+Mo;J}o^E6&OrGlL+W^Kb3#Bpdq!LIx32jPBAY+zAgqB+*Nj4e@D&c-= z>M=6%3GxfHGJfP-v`2@fB){!XEKPQgUvV+B&6UI} z#qU>Roo3S@J&=)<(YYBg7HfAU2BPm;@h9zCkg5*8fGwXwa}GyZ^Gi+wqA>e3{Nz&4 zrX1F{*U`nd%ftnZ;cM<(y95*q0x2Va8beTME_!_2p&5vki2oKz_y{N%MaB*& zTh?YHaFM=Xv3(hOPev&!e0h8-z5G1&X(N?OT%1Uv>iT#L`XCiO=5RcnXkwzOb^pb& z;8x0U(sXLc%Q7LUlTpSQnp-^t*`LH3rIbA3kVPP3c4T0px6E$a(EH86#PmFytx3SB zL;PgZk{wRud;qDQ^=Ey@+#XHD@%3_4me=LIy+n1Y@eUWQVHYD=zy|0Lj%qYBA#Y&Z zM_Le&bqCc+q;(P?p0C`^ppvzlk#R4PMy+TyTpM{^%P z`zFDezFTCgnRci)tw4EL;$}y(_#wNy`CV16+;>@-v;Nmqtr%(n(1^SoY5L4FO%tJV zw)AwKSqn3L6|8mks)D6{a8$s%kgo;lG2YCr-RG0_!Loh zX}FClHDv^JxgCQJb76juU-`3}FtKj{9X|0=KV;!z&%tPiT(v!-Nd`CmSnh)Vmzh}N za%?g*SQw#JQhJ7(?a7zRqe1%-FrUYBhem}lKv(`u{)Vg`Wkl12FL$sevyR*7kakZk z^=V1FKkuXnd()!|Hk$(&!)Dv_=2znQPP6W{w>QPlJsPPu*oHF%+HHB_m(1f2J-Ek~ z;(;iOc{jBAbrq99D%Xj)U_s2T`>rB%6L(mk4epkN;G`>oC1{2v8M={$X;uv=X#R5% zh&x-KO>u^;J6bq?dhZS<-RtVw#Y*}o*Pco^3c0`5So3MFl{kU~LY!(=HZJ#1sb{_B zm!7R1Z4;`-x@uvc4la+9t`Ie2_J_Tr9zP2J6Nt`Mx=XcuSxqSY?c?&lj!(Fw_uC~l z1nD)WB=8l03Jn^@4v?}H%5(>DX5a&e$tW5}&tGgk-hN;5%5E`85iue)Jt zw`9?p6K}LlBBDu-9KT*r-JjT#-S2tcJO~LPcLRa>jqB~_+p^x%Zce2W#?nz~oPpR? z|8;1%bQ0%^H%w}r=Smq}4tRE_|Hk4o(|jg;!kvbDVXE?FEbR+I&+F6%6JU26AqMzr z(m6T6&@2JzjHSHd-n&~jjx4EestY(GtS?CB^*Ra%a(aG&V+#tw$PTg_(o{+aXMtI4 zjMO=#k(%p+RhPAEgODp1U#JJvk-;{ualYIQmzi~)F>z|T-)S2#7 z0?70wlInii)VrGaaIB6&ye=!NB3LUT9lHq3>QsEDv@LfjR?qF#Z5%kqz6~oDU|+={ zeTeI>jzP;2L3zGFB}h7u!?AbX)$4GUB)bpk4J`tpL6rB&*zI0<7~XXjxTdLU6;ssn zY<{zRym&7#FY&-;CR?T^9t(~>d#_Wqc-!T{#)ld$v!&|_mhlvjeQDtSbv+toM9mh~ z0?FspA%gc11J&c#kgbl+UuG`VPi)^FzcbnT-l}N5*k>=G$H&^_{OBA1;EuYdq$;)Q zgr14(@ipV*ZIk}Cy=P-5C{PA#belkpy&U(^*~+B@_q=z+|E*bLZHg>gJ+2{rje9fZ zd82h9s@A{5z2+rg>Zt9PF5fqkD6#2913$8tcGX87rtLQ{{qa?h6(X#fr`11u`#)R&pXc%$U`u7s z+xR+Ga?|BlCdv^>se5jArQ#{lsv}SLe9>~EB1smTZw#G^%co3%+d&_;P|1B(w;Kn6 zkT%&tULd(2@Vdy8zz6T)ESit+oZBb6gaErF3Vhp zCm##7BMwp!V}3w=;}&jQOQZv94#)(Bb-b5QT1(6PeC%1IwdwlK-n}2`>z@qJ`-xmS znU*z9*4xe7-yCPyQa%Vv*Q>Nhk^pT7;^7sq0b75SHiK)K&!xusWvZ`yk@f8@kmtnJ zcC34j7kdSoOMBPPCsCQX)shHU#56J}B}fg64!li_F{nzd^aqZikvY9rs{`m$)vngD zZ11OH1C|B`iIcsA$w?!idq?uLpBS@{0UAT8{%EkM8|cZkoc>t9q0Ge|k+Nm{?E`d4 z=lNDX8mmC(=YH!TetQ40Rhyh*_}V4Ts_RC%Tnd(R6u)ByRte2h?~es*GJQ9imgw(% zzk2pn*r70KOSqTIdN$?opYu_f|EIn0ifVcb`c;G@!jayTP*qflSV@#F2%;i*R8WH; zaHNO?kZuSFN>iF*K|;5nq7*?|Na(~u6Qm_10fH1E2?!)eDBu3WvD~%3`*7E~kNCjy z;lH!X%%1X_*)!OL(c||Oo*zToErsvFgI+E=jY=oDS3Ft@W@IZTWd-S@$r@-X>Wr?g zwNCfbuv(M=U*hbO);8)r=UqdTlx(CY!eyx^)HgBRzwFb+hr!L54PR!9L+Z4nF>rl4 zvYX+a=oGmR)2o4~3-m{RiQ>%n06{jLLRrsX+$+wCD#7k)DR%dG)1mv7J>?s0i}%_<`p^57rqN6DAWYZ8;&ctHh5y6{?tA}>;)oznoPa~V_PN< z>kJ<>=Ps#hPNP9hqXQ*)hI9 zo<(yfh@XQK1G&S4OGV+CL!eS9HTe4Ial3L?5AMDn9cjT#C*jEMGdTRV4{hMS-RZum z#Lg-03ll76aTJDG@|4(83-Cr;g}pV34y4WZlv8MI<`5``n4UIJ5N78(#;DLl2eERA^K|W%m4t zd`Wr78(`%Q9iTkIg4dFHJ+76vM(8ELG{CyqP9>AM#&rZy##zHAYr^*j;okLWYxl&L zn{u0&L*ODJIbWYIJPZJOsn;285n8%EWLz9FR<1Gk-GRS5@{~cV*Hb7Y9@5Y23AVxn zC7=;?5mTV77-P5`B|dGAtw=|^e;)*8(rIjYfyCj&boa?uu6H(&$IicA2Nhs0;rK>1 zW#dfI;its8OrLnRlx-6ztTa*fi0<(cFXo0ezl}F?h=087`cSuQWA9DUCDnXjSEOJx z4Pz#i#c0tAB(P9t~bzQxnt$Ny`o-xbeZ_JXl{Pi?WDy z&r`gT3*M|Yd*{;*S z;Zrh-<`JB_h;dA>gVFx=fUl!ofxlfG7Q&{(CIGA4J0#5pP(E{iJs{Lc8 zx22Snr*We>Ra@Ob;qUcpdB#K&;xEMK6pAV42l8zCv7_V+sUpy!4meX8kPRGqBEsgb z!Pes7Z`r|Dc9pdS-waZ!>8U%b)2xm#zV=2t?dbHIYjtRSfK0IFXm8Qa(m^d}LB~xb zMXg3IbK9aFaF*~&K8}=~p*`rZP-)?;z6r=+b=CB`^MXKbJb~)!z|c{^QDfU#c}5{c zXSM}yFR0V-DNjdAocFtKQ31c#N#V@6f5g4+6RD3#Wy?nMm~rL{)|+yoknerS^$Sz} zha7~HP&K+Jj28MecPg3T)cWg9WSkt`^5)M)OU z^6Wi-uX+>ZHaxg05&P;F%%IEKX3dvl_v(a5Cm0Q#}4vvH{OyLIv0$rgFAZI_bQ zE(ZcVBm&EigfgF3p0XJ$j`mf$2RawRH_6l9ro$p91I_11O)Z2lBmZ3H*Fc-ef=2Zr zi}BBU(A3?FppNn(#594^Kvga@rJZ#v*fZ1W6_4(9RLf0Dnjhm@EvZPsoKoqNs`WR! zk$-E!yOA{p@Glbzn9 ziQC2}(3K0ev8WoA!oCT}9JcTE!X=!I%hsED39PDyPELiU;S6YZ63}nM(Cx9_)Zm9@ zYPcxhrTF_0PybV`B-W1Mdxe4(gnGi`*wjgi6S>7&0|AQ&f&Wn0Q^dK;lH-iuq5OO8 zDiil@*_^C?#br!`E#u;Aj$QP@052WgjMvPy8U+H@`g|jPbSbTF?lXop_$sb;wqmYf z|1{~4fY4;@oS$&~vx_C@a&<(#dytiGmPo5_8a~#_#=UI;GykTcqK8!w4HR&PfW0@U zj>mdb!v(I}pE`AgKPKBv)__YD23@KRyBf}=8wAx7xEe#LgqKuv+sjh0xm=-h4_~u_ zudP21?m?r^#fKD?7usiFyr5XtzD28N(rfDojuo$F>=aK3t9|RxKh7QSvI*aZUSyWgxhUKh zlnX_-scm?2=zhey$-|xjOJ>M60?OV%p z8N$8d_`}%xF`d_n)*gU*)MO9aI$F2u=_nf9Fo@>i;8_?f8yV4_d0seMLjKIwR_f`B zvY9s?l#0cfymXTe>b)1P;t!kZK{>}!w?bi1$04dG+Z^e+44Adx{mxmCbVZV(*0|o3 zcO?}rB#)V9d?)9rc_*ClR~Du+8O_+2{-d#X3cwvh2SL}jUN`rFT-VBN+8mPVa9V}Z zJozbiaFpG`eyG}Xd6)O&vaWKrXzfJ+=9Ou21I5z0pmh8`|J$c6blSh(r50EPb!LUd4YF$HoCYG66PLE z(B0KM*?sA%I7%>t>%Gw_&A77QuNHatOc(UU^omX?Yb>zc^iy;g3q4Z8$w=z=hvcP! zNeG~&`W0%*W2Qehk*VzB(c=t_vSh!T)1y#d+@(uGwjOXd^ql+tcIwDY;L&GC&-xuA<=iWlBPknugMP4D*5z#v{H`$ldHG=( zH-f&bbvPusmPb}2;a>dhu-bAV$*;a24r#9C!Kx&jIz`>Tg-!_yC4z3hf;6^`9~x^` z#0V*C!f&4vh?F_)uQM5xjZ8W*aBjw#^DUl$RIMk7V(8s1ZUy>BI3sC!Pt{@g2!H^8|LG^Z*`w?9{hkg1EFzSu7!dp7S{-SdgU1^+zHsMtHs zA(bD`%H>R>WzxG;=_AFXxSC9|Q}(de0n_ov`Qx|4n2h`Z8w}RDC7Jcrt5-P>JzsA( zU*GF|(1)5I0AVlh+8o)}A9$+sauzpQZ@Ne4iE|VKZ_46ML6{ED)*<(d3If`j4M};euX# z6U7-4CaC)DrH-|Ks_>UoSdlO=uYJUa8rz8u)!t z{%QA8o%^2WzjB}^4#H>i&4j5y7takd(JYqlO9!3 z<1rN_W0hCFd;7(`E7_TP{MJdm%7qvW!b?5$>%zWP^Y^*;-lxiu-x7g{)EGl&cL~4h z6Twt~j$$^m<7A|nhK~X^%3`Q~a5#9#w1V>vbVfX^pC~&xH`+x1-^!+cvFaLN(nfu^2WR+h2WC&vr{uUf&0yr9Ae5r2! zSaR=>V!0i{6n>H~k|ib=vGnRgOONmy3f&%MTQ^+tEuy(_xMy_o5IB|KO@j1z@?Q)N zA_ibvjymtdjBG|b7SIf&^c>1p0voQ-DT>`{8mg6>GK#x!Rm|w>t6p6|x~QB?mEqX* z$9$1JNd!>1$wS(Wg!$)sp55QJ3eP1Z;q9Q5c=^fey!ynsev!l?$%tASqY+dx25o=SV5@yIYDR?umB)e!99J z_c5}+5}X}C1rTt}0dMUQ-u+s|GF#N7f zQ9D-XZJ$Us)AoZXEsG^T3D^07PMaUcC1-F6zPj-XXGl%S(hU-hV^W99`672%zPV=H ze_70JUj12Ls~ozwGneurxZgWwPYL$2KNfxRM%7yW9SGQ(tkWWjmf?U%-=Y%SmW-S= zOl2+GZ5MTQIh%3G?0ygh;yor&3i?`+Q$T6$pGr=7;+ZDqrEBGfo+7iVi$|tooOK*? zF<2i^{_BB-om--*IW{0!&yI6*mR#{U7WJhlx~nXgM6TY?c%JQ@#~Atm9FB@1P&AxV zgM>6e%CL?za|UQAU!@_xrN|-B3S+Cdi2!BgF{(wU%-i)kbNi;5c7 zZ)%{di#_#pfsOG7?JOHpl^xvRs%dDAv|eB|U9|;h)CoI}%6Gj*AHFWj@59#KpcY~C zLb>@htv^)UkttB`K6vgrb{^>&XD1&Kg1zNVLYAB$iZS~u+eJf=()f`h!^w3tle&VB zlTgz*SI$vappjT7un>yZbQXZ;qf@|-_F%F!gFjP+UX|5I=I(Yc95+PYNq>&N?L0T; zryu%!p_n@z*iCfajjjODFp3#8tldKR!bL1{_AUDRTr2v(84|Rcttlqj2iYbD`%mo~ zI0Slh4^!W!FEJ|F@A4=?AR3UIrrG%cw43su(~&PiZ`coEt;4o=Dsw=*l_YN!jp9ZgDBu2G|N% z!dHM(ph$IKnf{9>LiSAcfmouGH-Ig!4?TDuz0@=d-2A-2h@hJAIOw~E)L6a1MA{SC~`acE}t-s+k=GCWtLHF=H4nA?uiQw{@e%uM@vZ=uw zd`>rZ`=BA|t`-AQaOo<23x$ zkx1XImuF^&`w`e`ji`G=aax$uqQv13JCW;w*ibO2c2s;Bp~lt(H0mwG;0R zbhPmRakL>S8b_v}a+_4G{puOm@<~6U&gj1#m1So+o&lizo_?1z)H` z!E`yATAI2qwlsH`^PAr@@={o!z#uQ7xV%K*af(_nTN0M#(&`&scdbItM+dZmPCx$d z5d|nzCMo&#f)ycwoSFVwr+%*B;@%#Q~Q zuvIWb``NtY`SCc-c_+sxVtinuV|kNQT8Z_0b&0Gw<`Yn7)9CM4ni~|bFIhfC)`seJ zI!Kr{5Y5?>8r+L^uRoD;zhC!XY}owS-v>S%ckt~;TLlK3Eu8|{RL#DhDO1GP$sBUKsvo}sx>7g7HU_o=D5hK7as4+#?H`fF_$=~Z!_vUH2m&si+ z&_30IK(JN3+y`)h#}F<6g1{+iCgwM)A~t~YlQsFbbc^JON~g8ib^Yl~;fqJRi%K>g zw5(HE^+l)=6egQG<%R{CTqs6K7Twa>jZ~9xo2sP9dgdU zx__O*Y*h^`Hn3D(iAq&=$+Q%6WHA@=nJsRQ87K3g@C@?4U?ky59>4V1_+C1Km;<&Q zuI!wm-+TbAj!XWxXR|mfw#LAJDVdOEK$Aoi4Al>PqR=5&&T8CV)Ny=qOxwRA1n?TW z_>|!AIPUff)|Gp+F>|EmxFtxh*Zg?&W~`7k1yExOIInSPqc(7UPXmvyW6AE-`a!o) zeZ`~MN&sO7q4L>SQ}L=DylkMd>jK~(isC$?in*2=)BDICy+T8$0Ag*0)33FfrZG(m z@P=1{#LKXa=@pvy>=TKT-sh{8wCZA~T8d+d{T)=tAniaIv#!#x0RU%re+%XYekY{f zi=mLG>d+4BsM~c%&5#UCFGqJttE?{{wRAQy9ivb)X3HrL4-`$OfV!CR4DbU3vrYjv zu~Q?a`sF3=y#$rpXDE&ozQ_?4XrUXzN^(f-D?QQ-h27q;sk6w*puO?VAq_Me&NPbfE?)NVI2f@P3imp@r5`Bwtn zpO+TyW&F4O+?Q-U-yF)Y!=Y(H%huZ!la=d-(?b@Ey15LG5VTsZ;;7o@8TUn9M%6kV zUiLPC?rZ9Ks2F4@FPj`jN5qfr;N`!s268SQ<=FqrOW1CCty;-sQ%TOKN*8-hM?i*Q z^3r=42~a2kz0&~AP@{IC%3;~(uNPtGnE{K>JZvVVu{NGjB3+}E=^!mZgT1?ZtT(E% z4030^lEdk{1%&S2k9#_9+fe!1QY`fK)YOF=7y83*?m`q{O?PF;tUMH_Ly^0KTaZcm_ga z0+AoK1CtQ%YM)YQ_9T-i>KK%Rh?}+gaW(8pc9>C7{Gc^FQ|pV}ydEKz%-P6B3=bke znvRrMQsHn~5qYe|ToYm3JNBLCQLUs$E)~1Xp1SbWYwH|cAnY>HUILcI65Lgq!NGpl zt`{lkipKBoJJdN7$oMk+mL}&@t&}@HGFNxu8ffdm7E)pt%c!dQwMe(nk+YNq$#OZJ z5kuh%GRJ9U<$3iNWYRys-6A%N+i6m7(j>ZcRUe#0UALcVP@@zqi*elc`@OP7XX4)J z06FWS`Xm@He~`67b@d4VB2@XWBz!n20AowY5FCR3t$ZKDOEew|^6w+xokm7bHk-{- zA>}vX&YiWAIE#v*kFfQFh9r z9_myZ5Mw`zKFcfAEpWCEC(p-_PtM2{s(jW$WxoA5SrCu8ni*eu;5hBGxw0O^)nd%8 zc!G3j-tqK!wAp-2-Gpu24*5C_XQ!494r0gw-1s}7M^_xB0ze-Z5nZa-2ShWvpG_f# z$Z+yIG(cU%)YtZS*U@L6FG8rs1Fh|Gi=C5!3<=^}+QN{>;!uid-P`mh9*F}8=W$Ga zyC@dbB6Sc)NE|LGuHP(Ol6~Q86#3l@WtZzU0<#4qh6SSaBQw2;i+^KERFie>Bz8Fn z-|N;^MqS*`#m%BewEd>woaLbGL%+`!CBVO_f8d5Ih??D1F;PRXzaG0Lbe)8yC&J)F zwz9#cC(vP=-a5xsuy8v_x{=IguWk3vE68zGkOxtt4C7D`Eg&?3nwna2$upkE`qzfW zibYjYw938|afr0V9_EwQqLL`rUA`}mAJCtBqF=ti`e;qK<~{w{6cpA4X-Pur1U8vs zj=hAU{%H2sV!%IRwR))Y@8jo>PSN!j>EtCSLurWAMPTQ0-R|r(N9%K*oA(}Ye_K!n z+z^`0@S!eO(oHXsM@D(SG9_m@+q2c&V`^?nrJm+4)( z>ki(4-|~8mEC?>L=H2gZ@5*KAbjB68wl3#QEGLdggY*l(4rGuO@C@Cs>|z75+xtlU z`F;lR5w7=ZGSqL8yRTxdBahlUKLn>So#ET#TQrA`y!I|=wjB_AG(;Z>D*|K4;b;tb;ME#n&oAY>pI6ewU~{OePYs4@do;cMDCVc@t% ztT?Xwh9XV@L!uvY_aYKE>HYV!%Y)ckGy3wkYk4ReWE@_R)ZyZ0M}ef?4oT|lOUBD9 zSpM{z=q!MQOYz%}>56uUM~4~drawChoXYf-Swhku`-4?q+6H0p&q7`QSX5OS?@2`e zEhrP?@YIdv<7&lucumxo@eCQz&Fj!wo=9lx3bsDim4D5@`()^c zvqFWR)pQ!K4C+{UzW3@juDBD^p&*Uj-Vf7gp~8n|nQ65LSELCiMXziuejqs5*Nf&&CoWih+vJ zA4DnO0WsX&vMDvSuN$iMtUM9tESy0ym-mV)z-rsYq<{&hyFX(m)B{4q>3KOGG7f`<7)v-#Uz5LdL!TW0r6>W{5^d3Xh&-3bcNEPoG+G3iqT zyQ}{PQ2iVO9Ospj7eo~;D_4ds25tk{cmg8OFn)BG5rR?Fp1cHdx_W206~BpTV8#P= z-NO3g$1bivv^6}kOk-R zO{Co_Yr`Y^Es)jr-A4s^cpH>fIGxxl2Bgggh-I(6M}M=ewzs%S{dHq^9}a~)sO2;^ z6#t^U+$c>fY5rA(ciM-fdC9N?rQ>>K3`z|yxb-vB&ZS;7lPW>grILkXx{V|G1Ky5#g6P(p27ZB4PCgcIqO2we~AyEj2; z!0Y6457&N65fI8jx|-cEpsY}@eE9wdpq?>Ay{$gAzeXzVRt<%ag(lFXaBAJ1^O^RX z`vO8Z_>>S2@3$2p`sLswW55RX&>`6KMi~9NfR`kAPF;=4stpznkH8D6BE~7r+8ab( z9AB0)mR&Hhjca+rAwM?!9~r6NQsxSu(7?C9!Xp+eg!v*DWDl)d%j3Hu+`lDhX92zJ z38BNDVzpa-$ra1gs^DCdr3<2T6xfrV51WmFLJ(Y02+_s=-h31cTFDL8^3Ihx00yZ*gJkaZ{~pxs5=yb9WVGfJ;sGyUiknY> zK`Zi0%Pmeu0}N7y2JLuY|6BERBSI-Q(4fV5>uURjhU?&3?)y)i?19GsX51(Z-R_je zi9c?4OqJ^L@~Fg@eVzfGK#XQwT~RGP-Zg+&FCb#&-8w1rTeaMlk|{4Y%l8Di#iOC=+h2qjO{azOug1jI@<&FFz zGSMsgb!+1BBwqeOi0{fofuj30b}i3AU>_W0Uf*Vrp|f((fm2mLroDhS8xDOYO0h#my z(4`gF6#h$K0IGzpF8DtIBC~qJEBv;~5v2#l_gdc$Es=u7{;nCv1+ZY znwKk{#;O7TpF6Yaar|`ER#{?|C05=1Ra^oj9S6IigCPLw+6pvjZCeXvEiB3c*kl)X*C1}0S~Ka{3;s1ipH;^@vCV3YVO2pvd6D1 znN{d@6?$ETURR;lRp@m!OK{aiT6K|DU8GePY1Kvg=^}l}l!B#!a?cO4QP6`A1aGKx zMIT>Nk8?VxdFSTlHrEXv0RPNlfIdKkqN)w`zV%&AjsEYcm)wdLw>hZJ zit||GA46i08Awl=l86zIi$0+nS_{2+3u+;60Y3{%uCiC3h?I(n`(p-0%Y++YMurLz znV=}c{VIDXtNr%e`;8H6LJRm|4Ltf%$Gv{PgaB_CtUxNiRjniVy&#Le-QQ2SX&`I@ z`oo)`@`tP=gQ8HVX}>AV%h(&#hf3i6Bh#Lj|Fxlk-j$6nPMY1QY7m53>0Nol7a6-G z!TVE8{(3s_^FObl7j|NaCVy=3cQ!uwsOluh-q#+TmHR)Y`QwxPd!R<}u@jqrze@P| z)Bk;>4QkwvKc$1ROn;2}V?9;Jzzk}8{(O=5=j#Db_#awgVjS)NI)fUR;dR~9<%9X( y=E#T27@s^83<_oc@4Hw%1i(`N9}b~vX}y%lC4rm*>3}uhpV^U<#<_dh+v literal 0 HcmV?d00001 diff --git a/_static/favicon-32x32.png b/_static/favicon-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..52c5a60587922b9433191bbcd20024cc55532fff GIT binary patch literal 1717 zcmV;m21@yfP)Px#L}ge>W=%~1DgXcg2mk?xX#fNO00031000^Q000001E2u_0{{R30RRC20H6W@ z1ONa40RR91AfN*P1ONa40RR91AOHXW0IY^$^8f$_A4x<(R9Fd>SIcf3MHH>-?wRq- z*v5&mP>9R~P^176vMg(4MHb>K<_p-u!doJ-=$`CwK7lp)kZe#^k=SMz%O=J|!T2#< zU7T}oS3f)sHZ4zg^{x9j_uQ)LYx~=;FMo2*JvMm}7p^cxUKpP{<7Uo!=gh1y=FG(A z6hA)COk~b&;_+_JZR`shS=)_G)QwEiOT6twHg3GfNo2c;G2O(QUdQ@QY;3m^`5tAl zG0~QNm|1T#Z;UgzJY0J3Eyvbi##)a7|KyT)nKLkfob&RGV5WBPLQ(6Y6OSA9=_}|) zc4FPQ^L17@H-!)&LMXcN0){|>OUi1`I?s0k56*)R4WgFdDY&SDLtrBs6U3w8R*Hxj zyytX8dAN!yGnEANumnRubV;9D)#EyoZDnxF$`S2GX*P9_ugZuD(B0~4^! z$+!T)oeY$@i8>h|I0BI^38`kS;S{)frXAl2>b(r8sxsk5J1Le+hL|oaJTibo5D#TV z!Dxx9U=#vo2o`+mHdMQ70nmu$ui{ICJkpUaUL+WSDp-A|iyH%=GMQK%ZKb*F4&9&>J%$8V$ZGESH{chR^5rbTJ5ybjvBQ2aVL1(m)%n%fzg=X27GSl|h)Td4I zeD8U`JM7M>?$b_gwS(8?@c8!^=G^3aW@)*P@VgMlTIq^vh>>amED4sOri~b34d@2q zd69JPE`Oirvmbr#V~>8op%?Jz36}6Q9~AxqPP89L%mVs>b5xDJLErY!8~Agy3wua^ z7v2+Q#BIocjK-`eL)O(!&@n-2Hs}Yw`q>xeDFj{MA3{9DcmO8@v?hyuhxGvZkeNZy zDfN1P9{)8(SZvYqD_SiyE00@XMCD=rT z3-kW)Ugitfmz`C?#YvzIKve>qzy%NWNk|YoOJgu$X-RqO91sw<#?m7qgd8@l3|I#IH2a!?0D~Ytx1ZoETF6&41QEC; zh)O2!Xla9ScoGEr0rrTH;n>ye!&Azj3x$5?E`s>feVj%S7ABY=SV9f@Ca|`$%{fj3>eb=%g{@OM&OYtAHS$J@eWA>Kg`s3>!4zO1ZM9 zdQyTngOeZf#dv?V5FMU%%;BkZhyS?Q;lM1tjw16Ng72{$RR$~of&4cHIkxW{eV8E# zn9yvN@BW@v{eS|;T~?NITgm1JWMb`Lg^L>f1k0fp zf$%2`$QHyo0H1ZuI4N9N>`9!BWj{bB)CJLB-WkDt`$-VDc5Y)$u;o_}QbH_3hG?Aw z@QM?STL_bgCKzz16ZQj+pJHEbF>%li_P)1?wXQc_$p`{5WnpUqjh2i|@c#n*e<=SU zUCxjohNCe)#Rl^r=7tOePy}H&SCC*K$y^CF0oFUbVX88A5qfc_Zd z;Lc8F3;*PF`BYDgMj+1LUw@by%)ec}n452?UO6O)w-0WoSYz40pB2%rh8S~F00000 LNkvXXu0mjfm literal 0 HcmV?d00001 diff --git a/_static/favicon.ico b/_static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..2aca5b035dd69fa9d7313ccbe0fa35d32e9e93ef GIT binary patch literal 1150 zcmbV~Jxjx25XVoDwlqsYa5L@F;#Uys3$$?)RK(d$!KI5JA~hY{ln&L!Ql#I*!NIMF zR4eqIRKI{jZK}16|4Z{o8Y5^EemU-PzvP}LA)3G$2oSCz+MOhtCZbt@5-P(w0uk;_ zNkh!sx@_e71SF6E^8QJi9+id!5?}_{+sTNYGZHwqTpwZzl-I5XK_gQ1z)68J2-H(#=)c%uW)0GjaK zT+5eN^8m6~~N53DapzCl=zaG1f7|~49&9!{~Misz&#p@NcN=mJuFXprGJKXCA`ehITK|xW` zs`hMrGg#P~ix$^RISR_I47Y|r1 z_CyJOe}D1){SET-^Amu_i71Lt6eYfZjRyw@I6OQAIXXHDfiX^GbOlHe=Ae4>0m)d(f|Me07*qoM6N<$f}vM^LjV8( literal 0 HcmV?d00001 diff --git a/_static/launch-logo.svg b/_static/launch-logo.svg new file mode 100644 index 00000000..1e9a4431 --- /dev/null +++ b/_static/launch-logo.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/_static/minus.png b/_static/minus.png new file mode 100644 index 0000000000000000000000000000000000000000..d96755fdaf8bb2214971e0db9c1fd3077d7c419d GIT binary patch literal 90 zcmeAS@N?(olHy`uVBq!ia0vp^+#t*WBp7;*Yy1LIik>cxAr*|t7R?Mi>2?kWtu=nj kDsEF_5m^0CR;1wuP-*O&G^0G}KYk!hp00i_>zopr08q^qX#fBK literal 0 HcmV?d00001 diff --git a/_static/plus.png b/_static/plus.png new file mode 100644 index 0000000000000000000000000000000000000000..7107cec93a979b9a5f64843235a16651d563ce2d GIT binary patch literal 90 zcmeAS@N?(olHy`uVBq!ia0vp^+#t*WBp7;*Yy1LIik>cxAr*|t7R?Mi>2?kWtu>-2 m3q%Vub%g%s<8sJhVPMczOq}xhg9DJoz~JfX=d#Wzp$Pyb1r*Kz literal 0 HcmV?d00001 diff --git a/api/client/index.html b/api/client/index.html new file mode 100644 index 00000000..9a163da8 --- /dev/null +++ b/api/client/index.html @@ -0,0 +1,2659 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Launch Client - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Launch Client

+ + +
+ + + +

+ LaunchClient + + +

+
LaunchClient(
+    api_key: str,
+    endpoint: Optional[str] = None,
+    self_hosted: bool = False,
+)
+
+ +
+ + +

Scale Launch Python Client.

+ + +

Initializes a Scale Launch Client.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
api_key + str +

Your Scale API key

 + required +
endpoint + Optional[str] +

The Scale Launch Endpoint (this should not need to be changed)

 + None +
self_hosted + bool +

True iff you are connecting to a self-hosted Scale Launch

 + False +
+ + + + +
+ + + + + + + + + +
+ + + +

+ batch_async_request + + +

+
batch_async_request(
+    *,
+    model_bundle: Union[ModelBundle, str],
+    urls: List[str] = None,
+    inputs: Optional[List[Dict[str, Any]]] = None,
+    batch_url_file_location: Optional[str] = None,
+    serialization_format: str = "JSON",
+    labels: Optional[Dict[str, str]] = None,
+    cpus: Optional[int] = None,
+    memory: Optional[str] = None,
+    gpus: Optional[int] = None,
+    gpu_type: Optional[str] = None,
+    storage: Optional[str] = None,
+    max_workers: Optional[int] = None,
+    per_worker: Optional[int] = None
+) -> Dict[str, Any]
+
+ +
+ +

Sends a batch inference request using a given bundle. Returns a key that can be used to +retrieve the results of inference at a later time.

+

Must have exactly one of urls or inputs passed in.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_bundle + Union[ModelBundle, str] +

The bundle or the name of a the bundle to use for inference.

 + required +
urls + List[str] +

A list of urls, each pointing to a file containing model input. Must be +accessible by Scale Launch, hence urls need to either be public or signedURLs.

 + None +
inputs + Optional[List[Dict[str, Any]]] +

A list of model inputs, if exists, we will upload the inputs and pass it in +to Launch.

 + None +
batch_url_file_location + Optional[str] +

In self-hosted mode, the input to the batch job will be +uploaded to this location if provided. Otherwise, one will be determined from +bundle_location_fn()

 + None +
serialization_format + str +

Serialization format of output, either 'PICKLE' or 'JSON'. +'pickle' corresponds to pickling results + returning

 + 'JSON' +
labels + Optional[Dict[str, str]] +

An optional dictionary of key/value pairs to associate with this endpoint.

 + None +
cpus + Optional[int] +

Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than +or equal to 1.

 + None +
memory + Optional[str] +

Amount of memory each worker should get, e.g. "4Gi", "512Mi", etc. This must be +a positive amount of memory.

 + None +
storage + Optional[str] +

Amount of local ephemeral storage each worker should get, e.g. "4Gi", "512Mi", +etc. This must be a positive amount of storage.

 + None +
gpus + Optional[int] +

Number of gpus each worker should get, e.g. 0, 1, etc.

 + None +
max_workers + Optional[int] +

The maximum number of workers. Must be greater than or equal to 0, and as +well as greater than or equal to min_workers.

 + None +
per_worker + Optional[int] +

The maximum number of concurrent requests that an individual worker can +service. Launch automatically scales the number of workers for the endpoint so that +each worker is processing per_worker requests:

+
    +
  • If the average number of concurrent requests per worker is lower than + per_worker, then the number of workers will be reduced.
    • +
  • Otherwise, if the average number of concurrent requests per worker is higher + than per_worker, then the number of workers will be increased to meet the + elevated traffic.
    • +
 + None +
gpu_type + Optional[str] +

If specifying a non-zero number of gpus, this controls the type of gpu +requested. Here are the supported values:

+
    +
  • nvidia-tesla-t4
    • +
  • nvidia-ampere-a10
    • +
 + None +
+ +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ Dict[str, Any] +

A dictionary that contains job_id as a key, and the ID as the value.

+ +
+ +
+ +
+ + + +

+ clone_model_bundle_with_changes + + +

+
clone_model_bundle_with_changes(
+    model_bundle: Union[ModelBundle, str],
+    app_config: Optional[Dict] = None,
+) -> ModelBundle
+
+ +
+ +

Clones an existing model bundle with changes to its app config. (More fields coming soon)

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_bundle + Union[ModelBundle, str] +

The existing bundle or its ID.

 + required +
app_config + Optional[Dict] +

The new bundle's app config, if not passed in, the new +bundle's app_config will be set to None

 + None +
+ +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ ModelBundle +

A ModelBundle object

+ +
+ +
+ +
+ + + +

+ create_model_bundle + + +

+
create_model_bundle(
+    model_bundle_name: str,
+    env_params: Dict[str, str],
+    *,
+    load_predict_fn: Optional[
+        Callable[[LaunchModel_T], Callable[[Any], Any]]
+    ] = None,
+    predict_fn_or_cls: Optional[
+        Callable[[Any], Any]
+    ] = None,
+    requirements: Optional[List[str]] = None,
+    model: Optional[LaunchModel_T] = None,
+    load_model_fn: Optional[
+        Callable[[], LaunchModel_T]
+    ] = None,
+    app_config: Optional[Union[Dict[str, Any], str]] = None,
+    globals_copy: Optional[Dict[str, Any]] = None,
+    request_schema: Optional[Type[BaseModel]] = None,
+    response_schema: Optional[Type[BaseModel]] = None
+) -> ModelBundle
+
+ +
+ +

Uploads and registers a model bundle to Scale Launch.

+

A model bundle consists of exactly one of the following:

+
    +
  • predict_fn_or_cls
    • +
  • load_predict_fn + model
    • +
  • load_predict_fn + load_model_fn
    • +
+

Pre/post-processing code can be included inside load_predict_fn/model or in +predict_fn_or_cls call.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_bundle_name + str +

The name of the model bundle you want to create. The name +must be unique across all bundles that you own.

 + required +
predict_fn_or_cls + Optional[Callable[[Any], Any]] +

Function or a Callable class that runs end-to-end +(pre/post processing and model inference) on the call. i.e. +predict_fn_or_cls(REQUEST) -> RESPONSE.

 + None +
model + Optional[LaunchModel_T] +

Typically a trained Neural Network, e.g. a Pytorch module.

+

Exactly one of model and load_model_fn must be provided.

 + None +
load_model_fn + Optional[Callable[[], LaunchModel_T]] +

A function that, when run, loads a model. This function is essentially +a deferred wrapper around the model argument.

+

Exactly one of model and load_model_fn must be provided.

 + None +
load_predict_fn + Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]] +

Function that, when called with a model, returns a function that +carries out inference.

+

If model is specified, then this is equivalent +to: + load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

+

Otherwise, if load_model_fn is specified, then this is equivalent to: +load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

+

In both cases, predict_fn is then the inference function, i.e.: + predict_fn(REQUEST) -> RESPONSE

 + None +
requirements + Optional[List[str]] +

A list of python package requirements, where each list element is of +the form <package_name>==<package_version>, e.g.

+

["tensorflow==2.3.0", "tensorflow-hub==0.11.0"]

+

If you do not pass in a value for requirements, then you must pass in +globals() for the globals_copy argument.

 + None +
app_config + Optional[Union[Dict[str, Any], str]] +

Either a Dictionary that represents a YAML file contents or a local path +to a YAML file.

 + None +
env_params + Dict[str, str] +

A dictionary that dictates environment information e.g. +the use of pytorch or tensorflow, which base image tag to use, etc. +Specifically, the dictionary should contain the following keys:

+
    +
  • +

    framework_type: either tensorflow or pytorch. - PyTorch fields: - +pytorch_image_tag: An image tag for the pytorch docker base image. The +list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - +Example:

    +

    .. code-block:: python

    +

    { + "framework_type": "pytorch", + "pytorch_image_tag": "1.10.0-cuda11.3-cudnn8-runtime" + }

    +
    • +
  • +

    Tensorflow fields:

    +
      +
    • tensorflow_version: Version of tensorflow, e.g. "2.3.0".
      • +
    +
    • +
 + required +
globals_copy + Optional[Dict[str, Any]] +

Dictionary of the global symbol table. Normally provided by +globals() built-in function.

 + None +
request_schema + Optional[Type[BaseModel]] +

A pydantic model that represents the request schema for the model +bundle. This is used to validate the request body for the model bundle's endpoint.

 + None +
response_schema + Optional[Type[BaseModel]] +

A pydantic model that represents the request schema for the model +bundle. This is used to validate the response for the model bundle's endpoint. +Note: If request_schema is specified, then response_schema must also be specified.

 + None +
+ +
+ +
+ +
+ + + +

+ create_model_bundle_from_dirs + + +

+
create_model_bundle_from_dirs(
+    *,
+    model_bundle_name: str,
+    base_paths: List[str],
+    requirements_path: str,
+    env_params: Dict[str, str],
+    load_predict_fn_module_path: str,
+    load_model_fn_module_path: str,
+    app_config: Optional[Union[Dict[str, Any], str]] = None,
+    request_schema: Optional[Type[BaseModel]] = None,
+    response_schema: Optional[Type[BaseModel]] = None
+) -> ModelBundle
+
+ +
+ +

Packages up code from one or more local filesystem folders and uploads them as a bundle +to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

+

For example, if you have a directory structure like so, and your current working +directory is also my_root:

+

.. code-block:: text

+

my_root/ + my_module1/ + init.py + ...files and directories + my_inference_file.py + my_module2/ + init.py + ...files and directories

+

then calling create_model_bundle_from_dirs with base_paths=["my_module1", +"my_module2"] essentially creates a zip file without the root directory, e.g.:

+

.. code-block:: text

+

my_module1/ + init.py + ...files and directories + my_inference_file.py + my_module2/ + init.py + ...files and directories

+

and these contents will be unzipped relative to the server side application root. Bear +these points in mind when referencing Python module paths for this bundle. For instance, +if my_inference_file.py has def f(...) as the desired inference loading function, +then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_bundle_name + str +

The name of the model bundle you want to create. The name +must be unique across all bundles that you own.

 + required +
base_paths + List[str] +

The paths on the local filesystem where the bundle code lives.

 + required +
requirements_path + str +

A path on the local filesystem where a requirements.txt file +lives.

 + required +
env_params + Dict[str, str] +

A dictionary that dictates environment information e.g. +the use of pytorch or tensorflow, which base image tag to use, etc. +Specifically, the dictionary should contain the following keys:

+
    +
  • +

    framework_type: either tensorflow or pytorch. - PyTorch fields: - +pytorch_image_tag: An image tag for the pytorch docker base image. The +list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - +Example:

    +

    .. code-block:: python

    +

    { + "framework_type": "pytorch", + "pytorch_image_tag": "1.10.0-cuda11.3-cudnn8-runtime" + }

    +
    • +
 + required +
load_predict_fn_module_path + str +

A python module path for a function that, when called +with the output of load_model_fn_module_path, returns a function that carries out +inference.

 + required +
load_model_fn_module_path + str +

A python module path for a function that returns a model. +The output feeds into the function located at load_predict_fn_module_path.

 + required +
app_config + Optional[Union[Dict[str, Any], str]] +

Either a Dictionary that represents a YAML file contents or a local path +to a YAML file.

 + None +
request_schema + Optional[Type[BaseModel]] +

A pydantic model that represents the request schema for the model +bundle. This is used to validate the request body for the model bundle's endpoint.

 + None +
response_schema + Optional[Type[BaseModel]] +

A pydantic model that represents the request schema for the model +bundle. This is used to validate the response for the model bundle's endpoint. +Note: If request_schema is specified, then response_schema must also be specified.

 + None +
+ +
+ +
+ +
+ + + +

+ create_model_endpoint + + +

+
create_model_endpoint(
+    *,
+    endpoint_name: str,
+    model_bundle: Union[ModelBundle, str],
+    cpus: int = 3,
+    memory: str = "8Gi",
+    storage: Optional[str] = None,
+    gpus: int = 0,
+    min_workers: int = 1,
+    max_workers: int = 1,
+    per_worker: int = 10,
+    gpu_type: Optional[str] = None,
+    endpoint_type: str = "sync",
+    post_inference_hooks: Optional[
+        List[PostInferenceHooks]
+    ] = None,
+    default_callback_url: Optional[str] = None,
+    update_if_exists: bool = False,
+    labels: Optional[Dict[str, str]] = None
+) -> Optional[Endpoint]
+
+ +
+ +

Creates and registers a model endpoint in Scale Launch. The returned object is an +instance of type Endpoint, which is a base class of either SyncEndpoint or +AsyncEndpoint. This is the object to which you sent inference requests.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
endpoint_name + str +

The name of the model endpoint you want to create. The name +must be unique across all endpoints that you own.

 + required +
model_bundle + Union[ModelBundle, str] +

The ModelBundle that the endpoint should serve.

 + required +
cpus + int +

Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater +than or equal to 1.

 + 3 +
memory + str +

Amount of memory each worker should get, e.g. "4Gi", "512Mi", etc. This must +be a positive amount of memory.

 + '8Gi' +
storage + Optional[str] +

Amount of local ephemeral storage each worker should get, e.g. "4Gi", +"512Mi", etc. This must be a positive amount of storage.

 + None +
gpus + int +

Number of gpus each worker should get, e.g. 0, 1, etc.

 + 0 +
min_workers + int +

The minimum number of workers. Must be greater than or equal to 0. This +should be determined by computing the minimum throughput of your workload and +dividing it by the throughput of a single worker. This field must be at least 1 +for synchronous endpoints.

 + 1 +
max_workers + int +

The maximum number of workers. Must be greater than or equal to 0, +and as well as greater than or equal to min_workers. This should be determined by +computing the maximum throughput of your workload and dividing it by the throughput +of a single worker.

 + 1 +
per_worker + int +

The maximum number of concurrent requests that an individual worker can +service. Launch automatically scales the number of workers for the endpoint so that +each worker is processing per_worker requests, subject to the limits defined by +min_workers and max_workers.

+
    +
  • If the average number of concurrent requests per worker is lower than +per_worker, then the number of workers will be reduced. - Otherwise, +if the average number of concurrent requests per worker is higher than +per_worker, then the number of workers will be increased to meet the elevated +traffic.
    • +
+

Here is our recommendation for computing per_worker:

+
    +
  1. Compute min_workers and max_workers per your minimum and maximum +throughput requirements. 2. Determine a value for the maximum number of +concurrent requests in the workload. Divide this number by max_workers. Doing +this ensures that the number of workers will "climb" to max_workers.
    2. +
 + 10 +
gpu_type + Optional[str] +

If specifying a non-zero number of gpus, this controls the type of gpu +requested. Here are the supported values:

+
    +
  • nvidia-tesla-t4
    • +
  • nvidia-ampere-a10
    • +
 + None +
endpoint_type + str +

Either "sync" or "async".

 + 'sync' +
post_inference_hooks + Optional[List[PostInferenceHooks]] +

List of hooks to trigger after inference tasks are served.

 + None +
default_callback_url + Optional[str] +

The default callback url to use for async endpoints. +This can be overridden in the task parameters for each individual task. +post_inference_hooks must contain "callback" for the callback to be triggered.

 + None +
update_if_exists + bool +

If True, will attempt to update the endpoint if it exists. +Otherwise, will unconditionally try to create a new endpoint. Note that endpoint +names for a given user must be unique, so attempting to call this function with +update_if_exists=False for an existing endpoint will raise an error.

 + False +
labels + Optional[Dict[str, str]] +

An optional dictionary of key/value pairs to associate with this endpoint.

 + None +
+ +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ Optional[Endpoint] +

A Endpoint object that can be used to make requests to the endpoint.

+ +
+ +
+ +
+ + + +

+ delete_model_endpoint + + +

+
delete_model_endpoint(
+    model_endpoint: Union[ModelEndpoint, str]
+)
+
+ +
+ +

Deletes a model endpoint.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_endpoint + Union[ModelEndpoint, str] +

A ModelEndpoint object.

 + required +
+ +
+ +
+ +
+ + + +

+ edit_model_endpoint + + +

+
edit_model_endpoint(
+    *,
+    model_endpoint: Union[ModelEndpoint, str],
+    model_bundle: Optional[Union[ModelBundle, str]] = None,
+    cpus: Optional[float] = None,
+    memory: Optional[str] = None,
+    storage: Optional[str] = None,
+    gpus: Optional[int] = None,
+    min_workers: Optional[int] = None,
+    max_workers: Optional[int] = None,
+    per_worker: Optional[int] = None,
+    gpu_type: Optional[str] = None,
+    post_inference_hooks: Optional[
+        List[PostInferenceHooks]
+    ] = None,
+    default_callback_url: Optional[str] = None
+) -> None
+
+ +
+ +

Edits an existing model endpoint. Here are the fields that cannot be edited on an +existing endpoint:

+
    +
  • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint +to an AsyncEndpoint or vice versa.
    • +
+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_endpoint + Union[ModelEndpoint, str] +

The model endpoint (or its name) you want to edit. The name +must be unique across all endpoints that you own.

 + required +
model_bundle + Optional[Union[ModelBundle, str]] +

The ModelBundle that the endpoint should serve.

 + None +
cpus + Optional[float] +

Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater +than or equal to 1.

 + None +
memory + Optional[str] +

Amount of memory each worker should get, e.g. "4Gi", "512Mi", etc. This must +be a positive amount of memory.

 + None +
storage + Optional[str] +

Amount of local ephemeral storage each worker should get, e.g. "4Gi", +"512Mi", etc. This must be a positive amount of storage.

 + None +
gpus + Optional[int] +

Number of gpus each worker should get, e.g. 0, 1, etc.

 + None +
min_workers + Optional[int] +

The minimum number of workers. Must be greater than or equal to 0.

 + None +
max_workers + Optional[int] +

The maximum number of workers. Must be greater than or equal to 0, +and as well as greater than or equal to min_workers.

 + None +
per_worker + Optional[int] +

The maximum number of concurrent requests that an individual worker can +service. Launch automatically scales the number of workers for the endpoint so that +each worker is processing per_worker requests:

+
    +
  • If the average number of concurrent requests per worker is lower than +per_worker, then the number of workers will be reduced. - Otherwise, +if the average number of concurrent requests per worker is higher than +per_worker, then the number of workers will be increased to meet the elevated +traffic.
    • +
 + None +
gpu_type + Optional[str] +

If specifying a non-zero number of gpus, this controls the type of gpu +requested. Here are the supported values:

+
    +
  • nvidia-tesla-t4
    • +
  • nvidia-ampere-a10
    • +
 + None +
post_inference_hooks + Optional[List[PostInferenceHooks]] +

List of hooks to trigger after inference tasks are served.

 + None +
default_callback_url + Optional[str] +

The default callback url to use for async endpoints. +This can be overridden in the task parameters for each individual task. +post_inference_hooks must contain "callback" for the callback to be triggered.

 + None +
+ +
+ +
+ +
+ + + +

+ get_batch_async_response + + +

+
get_batch_async_response(
+    batch_job_id: str,
+) -> Dict[str, Any]
+
+ +
+ +

Gets inference results from a previously created batch job.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
batch_job_id + str +

An id representing the batch task job. This id is the in the response from +calling batch_async_request.

 + required +
+ +

Returns:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeDescription
+ Dict[str, Any] +

A dictionary that contains the following fields:

+ Dict[str, Any] +
    +
  • status: The status of the job.
    • +
+ Dict[str, Any] +
    +
  • result: The url where the result is stored.
    • +
+ Dict[str, Any] +
    +
  • duration: A string representation of how long the job took to finish + or how long it has been running, for a job current in progress.
    • +
+ Dict[str, Any] +
    +
  • num_tasks_pending: The number of tasks that are still pending.
    • +
+ Dict[str, Any] +
    +
  • num_tasks_completed: The number of tasks that have completed.
    • +
+ +
+ +
+ +
+ + + +

+ get_model_bundle + + +

+
get_model_bundle(
+    model_bundle: Union[ModelBundle, str]
+) -> ModelBundle
+
+ +
+ +

Returns a model bundle specified by bundle_name that the user owns.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_bundle + Union[ModelBundle, str] +

The bundle or its name.

 + required +
+ +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ ModelBundle +

A ModelBundle object

+ +
+ +
+ +
+ + + +

+ get_model_endpoint + + +

+
get_model_endpoint(
+    endpoint_name: str,
+) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]
+
+ +
+ +

Gets a model endpoint associated with a name.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
endpoint_name + str +

The name of the endpoint to retrieve.

 + required +
+ +
+ +
+ +
+ + + +

+ list_model_bundles + + +

+
list_model_bundles() -> List[ModelBundle]
+
+ +
+ +

Returns a list of model bundles that the user owns.

+ +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ List[ModelBundle] +

A list of ModelBundle objects

+ +
+ +
+ +
+ + + +

+ list_model_endpoints + + +

+
list_model_endpoints() -> List[Endpoint]
+
+ +
+ +

Lists all model endpoints that the user owns.

+ +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ List[Endpoint] +

A list of ModelEndpoint objects.

+ +
+ +
+ +
+ + + +

+ read_endpoint_creation_logs + + +

+
read_endpoint_creation_logs(
+    model_endpoint: Union[ModelEndpoint, str]
+)
+
+ +
+ +

Retrieves the logs for the creation of the endpoint.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_endpoint + Union[ModelEndpoint, str] +

The endpoint or its name.

 + required +
+ +
+ +
+ +
+ + + +

+ register_batch_csv_location_fn + + +

+
register_batch_csv_location_fn(
+    batch_csv_location_fn: Callable[[], str]
+)
+
+ +
+ +

For self-hosted mode only. Registers a function that gives a location for batch CSV +inputs. Should give different locations each time. This function is called as +batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can +take.

+

Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is +that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes +in an object of type T as its second argument (i.e. batch_csv_url).

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
batch_csv_location_fn + Callable[[], str] +

Function that generates batch_csv_urls for upload_batch_csv_fn.

 + required +
+ +
+ +
+ +
+ + + +

+ register_bundle_location_fn + + +

+
register_bundle_location_fn(
+    bundle_location_fn: Callable[[], str]
+)
+
+ +
+ +

For self-hosted mode only. Registers a function that gives a location for a model bundle. +Should give different locations each time. This function is called as +bundle_location_fn(), and should return a bundle_url that +register_upload_bundle_fn can take.

+

Strictly, bundle_location_fn() does not need to return a str. The only +requirement is that if bundle_location_fn returns a value of type T, +then upload_bundle_fn() takes in an object of type T as its second argument (i.e. +bundle_url).

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
bundle_location_fn + Callable[[], str] +

Function that generates bundle_urls for upload_bundle_fn.

 + required +
+ +
+ +
+ +
+ + + +

+ register_upload_batch_csv_fn + + +

+
register_upload_batch_csv_fn(
+    upload_batch_csv_fn: Callable[[str, str], None]
+)
+
+ +
+ +

For self-hosted mode only. Registers a function that handles batch text upload. This +function is called as

+
upload_batch_csv_fn(csv_text, csv_url)
+
+

This function should directly write the contents of csv_text as a text string into +csv_url.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
upload_batch_csv_fn + Callable[[str, str], None] +

Function that takes in a csv text (string type), +and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

 + required +
+ +
+ +
+ +
+ + + +

+ register_upload_bundle_fn + + +

+
register_upload_bundle_fn(
+    upload_bundle_fn: Callable[[str, str], None]
+)
+
+ +
+ +

For self-hosted mode only. Registers a function that handles model bundle upload. This +function is called as

+
upload_bundle_fn(serialized_bundle, bundle_url)
+
+

This function should directly write the contents of serialized_bundle as a +binary string into bundle_url.

+

See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
upload_bundle_fn + Callable[[str, str], None] +

Function that takes in a serialized bundle (bytes type), +and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

 + required +
+ +
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/api/model_bundles/index.html b/api/model_bundles/index.html new file mode 100644 index 00000000..ae70d58b --- /dev/null +++ b/api/model_bundles/index.html @@ -0,0 +1,924 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Model Bundles - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Model Bundles

+ + +
+ + + +

+ ModelBundle + + + + dataclass + + +

+ + +
+ + +

Represents a ModelBundle.

+ + + + + +
+ + + + + + + +
+ + + +

+ app_config + + + + class-attribute + + +

+
app_config: Optional[Dict[Any, Any]] = None
+
+ +
+ +

An optional user-specified configuration mapping for the bundle.

+
+ +
+ +
+ + + +

+ env_params + + + + class-attribute + + +

+
env_params: Optional[Dict[str, str]] = None
+
+ +
+ +

A dictionary that dictates environment information. See LaunchClient.create_model_bundle +for more information.

+
+ +
+ +
+ + + +

+ id + + + + class-attribute + + +

+
id: Optional[str] = None
+
+ +
+ +

A globally unique identifier for the bundle.

+
+ +
+ +
+ + + +

+ location + + + + class-attribute + + +

+
location: Optional[str] = None
+
+ +
+ +

An opaque location for the bundle.

+
+ +
+ +
+ + + +

+ metadata + + + + class-attribute + + +

+
metadata: Optional[Dict[Any, Any]] = None
+
+ +
+ +

Arbitrary metadata for the bundle.

+
+ +
+ +
+ + + +

+ name + + + + class-attribute + + +

+
name: str
+
+ +
+ +

The name of the bundle. Must be unique across all bundles that the user owns.

+
+ +
+ +
+ + + +

+ packaging_type + + + + class-attribute + + +

+
packaging_type: Optional[str] = None
+
+ +
+ +

The packaging type for the bundle. Can be cloudpickle or zip.

+
+ +
+ +
+ + + +

+ requirements + + + + class-attribute + + +

+
requirements: Optional[List[str]] = None
+
+ +
+ +

A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle +for more information.

+
+ +
+ + + + + +
+ +
+ +
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/api/model_endpoints/index.html b/api/model_endpoints/index.html new file mode 100644 index 00000000..412dd6e2 --- /dev/null +++ b/api/model_endpoints/index.html @@ -0,0 +1,830 @@ + + + + + + + + + + + + + + + + + + + + + + + + Model Endpoints - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Model Endpoints

+

All classes here are returned by the +get_model_endpoint +method and provide a predict function.

+ + +
+ + + +

+ AsyncEndpoint + + +

+
AsyncEndpoint(model_endpoint: ModelEndpoint, client)
+
+ +
+

+ Bases: Endpoint

+ + +

An asynchronous model endpoint.

+ + + +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
model_endpoint + ModelEndpoint +

ModelEndpoint object.

 + required +
client +

A LaunchClient object

 + required +
+ + + + +
+ + + + + + + + + +
+ + + +

+ predict + + +

+
predict(request: EndpointRequest) -> EndpointResponseFuture
+
+ +
+ +

Runs an asynchronous prediction request.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
request + EndpointRequest +

The EndpointRequest object that contains the payload.

 + required +
+ +

Returns:

+ + + + + + + + + + + + + + + + + + + + + +
Name TypeDescription
+ EndpointResponseFuture +

An EndpointResponseFuture such the user can use to query the status of the request.

Example + EndpointResponseFuture +
+ EndpointResponseFuture +

.. code-block:: python

+

my_endpoint = AsyncEndpoint(...) +f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) +result = f.get() # blocks on completion

+ +
+ +
+ +
+ + + +

+ predict_batch + + +

+
predict_batch(
+    requests: Sequence[EndpointRequest],
+) -> AsyncEndpointBatchResponse
+
+ +
+ +

(deprecated) +Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
requests + Sequence[EndpointRequest] +

List of EndpointRequests. Request_ids must all be distinct.

 + required +
+ +

Returns:

+ + + + + + + + + + + + + +
TypeDescription
+ AsyncEndpointBatchResponse +

an AsyncEndpointResponse keeping track of the inference requests made

+ +
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/assets/_mkdocstrings.css b/assets/_mkdocstrings.css new file mode 100644 index 00000000..a65078d0 --- /dev/null +++ b/assets/_mkdocstrings.css @@ -0,0 +1,36 @@ + +/* Don't capitalize names. */ +h5.doc-heading { + text-transform: none !important; +} + +/* Avoid breaking parameters name, etc. in table cells. */ +.doc-contents td code { + word-break: normal !important; +} + +/* For pieces of Markdown rendered in table cells. */ +.doc-contents td p { + margin-top: 0 !important; + margin-bottom: 0 !important; +} + +/* Max width for docstring sections tables. */ +.doc .md-typeset__table, +.doc .md-typeset__table table { + display: table !important; + width: 100%; +} +.doc .md-typeset__table tr { + display: table-row; +} + +/* Avoid line breaks in rendered fields. */ +.field-body p { + display: inline; +} + +/* Defaults in Spacy table style. */ +.doc-param-default { + float: right; +} diff --git a/assets/images/favicon.png b/assets/images/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..1cf13b9f9d978896599290a74f77d5dbe7d1655c GIT binary patch literal 1870 zcmV-U2eJ5xP)Gc)JR9QMau)O=X#!i9;T z37kk-upj^(fsR36MHs_+1RCI)NNu9}lD0S{B^g8PN?Ww(5|~L#Ng*g{WsqleV}|#l zz8@ri&cTzw_h33bHI+12+kK6WN$h#n5cD8OQt`5kw6p~9H3()bUQ8OS4Q4HTQ=1Ol z_JAocz`fLbT2^{`8n~UAo=#AUOf=SOq4pYkt;XbC&f#7lb$*7=$na!mWCQ`dBQsO0 zLFBSPj*N?#u5&pf2t4XjEGH|=pPQ8xh7tpx;US5Cx_Ju;!O`ya-yF`)b%TEt5>eP1ZX~}sjjA%FJF?h7cX8=b!DZl<6%Cv z*G0uvvU+vmnpLZ2paivG-(cd*y3$hCIcsZcYOGh{$&)A6*XX&kXZd3G8m)G$Zz-LV z^GF3VAW^Mdv!)4OM8EgqRiz~*Cji;uzl2uC9^=8I84vNp;ltJ|q-*uQwGp2ma6cY7 z;`%`!9UXO@fr&Ebapfs34OmS9^u6$)bJxrucutf>`dKPKT%%*d3XlFVKunp9 zasduxjrjs>f8V=D|J=XNZp;_Zy^WgQ$9WDjgY=z@stwiEBm9u5*|34&1Na8BMjjgf3+SHcr`5~>oz1Y?SW^=K z^bTyO6>Gar#P_W2gEMwq)ot3; zREHn~U&Dp0l6YT0&k-wLwYjb?5zGK`W6S2v+K>AM(95m2C20L|3m~rN8dprPr@t)5lsk9Hu*W z?pS990s;Ez=+Rj{x7p``4>+c0G5^pYnB1^!TL=(?HLHZ+HicG{~4F1d^5Awl_2!1jICM-!9eoLhbbT^;yHcefyTAaqRcY zmuctDopPT!%k+}x%lZRKnzykr2}}XfG_ne?nRQO~?%hkzo;@RN{P6o`&mMUWBYMTe z6i8ChtjX&gXl`nvrU>jah)2iNM%JdjqoaeaU%yVn!^70x-flljp6Q5tK}5}&X8&&G zX3fpb3E(!rH=zVI_9Gjl45w@{(ITqngWFe7@9{mX;tO25Z_8 zQHEpI+FkTU#4xu>RkN>b3Tnc3UpWzPXWm#o55GKF09j^Mh~)K7{QqbO_~(@CVq! zS<8954|P8mXN2MRs86xZ&Q4EfM@JB94b=(YGuk)s&^jiSF=t3*oNK3`rD{H`yQ?d; ztE=laAUoZx5?RC8*WKOj`%LXEkgDd>&^Q4M^z`%u0rg-It=hLCVsq!Z%^6eB-OvOT zFZ28TN&cRmgU}Elrnk43)!>Z1FCPL2K$7}gwzIc48NX}#!A1BpJP?#v5wkNprhV** z?Cpalt1oH&{r!o3eSKc&ap)iz2BTn_VV`4>9M^b3;(YY}4>#ML6{~(4mH+?%07*qo IM6N<$f(jP3KmY&$ literal 0 HcmV?d00001 diff --git a/assets/javascripts/bundle.2a6f1dda.min.js b/assets/javascripts/bundle.2a6f1dda.min.js new file mode 100644 index 00000000..2f912a0b --- /dev/null +++ b/assets/javascripts/bundle.2a6f1dda.min.js @@ -0,0 +1,29 @@ +"use strict";(()=>{var Hi=Object.create;var xr=Object.defineProperty;var Pi=Object.getOwnPropertyDescriptor;var $i=Object.getOwnPropertyNames,Ht=Object.getOwnPropertySymbols,Ii=Object.getPrototypeOf,Er=Object.prototype.hasOwnProperty,an=Object.prototype.propertyIsEnumerable;var on=(e,t,r)=>t in e?xr(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r,P=(e,t)=>{for(var r in t||(t={}))Er.call(t,r)&&on(e,r,t[r]);if(Ht)for(var r of Ht(t))an.call(t,r)&&on(e,r,t[r]);return e};var sn=(e,t)=>{var r={};for(var n in e)Er.call(e,n)&&t.indexOf(n)<0&&(r[n]=e[n]);if(e!=null&&Ht)for(var n of Ht(e))t.indexOf(n)<0&&an.call(e,n)&&(r[n]=e[n]);return r};var Pt=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports);var Fi=(e,t,r,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of $i(t))!Er.call(e,o)&&o!==r&&xr(e,o,{get:()=>t[o],enumerable:!(n=Pi(t,o))||n.enumerable});return e};var yt=(e,t,r)=>(r=e!=null?Hi(Ii(e)):{},Fi(t||!e||!e.__esModule?xr(r,"default",{value:e,enumerable:!0}):r,e));var fn=Pt((wr,cn)=>{(function(e,t){typeof wr=="object"&&typeof cn!="undefined"?t():typeof define=="function"&&define.amd?define(t):t()})(wr,function(){"use strict";function e(r){var n=!0,o=!1,i=null,s={text:!0,search:!0,url:!0,tel:!0,email:!0,password:!0,number:!0,date:!0,month:!0,week:!0,time:!0,datetime:!0,"datetime-local":!0};function a(O){return!!(O&&O!==document&&O.nodeName!=="HTML"&&O.nodeName!=="BODY"&&"classList"in O&&"contains"in O.classList)}function f(O){var Ke=O.type,De=O.tagName;return!!(De==="INPUT"&&s[Ke]&&!O.readOnly||De==="TEXTAREA"&&!O.readOnly||O.isContentEditable)}function c(O){O.classList.contains("focus-visible")||(O.classList.add("focus-visible"),O.setAttribute("data-focus-visible-added",""))}function u(O){O.hasAttribute("data-focus-visible-added")&&(O.classList.remove("focus-visible"),O.removeAttribute("data-focus-visible-added"))}function p(O){O.metaKey||O.altKey||O.ctrlKey||(a(r.activeElement)&&c(r.activeElement),n=!0)}function m(O){n=!1}function d(O){a(O.target)&&(n||f(O.target))&&c(O.target)}function h(O){a(O.target)&&(O.target.classList.contains("focus-visible")||O.target.hasAttribute("data-focus-visible-added"))&&(o=!0,window.clearTimeout(i),i=window.setTimeout(function(){o=!1},100),u(O.target))}function v(O){document.visibilityState==="hidden"&&(o&&(n=!0),B())}function B(){document.addEventListener("mousemove",z),document.addEventListener("mousedown",z),document.addEventListener("mouseup",z),document.addEventListener("pointermove",z),document.addEventListener("pointerdown",z),document.addEventListener("pointerup",z),document.addEventListener("touchmove",z),document.addEventListener("touchstart",z),document.addEventListener("touchend",z)}function ne(){document.removeEventListener("mousemove",z),document.removeEventListener("mousedown",z),document.removeEventListener("mouseup",z),document.removeEventListener("pointermove",z),document.removeEventListener("pointerdown",z),document.removeEventListener("pointerup",z),document.removeEventListener("touchmove",z),document.removeEventListener("touchstart",z),document.removeEventListener("touchend",z)}function z(O){O.target.nodeName&&O.target.nodeName.toLowerCase()==="html"||(n=!1,ne())}document.addEventListener("keydown",p,!0),document.addEventListener("mousedown",m,!0),document.addEventListener("pointerdown",m,!0),document.addEventListener("touchstart",m,!0),document.addEventListener("visibilitychange",v,!0),B(),r.addEventListener("focus",d,!0),r.addEventListener("blur",h,!0),r.nodeType===Node.DOCUMENT_FRAGMENT_NODE&&r.host?r.host.setAttribute("data-js-focus-visible",""):r.nodeType===Node.DOCUMENT_NODE&&(document.documentElement.classList.add("js-focus-visible"),document.documentElement.setAttribute("data-js-focus-visible",""))}if(typeof window!="undefined"&&typeof document!="undefined"){window.applyFocusVisiblePolyfill=e;var t;try{t=new CustomEvent("focus-visible-polyfill-ready")}catch(r){t=document.createEvent("CustomEvent"),t.initCustomEvent("focus-visible-polyfill-ready",!1,!1,{})}window.dispatchEvent(t)}typeof document!="undefined"&&e(document)})});var un=Pt(Sr=>{(function(e){var t=function(){try{return!!Symbol.iterator}catch(c){return!1}},r=t(),n=function(c){var u={next:function(){var p=c.shift();return{done:p===void 0,value:p}}};return r&&(u[Symbol.iterator]=function(){return u}),u},o=function(c){return encodeURIComponent(c).replace(/%20/g,"+")},i=function(c){return decodeURIComponent(String(c).replace(/\+/g," "))},s=function(){var c=function(p){Object.defineProperty(this,"_entries",{writable:!0,value:{}});var m=typeof p;if(m!=="undefined")if(m==="string")p!==""&&this._fromString(p);else if(p instanceof c){var d=this;p.forEach(function(ne,z){d.append(z,ne)})}else if(p!==null&&m==="object")if(Object.prototype.toString.call(p)==="[object Array]")for(var h=0;hd[0]?1:0}),c._entries&&(c._entries={});for(var p=0;p1?i(d[1]):"")}})})(typeof global!="undefined"?global:typeof window!="undefined"?window:typeof self!="undefined"?self:Sr);(function(e){var t=function(){try{var o=new e.URL("b","http://a");return o.pathname="c d",o.href==="http://a/c%20d"&&o.searchParams}catch(i){return!1}},r=function(){var o=e.URL,i=function(f,c){typeof f!="string"&&(f=String(f)),c&&typeof c!="string"&&(c=String(c));var u=document,p;if(c&&(e.location===void 0||c!==e.location.href)){c=c.toLowerCase(),u=document.implementation.createHTMLDocument(""),p=u.createElement("base"),p.href=c,u.head.appendChild(p);try{if(p.href.indexOf(c)!==0)throw new Error(p.href)}catch(O){throw new Error("URL unable to set base "+c+" due to "+O)}}var m=u.createElement("a");m.href=f,p&&(u.body.appendChild(m),m.href=m.href);var d=u.createElement("input");if(d.type="url",d.value=f,m.protocol===":"||!/:/.test(m.href)||!d.checkValidity()&&!c)throw new TypeError("Invalid URL");Object.defineProperty(this,"_anchorElement",{value:m});var h=new e.URLSearchParams(this.search),v=!0,B=!0,ne=this;["append","delete","set"].forEach(function(O){var Ke=h[O];h[O]=function(){Ke.apply(h,arguments),v&&(B=!1,ne.search=h.toString(),B=!0)}}),Object.defineProperty(this,"searchParams",{value:h,enumerable:!0});var z=void 0;Object.defineProperty(this,"_updateSearchParams",{enumerable:!1,configurable:!1,writable:!1,value:function(){this.search!==z&&(z=this.search,B&&(v=!1,this.searchParams._fromString(this.search),v=!0))}})},s=i.prototype,a=function(f){Object.defineProperty(s,f,{get:function(){return this._anchorElement[f]},set:function(c){this._anchorElement[f]=c},enumerable:!0})};["hash","host","hostname","port","protocol"].forEach(function(f){a(f)}),Object.defineProperty(s,"search",{get:function(){return this._anchorElement.search},set:function(f){this._anchorElement.search=f,this._updateSearchParams()},enumerable:!0}),Object.defineProperties(s,{toString:{get:function(){var f=this;return function(){return f.href}}},href:{get:function(){return this._anchorElement.href.replace(/\?$/,"")},set:function(f){this._anchorElement.href=f,this._updateSearchParams()},enumerable:!0},pathname:{get:function(){return this._anchorElement.pathname.replace(/(^\/?)/,"/")},set:function(f){this._anchorElement.pathname=f},enumerable:!0},origin:{get:function(){var f={"http:":80,"https:":443,"ftp:":21}[this._anchorElement.protocol],c=this._anchorElement.port!=f&&this._anchorElement.port!=="";return this._anchorElement.protocol+"//"+this._anchorElement.hostname+(c?":"+this._anchorElement.port:"")},enumerable:!0},password:{get:function(){return""},set:function(f){},enumerable:!0},username:{get:function(){return""},set:function(f){},enumerable:!0}}),i.createObjectURL=function(f){return o.createObjectURL.apply(o,arguments)},i.revokeObjectURL=function(f){return o.revokeObjectURL.apply(o,arguments)},e.URL=i};if(t()||r(),e.location!==void 0&&!("origin"in e.location)){var n=function(){return e.location.protocol+"//"+e.location.hostname+(e.location.port?":"+e.location.port:"")};try{Object.defineProperty(e.location,"origin",{get:n,enumerable:!0})}catch(o){setInterval(function(){e.location.origin=n()},100)}}})(typeof global!="undefined"?global:typeof window!="undefined"?window:typeof self!="undefined"?self:Sr)});var Qr=Pt((Lt,Kr)=>{/*! + * clipboard.js v2.0.11 + * https://clipboardjs.com/ + * + * Licensed MIT © Zeno Rocha + */(function(t,r){typeof Lt=="object"&&typeof Kr=="object"?Kr.exports=r():typeof define=="function"&&define.amd?define([],r):typeof Lt=="object"?Lt.ClipboardJS=r():t.ClipboardJS=r()})(Lt,function(){return function(){var e={686:function(n,o,i){"use strict";i.d(o,{default:function(){return ki}});var s=i(279),a=i.n(s),f=i(370),c=i.n(f),u=i(817),p=i.n(u);function m(j){try{return document.execCommand(j)}catch(T){return!1}}var d=function(T){var w=p()(T);return m("cut"),w},h=d;function v(j){var T=document.documentElement.getAttribute("dir")==="rtl",w=document.createElement("textarea");w.style.fontSize="12pt",w.style.border="0",w.style.padding="0",w.style.margin="0",w.style.position="absolute",w.style[T?"right":"left"]="-9999px";var k=window.pageYOffset||document.documentElement.scrollTop;return w.style.top="".concat(k,"px"),w.setAttribute("readonly",""),w.value=j,w}var B=function(T,w){var k=v(T);w.container.appendChild(k);var F=p()(k);return m("copy"),k.remove(),F},ne=function(T){var w=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{container:document.body},k="";return typeof T=="string"?k=B(T,w):T instanceof HTMLInputElement&&!["text","search","url","tel","password"].includes(T==null?void 0:T.type)?k=B(T.value,w):(k=p()(T),m("copy")),k},z=ne;function O(j){return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?O=function(w){return typeof w}:O=function(w){return w&&typeof Symbol=="function"&&w.constructor===Symbol&&w!==Symbol.prototype?"symbol":typeof w},O(j)}var Ke=function(){var T=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{},w=T.action,k=w===void 0?"copy":w,F=T.container,q=T.target,Le=T.text;if(k!=="copy"&&k!=="cut")throw new Error('Invalid "action" value, use either "copy" or "cut"');if(q!==void 0)if(q&&O(q)==="object"&&q.nodeType===1){if(k==="copy"&&q.hasAttribute("disabled"))throw new Error('Invalid "target" attribute. Please use "readonly" instead of "disabled" attribute');if(k==="cut"&&(q.hasAttribute("readonly")||q.hasAttribute("disabled")))throw new Error(`Invalid "target" attribute. You can't cut text from elements with "readonly" or "disabled" attributes`)}else throw new Error('Invalid "target" value, use a valid Element');if(Le)return z(Le,{container:F});if(q)return k==="cut"?h(q):z(q,{container:F})},De=Ke;function Fe(j){return typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?Fe=function(w){return typeof w}:Fe=function(w){return w&&typeof Symbol=="function"&&w.constructor===Symbol&&w!==Symbol.prototype?"symbol":typeof w},Fe(j)}function Oi(j,T){if(!(j instanceof T))throw new TypeError("Cannot call a class as a function")}function nn(j,T){for(var w=0;w0&&arguments[0]!==void 0?arguments[0]:{};this.action=typeof F.action=="function"?F.action:this.defaultAction,this.target=typeof F.target=="function"?F.target:this.defaultTarget,this.text=typeof F.text=="function"?F.text:this.defaultText,this.container=Fe(F.container)==="object"?F.container:document.body}},{key:"listenClick",value:function(F){var q=this;this.listener=c()(F,"click",function(Le){return q.onClick(Le)})}},{key:"onClick",value:function(F){var q=F.delegateTarget||F.currentTarget,Le=this.action(q)||"copy",kt=De({action:Le,container:this.container,target:this.target(q),text:this.text(q)});this.emit(kt?"success":"error",{action:Le,text:kt,trigger:q,clearSelection:function(){q&&q.focus(),window.getSelection().removeAllRanges()}})}},{key:"defaultAction",value:function(F){return yr("action",F)}},{key:"defaultTarget",value:function(F){var q=yr("target",F);if(q)return document.querySelector(q)}},{key:"defaultText",value:function(F){return yr("text",F)}},{key:"destroy",value:function(){this.listener.destroy()}}],[{key:"copy",value:function(F){var q=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{container:document.body};return z(F,q)}},{key:"cut",value:function(F){return h(F)}},{key:"isSupported",value:function(){var F=arguments.length>0&&arguments[0]!==void 0?arguments[0]:["copy","cut"],q=typeof F=="string"?[F]:F,Le=!!document.queryCommandSupported;return q.forEach(function(kt){Le=Le&&!!document.queryCommandSupported(kt)}),Le}}]),w}(a()),ki=Ri},828:function(n){var o=9;if(typeof Element!="undefined"&&!Element.prototype.matches){var i=Element.prototype;i.matches=i.matchesSelector||i.mozMatchesSelector||i.msMatchesSelector||i.oMatchesSelector||i.webkitMatchesSelector}function s(a,f){for(;a&&a.nodeType!==o;){if(typeof a.matches=="function"&&a.matches(f))return a;a=a.parentNode}}n.exports=s},438:function(n,o,i){var s=i(828);function a(u,p,m,d,h){var v=c.apply(this,arguments);return u.addEventListener(m,v,h),{destroy:function(){u.removeEventListener(m,v,h)}}}function f(u,p,m,d,h){return typeof u.addEventListener=="function"?a.apply(null,arguments):typeof m=="function"?a.bind(null,document).apply(null,arguments):(typeof u=="string"&&(u=document.querySelectorAll(u)),Array.prototype.map.call(u,function(v){return a(v,p,m,d,h)}))}function c(u,p,m,d){return function(h){h.delegateTarget=s(h.target,p),h.delegateTarget&&d.call(u,h)}}n.exports=f},879:function(n,o){o.node=function(i){return i!==void 0&&i instanceof HTMLElement&&i.nodeType===1},o.nodeList=function(i){var s=Object.prototype.toString.call(i);return i!==void 0&&(s==="[object NodeList]"||s==="[object HTMLCollection]")&&"length"in i&&(i.length===0||o.node(i[0]))},o.string=function(i){return typeof i=="string"||i instanceof String},o.fn=function(i){var s=Object.prototype.toString.call(i);return s==="[object Function]"}},370:function(n,o,i){var s=i(879),a=i(438);function f(m,d,h){if(!m&&!d&&!h)throw new Error("Missing required arguments");if(!s.string(d))throw new TypeError("Second argument must be a String");if(!s.fn(h))throw new TypeError("Third argument must be a Function");if(s.node(m))return c(m,d,h);if(s.nodeList(m))return u(m,d,h);if(s.string(m))return p(m,d,h);throw new TypeError("First argument must be a String, HTMLElement, HTMLCollection, or NodeList")}function c(m,d,h){return m.addEventListener(d,h),{destroy:function(){m.removeEventListener(d,h)}}}function u(m,d,h){return Array.prototype.forEach.call(m,function(v){v.addEventListener(d,h)}),{destroy:function(){Array.prototype.forEach.call(m,function(v){v.removeEventListener(d,h)})}}}function p(m,d,h){return a(document.body,m,d,h)}n.exports=f},817:function(n){function o(i){var s;if(i.nodeName==="SELECT")i.focus(),s=i.value;else if(i.nodeName==="INPUT"||i.nodeName==="TEXTAREA"){var a=i.hasAttribute("readonly");a||i.setAttribute("readonly",""),i.select(),i.setSelectionRange(0,i.value.length),a||i.removeAttribute("readonly"),s=i.value}else{i.hasAttribute("contenteditable")&&i.focus();var f=window.getSelection(),c=document.createRange();c.selectNodeContents(i),f.removeAllRanges(),f.addRange(c),s=f.toString()}return s}n.exports=o},279:function(n){function o(){}o.prototype={on:function(i,s,a){var f=this.e||(this.e={});return(f[i]||(f[i]=[])).push({fn:s,ctx:a}),this},once:function(i,s,a){var f=this;function c(){f.off(i,c),s.apply(a,arguments)}return c._=s,this.on(i,c,a)},emit:function(i){var s=[].slice.call(arguments,1),a=((this.e||(this.e={}))[i]||[]).slice(),f=0,c=a.length;for(f;f{"use strict";/*! + * escape-html + * Copyright(c) 2012-2013 TJ Holowaychuk + * Copyright(c) 2015 Andreas Lubbe + * Copyright(c) 2015 Tiancheng "Timothy" Gu + * MIT Licensed + */var is=/["'&<>]/;Jo.exports=as;function as(e){var t=""+e,r=is.exec(t);if(!r)return t;var n,o="",i=0,s=0;for(i=r.index;i0&&i[i.length-1])&&(c[0]===6||c[0]===2)){r=0;continue}if(c[0]===3&&(!i||c[1]>i[0]&&c[1]=e.length&&(e=void 0),{value:e&&e[n++],done:!e}}};throw new TypeError(t?"Object is not iterable.":"Symbol.iterator is not defined.")}function W(e,t){var r=typeof Symbol=="function"&&e[Symbol.iterator];if(!r)return e;var n=r.call(e),o,i=[],s;try{for(;(t===void 0||t-- >0)&&!(o=n.next()).done;)i.push(o.value)}catch(a){s={error:a}}finally{try{o&&!o.done&&(r=n.return)&&r.call(n)}finally{if(s)throw s.error}}return i}function D(e,t,r){if(r||arguments.length===2)for(var n=0,o=t.length,i;n1||a(m,d)})})}function a(m,d){try{f(n[m](d))}catch(h){p(i[0][3],h)}}function f(m){m.value instanceof Ze?Promise.resolve(m.value.v).then(c,u):p(i[0][2],m)}function c(m){a("next",m)}function u(m){a("throw",m)}function p(m,d){m(d),i.shift(),i.length&&a(i[0][0],i[0][1])}}function mn(e){if(!Symbol.asyncIterator)throw new TypeError("Symbol.asyncIterator is not defined.");var t=e[Symbol.asyncIterator],r;return t?t.call(e):(e=typeof xe=="function"?xe(e):e[Symbol.iterator](),r={},n("next"),n("throw"),n("return"),r[Symbol.asyncIterator]=function(){return this},r);function n(i){r[i]=e[i]&&function(s){return new Promise(function(a,f){s=e[i](s),o(a,f,s.done,s.value)})}}function o(i,s,a,f){Promise.resolve(f).then(function(c){i({value:c,done:a})},s)}}function A(e){return typeof e=="function"}function at(e){var t=function(n){Error.call(n),n.stack=new Error().stack},r=e(t);return r.prototype=Object.create(Error.prototype),r.prototype.constructor=r,r}var It=at(function(e){return function(r){e(this),this.message=r?r.length+` errors occurred during unsubscription: +`+r.map(function(n,o){return o+1+") "+n.toString()}).join(` + `):"",this.name="UnsubscriptionError",this.errors=r}});function Ve(e,t){if(e){var r=e.indexOf(t);0<=r&&e.splice(r,1)}}var je=function(){function e(t){this.initialTeardown=t,this.closed=!1,this._parentage=null,this._finalizers=null}return e.prototype.unsubscribe=function(){var t,r,n,o,i;if(!this.closed){this.closed=!0;var s=this._parentage;if(s)if(this._parentage=null,Array.isArray(s))try{for(var a=xe(s),f=a.next();!f.done;f=a.next()){var c=f.value;c.remove(this)}}catch(v){t={error:v}}finally{try{f&&!f.done&&(r=a.return)&&r.call(a)}finally{if(t)throw t.error}}else s.remove(this);var u=this.initialTeardown;if(A(u))try{u()}catch(v){i=v instanceof It?v.errors:[v]}var p=this._finalizers;if(p){this._finalizers=null;try{for(var m=xe(p),d=m.next();!d.done;d=m.next()){var h=d.value;try{dn(h)}catch(v){i=i!=null?i:[],v instanceof It?i=D(D([],W(i)),W(v.errors)):i.push(v)}}}catch(v){n={error:v}}finally{try{d&&!d.done&&(o=m.return)&&o.call(m)}finally{if(n)throw n.error}}}if(i)throw new It(i)}},e.prototype.add=function(t){var r;if(t&&t!==this)if(this.closed)dn(t);else{if(t instanceof e){if(t.closed||t._hasParent(this))return;t._addParent(this)}(this._finalizers=(r=this._finalizers)!==null&&r!==void 0?r:[]).push(t)}},e.prototype._hasParent=function(t){var r=this._parentage;return r===t||Array.isArray(r)&&r.includes(t)},e.prototype._addParent=function(t){var r=this._parentage;this._parentage=Array.isArray(r)?(r.push(t),r):r?[r,t]:t},e.prototype._removeParent=function(t){var r=this._parentage;r===t?this._parentage=null:Array.isArray(r)&&Ve(r,t)},e.prototype.remove=function(t){var r=this._finalizers;r&&Ve(r,t),t instanceof e&&t._removeParent(this)},e.EMPTY=function(){var t=new e;return t.closed=!0,t}(),e}();var Tr=je.EMPTY;function Ft(e){return e instanceof je||e&&"closed"in e&&A(e.remove)&&A(e.add)&&A(e.unsubscribe)}function dn(e){A(e)?e():e.unsubscribe()}var Ae={onUnhandledError:null,onStoppedNotification:null,Promise:void 0,useDeprecatedSynchronousErrorHandling:!1,useDeprecatedNextContext:!1};var st={setTimeout:function(e,t){for(var r=[],n=2;n0},enumerable:!1,configurable:!0}),t.prototype._trySubscribe=function(r){return this._throwIfClosed(),e.prototype._trySubscribe.call(this,r)},t.prototype._subscribe=function(r){return this._throwIfClosed(),this._checkFinalizedStatuses(r),this._innerSubscribe(r)},t.prototype._innerSubscribe=function(r){var n=this,o=this,i=o.hasError,s=o.isStopped,a=o.observers;return i||s?Tr:(this.currentObservers=null,a.push(r),new je(function(){n.currentObservers=null,Ve(a,r)}))},t.prototype._checkFinalizedStatuses=function(r){var n=this,o=n.hasError,i=n.thrownError,s=n.isStopped;o?r.error(i):s&&r.complete()},t.prototype.asObservable=function(){var r=new U;return r.source=this,r},t.create=function(r,n){return new wn(r,n)},t}(U);var wn=function(e){ie(t,e);function t(r,n){var o=e.call(this)||this;return o.destination=r,o.source=n,o}return t.prototype.next=function(r){var n,o;(o=(n=this.destination)===null||n===void 0?void 0:n.next)===null||o===void 0||o.call(n,r)},t.prototype.error=function(r){var n,o;(o=(n=this.destination)===null||n===void 0?void 0:n.error)===null||o===void 0||o.call(n,r)},t.prototype.complete=function(){var r,n;(n=(r=this.destination)===null||r===void 0?void 0:r.complete)===null||n===void 0||n.call(r)},t.prototype._subscribe=function(r){var n,o;return(o=(n=this.source)===null||n===void 0?void 0:n.subscribe(r))!==null&&o!==void 0?o:Tr},t}(E);var Et={now:function(){return(Et.delegate||Date).now()},delegate:void 0};var wt=function(e){ie(t,e);function t(r,n,o){r===void 0&&(r=1/0),n===void 0&&(n=1/0),o===void 0&&(o=Et);var i=e.call(this)||this;return i._bufferSize=r,i._windowTime=n,i._timestampProvider=o,i._buffer=[],i._infiniteTimeWindow=!0,i._infiniteTimeWindow=n===1/0,i._bufferSize=Math.max(1,r),i._windowTime=Math.max(1,n),i}return t.prototype.next=function(r){var n=this,o=n.isStopped,i=n._buffer,s=n._infiniteTimeWindow,a=n._timestampProvider,f=n._windowTime;o||(i.push(r),!s&&i.push(a.now()+f)),this._trimBuffer(),e.prototype.next.call(this,r)},t.prototype._subscribe=function(r){this._throwIfClosed(),this._trimBuffer();for(var n=this._innerSubscribe(r),o=this,i=o._infiniteTimeWindow,s=o._buffer,a=s.slice(),f=0;f0?e.prototype.requestAsyncId.call(this,r,n,o):(r.actions.push(this),r._scheduled||(r._scheduled=ut.requestAnimationFrame(function(){return r.flush(void 0)})))},t.prototype.recycleAsyncId=function(r,n,o){var i;if(o===void 0&&(o=0),o!=null?o>0:this.delay>0)return e.prototype.recycleAsyncId.call(this,r,n,o);var s=r.actions;n!=null&&((i=s[s.length-1])===null||i===void 0?void 0:i.id)!==n&&(ut.cancelAnimationFrame(n),r._scheduled=void 0)},t}(Wt);var Tn=function(e){ie(t,e);function t(){return e!==null&&e.apply(this,arguments)||this}return t.prototype.flush=function(r){this._active=!0;var n=this._scheduled;this._scheduled=void 0;var o=this.actions,i;r=r||o.shift();do if(i=r.execute(r.state,r.delay))break;while((r=o[0])&&r.id===n&&o.shift());if(this._active=!1,i){for(;(r=o[0])&&r.id===n&&o.shift();)r.unsubscribe();throw i}},t}(Dt);var we=new Tn(On);var R=new U(function(e){return e.complete()});function Vt(e){return e&&A(e.schedule)}function kr(e){return e[e.length-1]}function Qe(e){return A(kr(e))?e.pop():void 0}function Se(e){return Vt(kr(e))?e.pop():void 0}function zt(e,t){return typeof kr(e)=="number"?e.pop():t}var pt=function(e){return e&&typeof e.length=="number"&&typeof e!="function"};function Nt(e){return A(e==null?void 0:e.then)}function qt(e){return A(e[ft])}function Kt(e){return Symbol.asyncIterator&&A(e==null?void 0:e[Symbol.asyncIterator])}function Qt(e){return new TypeError("You provided "+(e!==null&&typeof e=="object"?"an invalid object":"'"+e+"'")+" where a stream was expected. You can provide an Observable, Promise, ReadableStream, Array, AsyncIterable, or Iterable.")}function Ki(){return typeof Symbol!="function"||!Symbol.iterator?"@@iterator":Symbol.iterator}var Yt=Ki();function Gt(e){return A(e==null?void 0:e[Yt])}function Bt(e){return ln(this,arguments,function(){var r,n,o,i;return $t(this,function(s){switch(s.label){case 0:r=e.getReader(),s.label=1;case 1:s.trys.push([1,,9,10]),s.label=2;case 2:return[4,Ze(r.read())];case 3:return n=s.sent(),o=n.value,i=n.done,i?[4,Ze(void 0)]:[3,5];case 4:return[2,s.sent()];case 5:return[4,Ze(o)];case 6:return[4,s.sent()];case 7:return s.sent(),[3,2];case 8:return[3,10];case 9:return r.releaseLock(),[7];case 10:return[2]}})})}function Jt(e){return A(e==null?void 0:e.getReader)}function $(e){if(e instanceof U)return e;if(e!=null){if(qt(e))return Qi(e);if(pt(e))return Yi(e);if(Nt(e))return Gi(e);if(Kt(e))return _n(e);if(Gt(e))return Bi(e);if(Jt(e))return Ji(e)}throw Qt(e)}function Qi(e){return new U(function(t){var r=e[ft]();if(A(r.subscribe))return r.subscribe(t);throw new TypeError("Provided object does not correctly implement Symbol.observable")})}function Yi(e){return new U(function(t){for(var r=0;r=2;return function(n){return n.pipe(e?_(function(o,i){return e(o,i,n)}):de,Te(1),r?Pe(t):zn(function(){return new Zt}))}}function Nn(){for(var e=[],t=0;t=2,!0))}function ue(e){e===void 0&&(e={});var t=e.connector,r=t===void 0?function(){return new E}:t,n=e.resetOnError,o=n===void 0?!0:n,i=e.resetOnComplete,s=i===void 0?!0:i,a=e.resetOnRefCountZero,f=a===void 0?!0:a;return function(c){var u,p,m,d=0,h=!1,v=!1,B=function(){p==null||p.unsubscribe(),p=void 0},ne=function(){B(),u=m=void 0,h=v=!1},z=function(){var O=u;ne(),O==null||O.unsubscribe()};return g(function(O,Ke){d++,!v&&!h&&B();var De=m=m!=null?m:r();Ke.add(function(){d--,d===0&&!v&&!h&&(p=jr(z,f))}),De.subscribe(Ke),!u&&d>0&&(u=new tt({next:function(Fe){return De.next(Fe)},error:function(Fe){v=!0,B(),p=jr(ne,o,Fe),De.error(Fe)},complete:function(){h=!0,B(),p=jr(ne,s),De.complete()}}),$(O).subscribe(u))})(c)}}function jr(e,t){for(var r=[],n=2;ne.next(document)),e}function K(e,t=document){return Array.from(t.querySelectorAll(e))}function V(e,t=document){let r=ce(e,t);if(typeof r=="undefined")throw new ReferenceError(`Missing element: expected "${e}" to be present`);return r}function ce(e,t=document){return t.querySelector(e)||void 0}function _e(){return document.activeElement instanceof HTMLElement&&document.activeElement||void 0}function rr(e){return L(b(document.body,"focusin"),b(document.body,"focusout")).pipe(He(1),l(()=>{let t=_e();return typeof t!="undefined"?e.contains(t):!1}),N(e===_e()),G())}function Je(e){return{x:e.offsetLeft,y:e.offsetTop}}function Yn(e){return L(b(window,"load"),b(window,"resize")).pipe(Re(0,we),l(()=>Je(e)),N(Je(e)))}function nr(e){return{x:e.scrollLeft,y:e.scrollTop}}function dt(e){return L(b(e,"scroll"),b(window,"resize")).pipe(Re(0,we),l(()=>nr(e)),N(nr(e)))}var Bn=function(){if(typeof Map!="undefined")return Map;function e(t,r){var n=-1;return t.some(function(o,i){return o[0]===r?(n=i,!0):!1}),n}return function(){function t(){this.__entries__=[]}return Object.defineProperty(t.prototype,"size",{get:function(){return this.__entries__.length},enumerable:!0,configurable:!0}),t.prototype.get=function(r){var n=e(this.__entries__,r),o=this.__entries__[n];return o&&o[1]},t.prototype.set=function(r,n){var o=e(this.__entries__,r);~o?this.__entries__[o][1]=n:this.__entries__.push([r,n])},t.prototype.delete=function(r){var n=this.__entries__,o=e(n,r);~o&&n.splice(o,1)},t.prototype.has=function(r){return!!~e(this.__entries__,r)},t.prototype.clear=function(){this.__entries__.splice(0)},t.prototype.forEach=function(r,n){n===void 0&&(n=null);for(var o=0,i=this.__entries__;o0},e.prototype.connect_=function(){!zr||this.connected_||(document.addEventListener("transitionend",this.onTransitionEnd_),window.addEventListener("resize",this.refresh),xa?(this.mutationsObserver_=new MutationObserver(this.refresh),this.mutationsObserver_.observe(document,{attributes:!0,childList:!0,characterData:!0,subtree:!0})):(document.addEventListener("DOMSubtreeModified",this.refresh),this.mutationEventsAdded_=!0),this.connected_=!0)},e.prototype.disconnect_=function(){!zr||!this.connected_||(document.removeEventListener("transitionend",this.onTransitionEnd_),window.removeEventListener("resize",this.refresh),this.mutationsObserver_&&this.mutationsObserver_.disconnect(),this.mutationEventsAdded_&&document.removeEventListener("DOMSubtreeModified",this.refresh),this.mutationsObserver_=null,this.mutationEventsAdded_=!1,this.connected_=!1)},e.prototype.onTransitionEnd_=function(t){var r=t.propertyName,n=r===void 0?"":r,o=ya.some(function(i){return!!~n.indexOf(i)});o&&this.refresh()},e.getInstance=function(){return this.instance_||(this.instance_=new e),this.instance_},e.instance_=null,e}(),Jn=function(e,t){for(var r=0,n=Object.keys(t);r0},e}(),Zn=typeof WeakMap!="undefined"?new WeakMap:new Bn,eo=function(){function e(t){if(!(this instanceof e))throw new TypeError("Cannot call a class as a function.");if(!arguments.length)throw new TypeError("1 argument required, but only 0 present.");var r=Ea.getInstance(),n=new Ra(t,r,this);Zn.set(this,n)}return e}();["observe","unobserve","disconnect"].forEach(function(e){eo.prototype[e]=function(){var t;return(t=Zn.get(this))[e].apply(t,arguments)}});var ka=function(){return typeof or.ResizeObserver!="undefined"?or.ResizeObserver:eo}(),to=ka;var ro=new E,Ha=I(()=>H(new to(e=>{for(let t of e)ro.next(t)}))).pipe(x(e=>L(Oe,H(e)).pipe(C(()=>e.disconnect()))),J(1));function he(e){return{width:e.offsetWidth,height:e.offsetHeight}}function ge(e){return Ha.pipe(S(t=>t.observe(e)),x(t=>ro.pipe(_(({target:r})=>r===e),C(()=>t.unobserve(e)),l(()=>he(e)))),N(he(e)))}function bt(e){return{width:e.scrollWidth,height:e.scrollHeight}}function sr(e){let t=e.parentElement;for(;t&&(e.scrollWidth<=t.scrollWidth&&e.scrollHeight<=t.scrollHeight);)t=(e=t).parentElement;return t?e:void 0}var no=new E,Pa=I(()=>H(new IntersectionObserver(e=>{for(let t of e)no.next(t)},{threshold:0}))).pipe(x(e=>L(Oe,H(e)).pipe(C(()=>e.disconnect()))),J(1));function cr(e){return Pa.pipe(S(t=>t.observe(e)),x(t=>no.pipe(_(({target:r})=>r===e),C(()=>t.unobserve(e)),l(({isIntersecting:r})=>r))))}function oo(e,t=16){return dt(e).pipe(l(({y:r})=>{let n=he(e),o=bt(e);return r>=o.height-n.height-t}),G())}var fr={drawer:V("[data-md-toggle=drawer]"),search:V("[data-md-toggle=search]")};function io(e){return fr[e].checked}function qe(e,t){fr[e].checked!==t&&fr[e].click()}function Ue(e){let t=fr[e];return b(t,"change").pipe(l(()=>t.checked),N(t.checked))}function $a(e,t){switch(e.constructor){case HTMLInputElement:return e.type==="radio"?/^Arrow/.test(t):!0;case HTMLSelectElement:case HTMLTextAreaElement:return!0;default:return e.isContentEditable}}function Ia(){return L(b(window,"compositionstart").pipe(l(()=>!0)),b(window,"compositionend").pipe(l(()=>!1))).pipe(N(!1))}function ao(){let e=b(window,"keydown").pipe(_(t=>!(t.metaKey||t.ctrlKey)),l(t=>({mode:io("search")?"search":"global",type:t.key,claim(){t.preventDefault(),t.stopPropagation()}})),_(({mode:t,type:r})=>{if(t==="global"){let n=_e();if(typeof n!="undefined")return!$a(n,r)}return!0}),ue());return Ia().pipe(x(t=>t?R:e))}function Me(){return new URL(location.href)}function ot(e){location.href=e.href}function so(){return new E}function co(e,t){if(typeof t=="string"||typeof t=="number")e.innerHTML+=t.toString();else if(t instanceof Node)e.appendChild(t);else if(Array.isArray(t))for(let r of t)co(e,r)}function M(e,t,...r){let n=document.createElement(e);if(t)for(let o of Object.keys(t))typeof t[o]!="undefined"&&(typeof t[o]!="boolean"?n.setAttribute(o,t[o]):n.setAttribute(o,""));for(let o of r)co(n,o);return n}function ur(e){if(e>999){let t=+((e-950)%1e3>99);return`${((e+1e-6)/1e3).toFixed(t)}k`}else return e.toString()}function fo(){return location.hash.substring(1)}function uo(e){let t=M("a",{href:e});t.addEventListener("click",r=>r.stopPropagation()),t.click()}function Fa(){return b(window,"hashchange").pipe(l(fo),N(fo()),_(e=>e.length>0),J(1))}function po(){return Fa().pipe(l(e=>ce(`[id="${e}"]`)),_(e=>typeof e!="undefined"))}function Nr(e){let t=matchMedia(e);return er(r=>t.addListener(()=>r(t.matches))).pipe(N(t.matches))}function lo(){let e=matchMedia("print");return L(b(window,"beforeprint").pipe(l(()=>!0)),b(window,"afterprint").pipe(l(()=>!1))).pipe(N(e.matches))}function qr(e,t){return e.pipe(x(r=>r?t():R))}function pr(e,t={credentials:"same-origin"}){return pe(fetch(`${e}`,t)).pipe(fe(()=>R),x(r=>r.status!==200?Ot(()=>new Error(r.statusText)):H(r)))}function We(e,t){return pr(e,t).pipe(x(r=>r.json()),J(1))}function mo(e,t){let r=new DOMParser;return pr(e,t).pipe(x(n=>n.text()),l(n=>r.parseFromString(n,"text/xml")),J(1))}function lr(e){let t=M("script",{src:e});return I(()=>(document.head.appendChild(t),L(b(t,"load"),b(t,"error").pipe(x(()=>Ot(()=>new ReferenceError(`Invalid script: ${e}`))))).pipe(l(()=>{}),C(()=>document.head.removeChild(t)),Te(1))))}function ho(){return{x:Math.max(0,scrollX),y:Math.max(0,scrollY)}}function bo(){return L(b(window,"scroll",{passive:!0}),b(window,"resize",{passive:!0})).pipe(l(ho),N(ho()))}function vo(){return{width:innerWidth,height:innerHeight}}function go(){return b(window,"resize",{passive:!0}).pipe(l(vo),N(vo()))}function yo(){return Q([bo(),go()]).pipe(l(([e,t])=>({offset:e,size:t})),J(1))}function mr(e,{viewport$:t,header$:r}){let n=t.pipe(X("size")),o=Q([n,r]).pipe(l(()=>Je(e)));return Q([r,t,o]).pipe(l(([{height:i},{offset:s,size:a},{x:f,y:c}])=>({offset:{x:s.x-f,y:s.y-c+i},size:a})))}(()=>{function e(n,o){parent.postMessage(n,o||"*")}function t(...n){return n.reduce((o,i)=>o.then(()=>new Promise(s=>{let a=document.createElement("script");a.src=i,a.onload=s,document.body.appendChild(a)})),Promise.resolve())}var r=class extends EventTarget{constructor(n){super(),this.url=n,this.m=i=>{i.source===this.w&&(this.dispatchEvent(new MessageEvent("message",{data:i.data})),this.onmessage&&this.onmessage(i))},this.e=(i,s,a,f,c)=>{if(s===`${this.url}`){let u=new ErrorEvent("error",{message:i,filename:s,lineno:a,colno:f,error:c});this.dispatchEvent(u),this.onerror&&this.onerror(u)}};let o=document.createElement("iframe");o.hidden=!0,document.body.appendChild(this.iframe=o),this.w.document.open(),this.w.document.write(` + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

CLI

+ +

Launch comes with a CLI for listing bundles / endpoints, editing endpoints, +and sending tasks to endpoints.

+

The CLI can be used as scale-launch ....

+

Help

+

Run scale-launch --help for more options.

+
scale-launch --help
    This is the command line interface (CLI) package for Scale Launch.
+
+       ██╗      █████╗ ██╗   ██╗███╗   ██╗ ██████╗██╗  ██╗
+       ██║     ██╔â•â•â–ˆâ–ˆâ•—██║   ██║████╗  ██║██╔â•â•â•â•â•â–ˆâ–ˆâ•‘  ██║
+       ██║     ███████║██║   ██║██╔██╗ ██║██║     ███████║
+       ██║     ██╔â•â•â–ˆâ–ˆâ•‘██║   ██║██║╚██╗██║██║     ██╔â•â•â–ˆâ–ˆâ•‘
+       ███████╗██║  ██║╚██████╔â•â–ˆâ–ˆâ•‘ ╚████║╚██████╗██║  ██║
+       â•šâ•â•â•â•â•â•â•â•šâ•â•  â•šâ•â• â•šâ•â•â•â•â•â• â•šâ•â•  â•šâ•â•â•â• â•šâ•â•â•â•â•â•â•šâ•â•  â•šâ•â•
+
+Usage: scale-launch [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch
+  bundles     Bundles is a wrapper around model bundles in Scale Launch
+  config      Config is a wrapper around getting and setting your Scale...
+  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch
+  tasks       Tasks is a wrapper around sending requests to endpoints
+
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/concepts/batch_jobs/index.html b/concepts/batch_jobs/index.html new file mode 100644 index 00000000..25f6d761 --- /dev/null +++ b/concepts/batch_jobs/index.html @@ -0,0 +1,598 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Batch jobs - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Batch jobs

+ +

Batch Jobs

+

For predicting over a larger set of tasks (> 50) at once, it is recommended to +use batch jobs. Batch jobs are a way to send a large number of tasks to a model +bundle. The tasks are processed in parallel, and the results are returned as a +list of predictions.

+

Batch jobs are created using the +batch_async_request +method of the +LaunchClient.

+
Creating and Following a Batch Job
import os
+import time
+from launch import LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+batch_job = client.batch_async_request(
+    model_bundle="test-bundle",
+    inputs=[
+        {"x": 2, "y": "hello"},
+        {"x": 3, "y": "world"},
+    ],
+)
+
+status = "PENDING"
+res = None
+while status != "SUCCESS" and status != "FAILURE" and status != "CANCELLED":
+    time.sleep(30)
+    res = client.get_batch_async_response(batch_job["job_id"])
+    status = res["status"]
+    print(f"the batch job is {status}")
+
+print(res)
+
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/concepts/endpoint_predictions/index.html b/concepts/endpoint_predictions/index.html new file mode 100644 index 00000000..1af35a71 --- /dev/null +++ b/concepts/endpoint_predictions/index.html @@ -0,0 +1,1008 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Endpoint Predictions - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Endpoint Predictions

+

Once endpoints have been created, users can send tasks to them to make +predictions. The following code snippet shows how to send tasks to endpoints.

+
+
+
+
import os
+from launch import EndpointRequest, LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.get_model_endpoint("demo-endpoint-async")
+future = endpoint.predict(request=EndpointRequest(args={"x": 2, "y": "hello"}))
+response = future.get()
+print(response)
+
+
+
+
import os
+from launch import EndpointRequest, LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.get_model_endpoint("demo-endpoint-sync")
+response = endpoint.predict(request=EndpointRequest(args={"x": 2, "y": "hello"}))
+print(response)
+
+
+
+
+ + +
+ + + +

+ EndpointRequest + + +

+
EndpointRequest(
+    url: Optional[str] = None,
+    args: Optional[Dict] = None,
+    callback_url: Optional[str] = None,
+    return_pickled: Optional[bool] = False,
+    request_id: Optional[str] = None,
+)
+
+ +
+ + +

Represents a single request to either a SyncEndpoint or AsyncEndpoint.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
url + Optional[str] +

A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. +Note: the contents of the file located at url are opened as a sequence of bytes and passed +to the predict function. If you instead want to pass the url itself as an input to the predict function, +see args.

+

Exactly one of url and args must be specified.

 + None +
args + Optional[Dict] +

A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature +predict_fn(foo, bar), then the keys in the dictionary should be "foo" and "bar". +Values must be native Python objects.

+

Exactly one of url and args must be specified.

 + None +
return_pickled + Optional[bool] +

Whether the output should be a pickled python object, or directly returned serialized json.

 + False +
request_id + Optional[str] +

(deprecated) A user-specifiable id for requests. +Should be unique among EndpointRequests made in the same batch call. +If one isn't provided the client will generate its own.

 + None +
+ + + + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + +

+ EndpointResponseFuture + + +

+
EndpointResponseFuture(
+    client, endpoint_name: str, async_task_id: str
+)
+
+ +
+ + +

Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, +then its get method will return an actual instance of EndpointResponse.

+

This object should not be directly instantiated by the user.

+ + + +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
client +

An instance of LaunchClient.

 + required +
endpoint_name + str +

The name of the endpoint.

 + required +
async_task_id + str +

An async task id.

 + required +
+ + + + +
+ + + + + + + + + +
+ + + +

+ get + + +

+
get(timeout: Optional[float] = None) -> EndpointResponse
+
+ +
+ +

Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
timeout + Optional[float] +

The maximum number of seconds to wait for the response. If None, then +the method will block indefinitely until the response is ready.

 + None +
+ +
+ +
+ + + +
+ +
+ +
+ +
+ + + +

+ EndpointResponse + + +

+
EndpointResponse(
+    client,
+    status: str,
+    result_url: Optional[str] = None,
+    result: Optional[str] = None,
+    traceback: Optional[str] = None,
+)
+
+ +
+ + +

Represents a response received from a Endpoint.

+ + + +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
client +

An instance of LaunchClient.

 + required +
status + str +

A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

 + required +
result_url + Optional[str] +

A string that is a url containing the pickled python object from the +Endpoint's predict function.

+

Exactly one of result_url or result will be populated, +depending on the value of return_pickled in the request.

 + None +
result + Optional[str] +

A string that is the serialized return value (in json form) of the Endpoint's predict function. +Specifically, one can json.loads() the value of result to get the original python object back.

+

Exactly one of result_url or result will be populated, +depending on the value of return_pickled in the request.

 + None +
traceback + Optional[str] +

The stack trace if the inference endpoint raised an error. Can be used for debugging

 + None +
+ + + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/concepts/model_bundles/index.html b/concepts/model_bundles/index.html new file mode 100644 index 00000000..1dd31922 --- /dev/null +++ b/concepts/model_bundles/index.html @@ -0,0 +1,846 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Model Bundles - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Model Bundles

+

Model Bundles are deployable models that can be used to make predictions. They +are created by packaging a model up into a deployable format.

+

Creating Model Bundles

+

There are two methods for creating model bundles: +create_model_bundle +and +create_model_bundle_from_dirs. +The former directly pickles a user-specified load_predict_fn, a function which +loads the model and returns a predict_fn, a function which takes in a request. +The latter takes in directories containing a load_predict_fn and the +module path to the load_predict_fn.

+
+
+
+
import os
+from pydantic import BaseModel
+from launch import LaunchClient
+
+
+class MyRequestSchema(BaseModel):
+    x: int
+    y: str
+
+class MyResponseSchema(BaseModel):
+    __root__: int
+
+
+def my_load_predict_fn(model):
+    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:
+        """MyRequestSchema -> MyResponseSchema"""
+        assert isinstance(x, int) and isinstance(y, str)
+        return model(x) + len(y)
+
+    return returns_model_of_x_plus_len_of_y
+
+
+def my_model(x):
+    return x * 2
+
+ENV_PARAMS = {
+    "framework_type": "pytorch",
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+}
+
+BUNDLE_PARAMS = {
+    "model_bundle_name": "test-bundle",
+    "model": my_model,
+    "load_predict_fn": my_load_predict_fn,
+    "env_params": ENV_PARAMS,
+    "requirements": [],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+}
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+client.create_model_bundle(**BUNDLE_PARAMS)
+
+
+
+
import os
+import tempfile
+from pydantic import BaseModel
+from launch import LaunchClient
+
+directory = tempfile.mkdtemp()
+model_filename = os.path.join(directory, "model.py")
+with open(model_filename, "w") as f:
+    f.write("""
+def my_load_model_fn():
+    def my_model(x):
+        return x * 2
+
+    return my_model
+""")
+
+predict_filename = os.path.join(directory, "predict.py")
+with open(predict_filename, "w") as f:
+    f.write("""
+def my_load_predict_fn(model):
+    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:
+        assert isinstance(x, int) and isinstance(y, str)
+        return model(x) + len(y)
+
+    return returns_model_of_x_plus_len_of_y
+""")
+
+requirements_filename = os.path.join(directory, "requirements.txt")
+with open(predict_filename, "w") as f:
+    f.write("""
+numpy
+sklearn
+"""
+    )
+
+"""
+The directory structure should now look like
+
+directory/
+    model.py
+    predict.py
+    requirements.txt
+"""
+
+
+class MyRequestSchema(BaseModel):
+    x: int
+    y: str
+
+class MyResponseSchema(BaseModel):
+    __root__: int
+
+ENV_PARAMS = {
+    "framework_type": "pytorch",
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+}
+
+BUNDLE_PARAMS = {
+    "model_bundle_name": "test-bundle",
+    "base_paths": [directory],
+    "requirements_path": "requirements.txt",
+    "env_params": ENV_PARAMS,
+    "load_predict_fn": "predict.my_load_predict_fn",
+    "load_model_fn_module_path": "model.my_load_model_fn",
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+}
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+client.create_model_bundle(**BUNDLE_PARAMS)
+
+# Clean up files from demo
+os.remove(model_filename)
+os.remove(predict_filename)
+os.rmdir(directory)
+
+
+
+
+

Configuring Model Bundles

+

The app_config field of a model bundle is a dictionary that can be used to +configure the model bundle. If specified, the app_config is passed to the +load_predict_fn when the model bundle is deployed, alongside the model. This +can allow for more code reuse between multiple bundles that perform similar +tasks.

+
Creating Model Bundles with app_config
import os
+from launch import LaunchClient
+from pydantic import BaseModel
+from typing import List, Literal, Union
+
+
+class MyRequestSchemaSingle(BaseModel):
+    kind: Literal['single']
+    x: int
+    y: str
+
+class MyRequestSchemaBatched(BaseModel):
+    kind: Literal['batched']
+    x: List[int]
+    y: List[str]
+
+class MyRequestSchema(BaseModel):
+    __root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]
+
+class MyResponseSchema(BaseModel):
+    __root__: Union[int, List[int]]
+
+
+def my_load_predict_fn(app_config, model):
+    def returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:
+        """MyRequestSchema -> MyResponseSchema"""
+        if app_config["mode"] == "single":
+            assert isinstance(x, int) and isinstance(y, str)
+            return model(x) + len(y)
+
+        result = []
+        for x_i, y_i in zip(x, y):
+            result.append(model(x_i) + len(y_i))
+        return result
+
+    return returns_model_of_x_plus_len_of_y
+
+
+def my_load_model_fn(app_config):
+    def my_model_single(x: int):
+        return x * 2
+
+    def my_model_batched(x: List[int]):
+        return [my_model_single(x_i) for x_i in x]
+
+    if app_config["mode"] == "single":
+        return my_model_single
+
+    return my_model_batched
+
+
+ENV_PARAMS = {
+    "framework_type": "pytorch",
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+}
+
+BUNDLE_PARAMS_SINGLE = {
+    "model_bundle_name": "test-bundle-single",
+    "load_predict_fn": my_load_predict_fn,
+    "load_model_fn": my_load_model_fn,
+    "env_params": ENV_PARAMS,
+    "requirements": [],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "app_config": {"mode": "single"},
+}
+BUNDLE_PARAMS_BATCHED = {
+    "model_bundle_name": "test-bundle-batched",
+    "load_predict_fn": my_load_predict_fn,
+    "load_model_fn": my_load_model_fn,
+    "env_params": ENV_PARAMS,
+    "requirements": [],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "app_config": {"mode": "single"},
+}
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+bundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)
+bundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)
+
+

Updating Model Bundles

+

Model Bundles are immutable, meaning they cannot be edited once created. +However, it is possible to clone an existing model bundle with a new app_config +using +clone_model_bundle_with_changes.

+

Listing Model Bundles

+

To list all the model bundles you own, use +list_model_bundles.

+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/concepts/model_endpoints/index.html b/concepts/model_endpoints/index.html new file mode 100644 index 00000000..b0afcb58 --- /dev/null +++ b/concepts/model_endpoints/index.html @@ -0,0 +1,685 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Model Endpoints - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Model Endpoints

+

Model Endpoints are deployments of models that can receive requests and return +predictions containing the results of the model's inference. Each model endpoint +is associated with a model bundle, which contains the model's code. An endpoint +specifies deployment parameters, such as the minimum and maximum number of +workers, as well as the requested resources for each worker, such as the number +of CPUs, amount of memory, GPU count, and type of GPU.

+

Endpoints can be asynchronous or synchronous. Asynchronous endpoints return +a future immediately after receiving a request, and the future can be used to +retrieve the prediction once it is ready. Synchronous endpoints return the +prediction directly after receiving a request.

+

Creating Async Model Endpoints

+

Async model endpoints are the most cost-efficient way to perform inference on +tasks that are less latency-sensitive.

+
Creating an Async Model Endpoint
import os
+from launch import LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.create_model_endpoint(
+    endpoint_name="demo-endpoint-async",
+    model_bundle="test-bundle",
+    cpus=1,
+    min_workers=0,
+    endpoint_type="async",
+    update_if_exists=True,
+    labels={
+        "team": "MY_TEAM",
+        "product": "MY_PRODUCT"
+    },
+)
+
+

Creating Sync Model Endpoints

+

Sync model endpoints are useful for latency-sensitive tasks, such as real-time +inference. Sync endpoints are more expensive than async endpoints, and they +require at least 1 min_worker.

+
Creating a Sync Model Endpoint
import os
+from launch import LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.create_model_endpoint(
+    endpoint_name="demo-endpoint-sync",
+    model_bundle="test-bundle",
+    cpus=1,
+    min_workers=1,
+    endpoint_type="sync",
+    update_if_exists=True,
+    labels={
+        "team": "MY_TEAM",
+        "product": "MY_PRODUCT"
+    },
+)
+
+

Managing Model Endpoints

+

Model endpoints can be listed, updated, and deleted using the Launch API.

+
Listing Model Endpoints
import os
+from launch import LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoints = client.list_model_endpoints()
+
+
Updating a Model Endpoint
import os
+from launch import LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+client.edit_model_endpoint(
+    model_endpoint="demo-endpoint",
+    max_workers=2,
+)
+
+
Deleting a Model Endpoint
import os
+from launch import LaunchClient
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.create_model_endpoint(
+    endpoint_name="demo-endpoint-tmp",
+    model_bundle="test-bundle",
+    cpus=1,
+    min_workers=0,
+    endpoint_type="async",
+    update_if_exists=True,
+    labels={
+        "team": "MY_TEAM",
+        "product": "MY_PRODUCT"
+    },
+)
+client.delete_model_endpoint(model_endpoint="demo-endpoint-tmp")
+
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 00000000..248fb455 --- /dev/null +++ b/index.html @@ -0,0 +1,695 @@ + + + + + + + + + + + + + + + + + + + + + + + + Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Scale Launch

+

CI +pypi

+

Simple, scalable, and high performance ML service deployment in python.

+

Example

+
Launch Usage
import os
+import time
+from launch import LaunchClient
+from launch import EndpointRequest
+from pydantic import BaseModel
+from rich import print
+
+
+class MyRequestSchema(BaseModel):
+    x: int
+    y: str
+
+class MyResponseSchema(BaseModel):
+    __root__: int
+
+
+def my_load_predict_fn(model):
+    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:
+        """MyRequestSchema -> MyResponseSchema"""
+        assert isinstance(x, int) and isinstance(y, str)
+        return model(x) + len(y)
+
+    return returns_model_of_x_plus_len_of_y
+
+
+def my_model(x):
+    return x * 2
+
+ENV_PARAMS = {
+    "framework_type": "pytorch",
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+}
+
+BUNDLE_PARAMS = {
+    "model_bundle_name": "test-bundle",
+    "model": my_model,
+    "load_predict_fn": my_load_predict_fn,
+    "env_params": ENV_PARAMS,
+    "requirements": [],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+}
+
+ENDPOINT_PARAMS = {
+    "endpoint_name": "demo-endpoint",
+    "model_bundle": "test-bundle",
+    "cpus": 1,
+    "min_workers": 0,
+    "endpoint_type": "async",
+    "update_if_exists": True,
+    "labels": {
+        "team": "infra",
+        "product": "launch",
+    }
+}
+
+def predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:
+    # Wait for the endpoint to be ready first before submitting a task
+    endpoint = client.get_model_endpoint(endpoint_name="demo-endpoint")
+    while endpoint.status() != "READY":
+        time.sleep(10)
+
+    endpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)
+
+    future = endpoint.predict(request=endpoint_request)
+    raw_response = future.get()
+
+    response = MyResponseSchema.parse_raw(raw_response.result)
+    return response
+
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+
+client.create_model_bundle(**BUNDLE_PARAMS)
+endpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)
+
+request = MyRequestSchema(x=5, y="hello")
+response = predict_on_endpoint(request)
+print(response)
+"""
+MyResponseSchema(__root__=10)
+"""
+
+

What's going on here:

+
    +
  • First we use pydantic to define our request and response + schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API + documentation for our models.
    • +
  • Next we define the the model and the load_predict_fn, which tells Launch + how to load our model and how to make predictions with it. In this case, + we're just returning a function that adds the length of the string y to + model(x), where model doubles the integer x.
    • +
  • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the + response_schema. We also specify the env_params, which tell Launch environment settings like + the base image to use. In this case, we're using a PyTorch image.
    • +
  • Next, we create the model endpoint, which is the API that we'll use to make predictions. We + specify the model_bundle that we created above, and we specify the endpoint_type, which tells + Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an + asynchronous endpoint, which means that we can make predictions and return immediately with a + future object. We can then use the future object to get the prediction result later.
    • +
  • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. + This function first waits for the endpoint to be ready, then it submits a prediction request to + the endpoint. It then waits for the prediction result and returns it.
    • +
+

Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers +when it's not being used.

+

Installation

+

To use Scale Launch, first install it using pip:

+
Installation
pip install -U scale-launch
+
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/objects.inv b/objects.inv new file mode 100644 index 00000000..8de29452 --- /dev/null +++ b/objects.inv @@ -0,0 +1,6 @@ +# Sphinx inventory version 2 +# Project: Launch +# Version: 0.0.0 +# The remainder of this file is compressed using zlib. +xÚµÖÍNÄ ð»OÑD¯²ñêMžôâ Y˜¶D: Ð5ûöÒï­IWðHÒþ˜Âð§Z5Z!öÜúAa·Bƒ÷ÅMVí†Gv—WÅõ…þõ-¶‡ jþH‚;|oÑ÷R ÊÐy˜Ð†7F¢æû–¤Fþ¡BÍE T¡O€BXËY1^:Óp©\®‘¤5ŠÖ2*˜‘C©B6¬ÂÀ×ã­!ibžÝ]¤ôïÔʯËòY¬©°-öšœÞ·^4¸6UZÅ"Ñ{+ü!‚bKÊá‡-/ÚZmâb,5gUÇxØHžö +{é÷s;Ÿ†óª§6 ¬åÂP©ªƒÜß-¤·à ñé–’éÆÔéRƒ$H—LW,ˆ7¨U<m¯»¢•ÃÇô8C›Ò‚Ýu™ýxŽß;sN¨Í³.^1⧰M%‡£Oæëò‹3}uSimple, scalable, and high performance ML service deployment in python.

"},{"location":"#example","title":"Example","text":"Launch Usage
import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

What's going on here:

  • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
  • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
  • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
  • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
  • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

"},{"location":"#installation","title":"Installation","text":"

To use Scale Launch, first install it using pip:

Installation
pip install -U scale-launch\n
"},{"location":"cli/","title":"CLI","text":"

Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

The CLI can be used as scale-launch ....

"},{"location":"cli/#help","title":"Help","text":"

Run scale-launch --help for more options.

scale-launch --help
    This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your Scale...\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
"},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

Scale Launch Python Client.

Initializes a Scale Launch Client.

Parameters:

Name Type Description Default api_key str

Your Scale API key

required endpoint Optional[str]

The Scale Launch Endpoint (this should not need to be changed)

None self_hosted bool

True iff you are connecting to a self-hosted Scale Launch

False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

Must have exactly one of urls or inputs passed in.

Parameters:

Name Type Description Default model_bundle Union[ModelBundle, str]

The bundle or the name of a the bundle to use for inference.

required urls List[str]

A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

None inputs Optional[List[Dict[str, Any]]]

A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

None batch_url_file_location Optional[str]

In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

None serialization_format str

Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

'JSON' labels Optional[Dict[str, str]]

An optional dictionary of key/value pairs to associate with this endpoint.

None cpus Optional[int]

Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

None memory Optional[str]

Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

None storage Optional[str]

Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

None gpus Optional[int]

Number of gpus each worker should get, e.g. 0, 1, etc.

None max_workers Optional[int]

The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

None per_worker Optional[int]

The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

  • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
  • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
None gpu_type Optional[str]

If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

  • nvidia-tesla-t4
  • nvidia-ampere-a10
None

Returns:

Type Description Dict[str, Any]

A dictionary that contains job_id as a key, and the ID as the value.

"},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

Clones an existing model bundle with changes to its app config. (More fields coming soon)

Parameters:

Name Type Description Default model_bundle Union[ModelBundle, str]

The existing bundle or its ID.

required app_config Optional[Dict]

The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

None

Returns:

Type Description ModelBundle

A ModelBundle object

"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

Uploads and registers a model bundle to Scale Launch.

A model bundle consists of exactly one of the following:

  • predict_fn_or_cls
  • load_predict_fn + model
  • load_predict_fn + load_model_fn

Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

Parameters:

Name Type Description Default model_bundle_name str

The name of the model bundle you want to create. The name must be unique across all bundles that you own.

required predict_fn_or_cls Optional[Callable[[Any], Any]]

Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

None model Optional[LaunchModel_T]

Typically a trained Neural Network, e.g. a Pytorch module.

Exactly one of model and load_model_fn must be provided.

None load_model_fn Optional[Callable[[], LaunchModel_T]]

A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

Exactly one of model and load_model_fn must be provided.

None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

Function that, when called with a model, returns a function that carries out inference.

If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

None requirements Optional[List[str]]

A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

[\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

None app_config Optional[Union[Dict[str, Any], str]]

Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

None env_params Dict[str, str]

A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

  • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

    .. code-block:: python

    { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

  • Tensorflow fields:

    • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
required globals_copy Optional[Dict[str, Any]]

Dictionary of the global symbol table. Normally provided by globals() built-in function.

None request_schema Optional[Type[BaseModel]]

A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

None response_schema Optional[Type[BaseModel]]

A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

For example, if you have a directory structure like so, and your current working directory is also my_root:

.. code-block:: text

my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

.. code-block:: text

my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

Parameters:

Name Type Description Default model_bundle_name str

The name of the model bundle you want to create. The name must be unique across all bundles that you own.

required base_paths List[str]

The paths on the local filesystem where the bundle code lives.

required requirements_path str

A path on the local filesystem where a requirements.txt file lives.

required env_params Dict[str, str]

A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

  • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

    .. code-block:: python

    { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

required load_predict_fn_module_path str

A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

required load_model_fn_module_path str

A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

required app_config Optional[Union[Dict[str, Any], str]]

Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

None request_schema Optional[Type[BaseModel]]

A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

None response_schema Optional[Type[BaseModel]]

A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

Parameters:

Name Type Description Default endpoint_name str

The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

required model_bundle Union[ModelBundle, str]

The ModelBundle that the endpoint should serve.

required cpus int

Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

3 memory str

Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

'8Gi' storage Optional[str]

Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

None gpus int

Number of gpus each worker should get, e.g. 0, 1, etc.

0 min_workers int

The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

1 max_workers int

The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

1 per_worker int

The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

  • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

Here is our recommendation for computing per_worker:

  1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
10 gpu_type Optional[str]

If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

  • nvidia-tesla-t4
  • nvidia-ampere-a10
None endpoint_type str

Either \"sync\" or \"async\".

'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

List of hooks to trigger after inference tasks are served.

None default_callback_url Optional[str]

The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

None update_if_exists bool

If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

False labels Optional[Dict[str, str]]

An optional dictionary of key/value pairs to associate with this endpoint.

None

Returns:

Type Description Optional[Endpoint]

A Endpoint object that can be used to make requests to the endpoint.

"},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

Deletes a model endpoint.

Parameters:

Name Type Description Default model_endpoint Union[ModelEndpoint, str]

A ModelEndpoint object.

required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n

Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

  • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

Parameters:

Name Type Description Default model_endpoint Union[ModelEndpoint, str]

The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

required model_bundle Optional[Union[ModelBundle, str]]

The ModelBundle that the endpoint should serve.

None cpus Optional[float]

Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

None memory Optional[str]

Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

None storage Optional[str]

Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

None gpus Optional[int]

Number of gpus each worker should get, e.g. 0, 1, etc.

None min_workers Optional[int]

The minimum number of workers. Must be greater than or equal to 0.

None max_workers Optional[int]

The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

None per_worker Optional[int]

The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

  • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
None gpu_type Optional[str]

If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

  • nvidia-tesla-t4
  • nvidia-ampere-a10
None post_inference_hooks Optional[List[PostInferenceHooks]]

List of hooks to trigger after inference tasks are served.

None default_callback_url Optional[str]

The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

Gets inference results from a previously created batch job.

Parameters:

Name Type Description Default batch_job_id str

An id representing the batch task job. This id is the in the response from calling batch_async_request.

required

Returns:

Type Description Dict[str, Any]

A dictionary that contains the following fields:

Dict[str, Any]
  • status: The status of the job.
Dict[str, Any]
  • result: The url where the result is stored.
Dict[str, Any]
  • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
Dict[str, Any]
  • num_tasks_pending: The number of tasks that are still pending.
Dict[str, Any]
  • num_tasks_completed: The number of tasks that have completed.
"},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

Returns a model bundle specified by bundle_name that the user owns.

Parameters:

Name Type Description Default model_bundle Union[ModelBundle, str]

The bundle or its name.

required

Returns:

Type Description ModelBundle

A ModelBundle object

"},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

Gets a model endpoint associated with a name.

Parameters:

Name Type Description Default endpoint_name str

The name of the endpoint to retrieve.

required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
list_model_bundles() -> List[ModelBundle]\n

Returns a list of model bundles that the user owns.

Returns:

Type Description List[ModelBundle]

A list of ModelBundle objects

"},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
list_model_endpoints() -> List[Endpoint]\n

Lists all model endpoints that the user owns.

Returns:

Type Description List[Endpoint]

A list of ModelEndpoint objects.

"},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

Retrieves the logs for the creation of the endpoint.

Parameters:

Name Type Description Default model_endpoint Union[ModelEndpoint, str]

The endpoint or its name.

required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

Parameters:

Name Type Description Default batch_csv_location_fn Callable[[], str]

Function that generates batch_csv_urls for upload_batch_csv_fn.

required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

Parameters:

Name Type Description Default bundle_location_fn Callable[[], str]

Function that generates bundle_urls for upload_bundle_fn.

required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

upload_batch_csv_fn(csv_text, csv_url)\n

This function should directly write the contents of csv_text as a text string into csv_url.

Parameters:

Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

upload_bundle_fn(serialized_bundle, bundle_url)\n

This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

Parameters:

Name Type Description Default upload_bundle_fn Callable[[str, str], None]

Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

required"},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

Represents a ModelBundle.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
app_config: Optional[Dict[Any, Any]] = None\n

An optional user-specified configuration mapping for the bundle.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
env_params: Optional[Dict[str, str]] = None\n

A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
id: Optional[str] = None\n

A globally unique identifier for the bundle.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
location: Optional[str] = None\n

An opaque location for the bundle.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
metadata: Optional[Dict[Any, Any]] = None\n

Arbitrary metadata for the bundle.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
name: str\n

The name of the bundle. Must be unique across all bundles that the user owns.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
packaging_type: Optional[str] = None\n

The packaging type for the bundle. Can be cloudpickle or zip.

"},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
requirements: Optional[List[str]] = None\n

A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

"},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

All classes here are returned by the get_model_endpoint method and provide a predict function.

"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

Bases: Endpoint

An asynchronous model endpoint.

Parameters:

Name Type Description Default model_endpoint ModelEndpoint

ModelEndpoint object.

required client

A LaunchClient object

required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
predict(request: EndpointRequest) -> EndpointResponseFuture\n

Runs an asynchronous prediction request.

Parameters:

Name Type Description Default request EndpointRequest

The EndpointRequest object that contains the payload.

required

Returns:

Name Type Description EndpointResponseFuture

An EndpointResponseFuture such the user can use to query the status of the request.

Example EndpointResponseFuture EndpointResponseFuture

.. code-block:: python

my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

(deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

Parameters:

Name Type Description Default requests Sequence[EndpointRequest]

List of EndpointRequests. Request_ids must all be distinct.

required

Returns:

Type Description AsyncEndpointBatchResponse

an AsyncEndpointResponse keeping track of the inference requests made

"},{"location":"concepts/batch_jobs/","title":"Batch jobs","text":""},{"location":"concepts/batch_jobs/#batch-jobs","title":"Batch Jobs","text":"

For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

Batch jobs are created using the batch_async_request method of the LaunchClient.

Creating and Following a Batch Job
import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
"},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

Represents a single request to either a SyncEndpoint or AsyncEndpoint.

Parameters:

Name Type Description Default url Optional[str]

A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

Exactly one of url and args must be specified.

None args Optional[Dict]

A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

Exactly one of url and args must be specified.

None return_pickled Optional[bool]

Whether the output should be a pickled python object, or directly returned serialized json.

False request_id Optional[str]

(deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

This object should not be directly instantiated by the user.

Parameters:

Name Type Description Default client

An instance of LaunchClient.

required endpoint_name str

The name of the endpoint.

required async_task_id str

An async task id.

required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
get(timeout: Optional[float] = None) -> EndpointResponse\n

Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

Parameters:

Name Type Description Default timeout Optional[float]

The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

Represents a response received from a Endpoint.

Parameters:

Name Type Description Default client

An instance of LaunchClient.

required status str

A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

required result_url Optional[str]

A string that is a url containing the pickled python object from the Endpoint's predict function.

Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

None result Optional[str]

A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

None traceback Optional[str]

The stack trace if the inference endpoint raised an error. Can be used for debugging

None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

"},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
"},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

Creating Model Bundles with app_config
import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
"},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

"},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

To list all the model bundles you own, use list_model_bundles.

"},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

"},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

Creating an Async Model Endpoint
import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
"},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints, and they require at least 1 min_worker.

Creating a Sync Model Endpoint
import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
"},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

Model endpoints can be listed, updated, and deleted using the Launch API.

Listing Model Endpoints
import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
Updating a Model Endpoint
import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
Deleting a Model Endpoint
import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
"}]} \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml new file mode 100644 index 00000000..31a79a51 --- /dev/null +++ b/sitemap.xml @@ -0,0 +1,48 @@ + + + + https://example.com/ + 2023-02-24 + daily + + + https://example.com/cli/ + 2023-02-24 + daily + + + https://example.com/api/client/ + 2023-02-24 + daily + + + https://example.com/api/model_bundles/ + 2023-02-24 + daily + + + https://example.com/api/model_endpoints/ + 2023-02-24 + daily + + + https://example.com/concepts/batch_jobs/ + 2023-02-24 + daily + + + https://example.com/concepts/endpoint_predictions/ + 2023-02-24 + daily + + + https://example.com/concepts/model_bundles/ + 2023-02-24 + daily + + + https://example.com/concepts/model_endpoints/ + 2023-02-24 + daily + + \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..eeadd3e86d63b8e52b456d1786ecb43f2b7bcffb GIT binary patch literal 277 zcmV+w0qXuAiwFoNYWQOU|8r?{Wo=<_E_iKh0L_%mPQ)M-h4=Fm4f{f^F4VNl&La&x==+34zoAu4Bkaz`6dp#>+EoqFw6XClm*BWr$U8>U#_?i-P z44#GY8Ywu#I=0?yjUp!FByXGczHZvOeaM!Zwz0u1?1XOtKe!Z1e|Bu6TnX3@!e3!P ztFnNES1T0SR32A;2lX%>t<)Gx#X~E-Q#KU_2+jifOf`xG^YHG*QUSj|aPR@8f|RF5 bsi*}v Date: Fri, 24 Feb 2023 03:47:58 -0800 Subject: [PATCH 02/28] Deployed ebea46c with MkDocs version: 1.4.2 --- 404.html | 98 +- api/client/index.html | 66 +- api/endpoint_predictions/index.html | 1037 ++++++++++++++++++++++ api/hooks/index.html | 679 ++++++++++++++ api/model_bundles/index.html | 60 +- api/model_endpoints/index.html | 221 ++++- cli/index.html | 62 +- concepts/batch_jobs/index.html | 106 ++- concepts/callbacks/index.html | 642 ++++++++++++++ concepts/endpoint_predictions/index.html | 60 +- concepts/model_bundles/index.html | 62 +- concepts/model_endpoints/index.html | 67 +- concepts/overview/index.html | 598 +++++++++++++ index.html | 60 +- objects.inv | Bin 497 -> 533 bytes search/search_index.json | 2 +- sitemap.xml | 38 +- sitemap.xml.gz | Bin 277 -> 320 bytes 18 files changed, 3758 insertions(+), 100 deletions(-) create mode 100644 api/endpoint_predictions/index.html create mode 100644 api/hooks/index.html create mode 100644 concepts/callbacks/index.html create mode 100644 concepts/overview/index.html diff --git a/404.html b/404.html index 89dd6b76..e174bd99 100644 --- a/404.html +++ b/404.html @@ -12,7 +12,7 @@ - + @@ -21,10 +21,10 @@ - + - + @@ -40,9 +40,9 @@ - + - + @@ -80,9 +80,9 @@
@@ -1746,7 +1802,7 @@

Optional[List[PostInferenceHooks]] + Optional[List[PostInferenceHooks]]

List of hooks to trigger after inference tasks are served.

@@ -2029,7 +2085,7 @@

post_inference_hooks - Optional[List[PostInferenceHooks]] + Optional[List[PostInferenceHooks]]

List of hooks to trigger after inference tasks are served.

diff --git a/api/endpoint_predictions/index.html b/api/endpoint_predictions/index.html new file mode 100644 index 00000000..9147e194 --- /dev/null +++ b/api/endpoint_predictions/index.html @@ -0,0 +1,1037 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Endpoint Predictions - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Endpoint Predictions

+ + +
+ + + +

+ EndpointRequest + + +

+
EndpointRequest(
+    url: Optional[str] = None,
+    args: Optional[Dict] = None,
+    callback_url: Optional[str] = None,
+    return_pickled: Optional[bool] = False,
+    request_id: Optional[str] = None,
+)
+
+ +
+ + +

Represents a single request to either a SyncEndpoint or AsyncEndpoint.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
url + Optional[str] +

A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. +Note: the contents of the file located at url are opened as a sequence of bytes and passed +to the predict function. If you instead want to pass the url itself as an input to the predict function, +see args.

+

Exactly one of url and args must be specified.

 + None +
args + Optional[Dict] +

A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature +predict_fn(foo, bar), then the keys in the dictionary should be "foo" and "bar". +Values must be native Python objects.

+

Exactly one of url and args must be specified.

 + None +
return_pickled + Optional[bool] +

Whether the output should be a pickled python object, or directly returned serialized json.

 + False +
request_id + Optional[str] +

(deprecated) A user-specifiable id for requests. +Should be unique among EndpointRequests made in the same batch call. +If one isn't provided the client will generate its own.

 + None +
+ + + + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + +

+ EndpointResponse + + +

+
EndpointResponse(
+    client,
+    status: str,
+    result_url: Optional[str] = None,
+    result: Optional[str] = None,
+    traceback: Optional[str] = None,
+)
+
+ +
+ + +

Represents a response received from a Endpoint.

+ + + +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
client +

An instance of LaunchClient.

 + required +
status + str +

A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

 + required +
result_url + Optional[str] +

A string that is a url containing the pickled python object from the +Endpoint's predict function.

+

Exactly one of result_url or result will be populated, +depending on the value of return_pickled in the request.

 + None +
result + Optional[str] +

A string that is the serialized return value (in json form) of the Endpoint's predict function. +Specifically, one can json.loads() the value of result to get the original python object back.

+

Exactly one of result_url or result will be populated, +depending on the value of return_pickled in the request.

 + None +
traceback + Optional[str] +

The stack trace if the inference endpoint raised an error. Can be used for debugging

 + None +
+ + + + +
+ + + + + + + + + + + +
+ +
+ +
+ +
+ + + +

+ EndpointResponseFuture + + +

+
EndpointResponseFuture(
+    client, endpoint_name: str, async_task_id: str
+)
+
+ +
+ + +

Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, +then its get method will return an actual instance of EndpointResponse.

+

This object should not be directly instantiated by the user.

+ + + +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
client +

An instance of LaunchClient.

 + required +
endpoint_name + str +

The name of the endpoint.

 + required +
async_task_id + str +

An async task id.

 + required +
+ + + + +
+ + + + + + + + + +
+ + + +

+ get + + +

+
get(timeout: Optional[float] = None) -> EndpointResponse
+
+ +
+ +

Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

+ +

Parameters:

+ + + + + + + + + + + + + + + + + +
NameTypeDescriptionDefault
timeout + Optional[float] +

The maximum number of seconds to wait for the response. If None, then +the method will block indefinitely until the response is ready.

 + None +
+ +
+ +
+ + + +
+ +
+ +
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/api/hooks/index.html b/api/hooks/index.html new file mode 100644 index 00000000..9c22d181 --- /dev/null +++ b/api/hooks/index.html @@ -0,0 +1,679 @@ + + + + + + + + + + + + + + + + + + + + + + + + Hooks - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + +
+ + +
+ +
+ + + + + + +
+
+ + + +
+
+
+ + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + +

Hooks

+ +
+ + + +

+ PostInferenceHooks + + +

+ + +
+

+ Bases: str, Enum

+ + +

Post-inference hooks are functions that are called after inference is complete.

+ +

Attributes:

+ + + + + + + + + + + + + + + +
NameTypeDescription
CALLBACK + str +

The callback hook is called with the inference response and the task ID.

+ + + + + +
+ + + + + + + + + + + +
+ +
+ +
+ + + + + + +
+
+ + + + +
+ +
+ + + +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/api/model_bundles/index.html b/api/model_bundles/index.html index ae70d58b..cf3d560c 100644 --- a/api/model_bundles/index.html +++ b/api/model_bundles/index.html @@ -10,7 +10,7 @@ - + @@ -296,6 +296,20 @@ +
  • + + Overview + +
    • + + + + + + + + +
  • Model Bundles @@ -340,7 +354,21 @@
  • - Batch jobs + Batch Jobs + +
    • + + + + + + + + + +
  • + + Callbacks
    • @@ -529,6 +557,34 @@ + + + + + +
  • + + Endpoint Predictions + +
    • + + + + + + + + + +
  • + + Hooks + +
    • + + + + diff --git a/api/model_endpoints/index.html b/api/model_endpoints/index.html index 412dd6e2..47323783 100644 --- a/api/model_endpoints/index.html +++ b/api/model_endpoints/index.html @@ -10,12 +10,14 @@ - + + + @@ -294,6 +296,20 @@ +
  • + + Overview + +
    • + + + + + + + + +
  • Model Bundles @@ -338,7 +354,21 @@
  • - Batch jobs + Batch Jobs + +
    • + + + + + + + + + +
  • + + Callbacks
    • @@ -474,6 +504,26 @@ + + +
  • + + SyncEndpoint + + + +
    • @@ -485,6 +535,34 @@ + + + + + +
  • + + Endpoint Predictions + +
    • + + + + + + + + + +
  • + + Hooks + +
    • + + + + @@ -542,6 +620,26 @@ + + +
  • + + SyncEndpoint + + + +
    • @@ -776,6 +874,125 @@

    + + + +

    + SyncEndpoint + + +

    +
    SyncEndpoint(model_endpoint: ModelEndpoint, client)
+
    + +
    +

    + Bases: Endpoint

    + + +

    A synchronous model endpoint.

    + + + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    model_endpoint + ModelEndpoint +

    ModelEndpoint object.

    		 + required +
    client +

    A LaunchClient object

    		 + required +
    + + + + +
    + + + + + + + + + +
    + + + +

    + predict + + +

    +
    predict(request: EndpointRequest) -> EndpointResponse
+
    + +
    + +

    Runs a synchronous prediction request.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    request + EndpointRequest +

    The EndpointRequest object that contains the payload.

    		 + required +
    + +
    + +
    + + +
    diff --git a/cli/index.html b/cli/index.html index d39cfb20..4a624f5a 100644 --- a/cli/index.html +++ b/cli/index.html @@ -10,13 +10,13 @@ - + - + @@ -333,6 +333,20 @@ +
  • + + Overview + +
    • + + + + + + + + +
  • Model Bundles @@ -377,7 +391,21 @@
  • - Batch jobs + Batch Jobs + +
    • + + + + + + + + + +
  • + + Callbacks
    • @@ -461,6 +489,34 @@ + + + + + +
  • + + Endpoint Predictions + +
    • + + + + + + + + + +
  • + + Hooks + +
    • + + + + diff --git a/concepts/batch_jobs/index.html b/concepts/batch_jobs/index.html index 25f6d761..ed844795 100644 --- a/concepts/batch_jobs/index.html +++ b/concepts/batch_jobs/index.html @@ -10,20 +10,20 @@ - + - + - Batch jobs - Launch + Batch Jobs - Launch @@ -109,7 +109,7 @@
    - Batch jobs + Batch Jobs
    @@ -298,6 +298,20 @@ +
  • + + Overview + +
    • + + + + + + + + +
  • Model Bundles @@ -347,40 +361,27 @@ + - - Batch jobs + Batch Jobs - - -
    • + + + + + - - - - +
  • + + Callbacks +
    • @@ -463,6 +464,34 @@ + + + + + +
  • + + Endpoint Predictions + +
    • + + + + + + + + + +
  • + + Hooks + +
    • + + + + @@ -486,21 +515,8 @@ + - - @@ -515,9 +531,7 @@ -

    Batch jobs

    - -

    Batch Jobs

    +

    Batch Jobs

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a diff --git a/concepts/callbacks/index.html b/concepts/callbacks/index.html new file mode 100644 index 00000000..de680de9 --- /dev/null +++ b/concepts/callbacks/index.html @@ -0,0 +1,642 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Callbacks - Launch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    +
    + +
    + + + + +
    + + +
    + +
    + + + + + + +
    +
    + + + +
    +
    +
    + + + + +
    +
    +
    + + + +
    +
    +
    + + + +
    +
    +
    + + + +
    +
    + + + + +

    Callbacks

    +

    Async model endpoints can be configured to send callbacks to a user-defined +callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The +following code snippet shows how to create an async model endpoint with a +callback URL.

    +

    To configure an async endpoint to send callbacks, set the post_inference_hooks +field to include +launch.PostInferenceHooks.CALLBACK. +A callback URL also needs to be specified, and it can be configured as a default +using the default_callback_url argument to +launch.LaunchClient.create_model_endpoint. +or as a per-task override using the callback_url field of +launch.EndpointRequest.

    +
    +

    Note

    +

    Callbacks will not be sent if the endpoint does not have any post-inference +hooks specified, even if a default_callback_url is provided to the endpoint +creation method or if the prediction request has a callback_url override.

    +
    +
    Creating an Async Model Endpoint with a Callback URL
    import os
+from launch import EndpointRequest, LaunchClient, PostInferenceHooks
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.create_model_endpoint(
+    endpoint_name="demo-endpoint-callback",
+    model_bundle="test-bundle",
+    cpus=1,
+    min_workers=1,
+    endpoint_type="async",
+    update_if_exists=True,
+    labels={
+        "team": "MY_TEAM",
+        "product": "MY_PRODUCT"
+    },
+    post_inference_hooks=[PostInferenceHooks.CALLBACK],
+    default_callback_url="https://example.com",
+)
+
+future_default = endpoint.predict(request=EndpointRequest(args={"x": 2, "y": "hello"}))
+"""
+A callback is sent to https://example.com with the following JSON body:
+{
+    "task_id": "THE_TASK_ID",
+    "result": 7
+}
+"""
+
+
+future_custom_callback_url = endpoint.predict(
+    request=EndpointRequest(
+        args={"x": 3, "y": "hello"}, callback_url="https://example.com/custom"
+    ),
+)
+
+"""
+A callback is sent to https://example.com/custom with the following JSON body:
+{
+    "task_id": "THE_TASK_ID",
+    "result": 8
+}
+"""
+
    + + + + + + +
    +
    + + + + +
    + +
    + + + +
    +
    +
    +
    + + + + + + + + + \ No newline at end of file diff --git a/concepts/endpoint_predictions/index.html b/concepts/endpoint_predictions/index.html index 1af35a71..a9e8ed7c 100644 --- a/concepts/endpoint_predictions/index.html +++ b/concepts/endpoint_predictions/index.html @@ -10,7 +10,7 @@ - + @@ -298,6 +298,20 @@ +
  • + + Overview + +
    • + + + + + + + + +
  • Model Bundles @@ -410,7 +424,21 @@
  • - Batch jobs + Batch Jobs + +
    • + + + + + + + + + +
  • + + Callbacks
    • @@ -494,6 +522,34 @@ + + + + + +
  • + + Endpoint Predictions + +
    • + + + + + + + + + +
  • + + Hooks + +
    • + + + + diff --git a/concepts/model_bundles/index.html b/concepts/model_bundles/index.html index 1dd31922..dcfd4b95 100644 --- a/concepts/model_bundles/index.html +++ b/concepts/model_bundles/index.html @@ -10,10 +10,10 @@ - + - + @@ -297,6 +297,20 @@ + +
  • + + Overview + +
    • + + + + + + + + @@ -404,7 +418,21 @@
  • - Batch jobs + Batch Jobs + +
    • + + + + + + + + + +
  • + + Callbacks
    • @@ -488,6 +516,34 @@ + + + + + +
  • + + Endpoint Predictions + +
    • + + + + + + + + + +
  • + + Hooks + +
    • + + + + diff --git a/concepts/model_endpoints/index.html b/concepts/model_endpoints/index.html index b0afcb58..b524ad79 100644 --- a/concepts/model_endpoints/index.html +++ b/concepts/model_endpoints/index.html @@ -10,7 +10,7 @@ - + @@ -298,6 +298,20 @@ +
  • + + Overview + +
    • + + + + + + + + +
  • Model Bundles @@ -397,7 +411,21 @@
  • - Batch jobs + Batch Jobs + +
    • + + + + + + + + + +
  • + + Callbacks
    • @@ -481,6 +509,34 @@ + + + + + +
  • + + Endpoint Predictions + +
    • + + + + + + + + + +
  • + + Hooks + +
    • + + + + @@ -582,8 +638,11 @@

    Creating Async Model EndpointsCreating Sync Model Endpoints

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time -inference. Sync endpoints are more expensive than async endpoints, and they -require at least 1 min_worker.

    +inference. Sync endpoints are more expensive than async endpoints.

    +
    +

    Note

    +

    Sync model endpoints require at least 1 min_worker.

    +
    Creating a Sync Model Endpoint
    import os
 from launch import LaunchClient
 
diff --git a/concepts/overview/index.html b/concepts/overview/index.html
new file mode 100644
index 00000000..ed3c7516
--- /dev/null
+++ b/concepts/overview/index.html
@@ -0,0 +1,598 @@
+
+
+
+  
+    
+      
+      
+      
+        
+      
+      
+      
+        
+      
+      
+        
+      
+      
+        
+      
+      
+      
+    
+    
+      
+        Overview - Launch
+      
+    
+    
+      
+      
+        
+        
+      
+      
+
+    
+    
+    
+      
+        
+        
+        
+        
+        
+      
+    
+    
+      
+    
+    
+    
+      
+
+    
+    
+    
+  
+  
+  
+    
+    
+      
+    
+    
+    
+    
+    
+  
+    
+    
+      
+    
+    
+    
+    
+    
+    
    
+      
+    
    
+    
+    
+      
+
+
    
+  
+  
+
    
+    
+    
    
+      
+      
+        
+          
+        
+      
+      
    
+        
    
+          
+            
+              
+              
    
+                
    
+                  
    
+                    
+
+
+
+                  
    
+                
    
+              
    
+            
+            
+              
+              
    
+                
    
+                  
    
+                    
+
+
+                  
    
+                
    
+              
    
+            
+          
+          
+            
    
+              
    
+                
+                  
+
+
+
    Overview
    
+
    Creating deployments on Launch generally involves three steps:
    
+
      
+
    1. 
+
      Create and upload a ModelBundle. Pass your trained model
+   as well as pre-/post-processing code to the Scale Launch Python client, and 
+   we’ll create a model bundle based on the code and store it in our Bundle Store.
      
+
      2. 
+
    2. 
+
      Create a ModelEndpoint. Pass a ModelBundle as well as
+   infrastructure settings such as the desired number of GPUs to our client.
+   This provisions resources on Scale’s cluster dedicated to your ModelEndpoint.
      
+
      3. 
+
    3. 
+
      Make requests to the ModelEndpoint. You can make requests through the Python
+   client, or make HTTP requests directly to Scale.
      
+
      4. 
+
    
+
+
+
+
+
+                
+              
    
+            
    
+          
+          
+  
+
+        
    
+        
+      
    
+      
+        
+      
+    
    
+    
    
+      
    
+    
    
+    
+    
+    
+    
+      
+      
+    
+  
+
\ No newline at end of file
diff --git a/index.html b/index.html
index 248fb455..e0210afd 100644
--- a/index.html
+++ b/index.html
@@ -10,7 +10,7 @@
       
       
       
-        
+        
       
       
       
@@ -342,6 +342,20 @@
   
   
   
+    
  • 
+      
+        Overview
+      
+    
    • 
+  
+
+            
+          
+            
+              
+  
+  
+  
     
  • 
       
         Model Bundles
@@ -386,7 +400,21 @@
   
     
  • 
       
-        Batch jobs
+        Batch Jobs
+      
+    
    • 
+  
+
+            
+          
+            
+              
+  
+  
+  
+    
  • 
+      
+        Callbacks
       
     
    • 
   
@@ -470,6 +498,34 @@
 
             
           
+            
+              
+  
+  
+  
+    
  • 
+      
+        Endpoint Predictions
+      
+    
    • 
+  
+
+            
+          
+            
+              
+  
+  
+  
+    
  • 
+      
+        Hooks
+      
+    
    • 
+  
+
+            
+          
         
       
     
diff --git a/objects.inv b/objects.inv
index 8de294524e0cfe4484b8b9888381b42f3c89f4b1..e43ed22faff7f5befaa9ffd329ff598edc3168bd 100644
GIT binary patch
delta 420
zcmV;V0bBm@1C<1jdw*X~!ypg<@OM8&6MdD%SKrjkXrhU6AAsEGwI!nm0-f2nm;PC|
z&Y4ufx3R!)2ge;#(2%H@P!h?rmu~dX4KlO46$+foZZa@((N9@iUS-!8>c>@V4PI0n
z+>Qjd_|c&2q#{i5Qq%7+p)?_{b&iU!8_E^(Pw6WzDxeZw1b=K1{8Ubg@xtnw=hCK%
z5h*uXQp8yX1ru_ZOP>rY(evrWaYmEFI5ACj4^sR-m2}gvToP*^ScDetT0T0wyL!Rk
zN|#Y+8G=MRE_eCV|Gg
z#h656LydO6n}25>YA!kY*!af&C*79m$&t?gg-Yv=jaULvS5
zUNImxP)aJf-ztF~txoh9l<y3j~
OX|}wI*zp(6BO?+*Qq|o6

delta 383
zcmV-_0f7FM1n~oqdwP?NF^x_MUS(vs&Iv~(B`}WfBtshBO
z@JQ1Bn8NU<8d?%H6Ivk!_S}yi`axljw?c#W#Z3WL?S@BomsiF0h5q#|wuc~U4*p1j
zJA7}^_fi=j3ChrUn9zohH%6l7`#5@ummmknw%Zpw_Y^3HdPW@
z#Ujy-dn^1Qf?|+M%Ha-&Eic+yZQ^1qHD^`Fc-Tmu_6mFH2KRG2pN8|Qr#1-;tmVQ`
zsj3cx5ZvD_q<90j;Gps8mXhhl)DY=Xg8?K2NS6p2Ojj(3H>ear?5>WhohYaPv-0Li!*(7JX8X0e0`47ao#x5
dMsv9Rp4!kl{71UG+rBJAwi(VW`x9yp)~=?b!e9UZ

diff --git a/search/search_index.json b/search/search_index.json
index 26bf536e..82a01c37 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"
    Simple, scalable, and high performance ML service deployment in python.
    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n
     
    What's going on here:
     
       
    • First we use pydantic to define our request and response   schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API   documentation for our models.
      •  
    • Next we define the the model and the load_predict_fn, which tells Launch   how to load our model and how to make predictions with it. In this case,   we're just returning a function that adds the length of the string y to    model(x), where model doubles the integer x.
      •  
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the   response_schema. We also specify the env_params, which tell Launch environment settings like    the base image to use. In this case, we're using a PyTorch image.
      •  
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We   specify the model_bundle that we created above, and we specify the endpoint_type, which tells   Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an   asynchronous endpoint, which means that we can make predictions and return immediately with a   future object. We can then use the future object to get the prediction result later.
      •  
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object.   This function first waits for the endpoint to be ready, then it submits a prediction request to   the endpoint. It then waits for the prediction result and returns it.
      •  
     
    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.
    "},{"location":"#installation","title":"Installation","text":"
    To use Scale Launch, first install it using pip:
     Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"
    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.
     
    The CLI can be used as scale-launch ....
    "},{"location":"cli/#help","title":"Help","text":"
    Run scale-launch --help for more options.
     scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your Scale...\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n
     
    Scale Launch Python Client.
     
    Initializes a Scale Launch Client.
     
    Parameters:
     Name Type Description Default api_key str 
    Your Scale API key
     required endpoint Optional[str] 
    The Scale Launch Endpoint (this should not need to be changed)
     None self_hosted bool 
    True iff you are connecting to a self-hosted Scale Launch
     False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n
     
    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.
     
    Must have exactly one of urls or inputs passed in.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or the name of a the bundle to use for inference.
     required urls List[str] 
    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.
     None inputs Optional[List[Dict[str, Any]]] 
    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.
     None batch_url_file_location Optional[str] 
    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()
     None serialization_format str 
    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning
     'JSON' labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None cpus Optional[int] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than   per_worker, then the number of workers will be reduced.
      •  
    • Otherwise, if the average number of concurrent requests per worker is higher   than per_worker, then the number of workers will be increased to meet the   elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains job_id as a key, and the ID as the value.
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n
     
    Clones an existing model bundle with changes to its app config. (More fields coming soon)
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The existing bundle or its ID.
     required app_config Optional[Dict] 
    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None
     None 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Uploads and registers a model bundle to Scale Launch.
     
    A model bundle consists of exactly one of the following:
     
       
    • predict_fn_or_cls
      •  
    • load_predict_fn + model
      •  
    • load_predict_fn + load_model_fn
      •  
     
    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required predict_fn_or_cls Optional[Callable[[Any], Any]] 
    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.
     None model Optional[LaunchModel_T] 
    Typically a trained Neural Network, e.g. a Pytorch module.
     
    Exactly one of model and load_model_fn must be provided.
     None load_model_fn Optional[Callable[[], LaunchModel_T]] 
    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.
     
    Exactly one of model and load_model_fn must be provided.
     None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]] 
    Function that, when called with a model, returns a function that carries out inference.
     
    If model is specified, then this is equivalent to:     load_predict_fn(model, app_config=optional_app_config]) -> predict_fn
     
    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn
     
    In both cases, predict_fn is then the inference function, i.e.:     predict_fn(REQUEST) -> RESPONSE
     None requirements Optional[List[str]] 
    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.
     
    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]
     
    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.
     None app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
    •  
      Tensorflow fields:
       
         
      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
        •  
       
      •  
     required globals_copy Optional[Dict[str, Any]] 
    Dictionary of the global symbol table. Normally provided by globals() built-in function.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.
     
    For example, if you have a directory structure like so, and your current working directory is also my_root:
     
    .. code-block:: text
     
    my_root/        my_module1/            init.py            ...files and directories            my_inference_file.py        my_module2/            init.py            ...files and directories
     
    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:
     
    .. code-block:: text
     
    my_module1/        init.py        ...files and directories        my_inference_file.py    my_module2/        init.py        ...files and directories
     
    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required base_paths List[str] 
    The paths on the local filesystem where the bundle code lives.
     required requirements_path str 
    A path on the local filesystem where a requirements.txt file lives.
     required env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
     required load_predict_fn_module_path str 
    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.
     required load_model_fn_module_path str 
    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.
     required app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n
     
    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.
     required model_bundle Union[ModelBundle, str] 
    The ModelBundle that the endpoint should serve.
     required cpus int 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     3 memory str 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     '8Gi' storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus int 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     0 min_workers int 
    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.
     1 max_workers int 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.
     1 per_worker int 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     
    Here is our recommendation for computing per_worker:
     
       
    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      2.  
     10 gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None endpoint_type str 
    Either \"sync\" or \"async\".
     'sync' post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None update_if_exists bool 
    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.
     False labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None 
    Returns:
     Type Description Optional[Endpoint] 
    A Endpoint object that can be used to make requests to the endpoint.
    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Deletes a model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    A ModelEndpoint object.
     required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n
     
    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:
     
       
    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.
      •  
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.
     required model_bundle Optional[Union[ModelBundle, str]] 
    The ModelBundle that the endpoint should serve.
     None cpus Optional[float] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None min_workers Optional[int] 
    The minimum number of workers. Must be greater than or equal to 0.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n
     
    Gets inference results from a previously created batch job.
     
    Parameters:
     Name Type Description Default batch_job_id str 
    An id representing the batch task job. This id is the in the response from calling batch_async_request.
     required 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains the following fields:
     Dict[str, Any] 
       
    • status: The status of the job.
      •  
     Dict[str, Any] 
       
    • result: The url where the result is stored.
      •  
     Dict[str, Any] 
       
    • duration: A string representation of how long the job took to finish     or how long it has been running, for a job current in progress.
      •  
     Dict[str, Any] 
       
    • num_tasks_pending: The number of tasks that are still pending.
      •  
     Dict[str, Any] 
       
    • num_tasks_completed: The number of tasks that have completed.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n
     
    Returns a model bundle specified by bundle_name that the user owns.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or its name.
     required 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n
     
    Gets a model endpoint associated with a name.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the endpoint to retrieve.
     required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n
     
    Returns a list of model bundles that the user owns.
     
    Returns:
     Type Description List[ModelBundle] 
    A list of ModelBundle objects
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n
     
    Lists all model endpoints that the user owns.
     
    Returns:
     Type Description List[Endpoint] 
    A list of ModelEndpoint objects.
    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Retrieves the logs for the creation of the endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The endpoint or its name.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.
     
    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).
     
    Parameters:
     Name Type Description Default batch_csv_location_fn Callable[[], str] 
    Function that generates batch_csv_urls for upload_batch_csv_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.
     
    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).
     
    Parameters:
     Name Type Description Default bundle_location_fn Callable[[], str] 
    Function that generates bundle_urls for upload_bundle_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as
     
    upload_batch_csv_fn(csv_text, csv_url)\n
     
    This function should directly write the contents of csv_text as a text string into csv_url.
     
    Parameters:
     Name Type Description Default upload_batch_csv_fn Callable[[str, str], None] 
    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as
     
    upload_bundle_fn(serialized_bundle, bundle_url)\n
     
    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.
     
    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn
     
    Parameters:
     Name Type Description Default upload_bundle_fn Callable[[str, str], None] 
    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle  dataclass","text":"
    Represents a ModelBundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n
     
    An optional user-specified configuration mapping for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params  class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n
     
    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id  class-attribute","text":"
    id: Optional[str] = None\n
     
    A globally unique identifier for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location  class-attribute","text":"
    location: Optional[str] = None\n
     
    An opaque location for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata  class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n
     
    Arbitrary metadata for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name  class-attribute","text":"
    name: str\n
     
    The name of the bundle. Must be unique across all bundles that the user owns.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type  class-attribute","text":"
    packaging_type: Optional[str] = None\n
     
    The packaging type for the bundle. Can be cloudpickle or zip.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements  class-attribute","text":"
    requirements: Optional[List[str]] = None\n
     
    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"
    All classes here are returned by the  get_model_endpoint method and provide a predict function.
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    An asynchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n
     
    Runs an asynchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required 
    Returns:
     Name Type Description EndpointResponseFuture 
    An EndpointResponseFuture such the user can use to query the status of the request.
     Example EndpointResponseFuture EndpointResponseFuture 
    .. code-block:: python
     
    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get()  # blocks on completion
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n
     
    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.
     
    Parameters:
     Name Type Description Default requests Sequence[EndpointRequest] 
    List of EndpointRequests. Request_ids must all be distinct.
     required 
    Returns:
     Type Description AsyncEndpointBatchResponse 
    an AsyncEndpointResponse keeping track of the inference requests made
    "},{"location":"concepts/batch_jobs/","title":"Batch jobs","text":""},{"location":"concepts/batch_jobs/#batch-jobs","title":"Batch Jobs","text":"
    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.
     
    Batch jobs are created using the batch_async_request method of the LaunchClient.
     Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"
    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.
     Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
     
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"
    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format. 
    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"
    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.
     Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
     
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"
    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.
     Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"
    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.
    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"
    To list all the model bundles you own, use list_model_bundles.
    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"
    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.
     
    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"
    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.
     Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"
    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints, and they require at least 1 min_worker.
     Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"
    Model endpoints can be listed, updated, and deleted using the Launch API.
     Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
     Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
     Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"
    Simple, scalable, and high performance ML service deployment in python.
    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n
     
    What's going on here:
     
       
    • First we use pydantic to define our request and response   schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API   documentation for our models.
      •  
    • Next we define the the model and the load_predict_fn, which tells Launch   how to load our model and how to make predictions with it. In this case,   we're just returning a function that adds the length of the string y to    model(x), where model doubles the integer x.
      •  
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the   response_schema. We also specify the env_params, which tell Launch environment settings like    the base image to use. In this case, we're using a PyTorch image.
      •  
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We   specify the model_bundle that we created above, and we specify the endpoint_type, which tells   Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an   asynchronous endpoint, which means that we can make predictions and return immediately with a   future object. We can then use the future object to get the prediction result later.
      •  
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object.   This function first waits for the endpoint to be ready, then it submits a prediction request to   the endpoint. It then waits for the prediction result and returns it.
      •  
     
    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.
    "},{"location":"#installation","title":"Installation","text":"
    To use Scale Launch, first install it using pip:
     Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"
    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.
     
    The CLI can be used as scale-launch ....
    "},{"location":"cli/#help","title":"Help","text":"
    Run scale-launch --help for more options.
     scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your Scale...\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n
     
    Scale Launch Python Client.
     
    Initializes a Scale Launch Client.
     
    Parameters:
     Name Type Description Default api_key str 
    Your Scale API key
     required endpoint Optional[str] 
    The Scale Launch Endpoint (this should not need to be changed)
     None self_hosted bool 
    True iff you are connecting to a self-hosted Scale Launch
     False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n
     
    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.
     
    Must have exactly one of urls or inputs passed in.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or the name of a the bundle to use for inference.
     required urls List[str] 
    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.
     None inputs Optional[List[Dict[str, Any]]] 
    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.
     None batch_url_file_location Optional[str] 
    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()
     None serialization_format str 
    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning
     'JSON' labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None cpus Optional[int] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than   per_worker, then the number of workers will be reduced.
      •  
    • Otherwise, if the average number of concurrent requests per worker is higher   than per_worker, then the number of workers will be increased to meet the   elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains job_id as a key, and the ID as the value.
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n
     
    Clones an existing model bundle with changes to its app config. (More fields coming soon)
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The existing bundle or its ID.
     required app_config Optional[Dict] 
    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None
     None 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Uploads and registers a model bundle to Scale Launch.
     
    A model bundle consists of exactly one of the following:
     
       
    • predict_fn_or_cls
      •  
    • load_predict_fn + model
      •  
    • load_predict_fn + load_model_fn
      •  
     
    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required predict_fn_or_cls Optional[Callable[[Any], Any]] 
    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.
     None model Optional[LaunchModel_T] 
    Typically a trained Neural Network, e.g. a Pytorch module.
     
    Exactly one of model and load_model_fn must be provided.
     None load_model_fn Optional[Callable[[], LaunchModel_T]] 
    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.
     
    Exactly one of model and load_model_fn must be provided.
     None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]] 
    Function that, when called with a model, returns a function that carries out inference.
     
    If model is specified, then this is equivalent to:     load_predict_fn(model, app_config=optional_app_config]) -> predict_fn
     
    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn
     
    In both cases, predict_fn is then the inference function, i.e.:     predict_fn(REQUEST) -> RESPONSE
     None requirements Optional[List[str]] 
    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.
     
    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]
     
    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.
     None app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
    •  
      Tensorflow fields:
       
         
      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
        •  
       
      •  
     required globals_copy Optional[Dict[str, Any]] 
    Dictionary of the global symbol table. Normally provided by globals() built-in function.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.
     
    For example, if you have a directory structure like so, and your current working directory is also my_root:
     
    .. code-block:: text
     
    my_root/        my_module1/            init.py            ...files and directories            my_inference_file.py        my_module2/            init.py            ...files and directories
     
    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:
     
    .. code-block:: text
     
    my_module1/        init.py        ...files and directories        my_inference_file.py    my_module2/        init.py        ...files and directories
     
    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required base_paths List[str] 
    The paths on the local filesystem where the bundle code lives.
     required requirements_path str 
    A path on the local filesystem where a requirements.txt file lives.
     required env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
     required load_predict_fn_module_path str 
    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.
     required load_model_fn_module_path str 
    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.
     required app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n
     
    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.
     required model_bundle Union[ModelBundle, str] 
    The ModelBundle that the endpoint should serve.
     required cpus int 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     3 memory str 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     '8Gi' storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus int 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     0 min_workers int 
    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.
     1 max_workers int 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.
     1 per_worker int 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     
    Here is our recommendation for computing per_worker:
     
       
    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      2.  
     10 gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None endpoint_type str 
    Either \"sync\" or \"async\".
     'sync' post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None update_if_exists bool 
    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.
     False labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None 
    Returns:
     Type Description Optional[Endpoint] 
    A Endpoint object that can be used to make requests to the endpoint.
    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Deletes a model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    A ModelEndpoint object.
     required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n
     
    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:
     
       
    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.
      •  
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.
     required model_bundle Optional[Union[ModelBundle, str]] 
    The ModelBundle that the endpoint should serve.
     None cpus Optional[float] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None min_workers Optional[int] 
    The minimum number of workers. Must be greater than or equal to 0.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n
     
    Gets inference results from a previously created batch job.
     
    Parameters:
     Name Type Description Default batch_job_id str 
    An id representing the batch task job. This id is the in the response from calling batch_async_request.
     required 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains the following fields:
     Dict[str, Any] 
       
    • status: The status of the job.
      •  
     Dict[str, Any] 
       
    • result: The url where the result is stored.
      •  
     Dict[str, Any] 
       
    • duration: A string representation of how long the job took to finish     or how long it has been running, for a job current in progress.
      •  
     Dict[str, Any] 
       
    • num_tasks_pending: The number of tasks that are still pending.
      •  
     Dict[str, Any] 
       
    • num_tasks_completed: The number of tasks that have completed.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n
     
    Returns a model bundle specified by bundle_name that the user owns.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or its name.
     required 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n
     
    Gets a model endpoint associated with a name.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the endpoint to retrieve.
     required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n
     
    Returns a list of model bundles that the user owns.
     
    Returns:
     Type Description List[ModelBundle] 
    A list of ModelBundle objects
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n
     
    Lists all model endpoints that the user owns.
     
    Returns:
     Type Description List[Endpoint] 
    A list of ModelEndpoint objects.
    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Retrieves the logs for the creation of the endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The endpoint or its name.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.
     
    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).
     
    Parameters:
     Name Type Description Default batch_csv_location_fn Callable[[], str] 
    Function that generates batch_csv_urls for upload_batch_csv_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.
     
    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).
     
    Parameters:
     Name Type Description Default bundle_location_fn Callable[[], str] 
    Function that generates bundle_urls for upload_bundle_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as
     
    upload_batch_csv_fn(csv_text, csv_url)\n
     
    This function should directly write the contents of csv_text as a text string into csv_url.
     
    Parameters:
     Name Type Description Default upload_batch_csv_fn Callable[[str, str], None] 
    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as
     
    upload_bundle_fn(serialized_bundle, bundle_url)\n
     
    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.
     
    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn
     
    Parameters:
     Name Type Description Default upload_bundle_fn Callable[[str, str], None] 
    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"
             Bases: str, Enum
     
    Post-inference hooks are functions that are called after inference is complete.
     
    Attributes:
     Name Type Description CALLBACK str 
    The callback hook is called with the inference response and the task ID.
    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle  dataclass","text":"
    Represents a ModelBundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n
     
    An optional user-specified configuration mapping for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params  class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n
     
    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id  class-attribute","text":"
    id: Optional[str] = None\n
     
    A globally unique identifier for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location  class-attribute","text":"
    location: Optional[str] = None\n
     
    An opaque location for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata  class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n
     
    Arbitrary metadata for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name  class-attribute","text":"
    name: str\n
     
    The name of the bundle. Must be unique across all bundles that the user owns.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type  class-attribute","text":"
    packaging_type: Optional[str] = None\n
     
    The packaging type for the bundle. Can be cloudpickle or zip.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements  class-attribute","text":"
    requirements: Optional[List[str]] = None\n
     
    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"
    All classes here are returned by the  get_model_endpoint method and provide a predict function.
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    An asynchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n
     
    Runs an asynchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required 
    Returns:
     Name Type Description EndpointResponseFuture 
    An EndpointResponseFuture such the user can use to query the status of the request.
     Example EndpointResponseFuture EndpointResponseFuture 
    .. code-block:: python
     
    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get()  # blocks on completion
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n
     
    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.
     
    Parameters:
     Name Type Description Default requests Sequence[EndpointRequest] 
    List of EndpointRequests. Request_ids must all be distinct.
     required 
    Returns:
     Type Description AsyncEndpointBatchResponse 
    an AsyncEndpointResponse keeping track of the inference requests made
    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    A synchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n
     
    Runs a synchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"
    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.
     
    Batch jobs are created using the batch_async_request method of the LaunchClient.
     Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"
    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.
     
    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint. or as a per-task override using the callback_url field of launch.EndpointRequest.
     
    Note
     
    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.
     Creating an Async Model Endpoint with a Callback URL
    import os\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nfuture_default = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"
    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.
     Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
     
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"
    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format. 
    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"
    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.
     Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
     
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"
    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.
     Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"
    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.
    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"
    To list all the model bundles you own, use list_model_bundles.
    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"
    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.
     
    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"
    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.
     Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"
    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.
     
    Note
     
    Sync model endpoints require at least 1 min_worker.
     Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"
    Model endpoints can be listed, updated, and deleted using the Launch API.
     Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
     Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
     Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"
    Creating deployments on Launch generally involves three steps:
     
       
    1.  
      Create and upload a ModelBundle. Pass your trained model    as well as pre-/post-processing code to the Scale Launch Python client, and     we\u2019ll create a model bundle based on the code and store it in our Bundle Store.
       
      2.  
    2.  
      Create a ModelEndpoint. Pass a ModelBundle as well as    infrastructure settings such as the desired number of GPUs to our client.    This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.
       
      3.  
    3.  
      Make requests to the ModelEndpoint. You can make requests through the Python    client, or make HTTP requests directly to Scale.
       
      4.  
    "}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 31a79a51..e9e95501 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -1,47 +1,67 @@
 
 
     
-         https://example.com/
+         https://scaleapi.github.io/launch-python-client/
          2023-02-24
          daily
     
     
-         https://example.com/cli/
+         https://scaleapi.github.io/launch-python-client/cli/
          2023-02-24
          daily
     
     
-         https://example.com/api/client/
+         https://scaleapi.github.io/launch-python-client/api/client/
          2023-02-24
          daily
     
     
-         https://example.com/api/model_bundles/
+         https://scaleapi.github.io/launch-python-client/api/endpoint_predictions/
          2023-02-24
          daily
     
     
-         https://example.com/api/model_endpoints/
+         https://scaleapi.github.io/launch-python-client/api/hooks/
          2023-02-24
          daily
     
     
-         https://example.com/concepts/batch_jobs/
+         https://scaleapi.github.io/launch-python-client/api/model_bundles/
          2023-02-24
          daily
     
     
-         https://example.com/concepts/endpoint_predictions/
+         https://scaleapi.github.io/launch-python-client/api/model_endpoints/
          2023-02-24
          daily
     
     
-         https://example.com/concepts/model_bundles/
+         https://scaleapi.github.io/launch-python-client/concepts/batch_jobs/
          2023-02-24
          daily
     
     
-         https://example.com/concepts/model_endpoints/
+         https://scaleapi.github.io/launch-python-client/concepts/callbacks/
+         2023-02-24
+         daily
+    
+    
+         https://scaleapi.github.io/launch-python-client/concepts/endpoint_predictions/
+         2023-02-24
+         daily
+    
+    
+         https://scaleapi.github.io/launch-python-client/concepts/model_bundles/
+         2023-02-24
+         daily
+    
+    
+         https://scaleapi.github.io/launch-python-client/concepts/model_endpoints/
+         2023-02-24
+         daily
+    
+    
+         https://scaleapi.github.io/launch-python-client/concepts/overview/
          2023-02-24
          daily
     
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index eeadd3e86d63b8e52b456d1786ecb43f2b7bcffb..5e834fd03d8b342abb25796d96a3df62ab723fbc 100644
GIT binary patch
literal 320
zcmV-G0l)qqiwFqqqxfS2|8r?{Wo=<_E_iKh0M(YkZo?o9hVOlfhY<|Th?uSb

literal 277
zcmV+w0qXuAiwFoNYWQOU|8r?{Wo=<_E_iKh0L_%mPQ)M-h4=Fm4f{f^F4VNl&La&x==+34zoAu4Bkaz`6dp#>+EoqFw6XClm*BWr$U8>U#_?i-P
z44#GY8Ywu#I=0?yjUp!FByXGczHZvOeaM!Zwz0u1?1XOtKe!Z1e|Bu6TnX3@!e3!P
ztFnNES1T0SR32A;2lX%>t<)Gx#X~E-Q#KU_2+jifOf`xG^YHG*QUSj|aPR@8f|RF5
bsi*}v
Date: Fri, 24 Feb 2023 11:02:29 -0800
Subject: [PATCH 03/28] Deployed fc294c2 with MkDocs version: 1.4.2

---
 concepts/callbacks/index.html     |  12 ++++++------
 concepts/model_bundles/index.html |   4 ++--
 search/search_index.json          |   2 +-
 sitemap.xml.gz                    | Bin 320 -> 320 bytes
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/concepts/callbacks/index.html b/concepts/callbacks/index.html
index de680de9..90ec50b7 100644
--- a/concepts/callbacks/index.html
+++ b/concepts/callbacks/index.html
@@ -541,7 +541,7 @@ 
    Callbackslaunch.PostInferenceHooks.CALLBACK.
 A callback URL also needs to be specified, and it can be configured as a default
 using the default_callback_url argument to
-launch.LaunchClient.create_model_endpoint.
+launch.LaunchClient.create_model_endpoint
 or as a per-task override using the callback_url field of
 launch.EndpointRequest.
    
 
    
@@ -565,9 +565,9 @@ 
    Callbacks        "team": "MY_TEAM",
         "product": "MY_PRODUCT"
     },
-    post_inference_hooks=[PostInferenceHooks.CALLBACK],
-    default_callback_url="https://example.com",
-)
+    post_inference_hooks=[PostInferenceHooks.CALLBACK],
+    default_callback_url="https://example.com",
+)
 
 future_default = endpoint.predict(request=EndpointRequest(args={"x": 2, "y": "hello"}))
 """
@@ -581,8 +581,8 @@ 
    Callbacks
 future_custom_callback_url = endpoint.predict(
     request=EndpointRequest(
-        args={"x": 3, "y": "hello"}, callback_url="https://example.com/custom"
-    ),
+        args={"x": 3, "y": "hello"}, callback_url="https://example.com/custom"
+    ),
 )
 
 """
diff --git a/concepts/model_bundles/index.html b/concepts/model_bundles/index.html
index dcfd4b95..29fd1367 100644
--- a/concepts/model_bundles/index.html
+++ b/concepts/model_bundles/index.html
@@ -748,7 +748,7 @@ 
    Creating Model Bundles}
 
 client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
-client.create_model_bundle(**BUNDLE_PARAMS)
+client.create_model_bundle_from_dirs(**BUNDLE_PARAMS)
 
 # Clean up files from demo
 os.remove(model_filename)
@@ -838,7 +838,7 @@ 
    Configuring Model Bundles    "requirements": [],
     "request_schema": MyRequestSchema,
     "response_schema": MyResponseSchema,
-    "app_config": {"mode": "single"},
+    "app_config": {"mode": "batched"},
 }
 
 client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
diff --git a/search/search_index.json b/search/search_index.json
index 82a01c37..e76e3011 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"
    Simple, scalable, and high performance ML service deployment in python.
    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n
     
    What's going on here:
     
       
    • First we use pydantic to define our request and response   schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API   documentation for our models.
      •  
    • Next we define the the model and the load_predict_fn, which tells Launch   how to load our model and how to make predictions with it. In this case,   we're just returning a function that adds the length of the string y to    model(x), where model doubles the integer x.
      •  
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the   response_schema. We also specify the env_params, which tell Launch environment settings like    the base image to use. In this case, we're using a PyTorch image.
      •  
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We   specify the model_bundle that we created above, and we specify the endpoint_type, which tells   Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an   asynchronous endpoint, which means that we can make predictions and return immediately with a   future object. We can then use the future object to get the prediction result later.
      •  
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object.   This function first waits for the endpoint to be ready, then it submits a prediction request to   the endpoint. It then waits for the prediction result and returns it.
      •  
     
    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.
    "},{"location":"#installation","title":"Installation","text":"
    To use Scale Launch, first install it using pip:
     Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"
    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.
     
    The CLI can be used as scale-launch ....
    "},{"location":"cli/#help","title":"Help","text":"
    Run scale-launch --help for more options.
     scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your Scale...\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n
     
    Scale Launch Python Client.
     
    Initializes a Scale Launch Client.
     
    Parameters:
     Name Type Description Default api_key str 
    Your Scale API key
     required endpoint Optional[str] 
    The Scale Launch Endpoint (this should not need to be changed)
     None self_hosted bool 
    True iff you are connecting to a self-hosted Scale Launch
     False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n
     
    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.
     
    Must have exactly one of urls or inputs passed in.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or the name of a the bundle to use for inference.
     required urls List[str] 
    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.
     None inputs Optional[List[Dict[str, Any]]] 
    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.
     None batch_url_file_location Optional[str] 
    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()
     None serialization_format str 
    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning
     'JSON' labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None cpus Optional[int] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than   per_worker, then the number of workers will be reduced.
      •  
    • Otherwise, if the average number of concurrent requests per worker is higher   than per_worker, then the number of workers will be increased to meet the   elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains job_id as a key, and the ID as the value.
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n
     
    Clones an existing model bundle with changes to its app config. (More fields coming soon)
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The existing bundle or its ID.
     required app_config Optional[Dict] 
    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None
     None 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Uploads and registers a model bundle to Scale Launch.
     
    A model bundle consists of exactly one of the following:
     
       
    • predict_fn_or_cls
      •  
    • load_predict_fn + model
      •  
    • load_predict_fn + load_model_fn
      •  
     
    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required predict_fn_or_cls Optional[Callable[[Any], Any]] 
    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.
     None model Optional[LaunchModel_T] 
    Typically a trained Neural Network, e.g. a Pytorch module.
     
    Exactly one of model and load_model_fn must be provided.
     None load_model_fn Optional[Callable[[], LaunchModel_T]] 
    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.
     
    Exactly one of model and load_model_fn must be provided.
     None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]] 
    Function that, when called with a model, returns a function that carries out inference.
     
    If model is specified, then this is equivalent to:     load_predict_fn(model, app_config=optional_app_config]) -> predict_fn
     
    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn
     
    In both cases, predict_fn is then the inference function, i.e.:     predict_fn(REQUEST) -> RESPONSE
     None requirements Optional[List[str]] 
    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.
     
    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]
     
    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.
     None app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
    •  
      Tensorflow fields:
       
         
      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
        •  
       
      •  
     required globals_copy Optional[Dict[str, Any]] 
    Dictionary of the global symbol table. Normally provided by globals() built-in function.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.
     
    For example, if you have a directory structure like so, and your current working directory is also my_root:
     
    .. code-block:: text
     
    my_root/        my_module1/            init.py            ...files and directories            my_inference_file.py        my_module2/            init.py            ...files and directories
     
    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:
     
    .. code-block:: text
     
    my_module1/        init.py        ...files and directories        my_inference_file.py    my_module2/        init.py        ...files and directories
     
    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required base_paths List[str] 
    The paths on the local filesystem where the bundle code lives.
     required requirements_path str 
    A path on the local filesystem where a requirements.txt file lives.
     required env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
     required load_predict_fn_module_path str 
    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.
     required load_model_fn_module_path str 
    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.
     required app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n
     
    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.
     required model_bundle Union[ModelBundle, str] 
    The ModelBundle that the endpoint should serve.
     required cpus int 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     3 memory str 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     '8Gi' storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus int 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     0 min_workers int 
    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.
     1 max_workers int 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.
     1 per_worker int 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     
    Here is our recommendation for computing per_worker:
     
       
    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      2.  
     10 gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None endpoint_type str 
    Either \"sync\" or \"async\".
     'sync' post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None update_if_exists bool 
    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.
     False labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None 
    Returns:
     Type Description Optional[Endpoint] 
    A Endpoint object that can be used to make requests to the endpoint.
    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Deletes a model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    A ModelEndpoint object.
     required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n
     
    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:
     
       
    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.
      •  
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.
     required model_bundle Optional[Union[ModelBundle, str]] 
    The ModelBundle that the endpoint should serve.
     None cpus Optional[float] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None min_workers Optional[int] 
    The minimum number of workers. Must be greater than or equal to 0.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n
     
    Gets inference results from a previously created batch job.
     
    Parameters:
     Name Type Description Default batch_job_id str 
    An id representing the batch task job. This id is the in the response from calling batch_async_request.
     required 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains the following fields:
     Dict[str, Any] 
       
    • status: The status of the job.
      •  
     Dict[str, Any] 
       
    • result: The url where the result is stored.
      •  
     Dict[str, Any] 
       
    • duration: A string representation of how long the job took to finish     or how long it has been running, for a job current in progress.
      •  
     Dict[str, Any] 
       
    • num_tasks_pending: The number of tasks that are still pending.
      •  
     Dict[str, Any] 
       
    • num_tasks_completed: The number of tasks that have completed.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n
     
    Returns a model bundle specified by bundle_name that the user owns.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or its name.
     required 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n
     
    Gets a model endpoint associated with a name.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the endpoint to retrieve.
     required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n
     
    Returns a list of model bundles that the user owns.
     
    Returns:
     Type Description List[ModelBundle] 
    A list of ModelBundle objects
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n
     
    Lists all model endpoints that the user owns.
     
    Returns:
     Type Description List[Endpoint] 
    A list of ModelEndpoint objects.
    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Retrieves the logs for the creation of the endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The endpoint or its name.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.
     
    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).
     
    Parameters:
     Name Type Description Default batch_csv_location_fn Callable[[], str] 
    Function that generates batch_csv_urls for upload_batch_csv_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.
     
    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).
     
    Parameters:
     Name Type Description Default bundle_location_fn Callable[[], str] 
    Function that generates bundle_urls for upload_bundle_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as
     
    upload_batch_csv_fn(csv_text, csv_url)\n
     
    This function should directly write the contents of csv_text as a text string into csv_url.
     
    Parameters:
     Name Type Description Default upload_batch_csv_fn Callable[[str, str], None] 
    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as
     
    upload_bundle_fn(serialized_bundle, bundle_url)\n
     
    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.
     
    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn
     
    Parameters:
     Name Type Description Default upload_bundle_fn Callable[[str, str], None] 
    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"
             Bases: str, Enum
     
    Post-inference hooks are functions that are called after inference is complete.
     
    Attributes:
     Name Type Description CALLBACK str 
    The callback hook is called with the inference response and the task ID.
    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle  dataclass","text":"
    Represents a ModelBundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n
     
    An optional user-specified configuration mapping for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params  class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n
     
    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id  class-attribute","text":"
    id: Optional[str] = None\n
     
    A globally unique identifier for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location  class-attribute","text":"
    location: Optional[str] = None\n
     
    An opaque location for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata  class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n
     
    Arbitrary metadata for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name  class-attribute","text":"
    name: str\n
     
    The name of the bundle. Must be unique across all bundles that the user owns.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type  class-attribute","text":"
    packaging_type: Optional[str] = None\n
     
    The packaging type for the bundle. Can be cloudpickle or zip.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements  class-attribute","text":"
    requirements: Optional[List[str]] = None\n
     
    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"
    All classes here are returned by the  get_model_endpoint method and provide a predict function.
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    An asynchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n
     
    Runs an asynchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required 
    Returns:
     Name Type Description EndpointResponseFuture 
    An EndpointResponseFuture such the user can use to query the status of the request.
     Example EndpointResponseFuture EndpointResponseFuture 
    .. code-block:: python
     
    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get()  # blocks on completion
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n
     
    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.
     
    Parameters:
     Name Type Description Default requests Sequence[EndpointRequest] 
    List of EndpointRequests. Request_ids must all be distinct.
     required 
    Returns:
     Type Description AsyncEndpointBatchResponse 
    an AsyncEndpointResponse keeping track of the inference requests made
    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    A synchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n
     
    Runs a synchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"
    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.
     
    Batch jobs are created using the batch_async_request method of the LaunchClient.
     Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"
    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.
     
    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint. or as a per-task override using the callback_url field of launch.EndpointRequest.
     
    Note
     
    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.
     Creating an Async Model Endpoint with a Callback URL
    import os\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nfuture_default = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"
    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.
     Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
     
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"
    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format. 
    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"
    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.
     Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
     
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"
    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.
     Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"
    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.
    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"
    To list all the model bundles you own, use list_model_bundles.
    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"
    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.
     
    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"
    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.
     Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"
    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.
     
    Note
     
    Sync model endpoints require at least 1 min_worker.
     Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"
    Model endpoints can be listed, updated, and deleted using the Launch API.
     Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
     Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
     Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"
    Creating deployments on Launch generally involves three steps:
     
       
    1.  
      Create and upload a ModelBundle. Pass your trained model    as well as pre-/post-processing code to the Scale Launch Python client, and     we\u2019ll create a model bundle based on the code and store it in our Bundle Store.
       
      2.  
    2.  
      Create a ModelEndpoint. Pass a ModelBundle as well as    infrastructure settings such as the desired number of GPUs to our client.    This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.
       
      3.  
    3.  
      Make requests to the ModelEndpoint. You can make requests through the Python    client, or make HTTP requests directly to Scale.
       
      4.  
    "}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"
    Simple, scalable, and high performance ML service deployment in python.
    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n
     
    What's going on here:
     
       
    • First we use pydantic to define our request and response   schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API   documentation for our models.
      •  
    • Next we define the the model and the load_predict_fn, which tells Launch   how to load our model and how to make predictions with it. In this case,   we're just returning a function that adds the length of the string y to    model(x), where model doubles the integer x.
      •  
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the   response_schema. We also specify the env_params, which tell Launch environment settings like    the base image to use. In this case, we're using a PyTorch image.
      •  
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We   specify the model_bundle that we created above, and we specify the endpoint_type, which tells   Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an   asynchronous endpoint, which means that we can make predictions and return immediately with a   future object. We can then use the future object to get the prediction result later.
      •  
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object.   This function first waits for the endpoint to be ready, then it submits a prediction request to   the endpoint. It then waits for the prediction result and returns it.
      •  
     
    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.
    "},{"location":"#installation","title":"Installation","text":"
    To use Scale Launch, first install it using pip:
     Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"
    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.
     
    The CLI can be used as scale-launch ....
    "},{"location":"cli/#help","title":"Help","text":"
    Run scale-launch --help for more options.
     scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your Scale...\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n
     
    Scale Launch Python Client.
     
    Initializes a Scale Launch Client.
     
    Parameters:
     Name Type Description Default api_key str 
    Your Scale API key
     required endpoint Optional[str] 
    The Scale Launch Endpoint (this should not need to be changed)
     None self_hosted bool 
    True iff you are connecting to a self-hosted Scale Launch
     False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n
     
    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.
     
    Must have exactly one of urls or inputs passed in.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or the name of a the bundle to use for inference.
     required urls List[str] 
    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.
     None inputs Optional[List[Dict[str, Any]]] 
    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.
     None batch_url_file_location Optional[str] 
    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()
     None serialization_format str 
    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning
     'JSON' labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None cpus Optional[int] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than   per_worker, then the number of workers will be reduced.
      •  
    • Otherwise, if the average number of concurrent requests per worker is higher   than per_worker, then the number of workers will be increased to meet the   elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains job_id as a key, and the ID as the value.
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n
     
    Clones an existing model bundle with changes to its app config. (More fields coming soon)
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The existing bundle or its ID.
     required app_config Optional[Dict] 
    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None
     None 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Uploads and registers a model bundle to Scale Launch.
     
    A model bundle consists of exactly one of the following:
     
       
    • predict_fn_or_cls
      •  
    • load_predict_fn + model
      •  
    • load_predict_fn + load_model_fn
      •  
     
    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required predict_fn_or_cls Optional[Callable[[Any], Any]] 
    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.
     None model Optional[LaunchModel_T] 
    Typically a trained Neural Network, e.g. a Pytorch module.
     
    Exactly one of model and load_model_fn must be provided.
     None load_model_fn Optional[Callable[[], LaunchModel_T]] 
    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.
     
    Exactly one of model and load_model_fn must be provided.
     None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]] 
    Function that, when called with a model, returns a function that carries out inference.
     
    If model is specified, then this is equivalent to:     load_predict_fn(model, app_config=optional_app_config]) -> predict_fn
     
    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn
     
    In both cases, predict_fn is then the inference function, i.e.:     predict_fn(REQUEST) -> RESPONSE
     None requirements Optional[List[str]] 
    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.
     
    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]
     
    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.
     None app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
    •  
      Tensorflow fields:
       
         
      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
        •  
       
      •  
     required globals_copy Optional[Dict[str, Any]] 
    Dictionary of the global symbol table. Normally provided by globals() built-in function.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     
    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.
     
    For example, if you have a directory structure like so, and your current working directory is also my_root:
     
    .. code-block:: text
     
    my_root/        my_module1/            init.py            ...files and directories            my_inference_file.py        my_module2/            init.py            ...files and directories
     
    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:
     
    .. code-block:: text
     
    my_module1/        init.py        ...files and directories        my_inference_file.py    my_module2/        init.py        ...files and directories
     
    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required base_paths List[str] 
    The paths on the local filesystem where the bundle code lives.
     required requirements_path str 
    A path on the local filesystem where a requirements.txt file lives.
     required env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
     required load_predict_fn_module_path str 
    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.
     required load_model_fn_module_path str 
    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.
     required app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n
     
    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.
     required model_bundle Union[ModelBundle, str] 
    The ModelBundle that the endpoint should serve.
     required cpus int 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     3 memory str 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     '8Gi' storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus int 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     0 min_workers int 
    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.
     1 max_workers int 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.
     1 per_worker int 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     
    Here is our recommendation for computing per_worker:
     
       
    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      2.  
     10 gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None endpoint_type str 
    Either \"sync\" or \"async\".
     'sync' post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None update_if_exists bool 
    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.
     False labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None 
    Returns:
     Type Description Optional[Endpoint] 
    A Endpoint object that can be used to make requests to the endpoint.
    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Deletes a model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    A ModelEndpoint object.
     required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n
     
    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:
     
       
    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.
      •  
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.
     required model_bundle Optional[Union[ModelBundle, str]] 
    The ModelBundle that the endpoint should serve.
     None cpus Optional[float] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None min_workers Optional[int] 
    The minimum number of workers. Must be greater than or equal to 0.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n
     
    Gets inference results from a previously created batch job.
     
    Parameters:
     Name Type Description Default batch_job_id str 
    An id representing the batch task job. This id is the in the response from calling batch_async_request.
     required 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains the following fields:
     Dict[str, Any] 
       
    • status: The status of the job.
      •  
     Dict[str, Any] 
       
    • result: The url where the result is stored.
      •  
     Dict[str, Any] 
       
    • duration: A string representation of how long the job took to finish     or how long it has been running, for a job current in progress.
      •  
     Dict[str, Any] 
       
    • num_tasks_pending: The number of tasks that are still pending.
      •  
     Dict[str, Any] 
       
    • num_tasks_completed: The number of tasks that have completed.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n
     
    Returns a model bundle specified by bundle_name that the user owns.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or its name.
     required 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n
     
    Gets a model endpoint associated with a name.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the endpoint to retrieve.
     required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n
     
    Returns a list of model bundles that the user owns.
     
    Returns:
     Type Description List[ModelBundle] 
    A list of ModelBundle objects
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n
     
    Lists all model endpoints that the user owns.
     
    Returns:
     Type Description List[Endpoint] 
    A list of ModelEndpoint objects.
    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Retrieves the logs for the creation of the endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The endpoint or its name.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.
     
    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).
     
    Parameters:
     Name Type Description Default batch_csv_location_fn Callable[[], str] 
    Function that generates batch_csv_urls for upload_batch_csv_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.
     
    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).
     
    Parameters:
     Name Type Description Default bundle_location_fn Callable[[], str] 
    Function that generates bundle_urls for upload_bundle_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as
     
    upload_batch_csv_fn(csv_text, csv_url)\n
     
    This function should directly write the contents of csv_text as a text string into csv_url.
     
    Parameters:
     Name Type Description Default upload_batch_csv_fn Callable[[str, str], None] 
    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as
     
    upload_bundle_fn(serialized_bundle, bundle_url)\n
     
    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.
     
    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn
     
    Parameters:
     Name Type Description Default upload_bundle_fn Callable[[str, str], None] 
    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"
             Bases: str, Enum
     
    Post-inference hooks are functions that are called after inference is complete.
     
    Attributes:
     Name Type Description CALLBACK str 
    The callback hook is called with the inference response and the task ID.
    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle  dataclass","text":"
    Represents a ModelBundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n
     
    An optional user-specified configuration mapping for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params  class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n
     
    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id  class-attribute","text":"
    id: Optional[str] = None\n
     
    A globally unique identifier for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location  class-attribute","text":"
    location: Optional[str] = None\n
     
    An opaque location for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata  class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n
     
    Arbitrary metadata for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name  class-attribute","text":"
    name: str\n
     
    The name of the bundle. Must be unique across all bundles that the user owns.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type  class-attribute","text":"
    packaging_type: Optional[str] = None\n
     
    The packaging type for the bundle. Can be cloudpickle or zip.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements  class-attribute","text":"
    requirements: Optional[List[str]] = None\n
     
    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"
    All classes here are returned by the  get_model_endpoint method and provide a predict function.
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    An asynchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n
     
    Runs an asynchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required 
    Returns:
     Name Type Description EndpointResponseFuture 
    An EndpointResponseFuture such the user can use to query the status of the request.
     Example EndpointResponseFuture EndpointResponseFuture 
    .. code-block:: python
     
    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get()  # blocks on completion
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n
     
    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.
     
    Parameters:
     Name Type Description Default requests Sequence[EndpointRequest] 
    List of EndpointRequests. Request_ids must all be distinct.
     required 
    Returns:
     Type Description AsyncEndpointBatchResponse 
    an AsyncEndpointResponse keeping track of the inference requests made
    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    A synchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n
     
    Runs a synchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"
    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.
     
    Batch jobs are created using the batch_async_request method of the LaunchClient.
     Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"
    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.
     
    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.
     
    Note
     
    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.
     Creating an Async Model Endpoint with a Callback URL
    import os\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nfuture_default = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"
    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.
     Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
     
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"
    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format. 
    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"
    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.
     Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
     
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"
    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.
     Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"
    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.
    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"
    To list all the model bundles you own, use list_model_bundles.
    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"
    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.
     
    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"
    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.
     Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"
    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.
     
    Note
     
    Sync model endpoints require at least 1 min_worker.
     Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"
    Model endpoints can be listed, updated, and deleted using the Launch API.
     Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
     Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
     Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"
    Creating deployments on Launch generally involves three steps:
     
       
    1.  
      Create and upload a ModelBundle. Pass your trained model    as well as pre-/post-processing code to the Scale Launch Python client, and     we\u2019ll create a model bundle based on the code and store it in our Bundle Store.
       
      2.  
    2.  
      Create a ModelEndpoint. Pass a ModelBundle as well as    infrastructure settings such as the desired number of GPUs to our client.    This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.
       
      3.  
    3.  
      Make requests to the ModelEndpoint. You can make requests through the Python    client, or make HTTP requests directly to Scale.
       
      4.  
    "}]}
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 5e834fd03d8b342abb25796d96a3df62ab723fbc..46c73ab37155902cb454b649834bc213c932f3ed 100644
GIT binary patch
delta 16
XcmX@WbbyInzMF&N2
Date: Fri, 24 Feb 2023 12:56:45 -0800
Subject: [PATCH 04/28] Deployed fa35db0 with MkDocs version: 1.4.2

---
 cli/index.html           |   2 +-
 search/search_index.json |   2 +-
 sitemap.xml.gz           | Bin 320 -> 320 bytes
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cli/index.html b/cli/index.html
index 4a624f5a..7e221c0f 100644
--- a/cli/index.html
+++ b/cli/index.html
@@ -593,7 +593,7 @@ 
    Help&par
 Commands:
   batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch
   bundles     Bundles is a wrapper around model bundles in Scale Launch
-  config      Config is a wrapper around getting and setting your Scale...
+  config      Config is a wrapper around getting and setting your API key and other configuration options
   endpoints   Endpoints is a wrapper around model endpoints in Scale Launch
   tasks       Tasks is a wrapper around sending requests to endpoints
    diff --git a/search/search_index.json b/search/search_index.json index e76e3011..ef8cb31e 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your Scale...\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Uploads and registers a model bundle to Scale Launch.

    A model bundle consists of exactly one of the following:

    • predict_fn_or_cls
    • load_predict_fn + model
    • load_predict_fn + load_model_fn

    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is also my_root:

    .. code-block:: text

    my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

    .. code-block:: text

    my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nfuture_default = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Uploads and registers a model bundle to Scale Launch.

    A model bundle consists of exactly one of the following:

    • predict_fn_or_cls
    • load_predict_fn + model
    • load_predict_fn + load_model_fn

    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is also my_root:

    .. code-block:: text

    my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

    .. code-block:: text

    my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nfuture_default = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 46c73ab37155902cb454b649834bc213c932f3ed..49c6d680f4ea5c0b0f4f19944651c102b1db0ce6 100644 GIT binary patch delta 15 WcmX@WbbyIXzMF%iM`a_M6(ay6T?6L; delta 15 WcmX@WbbyIXzMF&N2 Date: Wed, 8 Mar 2023 00:50:15 -0800 Subject: [PATCH 05/28] Deployed b2a4b61 with MkDocs version: 1.4.2 --- api/client/index.html | 144 ++++++++++++++++- api/endpoint_predictions/index.html | 85 +++++++++- concepts/batch_jobs/index.html | 26 +-- concepts/callbacks/index.html | 191 ++++++++++++++++++----- concepts/endpoint_predictions/index.html | 85 +++++++++- concepts/model_bundles/index.html | 153 +++++++++--------- concepts/model_endpoints/index.html | 101 +++--------- index.html | 4 +- search/search_index.json | 2 +- sitemap.xml | 26 +-- sitemap.xml.gz | Bin 320 -> 319 bytes 11 files changed, 586 insertions(+), 231 deletions(-) diff --git a/api/client/index.html b/api/client/index.html index 1bedafc2..bdd3f96a 100644 --- a/api/client/index.html +++ b/api/client/index.html @@ -1635,9 +1635,16 @@

    List[PostInferenceHooks] ] = None, default_callback_url: Optional[str] = None, - update_if_exists: bool = False, - labels: Optional[Dict[str, str]] = None -) -> Optional[Endpoint] + default_callback_auth_kind: Optional[ + Literal[basic, mtls] + ] = None, + default_callback_auth_username: Optional[str] = None, + default_callback_auth_password: Optional[str] = None, + default_callback_auth_cert: Optional[str] = None, + default_callback_auth_key: Optional[str] = None, + update_if_exists: bool = False, + labels: Optional[Dict[str, str]] = None +) -> Optional[Endpoint]
    @@ -1821,6 +1828,66 @@

    Optional[Literal[basic, mtls]] + +

    The default callback auth kind to use for async endpoints. +Either "basic" or "mtls". This can be overridden in the task parameters for each +individual task.

    + + None + + + + default_callback_auth_username + + Optional[str] + +

    The default callback auth username to use. This only +applies if default_callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + default_callback_auth_password + + Optional[str] + +

    The default callback auth password to use. This only +applies if default_callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + default_callback_auth_cert + + Optional[str] + +

    The default callback auth cert to use. This only applies +if default_callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + + + default_callback_auth_key + + Optional[str] + +

    The default callback auth key to use. This only applies +if default_callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + update_if_exists @@ -1939,8 +2006,15 @@

    post_inference_hooks: Optional[ List[PostInferenceHooks] ] = None, - default_callback_url: Optional[str] = None -) -> None + default_callback_url: Optional[str] = None, + default_callback_auth_kind: Optional[ + Literal[basic, mtls] + ] = None, + default_callback_auth_username: Optional[str] = None, + default_callback_auth_password: Optional[str] = None, + default_callback_auth_cert: Optional[str] = None, + default_callback_auth_key: Optional[str] = None +) -> None

    @@ -2104,6 +2178,66 @@

    None + + default_callback_auth_kind + + Optional[Literal[basic, mtls]] + +

    The default callback auth kind to use for async endpoints. +Either "basic" or "mtls". This can be overridden in the task parameters for each +individual task.

    + + None + + + + default_callback_auth_username + + Optional[str] + +

    The default callback auth username to use. This only +applies if default_callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + default_callback_auth_password + + Optional[str] + +

    The default callback auth password to use. This only +applies if default_callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + default_callback_auth_cert + + Optional[str] + +

    The default callback auth cert to use. This only applies +if default_callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + + + default_callback_auth_key + + Optional[str] + +

    The default callback auth key to use. This only applies +if default_callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + diff --git a/api/endpoint_predictions/index.html b/api/endpoint_predictions/index.html index 9147e194..1f89282f 100644 --- a/api/endpoint_predictions/index.html +++ b/api/endpoint_predictions/index.html @@ -647,9 +647,16 @@

    url: Optional[str] = None, args: Optional[Dict] = None, callback_url: Optional[str] = None, - return_pickled: Optional[bool] = False, - request_id: Optional[str] = None, -) + callback_auth_kind: Optional[ + Literal[basic, mtls] + ] = None, + callback_auth_username: Optional[str] = None, + callback_auth_password: Optional[str] = None, + callback_auth_cert: Optional[str] = None, + callback_auth_key: Optional[str] = None, + return_pickled: Optional[bool] = False, + request_id: Optional[str] = None, +)

    @@ -705,6 +712,78 @@

    False + + callback_url + + Optional[str] + +

    The callback url to use for this task. If None, then the +default_callback_url of the endpoint is used. The endpoint must specify +"callback" as a post-inference hook for the callback to be triggered.

    + + None + + + + callback_auth_kind + + Optional[Literal[basic, mtls]] + +

    The default callback auth kind to use for async endpoints. +Either "basic" or "mtls". This can be overridden in the task parameters for each +individual task.

    + + None + + + + callback_auth_username + + Optional[str] + +

    The default callback auth username to use. This only +applies if callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + callback_auth_password + + Optional[str] + +

    The default callback auth password to use. This only +applies if callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + callback_auth_cert + + Optional[str] + +

    The default callback auth cert to use. This only applies +if callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + + + callback_auth_key + + Optional[str] + +

    The default callback auth key to use. This only applies +if callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + request_id diff --git a/concepts/batch_jobs/index.html b/concepts/batch_jobs/index.html index ed844795..07e046d4 100644 --- a/concepts/batch_jobs/index.html +++ b/concepts/batch_jobs/index.html @@ -551,17 +551,21 @@

    Batch Jobs {"x": 2, "y": "hello"}, {"x": 3, "y": "world"}, ], -) - -status = "PENDING" -res = None -while status != "SUCCESS" and status != "FAILURE" and status != "CANCELLED": - time.sleep(30) - res = client.get_batch_async_response(batch_job["job_id"]) - status = res["status"] - print(f"the batch job is {status}") - -print(res) + labels={ + "team": "MY_TEAM", + "product": "MY_PRODUCT", + } +) + +status = "PENDING" +res = None +while status != "SUCCESS" and status != "FAILURE" and status != "CANCELLED": + time.sleep(30) + res = client.get_batch_async_response(batch_job["job_id"]) + status = res["status"] + print(f"the batch job is {status}") + +print(res)

    diff --git a/concepts/callbacks/index.html b/concepts/callbacks/index.html index 90ec50b7..5ad2bcf1 100644 --- a/concepts/callbacks/index.html +++ b/concepts/callbacks/index.html @@ -378,10 +378,41 @@ + + Callbacks + + + + @@ -518,6 +549,21 @@ + + + @@ -551,47 +597,112 @@

    CallbacksCreating an Async Model Endpoint with a Callback URL
    import os
-from launch import EndpointRequest, LaunchClient, PostInferenceHooks
-
-client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
-endpoint = client.create_model_endpoint(
-    endpoint_name="demo-endpoint-callback",
-    model_bundle="test-bundle",
-    cpus=1,
-    min_workers=1,
-    endpoint_type="async",
-    update_if_exists=True,
-    labels={
-        "team": "MY_TEAM",
-        "product": "MY_PRODUCT"
-    },
-    post_inference_hooks=[PostInferenceHooks.CALLBACK],
-    default_callback_url="https://example.com",
-)
-
-future_default = endpoint.predict(request=EndpointRequest(args={"x": 2, "y": "hello"}))
-"""
-A callback is sent to https://example.com with the following JSON body:
-{
-    "task_id": "THE_TASK_ID",
-    "result": 7
-}
+import time
+from launch import EndpointRequest, LaunchClient, PostInferenceHooks
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.create_model_endpoint(
+    endpoint_name="demo-endpoint-callback",
+    model_bundle="test-bundle",
+    cpus=1,
+    min_workers=1,
+    endpoint_type="async",
+    update_if_exists=True,
+    labels={
+        "team": "MY_TEAM",
+        "product": "MY_PRODUCT",
+    },
+    post_inference_hooks=[PostInferenceHooks.CALLBACK],
+    default_callback_url="https://example.com",
+)
+
+while endpoint.status() != "READY":
+    time.sleep(10)
+
+future_default = endpoint.predict(
+    request=EndpointRequest(args={"x": 2, "y": "hello"})
+)
 """
-
-
-future_custom_callback_url = endpoint.predict(
-    request=EndpointRequest(
-        args={"x": 3, "y": "hello"}, callback_url="https://example.com/custom"
-    ),
-)
-
-"""
-A callback is sent to https://example.com/custom with the following JSON body:
-{
-    "task_id": "THE_TASK_ID",
-    "result": 8
-}
-"""
+A callback is sent to https://example.com with the following JSON body:
+{
+    "task_id": "THE_TASK_ID",
+    "result": 7
+}
+"""
+
+future_custom_callback_url = endpoint.predict(
+    request=EndpointRequest(
+        args={"x": 3, "y": "hello"}, callback_url="https://example.com/custom"
+    ),
+)
+
+"""
+A callback is sent to https://example.com/custom with the following JSON body:
+{
+    "task_id": "THE_TASK_ID",
+    "result": 8
+}
+"""
+
    +

    Authentication for callbacks

    +
    +

    Warning

    +

    This feature is currently in beta, and the API is likely to change.

    +
    +

    Callbacks can be authenticated using shared authentication headers. To enable authentication, +set either default_callback_auth_kind when creating the endpoint or callback_auth_kind +when making a prediction request.

    +

    Currently, the supported authentication methods are basic and mtls. If basic is used, +then the default_callback_auth_username and default_callback_auth_password fields must be +specified when creating the endpoint, or the callback_auth_username and callback_auth_password +fields must be specified when making a prediction request. If mtls is used, then the +same is true for the default_callback_auth_cert and default_callback_auth_key fields, +or the callback_auth_cert and callback_auth_key fields.

    +
    Creating an Async Model Endpoint with custom Callback auth
    import os
+import time
+from launch import EndpointRequest, LaunchClient, PostInferenceHooks
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+endpoint = client.create_model_endpoint(
+    endpoint_name="demo-endpoint-callback",
+    model_bundle="test-bundle",
+    cpus=1,
+    min_workers=1,
+    endpoint_type="async",
+    update_if_exists=True,
+    labels={
+        "team": "MY_TEAM",
+        "product": "MY_PRODUCT",
+    },
+    post_inference_hooks=[PostInferenceHooks.CALLBACK],
+    default_callback_url="https://example.com",
+    default_callback_auth_kind="basic",
+    default_callback_auth_username="user",
+    default_callback_auth_password="password",
+)
+
+while endpoint.status() != "READY":
+    time.sleep(10)
+
+future_default = endpoint.predict(
+    request=EndpointRequest(args={"x": 2, "y": "hello"})
+)
+"""
+A callback is sent to https://example.com with ("user", "password") as the basic auth.
+"""
+
+future_custom_callback_auth = endpoint.predict(
+    request=EndpointRequest(
+        args={"x": 3, "y": "hello"},
+        callback_auth_kind="mtls", 
+        callback_auth_cert="cert", 
+        callback_auth_key="key",
+    ),
+)
+
+"""
+A callback is sent with mTLS authentication.
+"""
    diff --git a/concepts/endpoint_predictions/index.html b/concepts/endpoint_predictions/index.html index a9e8ed7c..2a140022 100644 --- a/concepts/endpoint_predictions/index.html +++ b/concepts/endpoint_predictions/index.html @@ -674,9 +674,16 @@

    url: Optional[str] = None, args: Optional[Dict] = None, callback_url: Optional[str] = None, - return_pickled: Optional[bool] = False, - request_id: Optional[str] = None, -) + callback_auth_kind: Optional[ + Literal[basic, mtls] + ] = None, + callback_auth_username: Optional[str] = None, + callback_auth_password: Optional[str] = None, + callback_auth_cert: Optional[str] = None, + callback_auth_key: Optional[str] = None, + return_pickled: Optional[bool] = False, + request_id: Optional[str] = None, +)
    @@ -732,6 +739,78 @@

    False + + callback_url + + Optional[str] + +

    The callback url to use for this task. If None, then the +default_callback_url of the endpoint is used. The endpoint must specify +"callback" as a post-inference hook for the callback to be triggered.

    + + None + + + + callback_auth_kind + + Optional[Literal[basic, mtls]] + +

    The default callback auth kind to use for async endpoints. +Either "basic" or "mtls". This can be overridden in the task parameters for each +individual task.

    + + None + + + + callback_auth_username + + Optional[str] + +

    The default callback auth username to use. This only +applies if callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + callback_auth_password + + Optional[str] + +

    The default callback auth password to use. This only +applies if callback_auth_kind is "basic". This can be overridden in the task +parameters for each individual task.

    + + None + + + + callback_auth_cert + + Optional[str] + +

    The default callback auth cert to use. This only applies +if callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + + + callback_auth_key + + Optional[str] + +

    The default callback auth key to use. This only applies +if callback_auth_kind is "mtls". This can be overridden in the task +parameters for each individual task.

    + + None + + request_id diff --git a/concepts/model_bundles/index.html b/concepts/model_bundles/index.html index 29fd1367..339faeee 100644 --- a/concepts/model_bundles/index.html +++ b/concepts/model_bundles/index.html @@ -669,7 +669,7 @@

    Creating Model Bundles "model": my_model, "load_predict_fn": my_load_predict_fn, "env_params": ENV_PARAMS, - "requirements": [], + "requirements": ["pytest==7.2.1", "numpy"], # list your requirements here "request_schema": MyRequestSchema, "response_schema": MyResponseSchema, } @@ -709,8 +709,8 @@

    Creating Model Bundlesrequirements_filename = os.path.join(directory, "requirements.txt") with open(predict_filename, "w") as f: f.write(""" -numpy -sklearn +pytest==7.2.1 +numpy """ ) @@ -767,83 +767,84 @@

    Configuring Model BundlesCreating Model Bundles with app_config
    import os
 from launch import LaunchClient
 from pydantic import BaseModel
-from typing import List, Literal, Union
-
+from typing import List, Union
+from typing_extensions import Literal
 
-class MyRequestSchemaSingle(BaseModel):
-    kind: Literal['single']
-    x: int
-    y: str
-
-class MyRequestSchemaBatched(BaseModel):
-    kind: Literal['batched']
-    x: List[int]
-    y: List[str]
-
-class MyRequestSchema(BaseModel):
-    __root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]
-
-class MyResponseSchema(BaseModel):
-    __root__: Union[int, List[int]]
-
+
+class MyRequestSchemaSingle(BaseModel):
+    kind: Literal['single']
+    x: int
+    y: str
+
+class MyRequestSchemaBatched(BaseModel):
+    kind: Literal['batched']
+    x: List[int]
+    y: List[str]
+
+class MyRequestSchema(BaseModel):
+    __root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]
+
+class MyResponseSchema(BaseModel):
+    __root__: Union[int, List[int]]
 
-def my_load_predict_fn(app_config, model):
-    def returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:
-        """MyRequestSchema -> MyResponseSchema"""
-        if app_config["mode"] == "single":
-            assert isinstance(x, int) and isinstance(y, str)
-            return model(x) + len(y)
-
-        result = []
-        for x_i, y_i in zip(x, y):
-            result.append(model(x_i) + len(y_i))
-        return result
-
-    return returns_model_of_x_plus_len_of_y
-
+
+def my_load_predict_fn(app_config, model):
+    def returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:
+        """MyRequestSchema -> MyResponseSchema"""
+        if app_config["mode"] == "single":
+            assert isinstance(x, int) and isinstance(y, str)
+            return model(x) + len(y)
+
+        result = []
+        for x_i, y_i in zip(x, y):
+            result.append(model(x_i) + len(y_i))
+        return result
+
+    return returns_model_of_x_plus_len_of_y
 
-def my_load_model_fn(app_config):
-    def my_model_single(x: int):
-        return x * 2
-
-    def my_model_batched(x: List[int]):
-        return [my_model_single(x_i) for x_i in x]
-
-    if app_config["mode"] == "single":
-        return my_model_single
-
-    return my_model_batched
-
+
+def my_load_model_fn(app_config):
+    def my_model_single(x: int):
+        return x * 2
+
+    def my_model_batched(x: List[int]):
+        return [my_model_single(x_i) for x_i in x]
+
+    if app_config["mode"] == "single":
+        return my_model_single
+
+    return my_model_batched
 
-ENV_PARAMS = {
-    "framework_type": "pytorch",
-    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
-}
-
-BUNDLE_PARAMS_SINGLE = {
-    "model_bundle_name": "test-bundle-single",
-    "load_predict_fn": my_load_predict_fn,
-    "load_model_fn": my_load_model_fn,
-    "env_params": ENV_PARAMS,
-    "requirements": [],
-    "request_schema": MyRequestSchema,
-    "response_schema": MyResponseSchema,
-    "app_config": {"mode": "single"},
-}
-BUNDLE_PARAMS_BATCHED = {
-    "model_bundle_name": "test-bundle-batched",
-    "load_predict_fn": my_load_predict_fn,
-    "load_model_fn": my_load_model_fn,
-    "env_params": ENV_PARAMS,
-    "requirements": [],
-    "request_schema": MyRequestSchema,
-    "response_schema": MyResponseSchema,
-    "app_config": {"mode": "batched"},
-}
-
-client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
-bundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)
-bundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)
+
+ENV_PARAMS = {
+    "framework_type": "pytorch",
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+}
+
+BUNDLE_PARAMS_SINGLE = {
+    "model_bundle_name": "test-bundle-single",
+    "load_predict_fn": my_load_predict_fn,
+    "load_model_fn": my_load_model_fn,
+    "env_params": ENV_PARAMS,
+    "requirements": ["pytest==7.2.1", "numpy"],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "app_config": {"mode": "single"},
+}
+BUNDLE_PARAMS_BATCHED = {
+    "model_bundle_name": "test-bundle-batched",
+    "load_predict_fn": my_load_predict_fn,
+    "load_model_fn": my_load_model_fn,
+    "env_params": ENV_PARAMS,
+    "requirements": ["pytest==7.2.1", "numpy"],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "app_config": {"mode": "batched"},
+}
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+bundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)
+bundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)

    Updating Model Bundles

    Model Bundles are immutable, meaning they cannot be edited once created. diff --git a/concepts/model_endpoints/index.html b/concepts/model_endpoints/index.html index b524ad79..8b79291c 100644 --- a/concepts/model_endpoints/index.html +++ b/concepts/model_endpoints/index.html @@ -336,55 +336,10 @@ - - Model Endpoints - - -

    - @@ -563,35 +518,6 @@ - - - @@ -616,6 +542,27 @@

    Model Endpoints +

    Info

    +

    Choosing the right inference mode

    +

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    +

    A SyncEndpoint is good if:

    +
      +
    • You have strict latency requirements (e.g. on the order of seconds or less).
      • +
    • You are willing to have resources continually allocated.
      • +
    +

    An AsyncEndpoint is good if:

    +
      +
    • You want to save on compute costs.
      • +
    • Your inference code takes a long time to run.
      • +
    • Your latency requirements are on the order of minutes.
      • +
    +

    A BatchJob is good if:

    +
      +
    • You know there is a large batch of inputs ahead of time.
      • +
    • You want to optimize for throughput instead of latency.
      • +
    +

    Creating Async Model Endpoints

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    @@ -632,7 +579,7 @@

    Creating Async Model Endpoints update_if_exists=True, labels={ "team": "MY_TEAM", - "product": "MY_PRODUCT" + "product": "MY_PRODUCT", }, ) @@ -656,7 +603,7 @@

    Creating Sync Model Endpoints update_if_exists=True, labels={ "team": "MY_TEAM", - "product": "MY_PRODUCT" + "product": "MY_PRODUCT", }, ) @@ -690,7 +637,7 @@

    Managing Model Endpoints update_if_exists=True, labels={ "team": "MY_TEAM", - "product": "MY_PRODUCT" + "product": "MY_PRODUCT", }, ) client.delete_model_endpoint(model_endpoint="demo-endpoint-tmp") diff --git a/index.html b/index.html index e0210afd..70b8b52c 100644 --- a/index.html +++ b/index.html @@ -630,7 +630,7 @@

    Example "model": my_model, "load_predict_fn": my_load_predict_fn, "env_params": ENV_PARAMS, - "requirements": [], + "requirements": ["pytest==7.2.1", "numpy"], # list your requirements here "request_schema": MyRequestSchema, "response_schema": MyResponseSchema, } @@ -643,7 +643,7 @@

    Example "endpoint_type": "async", "update_if_exists": True, "labels": { - "team": "infra", + "team": "MY_TEAM", "product": "launch", } } diff --git a/search/search_index.json b/search/search_index.json index ef8cb31e..f6cdd813 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"infra\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Uploads and registers a model bundle to Scale Launch.

    A model bundle consists of exactly one of the following:

    • predict_fn_or_cls
    • load_predict_fn + model
    • load_predict_fn + load_model_fn

    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is also my_root:

    .. code-block:: text

    my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

    .. code-block:: text

    my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nfuture_default = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\nnumpy\nsklearn\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Literal, Union\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\"\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Uploads and registers a model bundle to Scale Launch.

    A model bundle consists of exactly one of the following:

    • predict_fn_or_cls
    • load_predict_fn + model
    • load_predict_fn + load_model_fn

    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is also my_root:

    .. code-block:: text

    my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

    .. code-block:: text

    my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml index e9e95501..e434d218 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,67 +2,67 @@ https://scaleapi.github.io/launch-python-client/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/cli/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/api/client/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/api/endpoint_predictions/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/api/hooks/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/api/model_bundles/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/api/model_endpoints/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/concepts/batch_jobs/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/concepts/callbacks/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/concepts/endpoint_predictions/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/concepts/model_bundles/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/concepts/model_endpoints/ - 2023-02-24 + 2023-03-08 daily https://scaleapi.github.io/launch-python-client/concepts/overview/ - 2023-02-24 + 2023-03-08 daily \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 49c6d680f4ea5c0b0f4f19944651c102b1db0ce6..4f65f5acd4907c53b4e05f4d143f454539fb0b93 100644 GIT binary patch literal 319 zcmV-F0l@wriwFotObBEG|8r?{Wo=<_E_iKh0M(YkZo?o9hVOlfh`mj zjtF>xm0%V~v%dYDwyUC@w*wZE!TPa2BaCPcUxO=7Fh+}h&mO9p72p+W>-#->dwnT) z>>xJHG`a|>AhC~oW>N~zoX_)IMVnxdAyzo{JSszr$ZtVjS5HiA3bG}^b#}4Fp%P1$ zNNJ*y4rH)ZZISW4VhA+F;Nn}g|YO$@ik7Z3`N0M7?B~wQF z9^Oaz6k1w$Zuly4rQ>(JzW^nd4=hP|ZU?+DgVzDAPi+`MTb1bk#=I&ph{w#3i2+>O zOt9g!u7_k|8r?{Wo=<_E_iKh0M(YkZo?o9hVOlfhY<|AvX{>Q From 4bf79b8776e2230d5599b2c409b6df39e131d207 Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Wed, 8 Mar 2023 15:27:59 -0800 Subject: [PATCH 06/28] Deployed 635fe4e with MkDocs version: 1.4.2 --- concepts/callbacks/index.html | 36 ++++++++++++++++++++++++++++++---- search/search_index.json | 2 +- sitemap.xml.gz | Bin 319 -> 319 bytes 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/concepts/callbacks/index.html b/concepts/callbacks/index.html index 5ad2bcf1..d27c19b0 100644 --- a/concepts/callbacks/index.html +++ b/concepts/callbacks/index.html @@ -699,10 +699,38 @@

    Authentication for callbacks callback_auth_key="key", ), ) - -""" -A callback is sent with mTLS authentication. -""" +""" +A callback is sent with mTLS authentication. +""" + +client.edit_model_endpoint( + model_endpoint=endpoint.model_endpoint, + default_callback_auth_kind="mtls", + default_callback_auth_cert="cert", + default_callback_auth_key="key", +) + +while endpoint.status() != "READY": + time.sleep(10) + +future_default = endpoint.predict( + request=EndpointRequest(args={"x": 2, "y": "hello"}) +) +""" +A callback is sent with mTLS auth. +""" + +future_custom_callback_auth = endpoint.predict( + request=EndpointRequest( + args={"x": 3, "y": "hello"}, + callback_auth_kind="basic", + callback_auth_username="user", + callback_auth_password="pass", + ), +) +""" +A callback is sent with ("user", "pass") as the basic auth. +""" diff --git a/search/search_index.json b/search/search_index.json index f6cdd813..3de1619b 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Uploads and registers a model bundle to Scale Launch.

    A model bundle consists of exactly one of the following:

    • predict_fn_or_cls
    • load_predict_fn + model
    • load_predict_fn + load_model_fn

    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is also my_root:

    .. code-block:: text

    my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

    .. code-block:: text

    my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Uploads and registers a model bundle to Scale Launch.

    A model bundle consists of exactly one of the following:

    • predict_fn_or_cls
    • load_predict_fn + model
    • load_predict_fn + load_model_fn

    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is also my_root:

    .. code-block:: text

    my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

    .. code-block:: text

    my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 4f65f5acd4907c53b4e05f4d143f454539fb0b93..fdbb5e46db3a16850d6701f2426b3096c770c5c2 100644 GIT binary patch delta 16 Xcmdnbw4aGxzMF&NpCso-b_+%TDM$oW delta 16 Xcmdnbw4aGxzMF%?&4*(ny9FZvC2a%G From da423be46107f21c930cb5cd2a9f0960c737d741 Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Tue, 14 Mar 2023 15:50:26 -0700 Subject: [PATCH 07/28] Deployed 3fbbc36 with MkDocs version: 1.4.2 --- 404.html | 2 +- api/client/index.html | 2 +- api/endpoint_predictions/index.html | 2 +- api/hooks/index.html | 2 +- api/model_bundles/index.html | 2 +- api/model_endpoints/index.html | 2 +- cli/index.html | 2 +- concepts/batch_jobs/index.html | 2 +- concepts/callbacks/index.html | 2 +- concepts/endpoint_predictions/index.html | 2 +- concepts/model_bundles/index.html | 2 +- concepts/model_endpoints/index.html | 2 +- concepts/overview/index.html | 2 +- index.html | 2 +- sitemap.xml | 26 +++++++++++------------ sitemap.xml.gz | Bin 319 -> 321 bytes 16 files changed, 27 insertions(+), 27 deletions(-) diff --git a/404.html b/404.html index e174bd99..1cc2c937 100644 --- a/404.html +++ b/404.html @@ -538,7 +538,7 @@

    404 - Not found

    - + diff --git a/api/client/index.html b/api/client/index.html index bdd3f96a..c566eec2 100644 --- a/api/client/index.html +++ b/api/client/index.html @@ -2839,7 +2839,7 @@

    - + diff --git a/api/endpoint_predictions/index.html b/api/endpoint_predictions/index.html index 1f89282f..2bc159d0 100644 --- a/api/endpoint_predictions/index.html +++ b/api/endpoint_predictions/index.html @@ -1106,7 +1106,7 @@

    - + diff --git a/api/hooks/index.html b/api/hooks/index.html index 9c22d181..76c675e5 100644 --- a/api/hooks/index.html +++ b/api/hooks/index.html @@ -669,7 +669,7 @@

    - + diff --git a/api/model_bundles/index.html b/api/model_bundles/index.html index cf3d560c..f0a89c8e 100644 --- a/api/model_bundles/index.html +++ b/api/model_bundles/index.html @@ -970,7 +970,7 @@

    - + diff --git a/api/model_endpoints/index.html b/api/model_endpoints/index.html index 47323783..0b8144b3 100644 --- a/api/model_endpoints/index.html +++ b/api/model_endpoints/index.html @@ -1037,7 +1037,7 @@

    - + diff --git a/cli/index.html b/cli/index.html index 7e221c0f..fe728f9b 100644 --- a/cli/index.html +++ b/cli/index.html @@ -636,7 +636,7 @@

    Help&par
    - + diff --git a/concepts/batch_jobs/index.html b/concepts/batch_jobs/index.html index 07e046d4..dcbd4fd6 100644 --- a/concepts/batch_jobs/index.html +++ b/concepts/batch_jobs/index.html @@ -606,7 +606,7 @@

    Batch Jobs - + diff --git a/concepts/callbacks/index.html b/concepts/callbacks/index.html index d27c19b0..002bbad3 100644 --- a/concepts/callbacks/index.html +++ b/concepts/callbacks/index.html @@ -771,7 +771,7 @@

    Authentication for callbacks - + diff --git a/concepts/endpoint_predictions/index.html b/concepts/endpoint_predictions/index.html index 2a140022..123e5b5b 100644 --- a/concepts/endpoint_predictions/index.html +++ b/concepts/endpoint_predictions/index.html @@ -1133,7 +1133,7 @@

    - + diff --git a/concepts/model_bundles/index.html b/concepts/model_bundles/index.html index 339faeee..943beb01 100644 --- a/concepts/model_bundles/index.html +++ b/concepts/model_bundles/index.html @@ -893,7 +893,7 @@

    Listing Model Bundles - + diff --git a/concepts/model_endpoints/index.html b/concepts/model_endpoints/index.html index 8b79291c..59c3b39a 100644 --- a/concepts/model_endpoints/index.html +++ b/concepts/model_endpoints/index.html @@ -681,7 +681,7 @@

    Managing Model Endpoints - + diff --git a/concepts/overview/index.html b/concepts/overview/index.html index ed3c7516..0e4fb450 100644 --- a/concepts/overview/index.html +++ b/concepts/overview/index.html @@ -588,7 +588,7 @@

    Overview - + diff --git a/index.html b/index.html index 70b8b52c..4fbea706 100644 --- a/index.html +++ b/index.html @@ -741,7 +741,7 @@

    Installation - + diff --git a/sitemap.xml b/sitemap.xml index e434d218..d411d5cb 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,67 +2,67 @@ https://scaleapi.github.io/launch-python-client/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/cli/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/api/client/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/api/endpoint_predictions/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/api/hooks/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/api/model_bundles/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/api/model_endpoints/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/concepts/batch_jobs/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/concepts/callbacks/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/concepts/endpoint_predictions/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/concepts/model_bundles/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/concepts/model_endpoints/ - 2023-03-08 + 2023-03-14 daily https://scaleapi.github.io/launch-python-client/concepts/overview/ - 2023-03-08 + 2023-03-14 daily \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index fdbb5e46db3a16850d6701f2426b3096c770c5c2..14dbab9beebf049f434492fd07eeabbda1c9dd64 100644 GIT binary patch delta 303 zcmV+~0nq-x0>J_YABzYGG5QdZ2OWRf?yyZsdfO9h=ZJt8SP5oGO{JXb#<}IqHLBdfzkviEC_Fk-b6ihg+YIA#pI;U z{W)rMxwHmcqk&VM(W>?Rs|aIm>4oKfNi_UDGN!W|8z}4@DiN_PsE*w zQnw%2r3g=Y+X;D0kv;9F=3XC0qQm=(KjjHV`e>aD%>R#mar9$rz5$JF9zWp;006)< Bj5`1T delta 301 zcmV+|0n+}#0>1(WABzYG{uv382OWRbUAifyw>`mjjtF>xm0%V~v%dYDwyUC@w*wZE z!TPa2BaCPcUxO=7Fh+}h&mO9p72p+W>-#->dwnT)>>xJHG`a|>AhC~oW>N~zoX_)I zMVnxdAyzo{JSszr$ZtVjS5HiA3bG}^b#}4Fp%P1$NNJ*y4rH)ZZHp2Fo?&@kck0Y+f80O$P)eM)?@^)&?^W;+{vU& z`+;4K@T7O0RL30I^L}dXWn>-NyubK!?qH;k*2%*B|L7M-KQ{3VjQ`Xl;Ryf${$!8b From 9776f6a100decb5e3fb75712aca054c1adbb3dca Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Mon, 27 Mar 2023 18:14:55 -0700 Subject: [PATCH 08/28] Deployed 21e34f3 with MkDocs version: 1.4.2 --- api/client/index.html | 768 +++++++++++++++++++++++++++++++++++---- objects.inv | Bin 533 -> 557 bytes search/search_index.json | 2 +- sitemap.xml | 26 +- sitemap.xml.gz | Bin 321 -> 320 bytes 5 files changed, 709 insertions(+), 87 deletions(-) diff --git a/api/client/index.html b/api/client/index.html index c566eec2..df1d7dea 100644 --- a/api/client/index.html +++ b/api/client/index.html @@ -478,6 +478,13 @@ create_model_bundle() + + +
  • + + create_model_bundle_from_callable_v2() + +
  • @@ -485,6 +492,20 @@ create_model_bundle_from_dirs() +
    • + +
  • + + create_model_bundle_from_dirs_v2() + + +
    • + +
  • + + create_model_bundle_from_runnable_image_v2() + +
  • @@ -707,6 +728,13 @@ create_model_bundle() +
    • + +
  • + + create_model_bundle_from_callable_v2() + +
  • @@ -714,6 +742,20 @@ create_model_bundle_from_dirs() +
    • + +
  • + + create_model_bundle_from_dirs_v2() + + +
    • + +
  • + + create_model_bundle_from_runnable_image_v2() + +
  • @@ -1242,16 +1284,12 @@

    -

    Uploads and registers a model bundle to Scale Launch.

    -

    A model bundle consists of exactly one of the following:

    -
      -
    • predict_fn_or_cls
      • -
    • load_predict_fn + model
      • -
    • load_predict_fn + load_model_fn
      • -
    -

    Pre/post-processing code can be included inside load_predict_fn/model or in -predict_fn_or_cls call.

    +
    + Warning +

    This method is deprecated. Use +create_model_bundle_from_callable_v2 instead.

    +

    Parameters:

    @@ -1428,6 +1466,217 @@

    +

    + create_model_bundle_from_callable_v2 + + +

    +
    create_model_bundle_from_callable_v2(
+    *,
+    model_bundle_name: str,
+    load_predict_fn: Callable[
+        [LaunchModel_T], Callable[[Any], Any]
+    ],
+    load_model_fn: Callable[[], LaunchModel_T],
+    request_schema: Type[BaseModel],
+    response_schema: Type[BaseModel],
+    requirements: Optional[List[str]] = None,
+    pytorch_image_tag: Optional[str] = None,
+    tensorflow_version: Optional[str] = None,
+    custom_base_image_repository: Optional[str] = None,
+    custom_base_image_tag: Optional[str] = None,
+    app_config: Optional[Union[Dict[str, Any], str]] = None
+) -> Dict[str, Any]
+
    + +
    + +

    Uploads and registers a model bundle to Scale Launch.

    + +

    Parameters:

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    model_bundle_name + str +

    Name of the model bundle.

    		 + required +
    load_predict_fn + Callable[[LaunchModel_T], Callable[[Any], Any]] +

    Function that takes in a model and returns a predict function. +When your model bundle is deployed, this predict function will be called as follows: +

    input = {"input": "some input"} # or whatever your request schema is.
+
+def load_model_fn():
+    # load model
+    return model
+
+def load_predict_fn(model, app_config=None):
+    def predict_fn(input):
+        # do pre-processing
+        output = model(input)
+        # do post-processing
+        return output
+    return predict_fn
+
+predict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)
+response = predict_fn(input)
+

    		 + required +
    load_model_fn + Callable[[], LaunchModel_T] +

    A function that, when run, loads a model.

    		 + required +
    request_schema + Type[BaseModel] +

    A pydantic model that represents the request schema for the model +bundle. This is used to validate the request body for the model bundle's endpoint.

    		 + required +
    response_schema + Type[BaseModel] +

    A pydantic model that represents the request schema for the model +bundle. This is used to validate the response for the model bundle's endpoint.

    		 + required +
    requirements + Optional[List[str]] +

    List of pip requirements.

    		 + None +
    pytorch_image_tag + Optional[str] +

    The image tag for the PyTorch image that will be used to run the +bundle. Exactly one of pytorch_image_tag, tensorflow_version, or +custom_base_image_repository must be specified.

    		 + None +
    tensorflow_version + Optional[str] +

    The version of TensorFlow that will be used to run the bundle. +If not specified, the default version will be used. Exactly one of +pytorch_image_tag, tensorflow_version, or custom_base_image_repository +must be specified.

    		 + None +
    custom_base_image_repository + Optional[str] +

    The repository for a custom base image that will be +used to run the bundle. If not specified, the default base image will be used. +Exactly one of pytorch_image_tag, tensorflow_version, or +custom_base_image_repository must be specified.

    		 + None +
    custom_base_image_tag + Optional[str] +

    The tag for a custom base image that will be used to run the +bundle. Must be specified if custom_base_image_repository is specified.

    		 + None +
    app_config + Optional[Union[Dict[str, Any], str]] +

    An optional dictionary of configuration values that will be passed to the +bundle when it is run. These values can be accessed by the bundle via the +app_config global variable.

    		 + None +
    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + Dict[str, Any] +

    A dictionary containing the following keys:

    +
      +
    • model_bundle_id: The ID of the created model bundle.
      • +
    + +
    + + + +
    + + +

    create_model_bundle_from_dirs @@ -1449,34 +1698,13 @@

    -

    Packages up code from one or more local filesystem folders and uploads them as a bundle -to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    -

    For example, if you have a directory structure like so, and your current working -directory is also my_root:

    -

    .. code-block:: text

    -

    my_root/ - my_module1/ - init.py - ...files and directories - my_inference_file.py - my_module2/ - init.py - ...files and directories

    -

    then calling create_model_bundle_from_dirs with base_paths=["my_module1", -"my_module2"] essentially creates a zip file without the root directory, e.g.:

    -

    .. code-block:: text

    -

    my_module1/ - init.py - ...files and directories - my_inference_file.py - my_module2/ - init.py - ...files and directories

    -

    and these contents will be unzipped relative to the server side application root. Bear -these points in mind when referencing Python module paths for this bundle. For instance, -if my_inference_file.py has def f(...) as the desired inference loading function, -then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    +
    + Warning +

    This method is deprecated. Use +create_model_bundle_from_dirs_v2 +instead.

    +

    Parameters:

    @@ -1529,18 +1757,19 @@

    +
  • framework_type: either tensorflow or pytorch.
    • +
  • PyTorch fields:
      +
    • pytorch_image_tag: An image tag for the pytorch docker base image. The + list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags
      • +
    • - + +

    Example: + 

    {
+    "framework_type": "pytorch",
+    "pytorch_image_tag": "1.10.0-cuda11.3-cudnn8-runtime",
+}
+

    @@ -1613,6 +1842,375 @@

    + create_model_bundle_from_dirs_v2 + + +

    +
    create_model_bundle_from_dirs_v2(
+    *,
+    model_bundle_name: str,
+    base_paths: List[str],
+    load_predict_fn_module_path: str,
+    load_model_fn_module_path: str,
+    request_schema: Type[BaseModel],
+    response_schema: Type[BaseModel],
+    requirements_path: str,
+    pytorch_image_tag: Optional[str] = None,
+    tensorflow_version: Optional[str] = None,
+    custom_base_image_repository: Optional[str] = None,
+    custom_base_image_tag: Optional[str] = None,
+    app_config: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]
+
    + +
    + +

    Packages up code from one or more local filesystem folders and uploads them as a bundle +to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    +

    For example, if you have a directory structure like so, and your current working +directory is my_root:

    +
       my_root/
+       my_module1/
+           __init__.py
+           ...files and directories
+           my_inference_file.py
+       my_module2/
+           __init__.py
+           ...files and directories
+
    +

    then calling create_model_bundle_from_dirs_v2 with base_paths=["my_module1", +"my_module2"] essentially creates a zip file without the root directory, e.g.:

    +
       my_module1/
+       __init__.py
+       ...files and directories
+       my_inference_file.py
+   my_module2/
+       __init__.py
+       ...files and directories
+
    +

    and these contents will be unzipped relative to the server side application root. Bear +these points in mind when referencing Python module paths for this bundle. For instance, +if my_inference_file.py has def f(...) as the desired inference loading function, +then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    + +

    Parameters:

    +
    required
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    model_bundle_name + str +

    The name of the model bundle you want to create.

    		 + required +
    base_paths + List[str] +

    A list of paths to directories that will be zipped up and uploaded +as a bundle. Each path must be relative to the current working directory.

    		 + required +
    load_predict_fn_module_path + str +

    The Python module path to the function that will be +used to load the model for inference. This function should take in a path to a +model directory, and return a model object. The model object should be pickleable.

    		 + required +
    load_model_fn_module_path + str +

    The Python module path to the function that will be +used to load the model for training. This function should take in a path to a +model directory, and return a model object. The model object should be pickleable.

    		 + required +
    request_schema + Type[BaseModel] +

    A Pydantic model that defines the request schema for the bundle.

    		 + required +
    response_schema + Type[BaseModel] +

    A Pydantic model that defines the response schema for the bundle.

    		 + required +
    requirements_path + str +

    Path to a requirements.txt file that will be used to install +dependencies for the bundle. This file must be relative to the current working +directory.

    		 + required +
    pytorch_image_tag + Optional[str] +

    The image tag for the PyTorch image that will be used to run the +bundle. Exactly one of pytorch_image_tag, tensorflow_version, or +custom_base_image_repository must be specified.

    		 + None +
    tensorflow_version + Optional[str] +

    The version of TensorFlow that will be used to run the bundle. +If not specified, the default version will be used. Exactly one of +pytorch_image_tag, tensorflow_version, or custom_base_image_repository +must be specified.

    		 + None +
    custom_base_image_repository + Optional[str] +

    The repository for a custom base image that will be +used to run the bundle. If not specified, the default base image will be used. +Exactly one of pytorch_image_tag, tensorflow_version, or +custom_base_image_repository must be specified.

    		 + None +
    custom_base_image_tag + Optional[str] +

    The tag for a custom base image that will be used to run the +bundle. Must be specified if custom_base_image_repository is specified.

    		 + None +
    app_config + Optional[Dict[str, Any]] +

    An optional dictionary of configuration values that will be passed to the +bundle when it is run. These values can be accessed by the bundle via the +app_config global variable.

    		 + None +
    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + Dict[str, Any] +

    A dictionary containing the following keys:

    +
      +
    • model_bundle_id: The ID of the created model bundle.
      • +
    + +

    + + + +
    + + + +

    + create_model_bundle_from_runnable_image_v2 + + +

    +
    create_model_bundle_from_runnable_image_v2(
+    *,
+    model_bundle_name: str,
+    request_schema: Type[BaseModel],
+    response_schema: Type[BaseModel],
+    repository: str,
+    tag: str,
+    command: List[str],
+    env: Dict[str, str]
+) -> Dict[str, Any]
+
    + +
    + +

    Create a model bundle from a runnable image. The specified command must start a process +that will listen for requests on port 5005 using HTTP.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    model_bundle_name + str +

    The name of the model bundle you want to create.

    		 + required +
    request_schema + Type[BaseModel] +

    A Pydantic model that defines the request schema for the bundle.

    		 + required +
    response_schema + Type[BaseModel] +

    A Pydantic model that defines the response schema for the bundle.

    		 + required +
    repository + str +

    The name of the Docker repository for the runnable image.

    		 + required +
    tag + str +

    The tag for the runnable image.

    		 + required +
    command + List[str] +

    The command that will be used to start the process that listens for requests.

    		 + required +
    env + Dict[str, str] +

    A dictionary of environment variables that will be passed to the bundle when it +is run.

    		 + required +
    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + Dict[str, Any] +

    A dictionary containing the following keys:

    +
      +
    • model_bundle_id: The ID of the created model bundle.
      • +
    + +
    + +
    + +
    + + +

    create_model_endpoint @@ -1631,20 +2229,21 @@

    per_worker: int = 10, gpu_type: Optional[str] = None, endpoint_type: str = "sync", - post_inference_hooks: Optional[ - List[PostInferenceHooks] - ] = None, - default_callback_url: Optional[str] = None, - default_callback_auth_kind: Optional[ - Literal[basic, mtls] - ] = None, - default_callback_auth_username: Optional[str] = None, - default_callback_auth_password: Optional[str] = None, - default_callback_auth_cert: Optional[str] = None, - default_callback_auth_key: Optional[str] = None, - update_if_exists: bool = False, - labels: Optional[Dict[str, str]] = None -) -> Optional[Endpoint] + high_priority: Optional[bool] = False, + post_inference_hooks: Optional[ + List[PostInferenceHooks] + ] = None, + default_callback_url: Optional[str] = None, + default_callback_auth_kind: Optional[ + Literal[basic, mtls] + ] = None, + default_callback_auth_username: Optional[str] = None, + default_callback_auth_password: Optional[str] = None, + default_callback_auth_cert: Optional[str] = None, + default_callback_auth_key: Optional[str] = None, + update_if_exists: bool = False, + labels: Optional[Dict[str, str]] = None +) -> Optional[Endpoint]

    @@ -1806,6 +2405,17 @@

    Optional[bool] + +

    Either True or False. Enabling this will allow the created +endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    + + False + + post_inference_hooks @@ -2003,18 +2613,19 @@

    max_workers: Optional[int] = None, per_worker: Optional[int] = None, gpu_type: Optional[str] = None, - post_inference_hooks: Optional[ - List[PostInferenceHooks] - ] = None, - default_callback_url: Optional[str] = None, - default_callback_auth_kind: Optional[ - Literal[basic, mtls] - ] = None, - default_callback_auth_username: Optional[str] = None, - default_callback_auth_password: Optional[str] = None, - default_callback_auth_cert: Optional[str] = None, - default_callback_auth_key: Optional[str] = None -) -> None + high_priority: Optional[bool] = None, + post_inference_hooks: Optional[ + List[PostInferenceHooks] + ] = None, + default_callback_url: Optional[str] = None, + default_callback_auth_kind: Optional[ + Literal[basic, mtls] + ] = None, + default_callback_auth_username: Optional[str] = None, + default_callback_auth_password: Optional[str] = None, + default_callback_auth_cert: Optional[str] = None, + default_callback_auth_key: Optional[str] = None +) -> None

    @@ -2156,6 +2767,17 @@

    None + + high_priority + + Optional[bool] + +

    Either True or False. Enabling this will allow the created +endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    + + None + + post_inference_hooks diff --git a/objects.inv b/objects.inv index e43ed22faff7f5befaa9ffd329ff598edc3168bd..e23a45dc4d4323f94d81beaf2a874e662ba434b2 100644 GIT binary patch delta 444 zcmV;t0Ym1*41Y#WK*-Ej%lCN~LavFN%iF0Yd73;FA)bPbM| z4D60LGx%AfZKd3IaYE7GFfJ7#t5k-Pt!v68vM=FE#!H|AZGQl45ZqLbim}|NigAz< z*1om9J%gSJW6#8%Cty<(waXzYC?YNjQl^!lfOF;*jL2aoTr{je$EHbTwIYjQVw{sd zO5yXA!cN1oQLKGn0h&K++2FAD=y`)lRRp1h4-!qkx7=<$$U8Zw7;cw&anV{!u%uxaa28@$W&~??i0frlxOA<6$T_ha)FBKTAkKYv!kri=x56IrD!D zY0ESisBE|;vT$Rcu;EgL4l-~ME)i51u4v#JCvGyWCp< delta 420 zcmV;V0bBm91eFAkdw*X~!ypg<@OM8&6MdD%SKrjkXrhU6AAsEGwI!nm0-f2nm;PC| z&Y4ufx3R!)2ge;#(2%H@P!h?rmu~dX4KlO46$+foZZa@((N9@iUS-!8>c>@V4PI0n z+>Qjd_|c&2q#{i5Qq%7+p)?_{b&iU!8_E^(Pw6WzDxeZw1b=K1{8Ubg@xtnw=hCK% z5h*uXQp8yX1ru_ZOP>rY(evrWaYmEFI5ACj4^sR-m2}gvToP*^ScDetT0T0wyL!Rk zN|#Y+8G=MRE_eCV|Gg z#h656LydO6n}25>YA!kY*!af&C*79m$&t?gg-Yv=jaULvS5 zUNImxP)aJf-ztF~txoh9l<y3j~ OX|}wI*zp(6BO?;jR@IsS diff --git a/search/search_index.json b/search/search_index.json index 3de1619b..58c75d30 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Uploads and registers a model bundle to Scale Launch.

    A model bundle consists of exactly one of the following:

    • predict_fn_or_cls
    • load_predict_fn + model
    • load_predict_fn + load_model_fn

    Pre/post-processing code can be included inside load_predict_fn/model or in predict_fn_or_cls call.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is also my_root:

    .. code-block:: text

    my_root/ my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    then calling create_model_bundle_from_dirs with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

    .. code-block:: text

    my_module1/ init.py ...files and directories my_inference_file.py my_module2/ init.py ...files and directories

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_callable_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
    create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> Dict[str, Any]\n

    Uploads and registers a model bundle to Scale Launch.

    Parameters:

    Name Type Description Default model_bundle_name str

    Name of the model bundle.

    required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]]

    Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 

    input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n

    required load_model_fn Callable[[], LaunchModel_T]

    A function that, when run, loads a model.

    required request_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    required response_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.

    required requirements Optional[List[str]]

    List of pip requirements.

    None pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Union[Dict[str, Any], str]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch.
    • PyTorch fields:
      • pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags

    Example: 

    {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
    create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> Dict[str, Any]\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is my_root:

       my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n

    then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

       my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required base_paths List[str]

    A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.

    required load_predict_fn_module_path str

    The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required load_model_fn_module_path str

    The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required requirements_path str

    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

    required pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Dict[str, Any]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
    create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> Dict[str, Any]\n

    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required repository str

    The name of the Docker repository for the runnable image.

    required tag str

    The tag for the runnable image.

    required command List[str]

    The command that will be used to start the process that listens for requests.

    required env Dict[str, str]

    A dictionary of environment variables that will be passed to the bundle when it is run.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    False post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml index d411d5cb..673827e7 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,67 +2,67 @@ https://scaleapi.github.io/launch-python-client/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/cli/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/api/client/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/api/endpoint_predictions/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/api/hooks/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/api/model_bundles/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/api/model_endpoints/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/concepts/batch_jobs/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/concepts/callbacks/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/concepts/endpoint_predictions/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/concepts/model_bundles/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/concepts/model_endpoints/ - 2023-03-14 + 2023-03-28 daily https://scaleapi.github.io/launch-python-client/concepts/overview/ - 2023-03-14 + 2023-03-28 daily \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 14dbab9beebf049f434492fd07eeabbda1c9dd64..fe7ecb745652149944546894991df571ef9b32b2 100644 GIT binary patch literal 320 zcmV-G0l)qqiwFpgKO$rT|8r?{Wo=<_E_iKh0M(YkZo?o9hVOlfh`mj zjtKDrE5R&~W_|lPZC6D*ZwGuaSU=WhWFxBM*I=_VjKQEguq`iG22NpTTz_D1uP^zY z9qUasjW$5cNbJIa=@|X95c53qU?L3C2aaPeg3`1J;vN(Qe`56}BU=*OR#&Sms(Q&1 zC{0k(g7k*>ChDo>28ETAQ#$vTsL|!h8gNlmVmY-=B10UotIML?<;5;9_awQ;Rx+if z>*0Nb&$^SwUMjJQ+~~vu?=L{fr2|V6k=g|>%-}kY#>K{u&>0n}e?wXoXv9-u$ix7) zX(!iNND}?0YZ8K2=oEM&Ze>)u`M@?sc+%Tes#A*WX+JfOG7=r$U;HUgFw#fsY+(L> S^sA#ETm22&1L(Tp2><{^WtXJ@ literal 321 zcmV-H0lxkpiwFoY`VeFS|8r?{Wo=<_E_iKh0M(YkZo?o9hVOlfhGO{JXb#<}IqHLBd zfzkviEC_Fk-b6ihg+Xq`+i* z3)jQ@2%n7-#-1y_id^aV9q%td$)y8J5}w)xFU;VSN8@7aM^Hva>fex71sd^~7&0+{ zZM(@S3rV8?bWK9=5}gE3#GQyzw;$N02v2(333*JBJ?*FFUPhwB`-?y22}b&8oea$X TkA89VV{5(vjcgu2;Ryf$7l4)j From d6b9b7d9a20ef651cf4a81e553d547ff0f73e7b5 Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Mon, 27 Mar 2023 18:27:51 -0700 Subject: [PATCH 09/28] Deployed 21e34f3 with MkDocs version: 1.4.2 --- api/client/index.html | 239 +++++++++++++++++++++++++++++++++++++++ objects.inv | Bin 557 -> 574 bytes search/search_index.json | 2 +- sitemap.xml.gz | Bin 320 -> 320 bytes 4 files changed, 240 insertions(+), 1 deletion(-) diff --git a/api/client/index.html b/api/client/index.html index df1d7dea..e4594f02 100644 --- a/api/client/index.html +++ b/api/client/index.html @@ -534,6 +534,13 @@ get_batch_async_response() +

    • + +
  • + + get_latest_model_bundle_v2() + +
  • @@ -541,6 +548,13 @@ get_model_bundle() +
    • + +
  • + + get_model_bundle_v2() + +
  • @@ -555,6 +569,13 @@ list_model_bundles() +
    • + +
  • + + list_model_bundles_v2() + +
  • @@ -784,6 +805,13 @@ get_batch_async_response() +
    • + +
  • + + get_latest_model_bundle_v2() + +
  • @@ -791,6 +819,13 @@ get_model_bundle() +
    • + +
  • + + get_model_bundle_v2() + +
  • @@ -805,6 +840,13 @@ list_model_bundles() +
    • + +
  • + + list_model_bundles_v2() + +
  • @@ -2977,6 +3019,81 @@

    + get_latest_model_bundle_v2 + + +

    +
    get_latest_model_bundle_v2(
+    model_bundle_name: str,
+) -> Dict[str, Any]
+
    + +
    + +

    Get the latest version of a model bundle.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    model_bundle_name + str +

    The name of the model bundle you want to get.

    		 + required +
    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + Dict[str, Any] +

    A dictionary containing the following keys:

    +
      +
    • id: The ID of the model bundle.
      • +
    • name: The name of the model bundle.
      • +
    • schema_location: The location of the schema for the model bundle.
      • +
    • flavor: The flavor of the model bundle. Either RunnableImage, + CloudpickleArtifact, or ZipArtifact.
      • +
    • created_at: The time the model bundle was created.
      • +
    • metadata: A dictionary of metadata associated with the model bundle.
      • +
    • model_artifact_ids: A list of IDs of model artifacts associated with the + bundle.
      • +
    + +
    + + + +
    + + +

    get_model_bundle @@ -3041,6 +3158,79 @@

    +

    + get_model_bundle_v2 + + +

    +
    get_model_bundle_v2(model_bundle_id: str) -> Dict[str, Any]
+
    + +
    + +

    Get a model bundle.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    model_bundle_id + str +

    The ID of the model bundle you want to get.

    		 + required +
    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + Dict[str, Any] +

    A dictionary containing the following keys:

    +
      +
    • id: The ID of the model bundle.
      • +
    • name: The name of the model bundle.
      • +
    • schema_location: The location of the schema for the model bundle.
      • +
    • flavor: The flavor of the model bundle. Either RunnableImage, + CloudpickleArtifact, or ZipArtifact.
      • +
    • created_at: The time the model bundle was created.
      • +
    • metadata: A dictionary of metadata associated with the model bundle.
      • +
    • model_artifact_ids: A list of IDs of model artifacts associated with the + bundle.
      • +
    + +
    + +
    + +
    + + +

    get_model_endpoint @@ -3125,6 +3315,55 @@

    +

    + list_model_bundles_v2 + + +

    +
    list_model_bundles_v2() -> List[Dict[str, Any]]
+
    + +
    + +

    List all model bundles.

    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + List[Dict[str, Any]] +

    A list of dictionaries containing the following keys:

    +
      +
    • id: The ID of the model bundle.
      • +
    • name: The name of the model bundle.
      • +
    • schema_location: The location of the schema for the model bundle.
      • +
    • flavor: The flavor of the model bundle. Either RunnableImage, + CloudpickleArtifact, or ZipArtifact.
      • +
    • created_at: The time the model bundle was created.
      • +
    • metadata: A dictionary of metadata associated with the model bundle.
      • +
    • model_artifact_ids: A list of IDs of model artifacts associated with the + bundle.
      • +
    + +
    + +
    + +
    + + +

    list_model_endpoints diff --git a/objects.inv b/objects.inv index e23a45dc4d4323f94d81beaf2a874e662ba434b2..815c81f8af2b479a7cd9d4454117f6f2a1795c4e 100644 GIT binary patch delta 461 zcmV;;0W$ur1il22dw*X~!ypg<@OM8&6MeOdKKiC+MiWhp`vBxduPqrp5a`Ulz4Xtz zwa%mpdm9V<3I}%}p(b9YTnZ$YzP7E$wvp)F9hYEja+81-i>}M!@+!H$kiU*f*Wh@` z!0w1MgP%3pR?2-BClvh+<5Cf_N@Xb7x~5Db`x35XyaX!H27kZ?!Hwmp7|V^S7zZg~ z?OWU1Gw7Kx_Dt+~CTwb=HXNdYB4SXGGOYv!oHMUrL=H3IqG1I(Hcl$568Rk9OfKV?Q7#qa=gFkvj6dMxMcU&HCMz@JG&S*Q5#>ATHnqy z4>cw_`c%8xoRe;{eiG98UnrH@*z{dl_d+=u^y8kJzoyTjw%>`lb(@;LHI0X%6dsQp zrTr`+1+AH1LN1C11Lw^DEu<~eV4$+$mdL`5eZq!I6-GM9z(Ke~P+_>Dfp4G?R4}*G z5xnL(EeuxJSNA_?L&n@UZ;0j5UHJ244mZebC+(BzL>&tyyV4s5ui`9u6|mxOF_7XY DOU>W! delta 444 zcmV;t0Ym=21g!*+dw)w#!!Qs4&^@P!1Y1(Uk{zHTkU&E80M^u*BvzfVv7Msc9_Q1f zDMbp{-NljpGXBgs5^CaQ%B4VZ>1*41Y#WK*-Ej%lCN~LavFN%iF0Yd73;FA)bPbM| z4D60LGx%AfZKd3IaYE7GFfJ7#t5k-Pt!v68vM=FE#!H|AZGQl45ZqLbim}|NigAz< z*1om9J%gSJW6#8%Cty<(waXzYC?YNjQl^!lfOF;*jL2aoTr{je$EHbTwIYjQVw{sd zO5yXA!cN1oQLKGn0h&K++2FAD=y`)lRRp1h4-!qkx7=<$$U8Zw7;cw&anV{!u%uxaa28@$W&~??i0frlxOA<6$T_ha)FBKTAkKYv!kri=x56IrD!D zY0ESisBE|;vT$Rcu;EgL4l-~ME)i51u4v#JCt1=G=q; diff --git a/search/search_index.json b/search/search_index.json index 58c75d30..58e77c1d 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_callable_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
    create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> Dict[str, Any]\n

    Uploads and registers a model bundle to Scale Launch.

    Parameters:

    Name Type Description Default model_bundle_name str

    Name of the model bundle.

    required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]]

    Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 

    input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n

    required load_model_fn Callable[[], LaunchModel_T]

    A function that, when run, loads a model.

    required request_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    required response_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.

    required requirements Optional[List[str]]

    List of pip requirements.

    None pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Union[Dict[str, Any], str]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch.
    • PyTorch fields:
      • pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags

    Example: 

    {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
    create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> Dict[str, Any]\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is my_root:

       my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n

    then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

       my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required base_paths List[str]

    A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.

    required load_predict_fn_module_path str

    The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required load_model_fn_module_path str

    The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required requirements_path str

    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

    required pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Dict[str, Any]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
    create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> Dict[str, Any]\n

    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required repository str

    The name of the Docker repository for the runnable image.

    required tag str

    The tag for the runnable image.

    required command List[str]

    The command that will be used to start the process that listens for requests.

    required env Dict[str, str]

    A dictionary of environment variables that will be passed to the bundle when it is run.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    False post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_callable_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
    create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> Dict[str, Any]\n

    Uploads and registers a model bundle to Scale Launch.

    Parameters:

    Name Type Description Default model_bundle_name str

    Name of the model bundle.

    required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]]

    Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 

    input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n

    required load_model_fn Callable[[], LaunchModel_T]

    A function that, when run, loads a model.

    required request_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    required response_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.

    required requirements Optional[List[str]]

    List of pip requirements.

    None pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Union[Dict[str, Any], str]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch.
    • PyTorch fields:
      • pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags

    Example: 

    {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
    create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> Dict[str, Any]\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is my_root:

       my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n

    then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

       my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required base_paths List[str]

    A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.

    required load_predict_fn_module_path str

    The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required load_model_fn_module_path str

    The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required requirements_path str

    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

    required pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Dict[str, Any]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
    create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> Dict[str, Any]\n

    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required repository str

    The name of the Docker repository for the runnable image.

    required tag str

    The tag for the runnable image.

    required command List[str]

    The command that will be used to start the process that listens for requests.

    required env Dict[str, str]

    A dictionary of environment variables that will be passed to the bundle when it is run.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    False post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_latest_model_bundle_v2","title":"get_latest_model_bundle_v2","text":"
    get_latest_model_bundle_v2(\nmodel_bundle_name: str,\n) -> Dict[str, Any]\n

    Get the latest version of a model bundle.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to get.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle_v2","title":"get_model_bundle_v2","text":"
    get_model_bundle_v2(model_bundle_id: str) -> Dict[str, Any]\n

    Get a model bundle.

    Parameters:

    Name Type Description Default model_bundle_id str

    The ID of the model bundle you want to get.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles_v2","title":"list_model_bundles_v2","text":"
    list_model_bundles_v2() -> List[Dict[str, Any]]\n

    List all model bundles.

    Returns:

    Type Description List[Dict[str, Any]]

    A list of dictionaries containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index fe7ecb745652149944546894991df571ef9b32b2..8854b4163a06e90c19f8d6bf1becc439a8c31de6 100644 GIT binary patch delta 15 WcmX@WbbyIXzMF$%n$t!$D@Fh$F9Z(& delta 15 WcmX@WbbyIXzMF%i&weAD6(ay6@B{<^ From 3b254d68ad2c72a55eef0cc4c30c0a7f4dcddae1 Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Mon, 27 Mar 2023 18:35:21 -0700 Subject: [PATCH 10/28] Deployed 3bff553 with MkDocs version: 1.4.2 --- api/client/index.html | 92 +++++++++++++++++++++++++++++++++++++++ objects.inv | Bin 574 -> 579 bytes search/search_index.json | 2 +- sitemap.xml.gz | Bin 320 -> 320 bytes 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/api/client/index.html b/api/client/index.html index e4594f02..2dea3927 100644 --- a/api/client/index.html +++ b/api/client/index.html @@ -471,6 +471,13 @@ clone_model_bundle_with_changes() +

    • + +
  • + + clone_model_bundle_with_changes_v2() + +
  • @@ -742,6 +749,13 @@ clone_model_bundle_with_changes() +
    • + +
  • + + clone_model_bundle_with_changes_v2() + +
  • @@ -1297,6 +1311,84 @@

    + clone_model_bundle_with_changes_v2 + + +

    +
    clone_model_bundle_with_changes_v2(
+    original_model_bundle_id: str,
+    new_app_config: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]
+
    + +
    + +

    Clone a model bundle with an optional new app_config.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    original_model_bundle_id + str +

    The ID of the model bundle you want to clone.

    		 + required +
    new_app_config + Optional[Dict[str, Any]] +

    A dictionary of new app config values to use for the cloned model.

    		 + None +
    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + Dict[str, Any] +

    A dictionary containing the following keys:

    +
      +
    • model_bundle_id: The ID of the cloned model bundle.
      • +
    + +
    + + + +
    + + +

    create_model_bundle diff --git a/objects.inv b/objects.inv index 815c81f8af2b479a7cd9d4454117f6f2a1795c4e..2b28fa7e1ff6f99124c9e7209cf14c780041cd23 100644 GIT binary patch delta 392 zcmV;30eAks1j7W7dw<l9IHBlo7?+BWRVqWt)-`1k*_Uu7<0Vjm_J3?J+ujD)2EmQ-s2IzQ zsu%|;VeMOyL+$iT7<(r6JOP`Us9g?GK@o9LkTR_V1)O28U_=fx;i6#$IyO#Ns})%c z6Qy51>&BZDvLTHbC!3~Hq_Dy_LW;?P*_c+)G>^FbJOhIBxs?qLYg5e|OsXOXEqst@ z`m@aK)`Ps0b8Cv>cBdsST5GBNVjNT;I{H+*+MJVavVO9p^S@9kwXx~DvhIa)H0Z}YH-Al^T5Z1*v2~l8zBP@9 mp%fmE9Hsp%AqB0OpOfDKE&{18lMezV3QTzwFyn8TrvvyjO}Z)o delta 364 zcmV-y0h9j21il22dw*X~!ypg<@OM8&6MeOdKKiC+MiWhp`vBxduPqrp5a`Ulz4Xtz zwa%mpdm9V<3I}%}p(b9YTnZ$YzP7E$wvp)F9hYEja+81-i>}M!@+!H$kiU*f*Wh@` z!0w1MgP%3pR?2-BClvh+<5Cf_N@Xb7x~5Db`x35XyaX!H27kZ?!Hwmp7|V^S7zZg~ z?OWU1Gw7Kx_Dt+~CTwb=HXNdYB4SXGGOYv!oHMUrL=H3IqG1I(Hcl$568Rk9OfKV?Q7#qa=gFkvj6dMxMcU&HItbE94`{m`CllN+Sv46 zS@%LY8ua6yo4=;dp|;;11 diff --git a/search/search_index.json b/search/search_index.json index 58e77c1d..9829d95a 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_callable_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
    create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> Dict[str, Any]\n

    Uploads and registers a model bundle to Scale Launch.

    Parameters:

    Name Type Description Default model_bundle_name str

    Name of the model bundle.

    required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]]

    Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 

    input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n

    required load_model_fn Callable[[], LaunchModel_T]

    A function that, when run, loads a model.

    required request_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    required response_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.

    required requirements Optional[List[str]]

    List of pip requirements.

    None pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Union[Dict[str, Any], str]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch.
    • PyTorch fields:
      • pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags

    Example: 

    {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
    create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> Dict[str, Any]\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is my_root:

       my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n

    then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

       my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required base_paths List[str]

    A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.

    required load_predict_fn_module_path str

    The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required load_model_fn_module_path str

    The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required requirements_path str

    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

    required pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Dict[str, Any]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
    create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> Dict[str, Any]\n

    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required repository str

    The name of the Docker repository for the runnable image.

    required tag str

    The tag for the runnable image.

    required command List[str]

    The command that will be used to start the process that listens for requests.

    required env Dict[str, str]

    A dictionary of environment variables that will be passed to the bundle when it is run.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    False post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_latest_model_bundle_v2","title":"get_latest_model_bundle_v2","text":"
    get_latest_model_bundle_v2(\nmodel_bundle_name: str,\n) -> Dict[str, Any]\n

    Get the latest version of a model bundle.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to get.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle_v2","title":"get_model_bundle_v2","text":"
    get_model_bundle_v2(model_bundle_id: str) -> Dict[str, Any]\n

    Get a model bundle.

    Parameters:

    Name Type Description Default model_bundle_id str

    The ID of the model bundle you want to get.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles_v2","title":"list_model_bundles_v2","text":"
    list_model_bundles_v2() -> List[Dict[str, Any]]\n

    List all model bundles.

    Returns:

    Type Description List[Dict[str, Any]]

    A list of dictionaries containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

    Simple, scalable, and high performance ML service deployment in python.

    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

    What's going on here:

    • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
    • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

    "},{"location":"#installation","title":"Installation","text":"

    To use Scale Launch, first install it using pip:

    Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"

    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

    The CLI can be used as scale-launch ....

    "},{"location":"cli/#help","title":"Help","text":"

    Run scale-launch --help for more options.

    scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

    Scale Launch Python Client.

    Initializes a Scale Launch Client.

    Parameters:

    Name Type Description Default api_key str

    Your Scale API key

    required endpoint Optional[str]

    The Scale Launch Endpoint (this should not need to be changed)

    None self_hosted bool

    True iff you are connecting to a self-hosted Scale Launch

    False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

    Must have exactly one of urls or inputs passed in.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or the name of a the bundle to use for inference.

    required urls List[str]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    None inputs Optional[List[Dict[str, Any]]]

    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

    None batch_url_file_location Optional[str]

    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

    None serialization_format str

    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

    'JSON' labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None cpus Optional[int]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
    • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains job_id as a key, and the ID as the value.

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The existing bundle or its ID.

    required app_config Optional[Dict]

    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

    None

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes_v2","title":"clone_model_bundle_with_changes_v2","text":"
    clone_model_bundle_with_changes_v2(\noriginal_model_bundle_id: str,\nnew_app_config: Optional[Dict[str, Any]] = None,\n) -> Dict[str, Any]\n

    Clone a model bundle with an optional new app_config.

    Parameters:

    Name Type Description Default original_model_bundle_id str

    The ID of the model bundle you want to clone.

    required new_app_config Optional[Dict[str, Any]]

    A dictionary of new app config values to use for the cloned model.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the cloned model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_callable_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required predict_fn_or_cls Optional[Callable[[Any], Any]]

    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

    None model Optional[LaunchModel_T]

    Typically a trained Neural Network, e.g. a Pytorch module.

    Exactly one of model and load_model_fn must be provided.

    None load_model_fn Optional[Callable[[], LaunchModel_T]]

    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

    Exactly one of model and load_model_fn must be provided.

    None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

    Function that, when called with a model, returns a function that carries out inference.

    If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

    In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

    None requirements Optional[List[str]]

    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

    None app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

      .. code-block:: python

      { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

    • Tensorflow fields:

      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
    required globals_copy Optional[Dict[str, Any]]

    Dictionary of the global symbol table. Normally provided by globals() built-in function.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
    create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> Dict[str, Any]\n

    Uploads and registers a model bundle to Scale Launch.

    Parameters:

    Name Type Description Default model_bundle_name str

    Name of the model bundle.

    required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]]

    Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 

    input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n

    required load_model_fn Callable[[], LaunchModel_T]

    A function that, when run, loads a model.

    required request_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    required response_schema Type[BaseModel]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.

    required requirements Optional[List[str]]

    List of pip requirements.

    None pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Union[Dict[str, Any], str]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
    Warning

    This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create. The name must be unique across all bundles that you own.

    required base_paths List[str]

    The paths on the local filesystem where the bundle code lives.

    required requirements_path str

    A path on the local filesystem where a requirements.txt file lives.

    required env_params Dict[str, str]

    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

    • framework_type: either tensorflow or pytorch.
    • PyTorch fields:
      • pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags

    Example: 

    {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n

    required load_predict_fn_module_path str

    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

    required load_model_fn_module_path str

    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

    required app_config Optional[Union[Dict[str, Any], str]]

    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

    None request_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

    None response_schema Optional[Type[BaseModel]]

    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

    None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
    create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> Dict[str, Any]\n

    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

    For example, if you have a directory structure like so, and your current working directory is my_root:

       my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n

    then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

       my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n

    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required base_paths List[str]

    A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.

    required load_predict_fn_module_path str

    The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required load_model_fn_module_path str

    The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required requirements_path str

    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

    required pytorch_image_tag Optional[str]

    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None tensorflow_version Optional[str]

    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_repository Optional[str]

    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

    None custom_base_image_tag Optional[str]

    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

    None app_config Optional[Dict[str, Any]]

    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

    None

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
    create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> Dict[str, Any]\n

    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to create.

    required request_schema Type[BaseModel]

    A Pydantic model that defines the request schema for the bundle.

    required response_schema Type[BaseModel]

    A Pydantic model that defines the response schema for the bundle.

    required repository str

    The name of the Docker repository for the runnable image.

    required tag str

    The tag for the runnable image.

    required command List[str]

    The command that will be used to start the process that listens for requests.

    required env Dict[str, str]

    A dictionary of environment variables that will be passed to the bundle when it is run.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

    required model_bundle Union[ModelBundle, str]

    The ModelBundle that the endpoint should serve.

    required cpus int

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    3 memory str

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    '8Gi' storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus int

    Number of gpus each worker should get, e.g. 0, 1, etc.

    0 min_workers int

    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

    1 max_workers int

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

    1 per_worker int

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

    Here is our recommendation for computing per_worker:

    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
    10 gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None endpoint_type str

    Either \"sync\" or \"async\".

    'sync' high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    False post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None update_if_exists bool

    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

    False labels Optional[Dict[str, str]]

    An optional dictionary of key/value pairs to associate with this endpoint.

    None

    Returns:

    Type Description Optional[Endpoint]

    A Endpoint object that can be used to make requests to the endpoint.

    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Deletes a model endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    A ModelEndpoint object.

    required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

    required model_bundle Optional[Union[ModelBundle, str]]

    The ModelBundle that the endpoint should serve.

    None cpus Optional[float]

    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

    None memory Optional[str]

    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

    None storage Optional[str]

    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

    None gpus Optional[int]

    Number of gpus each worker should get, e.g. 0, 1, etc.

    None min_workers Optional[int]

    The minimum number of workers. Must be greater than or equal to 0.

    None max_workers Optional[int]

    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

    None per_worker Optional[int]

    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
    None gpu_type Optional[str]

    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

    • nvidia-tesla-t4
    • nvidia-ampere-a10
    None high_priority Optional[bool]

    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

    None post_inference_hooks Optional[List[PostInferenceHooks]]

    List of hooks to trigger after inference tasks are served.

    None default_callback_url Optional[str]

    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

    None default_callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None default_callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

    Gets inference results from a previously created batch job.

    Parameters:

    Name Type Description Default batch_job_id str

    An id representing the batch task job. This id is the in the response from calling batch_async_request.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary that contains the following fields:

    Dict[str, Any]
    • status: The status of the job.
    Dict[str, Any]
    • result: The url where the result is stored.
    Dict[str, Any]
    • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
    Dict[str, Any]
    • num_tasks_pending: The number of tasks that are still pending.
    Dict[str, Any]
    • num_tasks_completed: The number of tasks that have completed.
    "},{"location":"api/client/#launch.client.LaunchClient.get_latest_model_bundle_v2","title":"get_latest_model_bundle_v2","text":"
    get_latest_model_bundle_v2(\nmodel_bundle_name: str,\n) -> Dict[str, Any]\n

    Get the latest version of a model bundle.

    Parameters:

    Name Type Description Default model_bundle_name str

    The name of the model bundle you want to get.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

    Returns a model bundle specified by bundle_name that the user owns.

    Parameters:

    Name Type Description Default model_bundle Union[ModelBundle, str]

    The bundle or its name.

    required

    Returns:

    Type Description ModelBundle

    A ModelBundle object

    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle_v2","title":"get_model_bundle_v2","text":"
    get_model_bundle_v2(model_bundle_id: str) -> Dict[str, Any]\n

    Get a model bundle.

    Parameters:

    Name Type Description Default model_bundle_id str

    The ID of the model bundle you want to get.

    required

    Returns:

    Type Description Dict[str, Any]

    A dictionary containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

    Gets a model endpoint associated with a name.

    Parameters:

    Name Type Description Default endpoint_name str

    The name of the endpoint to retrieve.

    required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n

    Returns a list of model bundles that the user owns.

    Returns:

    Type Description List[ModelBundle]

    A list of ModelBundle objects

    "},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles_v2","title":"list_model_bundles_v2","text":"
    list_model_bundles_v2() -> List[Dict[str, Any]]\n

    List all model bundles.

    Returns:

    Type Description List[Dict[str, Any]]

    A list of dictionaries containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
    • metadata: A dictionary of metadata associated with the model bundle.
    • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n

    Lists all model endpoints that the user owns.

    Returns:

    Type Description List[Endpoint]

    A list of ModelEndpoint objects.

    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

    Retrieves the logs for the creation of the endpoint.

    Parameters:

    Name Type Description Default model_endpoint Union[ModelEndpoint, str]

    The endpoint or its name.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

    Parameters:

    Name Type Description Default batch_csv_location_fn Callable[[], str]

    Function that generates batch_csv_urls for upload_batch_csv_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

    Parameters:

    Name Type Description Default bundle_location_fn Callable[[], str]

    Function that generates bundle_urls for upload_bundle_fn.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

    upload_batch_csv_fn(csv_text, csv_url)\n

    This function should directly write the contents of csv_text as a text string into csv_url.

    Parameters:

    Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

    upload_bundle_fn(serialized_bundle, bundle_url)\n

    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

    Parameters:

    Name Type Description Default upload_bundle_fn Callable[[str, str], None]

    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

    required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

    Bases: str, Enum

    Post-inference hooks are functions that are called after inference is complete.

    Attributes:

    Name Type Description CALLBACK str

    The callback hook is called with the inference response and the task ID.

    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

    Represents a ModelBundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n

    An optional user-specified configuration mapping for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n

    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
    id: Optional[str] = None\n

    A globally unique identifier for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
    location: Optional[str] = None\n

    An opaque location for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n

    Arbitrary metadata for the bundle.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
    name: str\n

    The name of the bundle. Must be unique across all bundles that the user owns.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
    packaging_type: Optional[str] = None\n

    The packaging type for the bundle. Can be cloudpickle or zip.

    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
    requirements: Optional[List[str]] = None\n

    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

    All classes here are returned by the get_model_endpoint method and provide a predict function.

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    An asynchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n

    Runs an asynchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required

    Returns:

    Name Type Description EndpointResponseFuture

    An EndpointResponseFuture such the user can use to query the status of the request.

    Example EndpointResponseFuture EndpointResponseFuture

    .. code-block:: python

    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

    Parameters:

    Name Type Description Default requests Sequence[EndpointRequest]

    List of EndpointRequests. Request_ids must all be distinct.

    required

    Returns:

    Type Description AsyncEndpointBatchResponse

    an AsyncEndpointResponse keeping track of the inference requests made

    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

    Bases: Endpoint

    A synchronous model endpoint.

    Parameters:

    Name Type Description Default model_endpoint ModelEndpoint

    ModelEndpoint object.

    required client

    A LaunchClient object

    required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n

    Runs a synchronous prediction request.

    Parameters:

    Name Type Description Default request EndpointRequest

    The EndpointRequest object that contains the payload.

    required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

    Batch jobs are created using the batch_async_request method of the LaunchClient.

    Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

    Note

    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

    Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

    Warning

    This feature is currently in beta, and the API is likely to change.

    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

    Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

    Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

    Represents a single request to either a SyncEndpoint or AsyncEndpoint.

    Parameters:

    Name Type Description Default url Optional[str]

    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

    Exactly one of url and args must be specified.

    None args Optional[Dict]

    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

    Exactly one of url and args must be specified.

    None return_pickled Optional[bool]

    Whether the output should be a pickled python object, or directly returned serialized json.

    False callback_url Optional[str]

    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

    None callback_auth_kind Optional[Literal[basic, mtls]]

    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_username Optional[str]

    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_password Optional[str]

    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

    None callback_auth_cert Optional[str]

    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None callback_auth_key Optional[str]

    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

    None request_id Optional[str]

    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

    This object should not be directly instantiated by the user.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required endpoint_name str

    The name of the endpoint.

    required async_task_id str

    An async task id.

    required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n

    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

    Parameters:

    Name Type Description Default timeout Optional[float]

    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

    None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

    Represents a response received from a Endpoint.

    Parameters:

    Name Type Description Default client

    An instance of LaunchClient.

    required status str

    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

    required result_url Optional[str]

    A string that is a url containing the pickled python object from the Endpoint's predict function.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None result Optional[str]

    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

    None traceback Optional[str]

    The stack trace if the inference endpoint raised an error. Can be used for debugging

    None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

    Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

    Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

    To list all the model bundles you own, use list_model_bundles.

    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

    Info

    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

    A SyncEndpoint is good if:

    • You have strict latency requirements (e.g. on the order of seconds or less).
    • You are willing to have resources continually allocated.

    An AsyncEndpoint is good if:

    • You want to save on compute costs.
    • Your inference code takes a long time to run.
    • Your latency requirements are on the order of minutes.

    A BatchJob is good if:

    • You know there is a large batch of inputs ahead of time.
    • You want to optimize for throughput instead of latency.
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

    Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

    Note

    Sync model endpoints require at least 1 min_worker.

    Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

    Model endpoints can be listed, updated, and deleted using the Launch API.

    Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
    Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
    Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"

    Creating deployments on Launch generally involves three steps:

    1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

    2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

    3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

    "}]} \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 8854b4163a06e90c19f8d6bf1becc439a8c31de6..b2789220c7adf9d04d2adec048c651132fab938f 100644 GIT binary patch delta 15 WcmX@WbbyIXzMF$1(q$u?6(ay5%LCK^ delta 15 WcmX@WbbyIXzMF$%n$t!$D@Fh$F9Z(& From 4cf1494644d483479a7d35d0340fdcf7149bb56e Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Mon, 27 Mar 2023 19:22:44 -0700 Subject: [PATCH 11/28] Deployed 7cf0303 with MkDocs version: 1.4.2 --- api/client/index.html | 61 +- api/model_bundles/index.html | 1725 ++++++++++++++++++++++++++++++++-- objects.inv | Bin 579 -> 842 bytes search/search_index.json | 2 +- sitemap.xml.gz | Bin 320 -> 320 bytes 5 files changed, 1673 insertions(+), 115 deletions(-) diff --git a/api/client/index.html b/api/client/index.html index 2dea3927..adbb8fc3 100644 --- a/api/client/index.html +++ b/api/client/index.html @@ -1248,8 +1248,12 @@

    -

    Clones an existing model bundle with changes to its app config. (More fields coming soon)

    +
    + Warning +

    This method is deprecated. Use +clone_model_bundle_with_changes_v2 instead.

    +

    Parameters:

    @@ -1319,7 +1323,7 @@ 

    clone_model_bundle_with_changes_v2(
     original_model_bundle_id: str,
     new_app_config: Optional[Dict[str, Any]] = None,
-) -> Dict[str, Any]
+) -> CreateModelBundleV2Response
    @@ -1371,9 +1375,9 @@

    Dict[str, Any] + CreateModelBundleV2Response -

    @@ -1620,7 +1624,7 @@

    custom_base_image_repository: Optional[str] = None, custom_base_image_tag: Optional[str] = None, app_config: Optional[Union[Dict[str, Any], str]] = None -) -> Dict[str, Any] +) -> CreateModelBundleV2Response
    @@ -1793,9 +1797,9 @@

    Dict[str, Any] + CreateModelBundleV2Response -

    @@ -1995,7 +1999,7 @@

    custom_base_image_repository: Optional[str] = None, custom_base_image_tag: Optional[str] = None, app_config: Optional[Dict[str, Any]] = None -) -> Dict[str, Any] +) -> CreateModelBundleV2Response
    @@ -2191,9 +2195,9 @@

    Dict[str, Any] + CreateModelBundleV2Response -

    @@ -2223,7 +2227,7 @@

    tag: str, command: List[str], env: Dict[str, str] -) -> Dict[str, Any] +) -> CreateModelBundleV2Response
    @@ -2327,9 +2331,9 @@

    - @@ -3118,7 +3122,7 @@ 

    get_latest_model_bundle_v2(
     model_bundle_name: str,
-) -> Dict[str, Any]
+) -> ModelBundleV2Response
    @@ -3160,9 +3164,9 @@

    Dict[str, Any] + ModelBundleV2Response -

    - - diff --git a/api/model_bundles/index.html b/api/model_bundles/index.html index f0a89c8e..1c47fb94 100644 --- a/api/model_bundles/index.html +++ b/api/model_bundles/index.html @@ -465,6 +465,121 @@ + + +
  • + + ModelBundleV2Response + + + + +
    • + +
  • + + PytorchFramework + + + + +
    • + +
  • + + RunnableImageFlavor + + + + +
    • + +
  • + + TensorflowFramework + + + + +
    • + +
  • + + ZipArtifactFlavor + + + +
    • @@ -618,65 +917,364 @@
    • - - ModelBundle + + CloudpickleArtifactFlavor -
      • - -
    • - - packaging_type + +
    • + + CreateModelBundleV2Response -
      • + + + + +
    • + + CustomFramework + + + + +
      • + +
    • + + ListModelBundlesV2Response + + + + +
      • + +
    • + + ModelBundle + + + + +
      • + +
    • + + ModelBundleV2Response + + + + +
      • + +
    • + + PytorchFramework + + + + +
      • + +
    • + + RunnableImageFlavor + + + + +
      • + +
    • + + TensorflowFramework + + + + +
      • + +
    • + + ZipArtifactFlavor + + + - - - - - - -
      -
      - - + +
      + + + + + +
      +
      + + + + +

      Model Bundles

      + + +
      + + + +

      + CloudpickleArtifactFlavor + + +

      + + +
      +

      + Bases: BaseModel

      + + + + + + +
      + + + + + + + +
      + + + +

      + app_config + + + + class-attribute + + +

      +
      app_config: Optional[Dict[str, Any]]
+
      + +
      + +

      Optional configuration for the application.

      +
      + +
      + +
      + + + +

      + framework + + + + class-attribute + + +

      +
      framework: Union[
+    PytorchFramework, TensorflowFramework, CustomFramework
+] = Field(Ellipsis, discriminator="framework_type")
+
      + +
      + +

      Machine Learning framework specification. Either +PytorchFramework, +TensorflowFramework, or +CustomFramework.

      +
      + +
      + +
      + + + +

      + load_model_fn + + + + class-attribute + + +

      +
      load_model_fn: str
+
      + +
      + +

      Function which, when called, returns the model object.

      +
      + +
      + +
      + + + +

      + load_predict_fn + + + + class-attribute + + +

      +
      load_predict_fn: str
+
      + +
      + +

      Function which, when called, returns the prediction function.

      +
      + +
      + +
      + + + +

      + requirements + + + + class-attribute + + +

      +
      requirements: List[str]
+
      + +
      + +

      List of requirements to install in the environment before running the model.

      +
      + +
      + + + + + +
      + +
      + +
      + +
      + + + +

      + CreateModelBundleV2Response + + +

      + + +
      +

      + Bases: BaseModel

      + + +

      Response object for creating a Model Bundle.

      + + + + + +
      + + + + + + + +
      + + + +

      + model_bundle_id + + + + class-attribute + + +

      +
      model_bundle_id: str
+
      + +
      + +

      ID of the Model Bundle.

      +
      + +
      + + + + + +
      + +
      + +
      + +
      + + + +

      + CustomFramework + + +

      + + +
      +

      + Bases: BaseModel

      + + + + + + +
      + + + + + + + +
      + + + +

      + image_repository + + + + class-attribute + + +

      +
      image_repository: str
+
      + +
      + +

      Docker image repository to use as the base image.

      +
      + +
      + +
      + + + +

      + image_tag + + + + class-attribute + + +

      +
      image_tag: str
+
      + +
      + +

      Docker image tag to use as the base image.

      +
      + +
      + + + + + +
      + +
      + +
      + +
      + + + +

      + ListModelBundlesV2Response + + +

      + + +
      +

      + Bases: BaseModel

      + + +

      Response object for listing Model Bundles.

      + + + + + +
      + + + + + + + +
      + + + +

      + model_bundles + + + + class-attribute + + +

      +
      model_bundles: List[ModelBundleV2Response]
+
      + +
      + +

      A list of Model Bundles.

      +
      + +
      + + + + + +
      + +
      + +
      + +
      + + + +

      + ModelBundle + + + + dataclass + + +

      + + +
      + + +

      Represents a ModelBundle.

      + + + + + +
      + + + + + + + +
      + + + +

      + app_config + + + + class-attribute + + +

      +
      app_config: Optional[Dict[Any, Any]] = None
+
      + +
      + +

      An optional user-specified configuration mapping for the bundle.

      +
      + +
      + +
      + + + +

      + env_params + + + + class-attribute + + +

      +
      env_params: Optional[Dict[str, str]] = None
+
      + +
      + +

      A dictionary that dictates environment information. See LaunchClient.create_model_bundle +for more information.

      +
      + +
      + +
      + + + +

      + id + + + + class-attribute + + +

      +
      id: Optional[str] = None
+
      + +
      + +

      A globally unique identifier for the bundle.

      +
      + +
      + +
      + + + +

      + location + + + + class-attribute + + +

      +
      location: Optional[str] = None
+
      + +
      + +

      An opaque location for the bundle.

      +
      + +
      + +
      + + + +

      + metadata + + + + class-attribute + + +

      +
      metadata: Optional[Dict[Any, Any]] = None
+
      + +
      + +

      Arbitrary metadata for the bundle.

      +
      + +
      + +
      + + + +

      + name + + + + class-attribute + + +

      +
      name: str
+
      + +
      + +

      The name of the bundle. Must be unique across all bundles that the user owns.

      +
      + +
      + +
      + + + +

      + packaging_type + + + + class-attribute + + +

      +
      packaging_type: Optional[str] = None
+
      + +
      + +

      The packaging type for the bundle. Can be cloudpickle or zip.

      +
      + +
      + +
      + + + +

      + requirements + + + + class-attribute + + +

      +
      requirements: Optional[List[str]] = None
+
      + +
      + +

      A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle +for more information.

      +
      + +
      + + + + + +
      + +
      + +
      + +
      + + + +

      + ModelBundleV2Response + + +

      + + +
      +

      + Bases: BaseModel

      + + +

      Response object for a single Model Bundle.

      + + + + + +
      + + + + + + + +
      + + + +

      + created_at + + + + class-attribute + + +

      +
      created_at: datetime.datetime
+
      + +
      + +

      Timestamp of when the Model Bundle was created.

      +
      + +
      + +
      + + + +

      + flavor + + + + class-attribute + + +

      +
      flavor: Union[
+    CloudpickleArtifactFlavor,
+    ZipArtifactFlavor,
+    RunnableImageFlavor,
+] = Field(Ellipsis, discriminator="flavor")
+
      + +
      + +

      Flavor of the Model Bundle, representing how the model bundle was packaged. Either +CloudpickleArtifactFlavor, +ZipArtifactFlavor, or +RunnableImageFlavor.

      +
      + +
      + +
      + + + +

      + id + + + + class-attribute + + +

      +
      id: str
+
      + +
      + +

      ID of the Model Bundle.

      +
      + +
      + +
      + + + +

      + metadata + + + + class-attribute + + +

      +
      metadata: Dict[str, Any]
+
      + +
      + +

      Metadata associated with the Model Bundle.

      +
      + +
      + +
      + + + +

      + model_artifact_ids + + + + class-attribute + +

      +
      model_artifact_ids: List[str]
+
      -

      Model Bundles

      +
      + +

      IDs of the Model Artifacts associated with the Model Bundle.

      +
      +
      -
      +
      -

      - ModelBundle +

      + name - dataclass + class-attribute -

      + +
      name: str
+
      + +
      + +

      Name of the Model Bundle.

      +
      + +
      + + + + + +
      + +
      + +
      + +
      + + + +

      + PytorchFramework + + +

      +

      + Bases: BaseModel

      + + + -

      Represents a ModelBundle.

      + +
      + + + + + + + +
      + + + +

      + pytorch_image_tag + + + + class-attribute + + +

      +
      pytorch_image_tag: str
+
      + +
      + +

      Image tag of the Pytorch image to use.

      +
      + +
      + + + + + +
      + +
      + +
      + +
      + + + +

      + RunnableImageFlavor + + +

      + + +
      +

      + Bases: BaseModel

      + @@ -740,21 +2176,21 @@

      -

      - app_config +

      + command class-attribute -

      -
      app_config: Optional[Dict[Any, Any]] = None
+
+
      command: List[str]
 
      
 
   
      
   
-      
      An optional user-specified configuration mapping for the bundle.
      
+      
      Command to run the image.
      @@ -763,22 +2199,21 @@

      -

      - env_params +

      + env class-attribute -

      -
      env_params: Optional[Dict[str, str]] = None
+
+
      env: Optional[Dict[str, str]]
 
      
 
   
      
   
-      
      A dictionary that dictates environment information. See LaunchClient.create_model_bundle
-for more information.
      
+      
      Environment variables to set when running the image.
      @@ -787,21 +2222,21 @@

      -

      - id +

      + repository class-attribute -

      -
      id: Optional[str] = None
+
+
      repository: str
 
      
 
   
      
   
-      
      A globally unique identifier for the bundle.
      
+      
      Docker repository of the image.
      @@ -810,44 +2245,143 @@

      -

      - location +

      + tag class-attribute -

      -
      location: Optional[str] = None
+
+
      tag: str
 
      
 
   
      
   
-      
      An opaque location for the bundle.
      
+      
      Docker tag of the image.
      
+  
      
+
+
      + + + + + +
      +
      +
      + + + +

      + TensorflowFramework + + +

      + + +
      +

      + Bases: BaseModel

      + + + + + + +
      + + + + + + +
      -

      - metadata +

      + tensorflow_version class-attribute -

      -
      metadata: Optional[Dict[Any, Any]] = None
+
+
      tensorflow_version: str
 
      
 
   
      
   
-      
      Arbitrary metadata for the bundle.
      
+      
      Tensorflow version to use.
      
+  
      
+
+
      + + + + + +
      + +
      + +
      + +
      + + + +

      + ZipArtifactFlavor + + +

      + + +
      +

      + Bases: BaseModel

      + + + + + + +
      + + + + + + + +
      + + + +

      + app_config + + + + class-attribute + + +

      +
      app_config: Optional[Dict[str, Any]]
+
      + +
      + +

      Optional configuration for the application.

      @@ -856,21 +2390,26 @@

      -

      - name +

      + framework class-attribute -

      -
      name: str
+
+
      framework: Union[
+    PytorchFramework, TensorflowFramework, CustomFramework
+] = Field(Ellipsis, discriminator="framework_type")
 
      
 
   
      
   
-      
      The name of the bundle. Must be unique across all bundles that the user owns.
      
+      
      Machine Learning framework specification. Either
+PytorchFramework,
+TensorflowFramework, or
+CustomFramework.
      @@ -879,21 +2418,21 @@

      -

      - packaging_type +

      + load_model_fn_module_path class-attribute -

      -
      packaging_type: Optional[str] = None
+
+
      load_model_fn_module_path: str
 
      
 
   
      
   
-      
      The packaging type for the bundle. Can be cloudpickle or zip.
      
+      
      Path to the module to load the model object.
      @@ -902,7 +2441,30 @@

      -

      +

      + load_predict_fn_module_path + + + + class-attribute + + +

      +
      load_predict_fn_module_path: str
+
      + +
      + +

      Path to the module to load the prediction function.

      +
      + +
      + +
      + + + +

      requirements @@ -910,14 +2472,13 @@

      class-attribute -

      -
      requirements: Optional[List[str]] = None
+
+
      requirements: List[str]
 
      
 
   
      
   
-      
      A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle
-for more information.
      
+      
      List of requirements to install in the environment before running the model.
      diff --git a/objects.inv b/objects.inv index 2b28fa7e1ff6f99124c9e7209cf14c780041cd23..33e012644310301a98924b5b9913177dbe721229 100644 GIT binary patch delta 732 zcmV<20wevy1j+`GdViQZZ`&{sfcO3i1MONG&DbpsVl+Ss1WD1M3vs3>i7-X7BxS?@ zzSP6EWGjH9J8u#}eY*D@si8KaWI`)xB71J%9@>|M+kY1tLRfrTfK$tE%kuM=#n(^z z-LKdhq9{3py%F5Yf3*xPDNkFXvgTtjp{c$;d8 z!L=<=&>{&beSaUzB5V^-bmZJnaJy)sh1O2nWQ>!{q}0APUWpw0mp%3mucron7zKni zLb@21M<+U;YEsJ&*D+5uh8()8W9@H=ZnFN&q}TtUw00BNiw*I~zniUtiadMa}6p^`i6H`L%Z z`n$bd@RP%P@7@64%*)ElO7&WVgrRT1&@12z{8g~%HlR%*17{=#vu|)$ns;0 zrXmtV?yz7`Mczdbg1*b87D4HAM1;TWQ%?{jw&|Q>nClievQNDBml;nD1~uAEkeIMl z1tudCt*ORzFl@`7V$OENZ)t+{xwboOs7#PpJXNmot@L4b|M;sM$vNPHgYMavMC9Cm z2^`J)%|o4LQ3tUk{lRJGzr`;QP4ReKy>Bk?9tRgumjQ6RzPXyidc7wu5wOVa-UA1I Oqfz+{q49r87B!4H?qkgW delta 467 zcmV;^0WAK?2Ezo9dVkj4PQxG+0N{I{qKV$xMK8TmGoy(n#ytQzqo*wyJs{ATy}k5j z-CAc-h24z>z8-$g0SPtnGUZYrx%9PdJ+_TR@9ww+Ym=J&@_*@{X7GL^SPA`4r^1*8%(Mq2rYb& zX!^6v?bd_5lYeuH;dZAbE?R4;{9+tpj;i*xaV9z5-*nmk_&8kP`=ON^95xI|E4xT1k? zpb%6rx6={4=BX|WR@hhfKWIb7+&6EC<rL3 diff --git a/search/search_index.json b/search/search_index.json index 9829d95a..c73a0521 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

      Simple, scalable, and high performance ML service deployment in python.

      "},{"location":"#example","title":"Example","text":"Launch Usage
      import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

      What's going on here:

      • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
      • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
      • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
      • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
      • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

      Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

      "},{"location":"#installation","title":"Installation","text":"

      To use Scale Launch, first install it using pip:

      Installation
      pip install -U scale-launch\n
      "},{"location":"cli/","title":"CLI","text":"

      Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

      The CLI can be used as scale-launch ....

      "},{"location":"cli/#help","title":"Help","text":"

      Run scale-launch --help for more options.

      scale-launch --help
          This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
      "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
      LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

      Scale Launch Python Client.

      Initializes a Scale Launch Client.

      Parameters:

      Name Type Description Default api_key str

      Your Scale API key

      required endpoint Optional[str]

      The Scale Launch Endpoint (this should not need to be changed)

      None self_hosted bool

      True iff you are connecting to a self-hosted Scale Launch

      False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
      batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

      Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

      Must have exactly one of urls or inputs passed in.

      Parameters:

      Name Type Description Default model_bundle Union[ModelBundle, str]

      The bundle or the name of a the bundle to use for inference.

      required urls List[str]

      A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

      None inputs Optional[List[Dict[str, Any]]]

      A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

      None batch_url_file_location Optional[str]

      In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

      None serialization_format str

      Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

      'JSON' labels Optional[Dict[str, str]]

      An optional dictionary of key/value pairs to associate with this endpoint.

      None cpus Optional[int]

      Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

      None memory Optional[str]

      Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

      None storage Optional[str]

      Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

      None gpus Optional[int]

      Number of gpus each worker should get, e.g. 0, 1, etc.

      None max_workers Optional[int]

      The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

      None per_worker Optional[int]

      The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

      • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
      • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      None gpu_type Optional[str]

      If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

      • nvidia-tesla-t4
      • nvidia-ampere-a10
      None

      Returns:

      Type Description Dict[str, Any]

      A dictionary that contains job_id as a key, and the ID as the value.

      "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
      clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n

      Clones an existing model bundle with changes to its app config. (More fields coming soon)

      Parameters:

      Name Type Description Default model_bundle Union[ModelBundle, str]

      The existing bundle or its ID.

      required app_config Optional[Dict]

      The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

      None

      Returns:

      Type Description ModelBundle

      A ModelBundle object

      "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes_v2","title":"clone_model_bundle_with_changes_v2","text":"
      clone_model_bundle_with_changes_v2(\noriginal_model_bundle_id: str,\nnew_app_config: Optional[Dict[str, Any]] = None,\n) -> Dict[str, Any]\n

      Clone a model bundle with an optional new app_config.

      Parameters:

      Name Type Description Default original_model_bundle_id str

      The ID of the model bundle you want to clone.

      required new_app_config Optional[Dict[str, Any]]

      A dictionary of new app config values to use for the cloned model.

      None

      Returns:

      Type Description Dict[str, Any]

      A dictionary containing the following keys:

      • model_bundle_id: The ID of the cloned model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
      create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
      Warning

      This method is deprecated. Use create_model_bundle_from_callable_v2 instead.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create. The name must be unique across all bundles that you own.

      required predict_fn_or_cls Optional[Callable[[Any], Any]]

      Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

      None model Optional[LaunchModel_T]

      Typically a trained Neural Network, e.g. a Pytorch module.

      Exactly one of model and load_model_fn must be provided.

      None load_model_fn Optional[Callable[[], LaunchModel_T]]

      A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

      Exactly one of model and load_model_fn must be provided.

      None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

      Function that, when called with a model, returns a function that carries out inference.

      If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

      Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

      In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

      None requirements Optional[List[str]]

      A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

      [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

      If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

      None app_config Optional[Union[Dict[str, Any], str]]

      Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

      None env_params Dict[str, str]

      A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

      • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

        .. code-block:: python

        { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

      • Tensorflow fields:

        • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
      required globals_copy Optional[Dict[str, Any]]

      Dictionary of the global symbol table. Normally provided by globals() built-in function.

      None request_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

      None response_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

      None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
      create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> Dict[str, Any]\n

      Uploads and registers a model bundle to Scale Launch.

      Parameters:

      Name Type Description Default model_bundle_name str

      Name of the model bundle.

      required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]]

      Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 

      input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n

      required load_model_fn Callable[[], LaunchModel_T]

      A function that, when run, loads a model.

      required request_schema Type[BaseModel]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

      required response_schema Type[BaseModel]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.

      required requirements Optional[List[str]]

      List of pip requirements.

      None pytorch_image_tag Optional[str]

      The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None tensorflow_version Optional[str]

      The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_repository Optional[str]

      The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_tag Optional[str]

      The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

      None app_config Optional[Union[Dict[str, Any], str]]

      An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

      None

      Returns:

      Type Description Dict[str, Any]

      A dictionary containing the following keys:

      • model_bundle_id: The ID of the created model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
      create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
      Warning

      This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create. The name must be unique across all bundles that you own.

      required base_paths List[str]

      The paths on the local filesystem where the bundle code lives.

      required requirements_path str

      A path on the local filesystem where a requirements.txt file lives.

      required env_params Dict[str, str]

      A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

      • framework_type: either tensorflow or pytorch.
      • PyTorch fields:
        • pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags

      Example: 

      {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n

      required load_predict_fn_module_path str

      A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

      required load_model_fn_module_path str

      A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

      required app_config Optional[Union[Dict[str, Any], str]]

      Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

      None request_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

      None response_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

      None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
      create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> Dict[str, Any]\n

      Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

      For example, if you have a directory structure like so, and your current working directory is my_root:

         my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n

      then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

         my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n

      and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create.

      required base_paths List[str]

      A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.

      required load_predict_fn_module_path str

      The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

      required load_model_fn_module_path str

      The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

      required request_schema Type[BaseModel]

      A Pydantic model that defines the request schema for the bundle.

      required response_schema Type[BaseModel]

      A Pydantic model that defines the response schema for the bundle.

      required requirements_path str

      Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

      required pytorch_image_tag Optional[str]

      The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None tensorflow_version Optional[str]

      The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_repository Optional[str]

      The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_tag Optional[str]

      The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

      None app_config Optional[Dict[str, Any]]

      An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

      None

      Returns:

      Type Description Dict[str, Any]

      A dictionary containing the following keys:

      • model_bundle_id: The ID of the created model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
      create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> Dict[str, Any]\n

      Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create.

      required request_schema Type[BaseModel]

      A Pydantic model that defines the request schema for the bundle.

      required response_schema Type[BaseModel]

      A Pydantic model that defines the response schema for the bundle.

      required repository str

      The name of the Docker repository for the runnable image.

      required tag str

      The tag for the runnable image.

      required command List[str]

      The command that will be used to start the process that listens for requests.

      required env Dict[str, str]

      A dictionary of environment variables that will be passed to the bundle when it is run.

      required

      Returns:

      Type Description Dict[str, Any]

      A dictionary containing the following keys:

      • model_bundle_id: The ID of the created model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
      create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

      Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

      Parameters:

      Name Type Description Default endpoint_name str

      The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

      required model_bundle Union[ModelBundle, str]

      The ModelBundle that the endpoint should serve.

      required cpus int

      Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

      3 memory str

      Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

      '8Gi' storage Optional[str]

      Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

      None gpus int

      Number of gpus each worker should get, e.g. 0, 1, etc.

      0 min_workers int

      The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

      1 max_workers int

      The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

      1 per_worker int

      The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

      • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

      Here is our recommendation for computing per_worker:

      1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      10 gpu_type Optional[str]

      If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

      • nvidia-tesla-t4
      • nvidia-ampere-a10
      None endpoint_type str

      Either \"sync\" or \"async\".

      'sync' high_priority Optional[bool]

      Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

      False post_inference_hooks Optional[List[PostInferenceHooks]]

      List of hooks to trigger after inference tasks are served.

      None default_callback_url Optional[str]

      The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

      None default_callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None update_if_exists bool

      If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

      False labels Optional[Dict[str, str]]

      An optional dictionary of key/value pairs to associate with this endpoint.

      None

      Returns:

      Type Description Optional[Endpoint]

      A Endpoint object that can be used to make requests to the endpoint.

      "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
      delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

      Deletes a model endpoint.

      Parameters:

      Name Type Description Default model_endpoint Union[ModelEndpoint, str]

      A ModelEndpoint object.

      required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
      edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

      Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

      • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

      Parameters:

      Name Type Description Default model_endpoint Union[ModelEndpoint, str]

      The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

      required model_bundle Optional[Union[ModelBundle, str]]

      The ModelBundle that the endpoint should serve.

      None cpus Optional[float]

      Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

      None memory Optional[str]

      Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

      None storage Optional[str]

      Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

      None gpus Optional[int]

      Number of gpus each worker should get, e.g. 0, 1, etc.

      None min_workers Optional[int]

      The minimum number of workers. Must be greater than or equal to 0.

      None max_workers Optional[int]

      The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

      None per_worker Optional[int]

      The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

      • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      None gpu_type Optional[str]

      If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

      • nvidia-tesla-t4
      • nvidia-ampere-a10
      None high_priority Optional[bool]

      Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

      None post_inference_hooks Optional[List[PostInferenceHooks]]

      List of hooks to trigger after inference tasks are served.

      None default_callback_url Optional[str]

      The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

      None default_callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
      get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

      Gets inference results from a previously created batch job.

      Parameters:

      Name Type Description Default batch_job_id str

      An id representing the batch task job. This id is the in the response from calling batch_async_request.

      required

      Returns:

      Type Description Dict[str, Any]

      A dictionary that contains the following fields:

      Dict[str, Any]
      • status: The status of the job.
      Dict[str, Any]
      • result: The url where the result is stored.
      Dict[str, Any]
      • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
      Dict[str, Any]
      • num_tasks_pending: The number of tasks that are still pending.
      Dict[str, Any]
      • num_tasks_completed: The number of tasks that have completed.
      "},{"location":"api/client/#launch.client.LaunchClient.get_latest_model_bundle_v2","title":"get_latest_model_bundle_v2","text":"
      get_latest_model_bundle_v2(\nmodel_bundle_name: str,\n) -> Dict[str, Any]\n

      Get the latest version of a model bundle.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to get.

      required

      Returns:

      Type Description Dict[str, Any]

      A dictionary containing the following keys:

      • id: The ID of the model bundle.
      • name: The name of the model bundle.
      • schema_location: The location of the schema for the model bundle.
      • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
      • created_at: The time the model bundle was created.
      • metadata: A dictionary of metadata associated with the model bundle.
      • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
      get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

      Returns a model bundle specified by bundle_name that the user owns.

      Parameters:

      Name Type Description Default model_bundle Union[ModelBundle, str]

      The bundle or its name.

      required

      Returns:

      Type Description ModelBundle

      A ModelBundle object

      "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle_v2","title":"get_model_bundle_v2","text":"
      get_model_bundle_v2(model_bundle_id: str) -> Dict[str, Any]\n

      Get a model bundle.

      Parameters:

      Name Type Description Default model_bundle_id str

      The ID of the model bundle you want to get.

      required

      Returns:

      Type Description Dict[str, Any]

      A dictionary containing the following keys:

      • id: The ID of the model bundle.
      • name: The name of the model bundle.
      • schema_location: The location of the schema for the model bundle.
      • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
      • created_at: The time the model bundle was created.
      • metadata: A dictionary of metadata associated with the model bundle.
      • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
      get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

      Gets a model endpoint associated with a name.

      Parameters:

      Name Type Description Default endpoint_name str

      The name of the endpoint to retrieve.

      required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
      list_model_bundles() -> List[ModelBundle]\n

      Returns a list of model bundles that the user owns.

      Returns:

      Type Description List[ModelBundle]

      A list of ModelBundle objects

      "},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles_v2","title":"list_model_bundles_v2","text":"
      list_model_bundles_v2() -> List[Dict[str, Any]]\n

      List all model bundles.

      Returns:

      Type Description List[Dict[str, Any]]

      A list of dictionaries containing the following keys:

      • id: The ID of the model bundle.
      • name: The name of the model bundle.
      • schema_location: The location of the schema for the model bundle.
      • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
      • created_at: The time the model bundle was created.
      • metadata: A dictionary of metadata associated with the model bundle.
      • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
      list_model_endpoints() -> List[Endpoint]\n

      Lists all model endpoints that the user owns.

      Returns:

      Type Description List[Endpoint]

      A list of ModelEndpoint objects.

      "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
      read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

      Retrieves the logs for the creation of the endpoint.

      Parameters:

      Name Type Description Default model_endpoint Union[ModelEndpoint, str]

      The endpoint or its name.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
      register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

      For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

      Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

      Parameters:

      Name Type Description Default batch_csv_location_fn Callable[[], str]

      Function that generates batch_csv_urls for upload_batch_csv_fn.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
      register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

      For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

      Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

      Parameters:

      Name Type Description Default bundle_location_fn Callable[[], str]

      Function that generates bundle_urls for upload_bundle_fn.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
      register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

      For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

      upload_batch_csv_fn(csv_text, csv_url)\n

      This function should directly write the contents of csv_text as a text string into csv_url.

      Parameters:

      Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

      Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
      register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

      For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

      upload_bundle_fn(serialized_bundle, bundle_url)\n

      This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

      See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

      Parameters:

      Name Type Description Default upload_bundle_fn Callable[[str, str], None]

      Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

      required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
      EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

      Represents a single request to either a SyncEndpoint or AsyncEndpoint.

      Parameters:

      Name Type Description Default url Optional[str]

      A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

      Exactly one of url and args must be specified.

      None args Optional[Dict]

      A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

      Exactly one of url and args must be specified.

      None return_pickled Optional[bool]

      Whether the output should be a pickled python object, or directly returned serialized json.

      False callback_url Optional[str]

      The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

      None callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None request_id Optional[str]

      (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

      None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
      EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

      Represents a response received from a Endpoint.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required status str

      A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

      required result_url Optional[str]

      A string that is a url containing the pickled python object from the Endpoint's predict function.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None result Optional[str]

      A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None traceback Optional[str]

      The stack trace if the inference endpoint raised an error. Can be used for debugging

      None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
      EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

      Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

      This object should not be directly instantiated by the user.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required endpoint_name str

      The name of the endpoint.

      required async_task_id str

      An async task id.

      required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
      get(timeout: Optional[float] = None) -> EndpointResponse\n

      Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

      Parameters:

      Name Type Description Default timeout Optional[float]

      The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

      None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

      Bases: str, Enum

      Post-inference hooks are functions that are called after inference is complete.

      Attributes:

      Name Type Description CALLBACK str

      The callback hook is called with the inference response and the task ID.

      "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

      Represents a ModelBundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
      app_config: Optional[Dict[Any, Any]] = None\n

      An optional user-specified configuration mapping for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
      env_params: Optional[Dict[str, str]] = None\n

      A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
      id: Optional[str] = None\n

      A globally unique identifier for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
      location: Optional[str] = None\n

      An opaque location for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
      metadata: Optional[Dict[Any, Any]] = None\n

      Arbitrary metadata for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
      name: str\n

      The name of the bundle. Must be unique across all bundles that the user owns.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
      packaging_type: Optional[str] = None\n

      The packaging type for the bundle. Can be cloudpickle or zip.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
      requirements: Optional[List[str]] = None\n

      A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

      "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

      All classes here are returned by the get_model_endpoint method and provide a predict function.

      "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
      AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

      Bases: Endpoint

      An asynchronous model endpoint.

      Parameters:

      Name Type Description Default model_endpoint ModelEndpoint

      ModelEndpoint object.

      required client

      A LaunchClient object

      required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
      predict(request: EndpointRequest) -> EndpointResponseFuture\n

      Runs an asynchronous prediction request.

      Parameters:

      Name Type Description Default request EndpointRequest

      The EndpointRequest object that contains the payload.

      required

      Returns:

      Name Type Description EndpointResponseFuture

      An EndpointResponseFuture such the user can use to query the status of the request.

      Example EndpointResponseFuture EndpointResponseFuture

      .. code-block:: python

      my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

      "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
      predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

      (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

      Parameters:

      Name Type Description Default requests Sequence[EndpointRequest]

      List of EndpointRequests. Request_ids must all be distinct.

      required

      Returns:

      Type Description AsyncEndpointBatchResponse

      an AsyncEndpointResponse keeping track of the inference requests made

      "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
      SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

      Bases: Endpoint

      A synchronous model endpoint.

      Parameters:

      Name Type Description Default model_endpoint ModelEndpoint

      ModelEndpoint object.

      required client

      A LaunchClient object

      required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
      predict(request: EndpointRequest) -> EndpointResponse\n

      Runs a synchronous prediction request.

      Parameters:

      Name Type Description Default request EndpointRequest

      The EndpointRequest object that contains the payload.

      required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

      For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

      Batch jobs are created using the batch_async_request method of the LaunchClient.

      Creating and Following a Batch Job
      import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
      "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

      Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

      To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

      Note

      Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

      Creating an Async Model Endpoint with a Callback URL
      import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
      "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

      Warning

      This feature is currently in beta, and the API is likely to change.

      Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

      Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

      Creating an Async Model Endpoint with custom Callback auth
      import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
      "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

      Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

      Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
      import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
      import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
      "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
      EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

      Represents a single request to either a SyncEndpoint or AsyncEndpoint.

      Parameters:

      Name Type Description Default url Optional[str]

      A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

      Exactly one of url and args must be specified.

      None args Optional[Dict]

      A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

      Exactly one of url and args must be specified.

      None return_pickled Optional[bool]

      Whether the output should be a pickled python object, or directly returned serialized json.

      False callback_url Optional[str]

      The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

      None callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None request_id Optional[str]

      (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

      None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
      EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

      Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

      This object should not be directly instantiated by the user.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required endpoint_name str

      The name of the endpoint.

      required async_task_id str

      An async task id.

      required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
      get(timeout: Optional[float] = None) -> EndpointResponse\n

      Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

      Parameters:

      Name Type Description Default timeout Optional[float]

      The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

      None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
      EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

      Represents a response received from a Endpoint.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required status str

      A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

      required result_url Optional[str]

      A string that is a url containing the pickled python object from the Endpoint's predict function.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None result Optional[str]

      A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None traceback Optional[str]

      The stack trace if the inference endpoint raised an error. Can be used for debugging

      None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

      Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

      "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

      There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

      Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
      import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
      import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
      "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

      The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

      Creating Model Bundles with app_config
      import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
      "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

      Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

      "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

      To list all the model bundles you own, use list_model_bundles.

      "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

      Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

      Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

      Info

      "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

      Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

      A SyncEndpoint is good if:

      • You have strict latency requirements (e.g. on the order of seconds or less).
      • You are willing to have resources continually allocated.

      An AsyncEndpoint is good if:

      • You want to save on compute costs.
      • Your inference code takes a long time to run.
      • Your latency requirements are on the order of minutes.

      A BatchJob is good if:

      • You know there is a large batch of inputs ahead of time.
      • You want to optimize for throughput instead of latency.
      "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

      Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

      Creating an Async Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
      "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

      Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

      Note

      Sync model endpoints require at least 1 min_worker.

      Creating a Sync Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
      "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

      Model endpoints can be listed, updated, and deleted using the Launch API.

      Listing Model Endpoints
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
      Updating a Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
      Deleting a Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
      "},{"location":"concepts/overview/","title":"Overview","text":"

      Creating deployments on Launch generally involves three steps:

      1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

      2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

      3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

      "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"

      Simple, scalable, and high performance ML service deployment in python.

      "},{"location":"#example","title":"Example","text":"Launch Usage
      import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n

      What's going on here:

      • First we use pydantic to define our request and response schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API documentation for our models.
      • Next we define the the model and the load_predict_fn, which tells Launch how to load our model and how to make predictions with it. In this case, we're just returning a function that adds the length of the string y to model(x), where model doubles the integer x.
      • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the response_schema. We also specify the env_params, which tell Launch environment settings like the base image to use. In this case, we're using a PyTorch image.
      • Next, we create the model endpoint, which is the API that we'll use to make predictions. We specify the model_bundle that we created above, and we specify the endpoint_type, which tells Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an asynchronous endpoint, which means that we can make predictions and return immediately with a future object. We can then use the future object to get the prediction result later.
      • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object. This function first waits for the endpoint to be ready, then it submits a prediction request to the endpoint. It then waits for the prediction result and returns it.

      Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.

      "},{"location":"#installation","title":"Installation","text":"

      To use Scale Launch, first install it using pip:

      Installation
      pip install -U scale-launch\n
      "},{"location":"cli/","title":"CLI","text":"

      Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.

      The CLI can be used as scale-launch ....

      "},{"location":"cli/#help","title":"Help","text":"

      Run scale-launch --help for more options.

      scale-launch --help
          This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
      "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
      LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n

      Scale Launch Python Client.

      Initializes a Scale Launch Client.

      Parameters:

      Name Type Description Default api_key str

      Your Scale API key

      required endpoint Optional[str]

      The Scale Launch Endpoint (this should not need to be changed)

      None self_hosted bool

      True iff you are connecting to a self-hosted Scale Launch

      False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
      batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n

      Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.

      Must have exactly one of urls or inputs passed in.

      Parameters:

      Name Type Description Default model_bundle Union[ModelBundle, str]

      The bundle or the name of a the bundle to use for inference.

      required urls List[str]

      A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

      None inputs Optional[List[Dict[str, Any]]]

      A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.

      None batch_url_file_location Optional[str]

      In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()

      None serialization_format str

      Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning

      'JSON' labels Optional[Dict[str, str]]

      An optional dictionary of key/value pairs to associate with this endpoint.

      None cpus Optional[int]

      Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

      None memory Optional[str]

      Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

      None storage Optional[str]

      Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

      None gpus Optional[int]

      Number of gpus each worker should get, e.g. 0, 1, etc.

      None max_workers Optional[int]

      The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

      None per_worker Optional[int]

      The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

      • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced.
      • Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      None gpu_type Optional[str]

      If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

      • nvidia-tesla-t4
      • nvidia-ampere-a10
      None

      Returns:

      Type Description Dict[str, Any]

      A dictionary that contains job_id as a key, and the ID as the value.

      "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
      clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n
      Warning

      This method is deprecated. Use clone_model_bundle_with_changes_v2 instead.

      Parameters:

      Name Type Description Default model_bundle Union[ModelBundle, str]

      The existing bundle or its ID.

      required app_config Optional[Dict]

      The new bundle's app config, if not passed in, the new bundle's app_config will be set to None

      None

      Returns:

      Type Description ModelBundle

      A ModelBundle object

      "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes_v2","title":"clone_model_bundle_with_changes_v2","text":"
      clone_model_bundle_with_changes_v2(\noriginal_model_bundle_id: str,\nnew_app_config: Optional[Dict[str, Any]] = None,\n) -> CreateModelBundleV2Response\n

      Clone a model bundle with an optional new app_config.

      Parameters:

      Name Type Description Default original_model_bundle_id str

      The ID of the model bundle you want to clone.

      required new_app_config Optional[Dict[str, Any]]

      A dictionary of new app config values to use for the cloned model.

      None

      Returns:

      Type Description CreateModelBundleV2Response

      An object containing the following keys:

      • model_bundle_id: The ID of the cloned model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
      create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
      Warning

      This method is deprecated. Use create_model_bundle_from_callable_v2 instead.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create. The name must be unique across all bundles that you own.

      required predict_fn_or_cls Optional[Callable[[Any], Any]]

      Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.

      None model Optional[LaunchModel_T]

      Typically a trained Neural Network, e.g. a Pytorch module.

      Exactly one of model and load_model_fn must be provided.

      None load_model_fn Optional[Callable[[], LaunchModel_T]]

      A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.

      Exactly one of model and load_model_fn must be provided.

      None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]]

      Function that, when called with a model, returns a function that carries out inference.

      If model is specified, then this is equivalent to: load_predict_fn(model, app_config=optional_app_config]) -> predict_fn

      Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn

      In both cases, predict_fn is then the inference function, i.e.: predict_fn(REQUEST) -> RESPONSE

      None requirements Optional[List[str]]

      A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.

      [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]

      If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.

      None app_config Optional[Union[Dict[str, Any], str]]

      Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

      None env_params Dict[str, str]

      A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

      • framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:

        .. code-block:: python

        { \"framework_type\": \"pytorch\", \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\" }

      • Tensorflow fields:

        • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
      required globals_copy Optional[Dict[str, Any]]

      Dictionary of the global symbol table. Normally provided by globals() built-in function.

      None request_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

      None response_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

      None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
      create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> CreateModelBundleV2Response\n

      Uploads and registers a model bundle to Scale Launch.

      Parameters:

      Name Type Description Default model_bundle_name str

      Name of the model bundle.

      required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]]

      Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 

      input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n

      required load_model_fn Callable[[], LaunchModel_T]

      A function that, when run, loads a model.

      required request_schema Type[BaseModel]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

      required response_schema Type[BaseModel]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.

      required requirements Optional[List[str]]

      List of pip requirements.

      None pytorch_image_tag Optional[str]

      The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None tensorflow_version Optional[str]

      The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_repository Optional[str]

      The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_tag Optional[str]

      The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

      None app_config Optional[Union[Dict[str, Any], str]]

      An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

      None

      Returns:

      Type Description CreateModelBundleV2Response

      An object containing the following keys:

      • model_bundle_id: The ID of the created model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
      create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
      Warning

      This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create. The name must be unique across all bundles that you own.

      required base_paths List[str]

      The paths on the local filesystem where the bundle code lives.

      required requirements_path str

      A path on the local filesystem where a requirements.txt file lives.

      required env_params Dict[str, str]

      A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:

      • framework_type: either tensorflow or pytorch.
      • PyTorch fields:
        • pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags

      Example: 

      {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n

      required load_predict_fn_module_path str

      A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.

      required load_model_fn_module_path str

      A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.

      required app_config Optional[Union[Dict[str, Any], str]]

      Either a Dictionary that represents a YAML file contents or a local path to a YAML file.

      None request_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.

      None response_schema Optional[Type[BaseModel]]

      A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.

      None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
      create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> CreateModelBundleV2Response\n

      Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.

      For example, if you have a directory structure like so, and your current working directory is my_root:

         my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n

      then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:

         my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n

      and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create.

      required base_paths List[str]

      A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.

      required load_predict_fn_module_path str

      The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

      required load_model_fn_module_path str

      The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.

      required request_schema Type[BaseModel]

      A Pydantic model that defines the request schema for the bundle.

      required response_schema Type[BaseModel]

      A Pydantic model that defines the response schema for the bundle.

      required requirements_path str

      Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

      required pytorch_image_tag Optional[str]

      The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None tensorflow_version Optional[str]

      The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_repository Optional[str]

      The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.

      None custom_base_image_tag Optional[str]

      The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.

      None app_config Optional[Dict[str, Any]]

      An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.

      None

      Returns:

      Type Description CreateModelBundleV2Response

      An object containing the following keys:

      • model_bundle_id: The ID of the created model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
      create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> CreateModelBundleV2Response\n

      Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to create.

      required request_schema Type[BaseModel]

      A Pydantic model that defines the request schema for the bundle.

      required response_schema Type[BaseModel]

      A Pydantic model that defines the response schema for the bundle.

      required repository str

      The name of the Docker repository for the runnable image.

      required tag str

      The tag for the runnable image.

      required command List[str]

      The command that will be used to start the process that listens for requests.

      required env Dict[str, str]

      A dictionary of environment variables that will be passed to the bundle when it is run.

      required

      Returns:

      Type Description CreateModelBundleV2Response

      An object containing the following keys:

      • model_bundle_id: The ID of the created model bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
      create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n

      Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.

      Parameters:

      Name Type Description Default endpoint_name str

      The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.

      required model_bundle Union[ModelBundle, str]

      The ModelBundle that the endpoint should serve.

      required cpus int

      Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

      3 memory str

      Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

      '8Gi' storage Optional[str]

      Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

      None gpus int

      Number of gpus each worker should get, e.g. 0, 1, etc.

      0 min_workers int

      The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.

      1 max_workers int

      The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.

      1 per_worker int

      The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.

      • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.

      Here is our recommendation for computing per_worker:

      1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      10 gpu_type Optional[str]

      If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

      • nvidia-tesla-t4
      • nvidia-ampere-a10
      None endpoint_type str

      Either \"sync\" or \"async\".

      'sync' high_priority Optional[bool]

      Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

      False post_inference_hooks Optional[List[PostInferenceHooks]]

      List of hooks to trigger after inference tasks are served.

      None default_callback_url Optional[str]

      The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

      None default_callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None update_if_exists bool

      If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.

      False labels Optional[Dict[str, str]]

      An optional dictionary of key/value pairs to associate with this endpoint.

      None

      Returns:

      Type Description Optional[Endpoint]

      A Endpoint object that can be used to make requests to the endpoint.

      "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
      delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

      Deletes a model endpoint.

      Parameters:

      Name Type Description Default model_endpoint Union[ModelEndpoint, str]

      A ModelEndpoint object.

      required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
      edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n

      Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:

      • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.

      Parameters:

      Name Type Description Default model_endpoint Union[ModelEndpoint, str]

      The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.

      required model_bundle Optional[Union[ModelBundle, str]]

      The ModelBundle that the endpoint should serve.

      None cpus Optional[float]

      Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.

      None memory Optional[str]

      Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.

      None storage Optional[str]

      Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.

      None gpus Optional[int]

      Number of gpus each worker should get, e.g. 0, 1, etc.

      None min_workers Optional[int]

      The minimum number of workers. Must be greater than or equal to 0.

      None max_workers Optional[int]

      The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.

      None per_worker Optional[int]

      The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:

      • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      None gpu_type Optional[str]

      If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:

      • nvidia-tesla-t4
      • nvidia-ampere-a10
      None high_priority Optional[bool]

      Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.

      None post_inference_hooks Optional[List[PostInferenceHooks]]

      List of hooks to trigger after inference tasks are served.

      None default_callback_url Optional[str]

      The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.

      None default_callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None default_callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
      get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n

      Gets inference results from a previously created batch job.

      Parameters:

      Name Type Description Default batch_job_id str

      An id representing the batch task job. This id is the in the response from calling batch_async_request.

      required

      Returns:

      Type Description Dict[str, Any]

      A dictionary that contains the following fields:

      Dict[str, Any]
      • status: The status of the job.
      Dict[str, Any]
      • result: The url where the result is stored.
      Dict[str, Any]
      • duration: A string representation of how long the job took to finish or how long it has been running, for a job current in progress.
      Dict[str, Any]
      • num_tasks_pending: The number of tasks that are still pending.
      Dict[str, Any]
      • num_tasks_completed: The number of tasks that have completed.
      "},{"location":"api/client/#launch.client.LaunchClient.get_latest_model_bundle_v2","title":"get_latest_model_bundle_v2","text":"
      get_latest_model_bundle_v2(\nmodel_bundle_name: str,\n) -> ModelBundleV2Response\n

      Get the latest version of a model bundle.

      Parameters:

      Name Type Description Default model_bundle_name str

      The name of the model bundle you want to get.

      required

      Returns:

      Type Description ModelBundleV2Response

      An object containing the following keys:

      • id: The ID of the model bundle.
      • name: The name of the model bundle.
      • schema_location: The location of the schema for the model bundle.
      • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
      • created_at: The time the model bundle was created.
      • metadata: A dictionary of metadata associated with the model bundle.
      • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
      get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n

      Returns a model bundle specified by bundle_name that the user owns.

      Parameters:

      Name Type Description Default model_bundle Union[ModelBundle, str]

      The bundle or its name.

      required

      Returns:

      Type Description ModelBundle

      A ModelBundle object

      "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle_v2","title":"get_model_bundle_v2","text":"
      get_model_bundle_v2(\nmodel_bundle_id: str,\n) -> ModelBundleV2Response\n

      Get a model bundle.

      Parameters:

      Name Type Description Default model_bundle_id str

      The ID of the model bundle you want to get.

      required

      Returns:

      Type Description ModelBundleV2Response

      An object containing the following fields:

      • id: The ID of the model bundle.
      • name: The name of the model bundle.
      • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
      • created_at: The time the model bundle was created.
      • metadata: A dictionary of metadata associated with the model bundle.
      • model_artifact_ids: A list of IDs of model artifacts associated with the bundle.
      "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
      get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n

      Gets a model endpoint associated with a name.

      Parameters:

      Name Type Description Default endpoint_name str

      The name of the endpoint to retrieve.

      required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
      list_model_bundles() -> List[ModelBundle]\n

      Returns a list of model bundles that the user owns.

      Returns:

      Type Description List[ModelBundle]

      A list of ModelBundle objects

      "},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles_v2","title":"list_model_bundles_v2","text":"
      list_model_bundles_v2() -> ListModelBundlesV2Response\n

      List all model bundles.

      Returns:

      Type Description ListModelBundlesV2Response

      An object containing the following keys:

      • model_bundles: A list of model bundles. Each model bundle is an object.
      "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
      list_model_endpoints() -> List[Endpoint]\n

      Lists all model endpoints that the user owns.

      Returns:

      Type Description List[Endpoint]

      A list of ModelEndpoint objects.

      "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
      read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n

      Retrieves the logs for the creation of the endpoint.

      Parameters:

      Name Type Description Default model_endpoint Union[ModelEndpoint, str]

      The endpoint or its name.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
      register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n

      For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.

      Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).

      Parameters:

      Name Type Description Default batch_csv_location_fn Callable[[], str]

      Function that generates batch_csv_urls for upload_batch_csv_fn.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
      register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n

      For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.

      Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).

      Parameters:

      Name Type Description Default bundle_location_fn Callable[[], str]

      Function that generates bundle_urls for upload_bundle_fn.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
      register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n

      For self-hosted mode only. Registers a function that handles batch text upload. This function is called as

      upload_batch_csv_fn(csv_text, csv_url)\n

      This function should directly write the contents of csv_text as a text string into csv_url.

      Parameters:

      Name Type Description Default upload_batch_csv_fn Callable[[str, str], None]

      Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

      required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
      register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n

      For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as

      upload_bundle_fn(serialized_bundle, bundle_url)\n

      This function should directly write the contents of serialized_bundle as a binary string into bundle_url.

      See register_bundle_location_fn for more notes on the signature of upload_bundle_fn

      Parameters:

      Name Type Description Default upload_bundle_fn Callable[[str, str], None]

      Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.

      required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
      EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

      Represents a single request to either a SyncEndpoint or AsyncEndpoint.

      Parameters:

      Name Type Description Default url Optional[str]

      A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

      Exactly one of url and args must be specified.

      None args Optional[Dict]

      A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

      Exactly one of url and args must be specified.

      None return_pickled Optional[bool]

      Whether the output should be a pickled python object, or directly returned serialized json.

      False callback_url Optional[str]

      The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

      None callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None request_id Optional[str]

      (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

      None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
      EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

      Represents a response received from a Endpoint.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required status str

      A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

      required result_url Optional[str]

      A string that is a url containing the pickled python object from the Endpoint's predict function.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None result Optional[str]

      A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None traceback Optional[str]

      The stack trace if the inference endpoint raised an error. Can be used for debugging

      None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
      EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

      Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

      This object should not be directly instantiated by the user.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required endpoint_name str

      The name of the endpoint.

      required async_task_id str

      An async task id.

      required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
      get(timeout: Optional[float] = None) -> EndpointResponse\n

      Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

      Parameters:

      Name Type Description Default timeout Optional[float]

      The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

      None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"

      Bases: str, Enum

      Post-inference hooks are functions that are called after inference is complete.

      Attributes:

      Name Type Description CALLBACK str

      The callback hook is called with the inference response and the task ID.

      "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor","title":"CloudpickleArtifactFlavor","text":"

      Bases: BaseModel

      "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.app_config","title":"app_config class-attribute","text":"
      app_config: Optional[Dict[str, Any]]\n

      Optional configuration for the application.

      "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.framework","title":"framework class-attribute","text":"
      framework: Union[\nPytorchFramework, TensorflowFramework, CustomFramework\n] = Field(Ellipsis, discriminator=\"framework_type\")\n

      Machine Learning framework specification. Either PytorchFramework, TensorflowFramework, or CustomFramework.

      "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.load_model_fn","title":"load_model_fn class-attribute","text":"
      load_model_fn: str\n

      Function which, when called, returns the model object.

      "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.load_predict_fn","title":"load_predict_fn class-attribute","text":"
      load_predict_fn: str\n

      Function which, when called, returns the prediction function.

      "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.requirements","title":"requirements class-attribute","text":"
      requirements: List[str]\n

      List of requirements to install in the environment before running the model.

      "},{"location":"api/model_bundles/#launch.model_bundle.CreateModelBundleV2Response","title":"CreateModelBundleV2Response","text":"

      Bases: BaseModel

      Response object for creating a Model Bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.CreateModelBundleV2Response.model_bundle_id","title":"model_bundle_id class-attribute","text":"
      model_bundle_id: str\n

      ID of the Model Bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework","title":"CustomFramework","text":"

      Bases: BaseModel

      "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework.image_repository","title":"image_repository class-attribute","text":"
      image_repository: str\n

      Docker image repository to use as the base image.

      "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework.image_tag","title":"image_tag class-attribute","text":"
      image_tag: str\n

      Docker image tag to use as the base image.

      "},{"location":"api/model_bundles/#launch.model_bundle.ListModelBundlesV2Response","title":"ListModelBundlesV2Response","text":"

      Bases: BaseModel

      Response object for listing Model Bundles.

      "},{"location":"api/model_bundles/#launch.model_bundle.ListModelBundlesV2Response.model_bundles","title":"model_bundles class-attribute","text":"
      model_bundles: List[ModelBundleV2Response]\n

      A list of Model Bundles.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle dataclass","text":"

      Represents a ModelBundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config class-attribute","text":"
      app_config: Optional[Dict[Any, Any]] = None\n

      An optional user-specified configuration mapping for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params class-attribute","text":"
      env_params: Optional[Dict[str, str]] = None\n

      A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id class-attribute","text":"
      id: Optional[str] = None\n

      A globally unique identifier for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location class-attribute","text":"
      location: Optional[str] = None\n

      An opaque location for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata class-attribute","text":"
      metadata: Optional[Dict[Any, Any]] = None\n

      Arbitrary metadata for the bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name class-attribute","text":"
      name: str\n

      The name of the bundle. Must be unique across all bundles that the user owns.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type class-attribute","text":"
      packaging_type: Optional[str] = None\n

      The packaging type for the bundle. Can be cloudpickle or zip.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements class-attribute","text":"
      requirements: Optional[List[str]] = None\n

      A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response","title":"ModelBundleV2Response","text":"

      Bases: BaseModel

      Response object for a single Model Bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.created_at","title":"created_at class-attribute","text":"
      created_at: datetime.datetime\n

      Timestamp of when the Model Bundle was created.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.flavor","title":"flavor class-attribute","text":"
      flavor: Union[\nCloudpickleArtifactFlavor,\nZipArtifactFlavor,\nRunnableImageFlavor,\n] = Field(Ellipsis, discriminator=\"flavor\")\n

      Flavor of the Model Bundle, representing how the model bundle was packaged. Either CloudpickleArtifactFlavor, ZipArtifactFlavor, or RunnableImageFlavor.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.id","title":"id class-attribute","text":"
      id: str\n

      ID of the Model Bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.metadata","title":"metadata class-attribute","text":"
      metadata: Dict[str, Any]\n

      Metadata associated with the Model Bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.model_artifact_ids","title":"model_artifact_ids class-attribute","text":"
      model_artifact_ids: List[str]\n

      IDs of the Model Artifacts associated with the Model Bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.name","title":"name class-attribute","text":"
      name: str\n

      Name of the Model Bundle.

      "},{"location":"api/model_bundles/#launch.model_bundle.PytorchFramework","title":"PytorchFramework","text":"

      Bases: BaseModel

      "},{"location":"api/model_bundles/#launch.model_bundle.PytorchFramework.pytorch_image_tag","title":"pytorch_image_tag class-attribute","text":"
      pytorch_image_tag: str\n

      Image tag of the Pytorch image to use.

      "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor","title":"RunnableImageFlavor","text":"

      Bases: BaseModel

      "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.command","title":"command class-attribute","text":"
      command: List[str]\n

      Command to run the image.

      "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.env","title":"env class-attribute","text":"
      env: Optional[Dict[str, str]]\n

      Environment variables to set when running the image.

      "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.repository","title":"repository class-attribute","text":"
      repository: str\n

      Docker repository of the image.

      "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.tag","title":"tag class-attribute","text":"
      tag: str\n

      Docker tag of the image.

      "},{"location":"api/model_bundles/#launch.model_bundle.TensorflowFramework","title":"TensorflowFramework","text":"

      Bases: BaseModel

      "},{"location":"api/model_bundles/#launch.model_bundle.TensorflowFramework.tensorflow_version","title":"tensorflow_version class-attribute","text":"
      tensorflow_version: str\n

      Tensorflow version to use.

      "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor","title":"ZipArtifactFlavor","text":"

      Bases: BaseModel

      "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.app_config","title":"app_config class-attribute","text":"
      app_config: Optional[Dict[str, Any]]\n

      Optional configuration for the application.

      "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.framework","title":"framework class-attribute","text":"
      framework: Union[\nPytorchFramework, TensorflowFramework, CustomFramework\n] = Field(Ellipsis, discriminator=\"framework_type\")\n

      Machine Learning framework specification. Either PytorchFramework, TensorflowFramework, or CustomFramework.

      "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.load_model_fn_module_path","title":"load_model_fn_module_path class-attribute","text":"
      load_model_fn_module_path: str\n

      Path to the module to load the model object.

      "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.load_predict_fn_module_path","title":"load_predict_fn_module_path class-attribute","text":"
      load_predict_fn_module_path: str\n

      Path to the module to load the prediction function.

      "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.requirements","title":"requirements class-attribute","text":"
      requirements: List[str]\n

      List of requirements to install in the environment before running the model.

      "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"

      All classes here are returned by the get_model_endpoint method and provide a predict function.

      "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
      AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n

      Bases: Endpoint

      An asynchronous model endpoint.

      Parameters:

      Name Type Description Default model_endpoint ModelEndpoint

      ModelEndpoint object.

      required client

      A LaunchClient object

      required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
      predict(request: EndpointRequest) -> EndpointResponseFuture\n

      Runs an asynchronous prediction request.

      Parameters:

      Name Type Description Default request EndpointRequest

      The EndpointRequest object that contains the payload.

      required

      Returns:

      Name Type Description EndpointResponseFuture

      An EndpointResponseFuture such the user can use to query the status of the request.

      Example EndpointResponseFuture EndpointResponseFuture

      .. code-block:: python

      my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get() # blocks on completion

      "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
      predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n

      (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.

      Parameters:

      Name Type Description Default requests Sequence[EndpointRequest]

      List of EndpointRequests. Request_ids must all be distinct.

      required

      Returns:

      Type Description AsyncEndpointBatchResponse

      an AsyncEndpointResponse keeping track of the inference requests made

      "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
      SyncEndpoint(model_endpoint: ModelEndpoint, client)\n

      Bases: Endpoint

      A synchronous model endpoint.

      Parameters:

      Name Type Description Default model_endpoint ModelEndpoint

      ModelEndpoint object.

      required client

      A LaunchClient object

      required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
      predict(request: EndpointRequest) -> EndpointResponse\n

      Runs a synchronous prediction request.

      Parameters:

      Name Type Description Default request EndpointRequest

      The EndpointRequest object that contains the payload.

      required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"

      For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.

      Batch jobs are created using the batch_async_request method of the LaunchClient.

      Creating and Following a Batch Job
      import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
      "},{"location":"concepts/callbacks/","title":"Callbacks","text":"

      Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.

      To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.

      Note

      Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.

      Creating an Async Model Endpoint with a Callback URL
      import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
      "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"

      Warning

      This feature is currently in beta, and the API is likely to change.

      Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.

      Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.

      Creating an Async Model Endpoint with custom Callback auth
      import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
      "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"

      Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.

      Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
      import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
      import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
      "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
      EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n

      Represents a single request to either a SyncEndpoint or AsyncEndpoint.

      Parameters:

      Name Type Description Default url Optional[str]

      A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.

      Exactly one of url and args must be specified.

      None args Optional[Dict]

      A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.

      Exactly one of url and args must be specified.

      None return_pickled Optional[bool]

      Whether the output should be a pickled python object, or directly returned serialized json.

      False callback_url Optional[str]

      The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.

      None callback_auth_kind Optional[Literal[basic, mtls]]

      The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_username Optional[str]

      The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_password Optional[str]

      The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.

      None callback_auth_cert Optional[str]

      The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None callback_auth_key Optional[str]

      The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.

      None request_id Optional[str]

      (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.

      None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
      EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n

      Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.

      This object should not be directly instantiated by the user.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required endpoint_name str

      The name of the endpoint.

      required async_task_id str

      An async task id.

      required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
      get(timeout: Optional[float] = None) -> EndpointResponse\n

      Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.

      Parameters:

      Name Type Description Default timeout Optional[float]

      The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.

      None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
      EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n

      Represents a response received from a Endpoint.

      Parameters:

      Name Type Description Default client

      An instance of LaunchClient.

      required status str

      A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING

      required result_url Optional[str]

      A string that is a url containing the pickled python object from the Endpoint's predict function.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None result Optional[str]

      A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.

      Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.

      None traceback Optional[str]

      The stack trace if the inference endpoint raised an error. Can be used for debugging

      None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"

      Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

      "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"

      There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.

      Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
      import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
      import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
      "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"

      The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.

      Creating Model Bundles with app_config
      import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
      "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"

      Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.

      "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"

      To list all the model bundles you own, use list_model_bundles.

      "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"

      Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.

      Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.

      Info

      "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"

      Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:

      A SyncEndpoint is good if:

      • You have strict latency requirements (e.g. on the order of seconds or less).
      • You are willing to have resources continually allocated.

      An AsyncEndpoint is good if:

      • You want to save on compute costs.
      • Your inference code takes a long time to run.
      • Your latency requirements are on the order of minutes.

      A BatchJob is good if:

      • You know there is a large batch of inputs ahead of time.
      • You want to optimize for throughput instead of latency.
      "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"

      Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.

      Creating an Async Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
      "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"

      Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.

      Note

      Sync model endpoints require at least 1 min_worker.

      Creating a Sync Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
      "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"

      Model endpoints can be listed, updated, and deleted using the Launch API.

      Listing Model Endpoints
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
      Updating a Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
      Deleting a Model Endpoint
      import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
      "},{"location":"concepts/overview/","title":"Overview","text":"

      Creating deployments on Launch generally involves three steps:

      1. Create and upload a ModelBundle. Pass your trained model as well as pre-/post-processing code to the Scale Launch Python client, and we\u2019ll create a model bundle based on the code and store it in our Bundle Store.

      2. Create a ModelEndpoint. Pass a ModelBundle as well as infrastructure settings such as the desired number of GPUs to our client. This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.

      3. Make requests to the ModelEndpoint. You can make requests through the Python client, or make HTTP requests directly to Scale.

      "}]} \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index b2789220c7adf9d04d2adec048c651132fab938f..9da3509e425d56c786b081cabe4da8f90624e755 100644 GIT binary patch delta 15 WcmX@WbbyIXzMF%i*ncCN6(ay6o&)&+ delta 15 WcmX@WbbyIXzMF$1(q$u?6(ay5%LCK^ From afb7af7c10c0be193ad8d54a136eae6f0a9391f4 Mon Sep 17 00:00:00 2001 From: Phil Chen Date: Tue, 28 Mar 2023 15:45:16 -0700 Subject: [PATCH 12/28] Deployed 467b4eb with MkDocs version: 1.4.2 --- api/client/index.html | 6 +- concepts/model_bundles/index.html | 368 ++++++++++++++++-------------- search/search_index.json | 2 +- sitemap.xml.gz | Bin 320 -> 320 bytes 4 files changed, 197 insertions(+), 179 deletions(-) diff --git a/api/client/index.html b/api/client/index.html index adbb8fc3..c6b46bee 100644 --- a/api/client/index.html +++ b/api/client/index.html @@ -1993,7 +1993,7 @@

      load_model_fn_module_path: str, request_schema: Type[BaseModel], response_schema: Type[BaseModel], - requirements_path: str, + requirements_path: Optional[str] = None, pytorch_image_tag: Optional[str] = None, tensorflow_version: Optional[str] = None, custom_base_image_repository: Optional[str] = None, @@ -2111,13 +2111,13 @@

      Optional[str]

    diff --git a/concepts/model_bundles/index.html b/concepts/model_bundles/index.html index 943beb01..7640dfdb 100644 --- a/concepts/model_bundles/index.html +++ b/concepts/model_bundles/index.html @@ -351,27 +351,6 @@ Creating Model Bundles - - -
  • - - Configuring Model Bundles - - -
    • - -
  • - - Updating Model Bundles - - -
    • - -
  • - - Listing Model Bundles - -
    • @@ -581,27 +560,6 @@ Creating Model Bundles - - -
  • - - Configuring Model Bundles - - -
    • - -
  • - - Updating Model Bundles - - -
    • - -
  • - - Listing Model Bundles - -
    • @@ -623,15 +581,42 @@

    Model BundlesModel Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format.

    Creating Model Bundles

    -

    There are two methods for creating model bundles: -create_model_bundle +

    There are three methods for creating model bundles: +create_model_bundle_from_callable_v2 and -create_model_bundle_from_dirs. -The former directly pickles a user-specified load_predict_fn, a function which +create_model_bundle_from_dirs_v2. +create_model_bundle_from_runnable_image_v2. +The first directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. -The latter takes in directories containing a load_predict_fn and the -module path to the load_predict_fn.

    -
    +The second takes in directories containing a load_predict_fn and the +module path to the load_predict_fn. +The third takes a Docker image that contains an entrypoint that starts a process +listening for requests at port 5005 using HTTP and exposes POST /predict and +GET /healthz endpoints.

    +

    Each of these modes of creating a model bundle is called a "Flavor".

    +
    +

    Info

    +

    Choosing the right model bundle flavor

    +

    Here are some tips for how to choose between the different flavors of ModelBundle:

    +

    A CloudpickleArtifactFlavor (creating from callable) is good if:

    +
      +
    • You are creating the model bundle from a Jupyter notebook.
      • +
    • The model bundle is small without too many dependencies.
      • +
    +

    A ZipArtifactFlavor (creating from directories) is good if:

    +
      +
    • You have a relatively constant set of dependencies.
      • +
    • You have a lot of custom code that you want to include in the model bundle.
      • +
    • You do not want to build a web server and Docker image to serve your model.
      • +
    +

    A RunnableImageArtifactFlavor (creating from runnable image) is good if:

    +
      +
    • You have a lot of dependencies.
      • +
    • You have a lot of custom code that you want to include in the model bundle.
      • +
    • You are comfortable with building a web server and Docker image to serve your model.
      • +
    +
    +
    import os
@@ -656,26 +641,24 @@ 
    Creating Model Bundles    return returns_model_of_x_plus_len_of_y
 
 
-def my_model(x):
-    return x * 2
-
-ENV_PARAMS = {
-    "framework_type": "pytorch",
-    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
-}
-
-BUNDLE_PARAMS = {
-    "model_bundle_name": "test-bundle",
-    "model": my_model,
-    "load_predict_fn": my_load_predict_fn,
-    "env_params": ENV_PARAMS,
-    "requirements": ["pytest==7.2.1", "numpy"],  # list your requirements here
-    "request_schema": MyRequestSchema,
-    "response_schema": MyResponseSchema,
-}
-
-client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
-client.create_model_bundle(**BUNDLE_PARAMS)
+def my_load_model_fn():
+    def my_model(x):
+        return x * 2
+
+    return my_model
+
+BUNDLE_PARAMS = {
+    "model_bundle_name": "test-bundle",
+    "load_model_fn": my_load_model_fn,
+    "load_predict_fn": my_load_predict_fn,
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "requirements": ["pytest==7.2.1", "numpy"],  # list your requirements here
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+}
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+client.create_model_bundle_from_callable_v2(**BUNDLE_PARAMS)
    @@ -731,29 +714,69 @@

    Creating Model Bundlesclass MyResponseSchema(BaseModel): __root__: int -ENV_PARAMS = { - "framework_type": "pytorch", - "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime", -} - -BUNDLE_PARAMS = { - "model_bundle_name": "test-bundle", - "base_paths": [directory], - "requirements_path": "requirements.txt", - "env_params": ENV_PARAMS, - "load_predict_fn": "predict.my_load_predict_fn", - "load_model_fn_module_path": "model.my_load_model_fn", - "request_schema": MyRequestSchema, - "response_schema": MyResponseSchema, -} - -client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY")) -client.create_model_bundle_from_dirs(**BUNDLE_PARAMS) - -# Clean up files from demo -os.remove(model_filename) -os.remove(predict_filename) -os.rmdir(directory) +BUNDLE_PARAMS = { + "model_bundle_name": "test-bundle", + "base_paths": [directory], + "load_predict_fn_module_path": "predict.my_load_predict_fn", + "load_model_fn_module_path": "model.my_load_model_fn", + "request_schema": MyRequestSchema, + "response_schema": MyResponseSchema, + "requirements_path": "requirements.txt", + "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime", +} + +client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY")) +client.create_model_bundle_from_dirs_v2(**BUNDLE_PARAMS) + +# Clean up files from demo +os.remove(model_filename) +os.remove(predict_filename) +os.rmdir(directory) +

    +
    +
    +
    import os
+from pydantic import BaseModel
+from launch import LaunchClient
+
+
+class MyRequestSchema(BaseModel):
+    x: int
+    y: str
+
+class MyResponseSchema(BaseModel):
+    __root__: int
+
+
+BUNDLE_PARAMS = {
+    "model_bundle_name": "test-bundle",
+    "load_model_fn": my_load_model_fn,
+    "load_predict_fn": my_load_predict_fn,
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "repository": "launch_rearch",
+    "tag": "12b9131c5a1489c76592cddd186962cce965f0f6-cpu",
+    "command": [
+        "dumb-init",
+        "--",
+        "ddtrace-run",
+        "run-service",
+        "--config",
+        "/install/launch_rearch/config/service--user_defined_code.yaml",
+        "--concurrency",
+        "1",
+        "--http",
+        "production",
+        "--port",
+        "5005",
+    ],
+    "env": {
+        "TEST_KEY": "test_value",
+    },
+}
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+client.create_model_bundle_from_runnable_image_v2(**BUNDLE_PARAMS)
    @@ -764,96 +787,91 @@

    Configuring Model BundlesCreating Model Bundles with app_config
    import os
-from launch import LaunchClient
-from pydantic import BaseModel
-from typing import List, Union
-from typing_extensions import Literal
-
-
-class MyRequestSchemaSingle(BaseModel):
-    kind: Literal['single']
-    x: int
-    y: str
-
-class MyRequestSchemaBatched(BaseModel):
-    kind: Literal['batched']
-    x: List[int]
-    y: List[str]
-
-class MyRequestSchema(BaseModel):
-    __root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]
-
-class MyResponseSchema(BaseModel):
-    __root__: Union[int, List[int]]
-
-
-def my_load_predict_fn(app_config, model):
-    def returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:
-        """MyRequestSchema -> MyResponseSchema"""
-        if app_config["mode"] == "single":
-            assert isinstance(x, int) and isinstance(y, str)
-            return model(x) + len(y)
-
-        result = []
-        for x_i, y_i in zip(x, y):
-            result.append(model(x_i) + len(y_i))
-        return result
-
-    return returns_model_of_x_plus_len_of_y
-
-
-def my_load_model_fn(app_config):
-    def my_model_single(x: int):
-        return x * 2
-
-    def my_model_batched(x: List[int]):
-        return [my_model_single(x_i) for x_i in x]
-
-    if app_config["mode"] == "single":
-        return my_model_single
-
-    return my_model_batched
-
-
-ENV_PARAMS = {
-    "framework_type": "pytorch",
-    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
-}
-
-BUNDLE_PARAMS_SINGLE = {
-    "model_bundle_name": "test-bundle-single",
-    "load_predict_fn": my_load_predict_fn,
-    "load_model_fn": my_load_model_fn,
-    "env_params": ENV_PARAMS,
-    "requirements": ["pytest==7.2.1", "numpy"],
-    "request_schema": MyRequestSchema,
-    "response_schema": MyResponseSchema,
-    "app_config": {"mode": "single"},
-}
-BUNDLE_PARAMS_BATCHED = {
-    "model_bundle_name": "test-bundle-batched",
-    "load_predict_fn": my_load_predict_fn,
-    "load_model_fn": my_load_model_fn,
-    "env_params": ENV_PARAMS,
-    "requirements": ["pytest==7.2.1", "numpy"],
-    "request_schema": MyRequestSchema,
-    "response_schema": MyResponseSchema,
-    "app_config": {"mode": "batched"},
-}
-
-client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
-bundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)
-bundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)
+
    Creating Model Bundles with app_config
    import os
+from launch import LaunchClient
+from pydantic import BaseModel
+from typing import List, Union
+from typing_extensions import Literal
+
+
+class MyRequestSchemaSingle(BaseModel):
+    kind: Literal['single']
+    x: int
+    y: str
+
+class MyRequestSchemaBatched(BaseModel):
+    kind: Literal['batched']
+    x: List[int]
+    y: List[str]
+
+class MyRequestSchema(BaseModel):
+    __root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]
+
+class MyResponseSchema(BaseModel):
+    __root__: Union[int, List[int]]
+
+
+def my_load_predict_fn(app_config, model):
+    def returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:
+        """MyRequestSchema -> MyResponseSchema"""
+        if app_config["mode"] == "single":
+            assert isinstance(x, int) and isinstance(y, str)
+            return model(x) + len(y)
+
+        result = []
+        for x_i, y_i in zip(x, y):
+            result.append(model(x_i) + len(y_i))
+        return result
+
+    return returns_model_of_x_plus_len_of_y
+
+
+def my_load_model_fn(app_config):
+    def my_model_single(x: int):
+        return x * 2
+
+    def my_model_batched(x: List[int]):
+        return [my_model_single(x_i) for x_i in x]
+
+    if app_config["mode"] == "single":
+        return my_model_single
+
+    return my_model_batched
+
+
+BUNDLE_PARAMS_SINGLE = {
+    "model_bundle_name": "test-bundle-single",
+    "load_predict_fn": my_load_predict_fn,
+    "load_model_fn": my_load_model_fn,
+    "requirements": ["pytest==7.2.1", "numpy"],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+    "app_config": {"mode": "single"},
+}
+BUNDLE_PARAMS_BATCHED = {
+    "model_bundle_name": "test-bundle-batched",
+    "load_predict_fn": my_load_predict_fn,
+    "load_model_fn": my_load_model_fn,
+    "requirements": ["pytest==7.2.1", "numpy"],
+    "request_schema": MyRequestSchema,
+    "response_schema": MyResponseSchema,
+    "pytorch_image_tag": "1.7.1-cuda11.0-cudnn8-runtime",
+    "app_config": {"mode": "batched"},
+}
+
+client = LaunchClient(api_key=os.getenv("LAUNCH_API_KEY"))
+bundle_single = client.create_model_bundle_from_callable_v2(**BUNDLE_PARAMS_SINGLE)
+bundle_batch = client.create_model_bundle_from_callable_v2(**BUNDLE_PARAMS_BATCHED)
 
    
 
    Updating Model Bundles
    
 
    Model Bundles are immutable, meaning they cannot be edited once created.
 However, it is possible to clone an existing model bundle with a new app_config
 using
-clone_model_bundle_with_changes.
    
+clone_model_bundle_with_changes_v2.
    
 
    Listing Model Bundles
    
 
    To list all the model bundles you own, use
-list_model_bundles.
    
+list_model_bundles_v2.
    
 
 
 
diff --git a/search/search_index.json b/search/search_index.json
index c73a0521..de67d37c 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"
    Simple, scalable, and high performance ML service deployment in python.
    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n
     
    What's going on here:
     
       
    • First we use pydantic to define our request and response   schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API   documentation for our models.
      •  
    • Next we define the the model and the load_predict_fn, which tells Launch   how to load our model and how to make predictions with it. In this case,   we're just returning a function that adds the length of the string y to    model(x), where model doubles the integer x.
      •  
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the   response_schema. We also specify the env_params, which tell Launch environment settings like    the base image to use. In this case, we're using a PyTorch image.
      •  
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We   specify the model_bundle that we created above, and we specify the endpoint_type, which tells   Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an   asynchronous endpoint, which means that we can make predictions and return immediately with a   future object. We can then use the future object to get the prediction result later.
      •  
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object.   This function first waits for the endpoint to be ready, then it submits a prediction request to   the endpoint. It then waits for the prediction result and returns it.
      •  
     
    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.
    "},{"location":"#installation","title":"Installation","text":"
    To use Scale Launch, first install it using pip:
     Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"
    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.
     
    The CLI can be used as scale-launch ....
    "},{"location":"cli/#help","title":"Help","text":"
    Run scale-launch --help for more options.
     scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n
     
    Scale Launch Python Client.
     
    Initializes a Scale Launch Client.
     
    Parameters:
     Name Type Description Default api_key str 
    Your Scale API key
     required endpoint Optional[str] 
    The Scale Launch Endpoint (this should not need to be changed)
     None self_hosted bool 
    True iff you are connecting to a self-hosted Scale Launch
     False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n
     
    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.
     
    Must have exactly one of urls or inputs passed in.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or the name of a the bundle to use for inference.
     required urls List[str] 
    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.
     None inputs Optional[List[Dict[str, Any]]] 
    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.
     None batch_url_file_location Optional[str] 
    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()
     None serialization_format str 
    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning
     'JSON' labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None cpus Optional[int] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than   per_worker, then the number of workers will be reduced.
      •  
    • Otherwise, if the average number of concurrent requests per worker is higher   than per_worker, then the number of workers will be increased to meet the   elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains job_id as a key, and the ID as the value.
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n
     Warning 
    This method is deprecated. Use clone_model_bundle_with_changes_v2 instead.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The existing bundle or its ID.
     required app_config Optional[Dict] 
    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None
     None 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes_v2","title":"clone_model_bundle_with_changes_v2","text":"
    clone_model_bundle_with_changes_v2(\noriginal_model_bundle_id: str,\nnew_app_config: Optional[Dict[str, Any]] = None,\n) -> CreateModelBundleV2Response\n
     
    Clone a model bundle with an optional new app_config.
     
    Parameters:
     Name Type Description Default original_model_bundle_id str 
    The ID of the model bundle you want to clone.
     required new_app_config Optional[Dict[str, Any]] 
    A dictionary of new app config values to use for the cloned model.
     None 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the cloned model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     Warning 
    This method is deprecated. Use create_model_bundle_from_callable_v2 instead.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required predict_fn_or_cls Optional[Callable[[Any], Any]] 
    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.
     None model Optional[LaunchModel_T] 
    Typically a trained Neural Network, e.g. a Pytorch module.
     
    Exactly one of model and load_model_fn must be provided.
     None load_model_fn Optional[Callable[[], LaunchModel_T]] 
    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.
     
    Exactly one of model and load_model_fn must be provided.
     None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]] 
    Function that, when called with a model, returns a function that carries out inference.
     
    If model is specified, then this is equivalent to:     load_predict_fn(model, app_config=optional_app_config]) -> predict_fn
     
    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn
     
    In both cases, predict_fn is then the inference function, i.e.:     predict_fn(REQUEST) -> RESPONSE
     None requirements Optional[List[str]] 
    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.
     
    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]
     
    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.
     None app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
    •  
      Tensorflow fields:
       
         
      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
        •  
       
      •  
     required globals_copy Optional[Dict[str, Any]] 
    Dictionary of the global symbol table. Normally provided by globals() built-in function.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
    create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> CreateModelBundleV2Response\n
     
    Uploads and registers a model bundle to Scale Launch.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    Name of the model bundle.
     required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]] 
    Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 
    input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n
     required load_model_fn Callable[[], LaunchModel_T] 
    A function that, when run, loads a model.
     required request_schema Type[BaseModel] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     required response_schema Type[BaseModel] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.
     required requirements Optional[List[str]] 
    List of pip requirements.
     None pytorch_image_tag Optional[str] 
    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None tensorflow_version Optional[str] 
    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_repository Optional[str] 
    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_tag Optional[str] 
    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.
     None app_config Optional[Union[Dict[str, Any], str]] 
    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.
     None 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the created model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     Warning 
    This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required base_paths List[str] 
    The paths on the local filesystem where the bundle code lives.
     required requirements_path str 
    A path on the local filesystem where a requirements.txt file lives.
     required env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    • framework_type: either tensorflow or pytorch.
      •  
    • PyTorch fields:
         
      • pytorch_image_tag: An image tag for the pytorch docker base image. The     list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags
        •  
       
      •  
     
    Example:    
    {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n
     required load_predict_fn_module_path str 
    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.
     required load_model_fn_module_path str 
    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.
     required app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
    create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: str,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> CreateModelBundleV2Response\n
     
    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.
     
    For example, if you have a directory structure like so, and your current working directory is my_root:
     
       my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n
     
    then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:
     
       my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n
     
    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create.
     required base_paths List[str] 
    A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.
     required load_predict_fn_module_path str 
    The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.
     required load_model_fn_module_path str 
    The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.
     required request_schema Type[BaseModel] 
    A Pydantic model that defines the request schema for the bundle.
     required response_schema Type[BaseModel] 
    A Pydantic model that defines the response schema for the bundle.
     required requirements_path str 
    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.
     required pytorch_image_tag Optional[str] 
    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None tensorflow_version Optional[str] 
    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_repository Optional[str] 
    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_tag Optional[str] 
    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.
     None app_config Optional[Dict[str, Any]] 
    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.
     None 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the created model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
    create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> CreateModelBundleV2Response\n
     
    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create.
     required request_schema Type[BaseModel] 
    A Pydantic model that defines the request schema for the bundle.
     required response_schema Type[BaseModel] 
    A Pydantic model that defines the response schema for the bundle.
     required repository str 
    The name of the Docker repository for the runnable image.
     required tag str 
    The tag for the runnable image.
     required command List[str] 
    The command that will be used to start the process that listens for requests.
     required env Dict[str, str] 
    A dictionary of environment variables that will be passed to the bundle when it is run.
     required 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the created model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n
     
    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.
     required model_bundle Union[ModelBundle, str] 
    The ModelBundle that the endpoint should serve.
     required cpus int 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     3 memory str 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     '8Gi' storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus int 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     0 min_workers int 
    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.
     1 max_workers int 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.
     1 per_worker int 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     
    Here is our recommendation for computing per_worker:
     
       
    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      2.  
     10 gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None endpoint_type str 
    Either \"sync\" or \"async\".
     'sync' high_priority Optional[bool] 
    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.
     False post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None default_callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None update_if_exists bool 
    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.
     False labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None 
    Returns:
     Type Description Optional[Endpoint] 
    A Endpoint object that can be used to make requests to the endpoint.
    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Deletes a model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    A ModelEndpoint object.
     required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n
     
    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:
     
       
    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.
      •  
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.
     required model_bundle Optional[Union[ModelBundle, str]] 
    The ModelBundle that the endpoint should serve.
     None cpus Optional[float] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None min_workers Optional[int] 
    The minimum number of workers. Must be greater than or equal to 0.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None high_priority Optional[bool] 
    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.
     None post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None default_callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n
     
    Gets inference results from a previously created batch job.
     
    Parameters:
     Name Type Description Default batch_job_id str 
    An id representing the batch task job. This id is the in the response from calling batch_async_request.
     required 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains the following fields:
     Dict[str, Any] 
       
    • status: The status of the job.
      •  
     Dict[str, Any] 
       
    • result: The url where the result is stored.
      •  
     Dict[str, Any] 
       
    • duration: A string representation of how long the job took to finish     or how long it has been running, for a job current in progress.
      •  
     Dict[str, Any] 
       
    • num_tasks_pending: The number of tasks that are still pending.
      •  
     Dict[str, Any] 
       
    • num_tasks_completed: The number of tasks that have completed.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_latest_model_bundle_v2","title":"get_latest_model_bundle_v2","text":"
    get_latest_model_bundle_v2(\nmodel_bundle_name: str,\n) -> ModelBundleV2Response\n
     
    Get the latest version of a model bundle.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to get.
     required 
    Returns:
     Type Description ModelBundleV2Response 
    An object containing the following keys:
     
       
    • id: The ID of the model bundle.
      •  
    • name: The name of the model bundle.
      •  
    • schema_location: The location of the schema for the model bundle.
      •  
    • flavor: The flavor of the model bundle. Either RunnableImage,     CloudpickleArtifact, or ZipArtifact.
      •  
    • created_at: The time the model bundle was created.
      •  
    • metadata: A dictionary of metadata associated with the model bundle.
      •  
    • model_artifact_ids: A list of IDs of model artifacts associated with the     bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n
     
    Returns a model bundle specified by bundle_name that the user owns.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or its name.
     required 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle_v2","title":"get_model_bundle_v2","text":"
    get_model_bundle_v2(\nmodel_bundle_id: str,\n) -> ModelBundleV2Response\n
     
    Get a model bundle.
     
    Parameters:
     Name Type Description Default model_bundle_id str 
    The ID of the model bundle you want to get.
     required 
    Returns:
     Type Description ModelBundleV2Response 
    An object containing the following fields:
     
       
    • id: The ID of the model bundle.
      •  
    • name: The name of the model bundle.
      •  
    • flavor: The flavor of the model bundle. Either RunnableImage,     CloudpickleArtifact, or ZipArtifact.
      •  
    • created_at: The time the model bundle was created.
      •  
    • metadata: A dictionary of metadata associated with the model bundle.
      •  
    • model_artifact_ids: A list of IDs of model artifacts associated with the     bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n
     
    Gets a model endpoint associated with a name.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the endpoint to retrieve.
     required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n
     
    Returns a list of model bundles that the user owns.
     
    Returns:
     Type Description List[ModelBundle] 
    A list of ModelBundle objects
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles_v2","title":"list_model_bundles_v2","text":"
    list_model_bundles_v2() -> ListModelBundlesV2Response\n
     
    List all model bundles.
     
    Returns:
     Type Description ListModelBundlesV2Response 
    An object containing the following keys:
     
       
    • model_bundles: A list of model bundles. Each model bundle is an object.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n
     
    Lists all model endpoints that the user owns.
     
    Returns:
     Type Description List[Endpoint] 
    A list of ModelEndpoint objects.
    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Retrieves the logs for the creation of the endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The endpoint or its name.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.
     
    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).
     
    Parameters:
     Name Type Description Default batch_csv_location_fn Callable[[], str] 
    Function that generates batch_csv_urls for upload_batch_csv_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.
     
    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).
     
    Parameters:
     Name Type Description Default bundle_location_fn Callable[[], str] 
    Function that generates bundle_urls for upload_bundle_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as
     
    upload_batch_csv_fn(csv_text, csv_url)\n
     
    This function should directly write the contents of csv_text as a text string into csv_url.
     
    Parameters:
     Name Type Description Default upload_batch_csv_fn Callable[[str, str], None] 
    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as
     
    upload_bundle_fn(serialized_bundle, bundle_url)\n
     
    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.
     
    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn
     
    Parameters:
     Name Type Description Default upload_bundle_fn Callable[[str, str], None] 
    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False callback_url Optional[str] 
    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.
     None callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"
             Bases: str, Enum
     
    Post-inference hooks are functions that are called after inference is complete.
     
    Attributes:
     Name Type Description CALLBACK str 
    The callback hook is called with the inference response and the task ID.
    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor","title":"CloudpickleArtifactFlavor","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[str, Any]]\n
     
    Optional configuration for the application.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.framework","title":"framework  class-attribute","text":"
    framework: Union[\nPytorchFramework, TensorflowFramework, CustomFramework\n] = Field(Ellipsis, discriminator=\"framework_type\")\n
     
    Machine Learning framework specification. Either PytorchFramework, TensorflowFramework, or CustomFramework.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.load_model_fn","title":"load_model_fn  class-attribute","text":"
    load_model_fn: str\n
     
    Function which, when called, returns the model object.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.load_predict_fn","title":"load_predict_fn  class-attribute","text":"
    load_predict_fn: str\n
     
    Function which, when called, returns the prediction function.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.requirements","title":"requirements  class-attribute","text":"
    requirements: List[str]\n
     
    List of requirements to install in the environment before running the model.
    "},{"location":"api/model_bundles/#launch.model_bundle.CreateModelBundleV2Response","title":"CreateModelBundleV2Response","text":"
             Bases: BaseModel
     
    Response object for creating a Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.CreateModelBundleV2Response.model_bundle_id","title":"model_bundle_id  class-attribute","text":"
    model_bundle_id: str\n
     
    ID of the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework","title":"CustomFramework","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework.image_repository","title":"image_repository  class-attribute","text":"
    image_repository: str\n
     
    Docker image repository to use as the base image.
    "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework.image_tag","title":"image_tag  class-attribute","text":"
    image_tag: str\n
     
    Docker image tag to use as the base image.
    "},{"location":"api/model_bundles/#launch.model_bundle.ListModelBundlesV2Response","title":"ListModelBundlesV2Response","text":"
             Bases: BaseModel
     
    Response object for listing Model Bundles.
    "},{"location":"api/model_bundles/#launch.model_bundle.ListModelBundlesV2Response.model_bundles","title":"model_bundles  class-attribute","text":"
    model_bundles: List[ModelBundleV2Response]\n
     
    A list of Model Bundles.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle  dataclass","text":"
    Represents a ModelBundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n
     
    An optional user-specified configuration mapping for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params  class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n
     
    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id  class-attribute","text":"
    id: Optional[str] = None\n
     
    A globally unique identifier for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location  class-attribute","text":"
    location: Optional[str] = None\n
     
    An opaque location for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata  class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n
     
    Arbitrary metadata for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name  class-attribute","text":"
    name: str\n
     
    The name of the bundle. Must be unique across all bundles that the user owns.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type  class-attribute","text":"
    packaging_type: Optional[str] = None\n
     
    The packaging type for the bundle. Can be cloudpickle or zip.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements  class-attribute","text":"
    requirements: Optional[List[str]] = None\n
     
    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response","title":"ModelBundleV2Response","text":"
             Bases: BaseModel
     
    Response object for a single Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.created_at","title":"created_at  class-attribute","text":"
    created_at: datetime.datetime\n
     
    Timestamp of when the Model Bundle was created.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.flavor","title":"flavor  class-attribute","text":"
    flavor: Union[\nCloudpickleArtifactFlavor,\nZipArtifactFlavor,\nRunnableImageFlavor,\n] = Field(Ellipsis, discriminator=\"flavor\")\n
     
    Flavor of the Model Bundle, representing how the model bundle was packaged. Either CloudpickleArtifactFlavor, ZipArtifactFlavor, or RunnableImageFlavor.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.id","title":"id  class-attribute","text":"
    id: str\n
     
    ID of the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.metadata","title":"metadata  class-attribute","text":"
    metadata: Dict[str, Any]\n
     
    Metadata associated with the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.model_artifact_ids","title":"model_artifact_ids  class-attribute","text":"
    model_artifact_ids: List[str]\n
     
    IDs of the Model Artifacts associated with the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.name","title":"name  class-attribute","text":"
    name: str\n
     
    Name of the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.PytorchFramework","title":"PytorchFramework","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.PytorchFramework.pytorch_image_tag","title":"pytorch_image_tag  class-attribute","text":"
    pytorch_image_tag: str\n
     
    Image tag of the Pytorch image to use.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor","title":"RunnableImageFlavor","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.command","title":"command  class-attribute","text":"
    command: List[str]\n
     
    Command to run the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.env","title":"env  class-attribute","text":"
    env: Optional[Dict[str, str]]\n
     
    Environment variables to set when running the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.repository","title":"repository  class-attribute","text":"
    repository: str\n
     
    Docker repository of the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.tag","title":"tag  class-attribute","text":"
    tag: str\n
     
    Docker tag of the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.TensorflowFramework","title":"TensorflowFramework","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.TensorflowFramework.tensorflow_version","title":"tensorflow_version  class-attribute","text":"
    tensorflow_version: str\n
     
    Tensorflow version to use.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor","title":"ZipArtifactFlavor","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[str, Any]]\n
     
    Optional configuration for the application.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.framework","title":"framework  class-attribute","text":"
    framework: Union[\nPytorchFramework, TensorflowFramework, CustomFramework\n] = Field(Ellipsis, discriminator=\"framework_type\")\n
     
    Machine Learning framework specification. Either PytorchFramework, TensorflowFramework, or CustomFramework.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.load_model_fn_module_path","title":"load_model_fn_module_path  class-attribute","text":"
    load_model_fn_module_path: str\n
     
    Path to the module to load the model object.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.load_predict_fn_module_path","title":"load_predict_fn_module_path  class-attribute","text":"
    load_predict_fn_module_path: str\n
     
    Path to the module to load the prediction function.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.requirements","title":"requirements  class-attribute","text":"
    requirements: List[str]\n
     
    List of requirements to install in the environment before running the model.
    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"
    All classes here are returned by the  get_model_endpoint method and provide a predict function.
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    An asynchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n
     
    Runs an asynchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required 
    Returns:
     Name Type Description EndpointResponseFuture 
    An EndpointResponseFuture such the user can use to query the status of the request.
     Example EndpointResponseFuture EndpointResponseFuture 
    .. code-block:: python
     
    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get()  # blocks on completion
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n
     
    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.
     
    Parameters:
     Name Type Description Default requests Sequence[EndpointRequest] 
    List of EndpointRequests. Request_ids must all be distinct.
     required 
    Returns:
     Type Description AsyncEndpointBatchResponse 
    an AsyncEndpointResponse keeping track of the inference requests made
    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    A synchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n
     
    Runs a synchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"
    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.
     
    Batch jobs are created using the batch_async_request method of the LaunchClient.
     Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"
    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.
     
    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.
     
    Note
     
    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.
     Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"
    Warning
     
    This feature is currently in beta, and the API is likely to change.
     
    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.
     
    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the  same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.
     Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"
    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.
     Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
     
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False callback_url Optional[str] 
    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.
     None callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"
    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format. 
    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"
    There are two methods for creating model bundles: create_model_bundle and create_model_bundle_from_dirs. The former directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The latter takes in directories containing a load_predict_fn and the module path to the load_predict_fn.
     Creating a Model Bundle DirectlyCreating a Model Bundle from Directories 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\n
     
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"requirements_path\": \"requirements.txt\",\n\"env_params\": ENV_PARAMS,\n\"load_predict_fn\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"
    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.
     Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"
    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes.
    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"
    To list all the model bundles you own, use list_model_bundles.
    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"
    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.
     
    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.
     
    Info
    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"
    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:
     
    A SyncEndpoint is good if:
     
       
    • You have strict latency requirements (e.g. on the order of seconds or less).
      •  
    • You are willing to have resources continually allocated.
      •  
     
    An AsyncEndpoint is good if:
     
       
    • You want to save on compute costs.
      •  
    • Your inference code takes a long time to run.
      •  
    • Your latency requirements are on the order of minutes.
      •  
     
    A BatchJob is good if:
     
       
    • You know there is a large batch of inputs ahead of time.
      •  
    • You want to optimize for throughput instead of latency.
      •  
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"
    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.
     Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"
    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.
     
    Note
     
    Sync model endpoints require at least 1 min_worker.
     Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"
    Model endpoints can be listed, updated, and deleted using the Launch API.
     Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
     Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
     Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"
    Creating deployments on Launch generally involves three steps:
     
       
    1.  
      Create and upload a ModelBundle. Pass your trained model    as well as pre-/post-processing code to the Scale Launch Python client, and     we\u2019ll create a model bundle based on the code and store it in our Bundle Store.
       
      2.  
    2.  
      Create a ModelEndpoint. Pass a ModelBundle as well as    infrastructure settings such as the desired number of GPUs to our client.    This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.
       
      3.  
    3.  
      Make requests to the ModelEndpoint. You can make requests through the Python    client, or make HTTP requests directly to Scale.
       
      4.  
    "}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Scale Launch","text":"
    Simple, scalable, and high performance ML service deployment in python.
    "},{"location":"#example","title":"Example","text":"Launch Usage
    import os\nimport time\nfrom launch import LaunchClient\nfrom launch import EndpointRequest\nfrom pydantic import BaseModel\nfrom rich import print\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_model(x):\nreturn x * 2\nENV_PARAMS = {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"model\": my_model,\n\"load_predict_fn\": my_load_predict_fn,\n\"env_params\": ENV_PARAMS,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n}\nENDPOINT_PARAMS = {\n\"endpoint_name\": \"demo-endpoint\",\n\"model_bundle\": \"test-bundle\",\n\"cpus\": 1,\n\"min_workers\": 0,\n\"endpoint_type\": \"async\",\n\"update_if_exists\": True,\n\"labels\": {\n\"team\": \"MY_TEAM\",\n\"product\": \"launch\",\n}\n}\ndef predict_on_endpoint(request: MyRequestSchema) -> MyResponseSchema:\n# Wait for the endpoint to be ready first before submitting a task\nendpoint = client.get_model_endpoint(endpoint_name=\"demo-endpoint\")\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nendpoint_request = EndpointRequest(args=request.dict(), return_pickled=False)\nfuture = endpoint.predict(request=endpoint_request)\nraw_response = future.get()\nresponse = MyResponseSchema.parse_raw(raw_response.result)\nreturn response\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle(**BUNDLE_PARAMS)\nendpoint = client.create_model_endpoint(**ENDPOINT_PARAMS)\nrequest = MyRequestSchema(x=5, y=\"hello\")\nresponse = predict_on_endpoint(request)\nprint(response)\n\"\"\"\nMyResponseSchema(__root__=10)\n\"\"\"\n
     
    What's going on here:
     
       
    • First we use pydantic to define our request and response   schemas, MyRequestSchema and MyResponseSchema. These schemas are used to generate the API   documentation for our models.
      •  
    • Next we define the the model and the load_predict_fn, which tells Launch   how to load our model and how to make predictions with it. In this case,   we're just returning a function that adds the length of the string y to    model(x), where model doubles the integer x.
      •  
    • We then define the model bundle by specifying the load_predict_fn, the request_schema, and the   response_schema. We also specify the env_params, which tell Launch environment settings like    the base image to use. In this case, we're using a PyTorch image.
      •  
    • Next, we create the model endpoint, which is the API that we'll use to make predictions. We   specify the model_bundle that we created above, and we specify the endpoint_type, which tells   Launch whether to use a synchronous or asynchronous endpoint. In this case, we're using an   asynchronous endpoint, which means that we can make predictions and return immediately with a   future object. We can then use the future object to get the prediction result later.
      •  
    • Finally, we make a prediction by calling predict_on_endpoint with a MyRequestSchema object.   This function first waits for the endpoint to be ready, then it submits a prediction request to   the endpoint. It then waits for the prediction result and returns it.
      •  
     
    Notice that we specified min_workers=0, meaning that the endpoint will scale down to 0 workers when it's not being used.
    "},{"location":"#installation","title":"Installation","text":"
    To use Scale Launch, first install it using pip:
     Installation
    pip install -U scale-launch\n
    "},{"location":"cli/","title":"CLI","text":"
    Launch comes with a CLI for listing bundles / endpoints, editing endpoints, and sending tasks to endpoints.
     
    The CLI can be used as scale-launch ....
    "},{"location":"cli/#help","title":"Help","text":"
    Run scale-launch --help for more options.
     scale-launch --help
        This is the command line interface (CLI) package for Scale Launch.\n\n       \u2588\u2588\u2557      \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2588\u2557   \u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\n       \u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\n       \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2588\u2551\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\n       \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d  \u255a\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d\u255a\u2550\u255d  \u255a\u2550\u255d\n\nUsage: scale-launch [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n  --help  Show this message and exit.\n\nCommands:\n  batch-jobs  Batch Jobs is a wrapper around batch jobs in Scale Launch\n  bundles     Bundles is a wrapper around model bundles in Scale Launch\n  config      Config is a wrapper around getting and setting your API key and other configuration options\n  endpoints   Endpoints is a wrapper around model endpoints in Scale Launch\n  tasks       Tasks is a wrapper around sending requests to endpoints\n
    "},{"location":"api/client/","title":"Launch Client","text":""},{"location":"api/client/#launch.client.LaunchClient","title":"LaunchClient","text":"
    LaunchClient(\napi_key: str,\nendpoint: Optional[str] = None,\nself_hosted: bool = False,\n)\n
     
    Scale Launch Python Client.
     
    Initializes a Scale Launch Client.
     
    Parameters:
     Name Type Description Default api_key str 
    Your Scale API key
     required endpoint Optional[str] 
    The Scale Launch Endpoint (this should not need to be changed)
     None self_hosted bool 
    True iff you are connecting to a self-hosted Scale Launch
     False"},{"location":"api/client/#launch.client.LaunchClient.batch_async_request","title":"batch_async_request","text":"
    batch_async_request(\n*,\nmodel_bundle: Union[ModelBundle, str],\nurls: List[str] = None,\ninputs: Optional[List[Dict[str, Any]]] = None,\nbatch_url_file_location: Optional[str] = None,\nserialization_format: str = \"JSON\",\nlabels: Optional[Dict[str, str]] = None,\ncpus: Optional[int] = None,\nmemory: Optional[str] = None,\ngpus: Optional[int] = None,\ngpu_type: Optional[str] = None,\nstorage: Optional[str] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None\n) -> Dict[str, Any]\n
     
    Sends a batch inference request using a given bundle. Returns a key that can be used to retrieve the results of inference at a later time.
     
    Must have exactly one of urls or inputs passed in.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or the name of a the bundle to use for inference.
     required urls List[str] 
    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.
     None inputs Optional[List[Dict[str, Any]]] 
    A list of model inputs, if exists, we will upload the inputs and pass it in to Launch.
     None batch_url_file_location Optional[str] 
    In self-hosted mode, the input to the batch job will be uploaded to this location if provided. Otherwise, one will be determined from bundle_location_fn()
     None serialization_format str 
    Serialization format of output, either 'PICKLE' or 'JSON'. 'pickle' corresponds to pickling results + returning
     'JSON' labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None cpus Optional[int] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than   per_worker, then the number of workers will be reduced.
      •  
    • Otherwise, if the average number of concurrent requests per worker is higher   than per_worker, then the number of workers will be increased to meet the   elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains job_id as a key, and the ID as the value.
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes","title":"clone_model_bundle_with_changes","text":"
    clone_model_bundle_with_changes(\nmodel_bundle: Union[ModelBundle, str],\napp_config: Optional[Dict] = None,\n) -> ModelBundle\n
     Warning 
    This method is deprecated. Use clone_model_bundle_with_changes_v2 instead.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The existing bundle or its ID.
     required app_config Optional[Dict] 
    The new bundle's app config, if not passed in, the new bundle's app_config will be set to None
     None 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.clone_model_bundle_with_changes_v2","title":"clone_model_bundle_with_changes_v2","text":"
    clone_model_bundle_with_changes_v2(\noriginal_model_bundle_id: str,\nnew_app_config: Optional[Dict[str, Any]] = None,\n) -> CreateModelBundleV2Response\n
     
    Clone a model bundle with an optional new app_config.
     
    Parameters:
     Name Type Description Default original_model_bundle_id str 
    The ID of the model bundle you want to clone.
     required new_app_config Optional[Dict[str, Any]] 
    A dictionary of new app config values to use for the cloned model.
     None 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the cloned model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle","title":"create_model_bundle","text":"
    create_model_bundle(\nmodel_bundle_name: str,\nenv_params: Dict[str, str],\n*,\nload_predict_fn: Optional[\nCallable[[LaunchModel_T], Callable[[Any], Any]]\n] = None,\npredict_fn_or_cls: Optional[\nCallable[[Any], Any]\n] = None,\nrequirements: Optional[List[str]] = None,\nmodel: Optional[LaunchModel_T] = None,\nload_model_fn: Optional[\nCallable[[], LaunchModel_T]\n] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nglobals_copy: Optional[Dict[str, Any]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     Warning 
    This method is deprecated. Use create_model_bundle_from_callable_v2 instead.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required predict_fn_or_cls Optional[Callable[[Any], Any]] 
    Function or a Callable class that runs end-to-end (pre/post processing and model inference) on the call. i.e. predict_fn_or_cls(REQUEST) -> RESPONSE.
     None model Optional[LaunchModel_T] 
    Typically a trained Neural Network, e.g. a Pytorch module.
     
    Exactly one of model and load_model_fn must be provided.
     None load_model_fn Optional[Callable[[], LaunchModel_T]] 
    A function that, when run, loads a model. This function is essentially a deferred wrapper around the model argument.
     
    Exactly one of model and load_model_fn must be provided.
     None load_predict_fn Optional[Callable[[LaunchModel_T], Callable[[Any], Any]]] 
    Function that, when called with a model, returns a function that carries out inference.
     
    If model is specified, then this is equivalent to:     load_predict_fn(model, app_config=optional_app_config]) -> predict_fn
     
    Otherwise, if load_model_fn is specified, then this is equivalent to: load_predict_fn(load_model_fn(), app_config=optional_app_config]) -> predict_fn
     
    In both cases, predict_fn is then the inference function, i.e.:     predict_fn(REQUEST) -> RESPONSE
     None requirements Optional[List[str]] 
    A list of python package requirements, where each list element is of the form <package_name>==<package_version>, e.g.
     
    [\"tensorflow==2.3.0\", \"tensorflow-hub==0.11.0\"]
     
    If you do not pass in a value for requirements, then you must pass in globals() for the globals_copy argument.
     None app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    •  
      framework_type: either tensorflow or pytorch. - PyTorch fields: - pytorch_image_tag: An image tag for the pytorch docker base image. The list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags. - Example:
       
      .. code-block:: python
       
      {        \"framework_type\": \"pytorch\",        \"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\"    }
       
      •  
    •  
      Tensorflow fields:
       
         
      • tensorflow_version: Version of tensorflow, e.g. \"2.3.0\".
        •  
       
      •  
     required globals_copy Optional[Dict[str, Any]] 
    Dictionary of the global symbol table. Normally provided by globals() built-in function.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_callable_v2","title":"create_model_bundle_from_callable_v2","text":"
    create_model_bundle_from_callable_v2(\n*,\nmodel_bundle_name: str,\nload_predict_fn: Callable[\n[LaunchModel_T], Callable[[Any], Any]\n],\nload_model_fn: Callable[[], LaunchModel_T],\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements: Optional[List[str]] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Union[Dict[str, Any], str]] = None\n) -> CreateModelBundleV2Response\n
     
    Uploads and registers a model bundle to Scale Launch.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    Name of the model bundle.
     required load_predict_fn Callable[[LaunchModel_T], Callable[[Any], Any]] 
    Function that takes in a model and returns a predict function. When your model bundle is deployed, this predict function will be called as follows: 
    input = {\"input\": \"some input\"} # or whatever your request schema is.\n\ndef load_model_fn():\n    # load model\n    return model\n\ndef load_predict_fn(model, app_config=None):\n    def predict_fn(input):\n        # do pre-processing\n        output = model(input)\n        # do post-processing\n        return output\n    return predict_fn\n\npredict_fn = load_predict_fn(load_model_fn(), app_config=optional_app_config)\nresponse = predict_fn(input)\n
     required load_model_fn Callable[[], LaunchModel_T] 
    A function that, when run, loads a model.
     required request_schema Type[BaseModel] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     required response_schema Type[BaseModel] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint.
     required requirements Optional[List[str]] 
    List of pip requirements.
     None pytorch_image_tag Optional[str] 
    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None tensorflow_version Optional[str] 
    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_repository Optional[str] 
    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_tag Optional[str] 
    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.
     None app_config Optional[Union[Dict[str, Any], str]] 
    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.
     None 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the created model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs","title":"create_model_bundle_from_dirs","text":"
    create_model_bundle_from_dirs(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nrequirements_path: str,\nenv_params: Dict[str, str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\napp_config: Optional[Union[Dict[str, Any], str]] = None,\nrequest_schema: Optional[Type[BaseModel]] = None,\nresponse_schema: Optional[Type[BaseModel]] = None\n) -> ModelBundle\n
     Warning 
    This method is deprecated. Use create_model_bundle_from_dirs_v2 instead.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create. The name must be unique across all bundles that you own.
     required base_paths List[str] 
    The paths on the local filesystem where the bundle code lives.
     required requirements_path str 
    A path on the local filesystem where a requirements.txt file lives.
     required env_params Dict[str, str] 
    A dictionary that dictates environment information e.g. the use of pytorch or tensorflow, which base image tag to use, etc. Specifically, the dictionary should contain the following keys:
     
       
    • framework_type: either tensorflow or pytorch.
      •  
    • PyTorch fields:
         
      • pytorch_image_tag: An image tag for the pytorch docker base image. The     list of tags can be found from https://hub.docker.com/r/pytorch/pytorch/tags
        •  
       
      •  
     
    Example:    
    {\n\"framework_type\": \"pytorch\",\n\"pytorch_image_tag\": \"1.10.0-cuda11.3-cudnn8-runtime\",\n}\n
     required load_predict_fn_module_path str 
    A python module path for a function that, when called with the output of load_model_fn_module_path, returns a function that carries out inference.
     required load_model_fn_module_path str 
    A python module path for a function that returns a model. The output feeds into the function located at load_predict_fn_module_path.
     required app_config Optional[Union[Dict[str, Any], str]] 
    Either a Dictionary that represents a YAML file contents or a local path to a YAML file.
     None request_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the request body for the model bundle's endpoint.
     None response_schema Optional[Type[BaseModel]] 
    A pydantic model that represents the request schema for the model bundle. This is used to validate the response for the model bundle's endpoint. Note: If request_schema is specified, then response_schema must also be specified.
     None"},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_dirs_v2","title":"create_model_bundle_from_dirs_v2","text":"
    create_model_bundle_from_dirs_v2(\n*,\nmodel_bundle_name: str,\nbase_paths: List[str],\nload_predict_fn_module_path: str,\nload_model_fn_module_path: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrequirements_path: Optional[str] = None,\npytorch_image_tag: Optional[str] = None,\ntensorflow_version: Optional[str] = None,\ncustom_base_image_repository: Optional[str] = None,\ncustom_base_image_tag: Optional[str] = None,\napp_config: Optional[Dict[str, Any]] = None\n) -> CreateModelBundleV2Response\n
     
    Packages up code from one or more local filesystem folders and uploads them as a bundle to Scale Launch. In this mode, a bundle is just local code instead of a serialized object.
     
    For example, if you have a directory structure like so, and your current working directory is my_root:
     
       my_root/\n       my_module1/\n           __init__.py\n           ...files and directories\n           my_inference_file.py\n       my_module2/\n           __init__.py\n           ...files and directories\n
     
    then calling create_model_bundle_from_dirs_v2 with base_paths=[\"my_module1\", \"my_module2\"] essentially creates a zip file without the root directory, e.g.:
     
       my_module1/\n       __init__.py\n       ...files and directories\n       my_inference_file.py\n   my_module2/\n       __init__.py\n       ...files and directories\n
     
    and these contents will be unzipped relative to the server side application root. Bear these points in mind when referencing Python module paths for this bundle. For instance, if my_inference_file.py has def f(...) as the desired inference loading function, then the load_predict_fn_module_path argument should be my_module1.my_inference_file.f.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create.
     required base_paths List[str] 
    A list of paths to directories that will be zipped up and uploaded as a bundle. Each path must be relative to the current working directory.
     required load_predict_fn_module_path str 
    The Python module path to the function that will be used to load the model for inference. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.
     required load_model_fn_module_path str 
    The Python module path to the function that will be used to load the model for training. This function should take in a path to a model directory, and return a model object. The model object should be pickleable.
     required request_schema Type[BaseModel] 
    A Pydantic model that defines the request schema for the bundle.
     required response_schema Type[BaseModel] 
    A Pydantic model that defines the response schema for the bundle.
     required requirements_path Optional[str] 
    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.
     None pytorch_image_tag Optional[str] 
    The image tag for the PyTorch image that will be used to run the bundle. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None tensorflow_version Optional[str] 
    The version of TensorFlow that will be used to run the bundle. If not specified, the default version will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_repository Optional[str] 
    The repository for a custom base image that will be used to run the bundle. If not specified, the default base image will be used. Exactly one of pytorch_image_tag, tensorflow_version, or custom_base_image_repository must be specified.
     None custom_base_image_tag Optional[str] 
    The tag for a custom base image that will be used to run the bundle. Must be specified if custom_base_image_repository is specified.
     None app_config Optional[Dict[str, Any]] 
    An optional dictionary of configuration values that will be passed to the bundle when it is run. These values can be accessed by the bundle via the app_config global variable.
     None 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the created model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_bundle_from_runnable_image_v2","title":"create_model_bundle_from_runnable_image_v2","text":"
    create_model_bundle_from_runnable_image_v2(\n*,\nmodel_bundle_name: str,\nrequest_schema: Type[BaseModel],\nresponse_schema: Type[BaseModel],\nrepository: str,\ntag: str,\ncommand: List[str],\nenv: Dict[str, str]\n) -> CreateModelBundleV2Response\n
     
    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to create.
     required request_schema Type[BaseModel] 
    A Pydantic model that defines the request schema for the bundle.
     required response_schema Type[BaseModel] 
    A Pydantic model that defines the response schema for the bundle.
     required repository str 
    The name of the Docker repository for the runnable image.
     required tag str 
    The tag for the runnable image.
     required command List[str] 
    The command that will be used to start the process that listens for requests.
     required env Dict[str, str] 
    A dictionary of environment variables that will be passed to the bundle when it is run.
     required 
    Returns:
     Type Description CreateModelBundleV2Response 
    An object containing the following keys:
     
       
    • model_bundle_id: The ID of the created model bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.create_model_endpoint","title":"create_model_endpoint","text":"
    create_model_endpoint(\n*,\nendpoint_name: str,\nmodel_bundle: Union[ModelBundle, str],\ncpus: int = 3,\nmemory: str = \"8Gi\",\nstorage: Optional[str] = None,\ngpus: int = 0,\nmin_workers: int = 1,\nmax_workers: int = 1,\nper_worker: int = 10,\ngpu_type: Optional[str] = None,\nendpoint_type: str = \"sync\",\nhigh_priority: Optional[bool] = False,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None,\nupdate_if_exists: bool = False,\nlabels: Optional[Dict[str, str]] = None\n) -> Optional[Endpoint]\n
     
    Creates and registers a model endpoint in Scale Launch. The returned object is an instance of type Endpoint, which is a base class of either SyncEndpoint or AsyncEndpoint. This is the object to which you sent inference requests.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the model endpoint you want to create. The name must be unique across all endpoints that you own.
     required model_bundle Union[ModelBundle, str] 
    The ModelBundle that the endpoint should serve.
     required cpus int 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     3 memory str 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     '8Gi' storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus int 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     0 min_workers int 
    The minimum number of workers. Must be greater than or equal to 0. This should be determined by computing the minimum throughput of your workload and dividing it by the throughput of a single worker. This field must be at least 1 for synchronous endpoints.
     1 max_workers int 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers. This should be determined by computing the maximum throughput of your workload and dividing it by the throughput of a single worker.
     1 per_worker int 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests, subject to the limits defined by min_workers and max_workers.
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     
    Here is our recommendation for computing per_worker:
     
       
    1. Compute min_workers and max_workers per your minimum and maximum throughput requirements. 2. Determine a value for the maximum number of concurrent requests in the workload. Divide this number by max_workers. Doing this ensures that the number of workers will \"climb\" to max_workers.
      2.  
     10 gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None endpoint_type str 
    Either \"sync\" or \"async\".
     'sync' high_priority Optional[bool] 
    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.
     False post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None default_callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None update_if_exists bool 
    If True, will attempt to update the endpoint if it exists. Otherwise, will unconditionally try to create a new endpoint. Note that endpoint names for a given user must be unique, so attempting to call this function with update_if_exists=False for an existing endpoint will raise an error.
     False labels Optional[Dict[str, str]] 
    An optional dictionary of key/value pairs to associate with this endpoint.
     None 
    Returns:
     Type Description Optional[Endpoint] 
    A Endpoint object that can be used to make requests to the endpoint.
    "},{"location":"api/client/#launch.client.LaunchClient.delete_model_endpoint","title":"delete_model_endpoint","text":"
    delete_model_endpoint(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Deletes a model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    A ModelEndpoint object.
     required"},{"location":"api/client/#launch.client.LaunchClient.edit_model_endpoint","title":"edit_model_endpoint","text":"
    edit_model_endpoint(\n*,\nmodel_endpoint: Union[ModelEndpoint, str],\nmodel_bundle: Optional[Union[ModelBundle, str]] = None,\ncpus: Optional[float] = None,\nmemory: Optional[str] = None,\nstorage: Optional[str] = None,\ngpus: Optional[int] = None,\nmin_workers: Optional[int] = None,\nmax_workers: Optional[int] = None,\nper_worker: Optional[int] = None,\ngpu_type: Optional[str] = None,\nhigh_priority: Optional[bool] = None,\npost_inference_hooks: Optional[\nList[PostInferenceHooks]\n] = None,\ndefault_callback_url: Optional[str] = None,\ndefault_callback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ndefault_callback_auth_username: Optional[str] = None,\ndefault_callback_auth_password: Optional[str] = None,\ndefault_callback_auth_cert: Optional[str] = None,\ndefault_callback_auth_key: Optional[str] = None\n) -> None\n
     
    Edits an existing model endpoint. Here are the fields that cannot be edited on an existing endpoint:
     
       
    • The endpoint's name. - The endpoint's type (i.e. you cannot go from a SyncEnpdoint to an AsyncEndpoint or vice versa.
      •  
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The model endpoint (or its name) you want to edit. The name must be unique across all endpoints that you own.
     required model_bundle Optional[Union[ModelBundle, str]] 
    The ModelBundle that the endpoint should serve.
     None cpus Optional[float] 
    Number of cpus each worker should get, e.g. 1, 2, etc. This must be greater than or equal to 1.
     None memory Optional[str] 
    Amount of memory each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of memory.
     None storage Optional[str] 
    Amount of local ephemeral storage each worker should get, e.g. \"4Gi\", \"512Mi\", etc. This must be a positive amount of storage.
     None gpus Optional[int] 
    Number of gpus each worker should get, e.g. 0, 1, etc.
     None min_workers Optional[int] 
    The minimum number of workers. Must be greater than or equal to 0.
     None max_workers Optional[int] 
    The maximum number of workers. Must be greater than or equal to 0, and as well as greater than or equal to min_workers.
     None per_worker Optional[int] 
    The maximum number of concurrent requests that an individual worker can service. Launch automatically scales the number of workers for the endpoint so that each worker is processing per_worker requests:
     
       
    • If the average number of concurrent requests per worker is lower than per_worker, then the number of workers will be reduced. - Otherwise, if the average number of concurrent requests per worker is higher than per_worker, then the number of workers will be increased to meet the elevated traffic.
      •  
     None gpu_type Optional[str] 
    If specifying a non-zero number of gpus, this controls the type of gpu requested. Here are the supported values:
     
       
    • nvidia-tesla-t4
      •  
    • nvidia-ampere-a10
      •  
     None high_priority Optional[bool] 
    Either True or False. Enabling this will allow the created endpoint to leverage the shared pool of prewarmed nodes for faster spinup time.
     None post_inference_hooks Optional[List[PostInferenceHooks]] 
    List of hooks to trigger after inference tasks are served.
     None default_callback_url Optional[str] 
    The default callback url to use for async endpoints. This can be overridden in the task parameters for each individual task. post_inference_hooks must contain \"callback\" for the callback to be triggered.
     None default_callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if default_callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None default_callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if default_callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None"},{"location":"api/client/#launch.client.LaunchClient.get_batch_async_response","title":"get_batch_async_response","text":"
    get_batch_async_response(\nbatch_job_id: str,\n) -> Dict[str, Any]\n
     
    Gets inference results from a previously created batch job.
     
    Parameters:
     Name Type Description Default batch_job_id str 
    An id representing the batch task job. This id is the in the response from calling batch_async_request.
     required 
    Returns:
     Type Description Dict[str, Any] 
    A dictionary that contains the following fields:
     Dict[str, Any] 
       
    • status: The status of the job.
      •  
     Dict[str, Any] 
       
    • result: The url where the result is stored.
      •  
     Dict[str, Any] 
       
    • duration: A string representation of how long the job took to finish     or how long it has been running, for a job current in progress.
      •  
     Dict[str, Any] 
       
    • num_tasks_pending: The number of tasks that are still pending.
      •  
     Dict[str, Any] 
       
    • num_tasks_completed: The number of tasks that have completed.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_latest_model_bundle_v2","title":"get_latest_model_bundle_v2","text":"
    get_latest_model_bundle_v2(\nmodel_bundle_name: str,\n) -> ModelBundleV2Response\n
     
    Get the latest version of a model bundle.
     
    Parameters:
     Name Type Description Default model_bundle_name str 
    The name of the model bundle you want to get.
     required 
    Returns:
     Type Description ModelBundleV2Response 
    An object containing the following keys:
     
       
    • id: The ID of the model bundle.
      •  
    • name: The name of the model bundle.
      •  
    • schema_location: The location of the schema for the model bundle.
      •  
    • flavor: The flavor of the model bundle. Either RunnableImage,     CloudpickleArtifact, or ZipArtifact.
      •  
    • created_at: The time the model bundle was created.
      •  
    • metadata: A dictionary of metadata associated with the model bundle.
      •  
    • model_artifact_ids: A list of IDs of model artifacts associated with the     bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle","title":"get_model_bundle","text":"
    get_model_bundle(\nmodel_bundle: Union[ModelBundle, str]\n) -> ModelBundle\n
     
    Returns a model bundle specified by bundle_name that the user owns.
     
    Parameters:
     Name Type Description Default model_bundle Union[ModelBundle, str] 
    The bundle or its name.
     required 
    Returns:
     Type Description ModelBundle 
    A ModelBundle object
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_bundle_v2","title":"get_model_bundle_v2","text":"
    get_model_bundle_v2(\nmodel_bundle_id: str,\n) -> ModelBundleV2Response\n
     
    Get a model bundle.
     
    Parameters:
     Name Type Description Default model_bundle_id str 
    The ID of the model bundle you want to get.
     required 
    Returns:
     Type Description ModelBundleV2Response 
    An object containing the following fields:
     
       
    • id: The ID of the model bundle.
      •  
    • name: The name of the model bundle.
      •  
    • flavor: The flavor of the model bundle. Either RunnableImage,     CloudpickleArtifact, or ZipArtifact.
      •  
    • created_at: The time the model bundle was created.
      •  
    • metadata: A dictionary of metadata associated with the model bundle.
      •  
    • model_artifact_ids: A list of IDs of model artifacts associated with the     bundle.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.get_model_endpoint","title":"get_model_endpoint","text":"
    get_model_endpoint(\nendpoint_name: str,\n) -> Optional[Union[AsyncEndpoint, SyncEndpoint]]\n
     
    Gets a model endpoint associated with a name.
     
    Parameters:
     Name Type Description Default endpoint_name str 
    The name of the endpoint to retrieve.
     required"},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles","title":"list_model_bundles","text":"
    list_model_bundles() -> List[ModelBundle]\n
     
    Returns a list of model bundles that the user owns.
     
    Returns:
     Type Description List[ModelBundle] 
    A list of ModelBundle objects
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_bundles_v2","title":"list_model_bundles_v2","text":"
    list_model_bundles_v2() -> ListModelBundlesV2Response\n
     
    List all model bundles.
     
    Returns:
     Type Description ListModelBundlesV2Response 
    An object containing the following keys:
     
       
    • model_bundles: A list of model bundles. Each model bundle is an object.
      •  
    "},{"location":"api/client/#launch.client.LaunchClient.list_model_endpoints","title":"list_model_endpoints","text":"
    list_model_endpoints() -> List[Endpoint]\n
     
    Lists all model endpoints that the user owns.
     
    Returns:
     Type Description List[Endpoint] 
    A list of ModelEndpoint objects.
    "},{"location":"api/client/#launch.client.LaunchClient.read_endpoint_creation_logs","title":"read_endpoint_creation_logs","text":"
    read_endpoint_creation_logs(\nmodel_endpoint: Union[ModelEndpoint, str]\n)\n
     
    Retrieves the logs for the creation of the endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint Union[ModelEndpoint, str] 
    The endpoint or its name.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_batch_csv_location_fn","title":"register_batch_csv_location_fn","text":"
    register_batch_csv_location_fn(\nbatch_csv_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for batch CSV inputs. Should give different locations each time. This function is called as batch_csv_location_fn(), and should return a batch_csv_url that upload_batch_csv_fn can take.
     
    Strictly, batch_csv_location_fn() does not need to return a str. The only requirement is that if batch_csv_location_fn returns a value of type T, then upload_batch_csv_fn() takes in an object of type T as its second argument (i.e. batch_csv_url).
     
    Parameters:
     Name Type Description Default batch_csv_location_fn Callable[[], str] 
    Function that generates batch_csv_urls for upload_batch_csv_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_bundle_location_fn","title":"register_bundle_location_fn","text":"
    register_bundle_location_fn(\nbundle_location_fn: Callable[[], str]\n)\n
     
    For self-hosted mode only. Registers a function that gives a location for a model bundle. Should give different locations each time. This function is called as bundle_location_fn(), and should return a bundle_url that register_upload_bundle_fn can take.
     
    Strictly, bundle_location_fn() does not need to return a str. The only requirement is that if bundle_location_fn returns a value of type T, then upload_bundle_fn() takes in an object of type T as its second argument (i.e. bundle_url).
     
    Parameters:
     Name Type Description Default bundle_location_fn Callable[[], str] 
    Function that generates bundle_urls for upload_bundle_fn.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_batch_csv_fn","title":"register_upload_batch_csv_fn","text":"
    register_upload_batch_csv_fn(\nupload_batch_csv_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles batch text upload. This function is called as
     
    upload_batch_csv_fn(csv_text, csv_url)\n
     
    This function should directly write the contents of csv_text as a text string into csv_url.
     
    Parameters:
     Name Type Description Default upload_batch_csv_fn Callable[[str, str], None] 
    Function that takes in a csv text (string type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/client/#launch.client.LaunchClient.register_upload_bundle_fn","title":"register_upload_bundle_fn","text":"
    register_upload_bundle_fn(\nupload_bundle_fn: Callable[[str, str], None]\n)\n
     
    For self-hosted mode only. Registers a function that handles model bundle upload. This function is called as
     
    upload_bundle_fn(serialized_bundle, bundle_url)\n
     
    This function should directly write the contents of serialized_bundle as a binary string into bundle_url.
     
    See register_bundle_location_fn for more notes on the signature of upload_bundle_fn
     
    Parameters:
     Name Type Description Default upload_bundle_fn Callable[[str, str], None] 
    Function that takes in a serialized bundle (bytes type), and uploads that bundle to an appropriate location. Only needed for self-hosted mode.
     required"},{"location":"api/endpoint_predictions/","title":"Endpoint Predictions","text":""},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False callback_url Optional[str] 
    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.
     None callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"api/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"api/hooks/","title":"Hooks","text":""},{"location":"api/hooks/#launch.hooks.PostInferenceHooks","title":"PostInferenceHooks","text":"
             Bases: str, Enum
     
    Post-inference hooks are functions that are called after inference is complete.
     
    Attributes:
     Name Type Description CALLBACK str 
    The callback hook is called with the inference response and the task ID.
    "},{"location":"api/model_bundles/","title":"Model Bundles","text":""},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor","title":"CloudpickleArtifactFlavor","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[str, Any]]\n
     
    Optional configuration for the application.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.framework","title":"framework  class-attribute","text":"
    framework: Union[\nPytorchFramework, TensorflowFramework, CustomFramework\n] = Field(Ellipsis, discriminator=\"framework_type\")\n
     
    Machine Learning framework specification. Either PytorchFramework, TensorflowFramework, or CustomFramework.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.load_model_fn","title":"load_model_fn  class-attribute","text":"
    load_model_fn: str\n
     
    Function which, when called, returns the model object.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.load_predict_fn","title":"load_predict_fn  class-attribute","text":"
    load_predict_fn: str\n
     
    Function which, when called, returns the prediction function.
    "},{"location":"api/model_bundles/#launch.model_bundle.CloudpickleArtifactFlavor.requirements","title":"requirements  class-attribute","text":"
    requirements: List[str]\n
     
    List of requirements to install in the environment before running the model.
    "},{"location":"api/model_bundles/#launch.model_bundle.CreateModelBundleV2Response","title":"CreateModelBundleV2Response","text":"
             Bases: BaseModel
     
    Response object for creating a Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.CreateModelBundleV2Response.model_bundle_id","title":"model_bundle_id  class-attribute","text":"
    model_bundle_id: str\n
     
    ID of the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework","title":"CustomFramework","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework.image_repository","title":"image_repository  class-attribute","text":"
    image_repository: str\n
     
    Docker image repository to use as the base image.
    "},{"location":"api/model_bundles/#launch.model_bundle.CustomFramework.image_tag","title":"image_tag  class-attribute","text":"
    image_tag: str\n
     
    Docker image tag to use as the base image.
    "},{"location":"api/model_bundles/#launch.model_bundle.ListModelBundlesV2Response","title":"ListModelBundlesV2Response","text":"
             Bases: BaseModel
     
    Response object for listing Model Bundles.
    "},{"location":"api/model_bundles/#launch.model_bundle.ListModelBundlesV2Response.model_bundles","title":"model_bundles  class-attribute","text":"
    model_bundles: List[ModelBundleV2Response]\n
     
    A list of Model Bundles.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle","title":"ModelBundle  dataclass","text":"
    Represents a ModelBundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[Any, Any]] = None\n
     
    An optional user-specified configuration mapping for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.env_params","title":"env_params  class-attribute","text":"
    env_params: Optional[Dict[str, str]] = None\n
     
    A dictionary that dictates environment information. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.id","title":"id  class-attribute","text":"
    id: Optional[str] = None\n
     
    A globally unique identifier for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.location","title":"location  class-attribute","text":"
    location: Optional[str] = None\n
     
    An opaque location for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.metadata","title":"metadata  class-attribute","text":"
    metadata: Optional[Dict[Any, Any]] = None\n
     
    Arbitrary metadata for the bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.name","title":"name  class-attribute","text":"
    name: str\n
     
    The name of the bundle. Must be unique across all bundles that the user owns.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.packaging_type","title":"packaging_type  class-attribute","text":"
    packaging_type: Optional[str] = None\n
     
    The packaging type for the bundle. Can be cloudpickle or zip.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundle.requirements","title":"requirements  class-attribute","text":"
    requirements: Optional[List[str]] = None\n
     
    A list of Python package requirements for the bundle. See LaunchClient.create_model_bundle for more information.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response","title":"ModelBundleV2Response","text":"
             Bases: BaseModel
     
    Response object for a single Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.created_at","title":"created_at  class-attribute","text":"
    created_at: datetime.datetime\n
     
    Timestamp of when the Model Bundle was created.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.flavor","title":"flavor  class-attribute","text":"
    flavor: Union[\nCloudpickleArtifactFlavor,\nZipArtifactFlavor,\nRunnableImageFlavor,\n] = Field(Ellipsis, discriminator=\"flavor\")\n
     
    Flavor of the Model Bundle, representing how the model bundle was packaged. Either CloudpickleArtifactFlavor, ZipArtifactFlavor, or RunnableImageFlavor.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.id","title":"id  class-attribute","text":"
    id: str\n
     
    ID of the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.metadata","title":"metadata  class-attribute","text":"
    metadata: Dict[str, Any]\n
     
    Metadata associated with the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.model_artifact_ids","title":"model_artifact_ids  class-attribute","text":"
    model_artifact_ids: List[str]\n
     
    IDs of the Model Artifacts associated with the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.ModelBundleV2Response.name","title":"name  class-attribute","text":"
    name: str\n
     
    Name of the Model Bundle.
    "},{"location":"api/model_bundles/#launch.model_bundle.PytorchFramework","title":"PytorchFramework","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.PytorchFramework.pytorch_image_tag","title":"pytorch_image_tag  class-attribute","text":"
    pytorch_image_tag: str\n
     
    Image tag of the Pytorch image to use.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor","title":"RunnableImageFlavor","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.command","title":"command  class-attribute","text":"
    command: List[str]\n
     
    Command to run the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.env","title":"env  class-attribute","text":"
    env: Optional[Dict[str, str]]\n
     
    Environment variables to set when running the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.repository","title":"repository  class-attribute","text":"
    repository: str\n
     
    Docker repository of the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.RunnableImageFlavor.tag","title":"tag  class-attribute","text":"
    tag: str\n
     
    Docker tag of the image.
    "},{"location":"api/model_bundles/#launch.model_bundle.TensorflowFramework","title":"TensorflowFramework","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.TensorflowFramework.tensorflow_version","title":"tensorflow_version  class-attribute","text":"
    tensorflow_version: str\n
     
    Tensorflow version to use.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor","title":"ZipArtifactFlavor","text":"
             Bases: BaseModel
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.app_config","title":"app_config  class-attribute","text":"
    app_config: Optional[Dict[str, Any]]\n
     
    Optional configuration for the application.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.framework","title":"framework  class-attribute","text":"
    framework: Union[\nPytorchFramework, TensorflowFramework, CustomFramework\n] = Field(Ellipsis, discriminator=\"framework_type\")\n
     
    Machine Learning framework specification. Either PytorchFramework, TensorflowFramework, or CustomFramework.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.load_model_fn_module_path","title":"load_model_fn_module_path  class-attribute","text":"
    load_model_fn_module_path: str\n
     
    Path to the module to load the model object.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.load_predict_fn_module_path","title":"load_predict_fn_module_path  class-attribute","text":"
    load_predict_fn_module_path: str\n
     
    Path to the module to load the prediction function.
    "},{"location":"api/model_bundles/#launch.model_bundle.ZipArtifactFlavor.requirements","title":"requirements  class-attribute","text":"
    requirements: List[str]\n
     
    List of requirements to install in the environment before running the model.
    "},{"location":"api/model_endpoints/","title":"Model Endpoints","text":"
    All classes here are returned by the  get_model_endpoint method and provide a predict function.
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint","title":"AsyncEndpoint","text":"
    AsyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    An asynchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponseFuture\n
     
    Runs an asynchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required 
    Returns:
     Name Type Description EndpointResponseFuture 
    An EndpointResponseFuture such the user can use to query the status of the request.
     Example EndpointResponseFuture EndpointResponseFuture 
    .. code-block:: python
     
    my_endpoint = AsyncEndpoint(...) f: EndpointResponseFuture = my_endpoint.predict(EndpointRequest(...)) result = f.get()  # blocks on completion
    "},{"location":"api/model_endpoints/#launch.model_endpoint.AsyncEndpoint.predict_batch","title":"predict_batch","text":"
    predict_batch(\nrequests: Sequence[EndpointRequest],\n) -> AsyncEndpointBatchResponse\n
     
    (deprecated) Runs inference on the data items specified by urls. Returns a AsyncEndpointResponse.
     
    Parameters:
     Name Type Description Default requests Sequence[EndpointRequest] 
    List of EndpointRequests. Request_ids must all be distinct.
     required 
    Returns:
     Type Description AsyncEndpointBatchResponse 
    an AsyncEndpointResponse keeping track of the inference requests made
    "},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint","title":"SyncEndpoint","text":"
    SyncEndpoint(model_endpoint: ModelEndpoint, client)\n
     
             Bases: Endpoint
     
    A synchronous model endpoint.
     
    Parameters:
     Name Type Description Default model_endpoint ModelEndpoint 
    ModelEndpoint object.
     required client 
    A LaunchClient object
     required"},{"location":"api/model_endpoints/#launch.model_endpoint.SyncEndpoint.predict","title":"predict","text":"
    predict(request: EndpointRequest) -> EndpointResponse\n
     
    Runs a synchronous prediction request.
     
    Parameters:
     Name Type Description Default request EndpointRequest 
    The EndpointRequest object that contains the payload.
     required"},{"location":"concepts/batch_jobs/","title":"Batch Jobs","text":"
    For predicting over a larger set of tasks (> 50) at once, it is recommended to use batch jobs. Batch jobs are a way to send a large number of tasks to a model bundle. The tasks are processed in parallel, and the results are returned as a list of predictions.
     
    Batch jobs are created using the batch_async_request method of the LaunchClient.
     Creating and Following a Batch Job
    import os\nimport time\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbatch_job = client.batch_async_request(\nmodel_bundle=\"test-bundle\",\ninputs=[\n{\"x\": 2, \"y\": \"hello\"},\n{\"x\": 3, \"y\": \"world\"},\n],\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n}\n)\nstatus = \"PENDING\"\nres = None\nwhile status != \"SUCCESS\" and status != \"FAILURE\" and status != \"CANCELLED\":\ntime.sleep(30)\nres = client.get_batch_async_response(batch_job[\"job_id\"])\nstatus = res[\"status\"]\nprint(f\"the batch job is {status}\")\nprint(res)\n
    "},{"location":"concepts/callbacks/","title":"Callbacks","text":"
    Async model endpoints can be configured to send callbacks to a user-defined callback URL. Callbacks are sent as HTTP POST requests with a JSON body. The following code snippet shows how to create an async model endpoint with a callback URL.
     
    To configure an async endpoint to send callbacks, set the post_inference_hooks field to include launch.PostInferenceHooks.CALLBACK. A callback URL also needs to be specified, and it can be configured as a default using the default_callback_url argument to launch.LaunchClient.create_model_endpoint or as a per-task override using the callback_url field of launch.EndpointRequest.
     
    Note
     
    Callbacks will not be sent if the endpoint does not have any post-inference hooks specified, even if a default_callback_url is provided to the endpoint creation method or if the prediction request has a callback_url override.
     Creating an Async Model Endpoint with a Callback URL
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 7\n}\n\"\"\"\nfuture_custom_callback_url = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"}, callback_url=\"https://example.com/custom\"\n),\n)\n\"\"\"\nA callback is sent to https://example.com/custom with the following JSON body:\n{\n    \"task_id\": \"THE_TASK_ID\",\n    \"result\": 8\n}\n\"\"\"\n
    "},{"location":"concepts/callbacks/#authentication-for-callbacks","title":"Authentication for callbacks","text":"
    Warning
     
    This feature is currently in beta, and the API is likely to change.
     
    Callbacks can be authenticated using shared authentication headers. To enable authentication, set either default_callback_auth_kind when creating the endpoint or callback_auth_kind when making a prediction request.
     
    Currently, the supported authentication methods are basic and mtls. If basic is used, then the default_callback_auth_username and default_callback_auth_password fields must be specified when creating the endpoint, or the callback_auth_username and callback_auth_password fields must be specified when making a prediction request. If mtls is used, then the  same is true for the default_callback_auth_cert and default_callback_auth_key fields, or the callback_auth_cert and callback_auth_key fields.
     Creating an Async Model Endpoint with custom Callback auth
    import os\nimport time\nfrom launch import EndpointRequest, LaunchClient, PostInferenceHooks\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-callback\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\npost_inference_hooks=[PostInferenceHooks.CALLBACK],\ndefault_callback_url=\"https://example.com\",\ndefault_callback_auth_kind=\"basic\",\ndefault_callback_auth_username=\"user\",\ndefault_callback_auth_password=\"password\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent to https://example.com with (\"user\", \"password\") as the basic auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"mtls\", \ncallback_auth_cert=\"cert\", \ncallback_auth_key=\"key\",\n),\n)\n\"\"\"\nA callback is sent with mTLS authentication.\n\"\"\"\nclient.edit_model_endpoint(\nmodel_endpoint=endpoint.model_endpoint,\ndefault_callback_auth_kind=\"mtls\",\ndefault_callback_auth_cert=\"cert\",\ndefault_callback_auth_key=\"key\",\n)\nwhile endpoint.status() != \"READY\":\ntime.sleep(10)\nfuture_default = endpoint.predict(\nrequest=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"})\n)\n\"\"\"\nA callback is sent with mTLS auth.\n\"\"\"\nfuture_custom_callback_auth = endpoint.predict(\nrequest=EndpointRequest(\nargs={\"x\": 3, \"y\": \"hello\"},\ncallback_auth_kind=\"basic\",\ncallback_auth_username=\"user\",\ncallback_auth_password=\"pass\",\n),\n)\n\"\"\"\nA callback is sent with (\"user\", \"pass\") as the basic auth.\n\"\"\"\n
    "},{"location":"concepts/endpoint_predictions/","title":"Endpoint Predictions","text":"
    Once endpoints have been created, users can send tasks to them to make predictions. The following code snippet shows how to send tasks to endpoints.
     Sending a Task to an Async EndpointSending a Task to a Sync Endpoint 
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-async\")\nfuture = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nresponse = future.get()\nprint(response)\n
     
    import os\nfrom launch import EndpointRequest, LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.get_model_endpoint(\"demo-endpoint-sync\")\nresponse = endpoint.predict(request=EndpointRequest(args={\"x\": 2, \"y\": \"hello\"}))\nprint(response)\n
    "},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointRequest","title":"EndpointRequest","text":"
    EndpointRequest(\nurl: Optional[str] = None,\nargs: Optional[Dict] = None,\ncallback_url: Optional[str] = None,\ncallback_auth_kind: Optional[\nLiteral[basic, mtls]\n] = None,\ncallback_auth_username: Optional[str] = None,\ncallback_auth_password: Optional[str] = None,\ncallback_auth_cert: Optional[str] = None,\ncallback_auth_key: Optional[str] = None,\nreturn_pickled: Optional[bool] = False,\nrequest_id: Optional[str] = None,\n)\n
     
    Represents a single request to either a SyncEndpoint or AsyncEndpoint.
     
    Parameters:
     Name Type Description Default url Optional[str] 
    A url to some file that can be read in to a ModelBundle's predict function. Can be an image, raw text, etc. Note: the contents of the file located at url are opened as a sequence of bytes and passed to the predict function. If you instead want to pass the url itself as an input to the predict function, see args.
     
    Exactly one of url and args must be specified.
     None args Optional[Dict] 
    A Dictionary with arguments to a ModelBundle's predict function. If the predict function has signature predict_fn(foo, bar), then the keys in the dictionary should be \"foo\" and \"bar\". Values must be native Python objects.
     
    Exactly one of url and args must be specified.
     None return_pickled Optional[bool] 
    Whether the output should be a pickled python object, or directly returned serialized json.
     False callback_url Optional[str] 
    The callback url to use for this task. If None, then the default_callback_url of the endpoint is used. The endpoint must specify \"callback\" as a post-inference hook for the callback to be triggered.
     None callback_auth_kind Optional[Literal[basic, mtls]] 
    The default callback auth kind to use for async endpoints. Either \"basic\" or \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_username Optional[str] 
    The default callback auth username to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_password Optional[str] 
    The default callback auth password to use. This only applies if callback_auth_kind is \"basic\". This can be overridden in the task parameters for each individual task.
     None callback_auth_cert Optional[str] 
    The default callback auth cert to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None callback_auth_key Optional[str] 
    The default callback auth key to use. This only applies if callback_auth_kind is \"mtls\". This can be overridden in the task parameters for each individual task.
     None request_id Optional[str] 
    (deprecated) A user-specifiable id for requests. Should be unique among EndpointRequests made in the same batch call. If one isn't provided the client will generate its own.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture","title":"EndpointResponseFuture","text":"
    EndpointResponseFuture(\nclient, endpoint_name: str, async_task_id: str\n)\n
     
    Represents a future response from an Endpoint. Specifically, when the EndpointResponseFuture is ready, then its get method will return an actual instance of EndpointResponse.
     
    This object should not be directly instantiated by the user.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required endpoint_name str 
    The name of the endpoint.
     required async_task_id str 
    An async task id.
     required"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponseFuture.get","title":"get","text":"
    get(timeout: Optional[float] = None) -> EndpointResponse\n
     
    Retrieves the EndpointResponse for the prediction request after it completes. This method blocks.
     
    Parameters:
     Name Type Description Default timeout Optional[float] 
    The maximum number of seconds to wait for the response. If None, then the method will block indefinitely until the response is ready.
     None"},{"location":"concepts/endpoint_predictions/#launch.model_endpoint.EndpointResponse","title":"EndpointResponse","text":"
    EndpointResponse(\nclient,\nstatus: str,\nresult_url: Optional[str] = None,\nresult: Optional[str] = None,\ntraceback: Optional[str] = None,\n)\n
     
    Represents a response received from a Endpoint.
     
    Parameters:
     Name Type Description Default client 
    An instance of LaunchClient.
     required status str 
    A string representing the status of the request, i.e. SUCCESS, FAILURE, or PENDING
     required result_url Optional[str] 
    A string that is a url containing the pickled python object from the Endpoint's predict function.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None result Optional[str] 
    A string that is the serialized return value (in json form) of the Endpoint's predict function. Specifically, one can json.loads() the value of result to get the original python object back.
     
    Exactly one of result_url or result will be populated, depending on the value of return_pickled in the request.
     None traceback Optional[str] 
    The stack trace if the inference endpoint raised an error. Can be used for debugging
     None"},{"location":"concepts/model_bundles/","title":"Model Bundles","text":"
    Model Bundles are deployable models that can be used to make predictions. They are created by packaging a model up into a deployable format. 
    "},{"location":"concepts/model_bundles/#creating-model-bundles","title":"Creating Model Bundles","text":"
    There are three methods for creating model bundles: create_model_bundle_from_callable_v2 and create_model_bundle_from_dirs_v2. create_model_bundle_from_runnable_image_v2. The first directly pickles a user-specified load_predict_fn, a function which loads the model and returns a predict_fn, a function which takes in a request. The second takes in directories containing a load_predict_fn and the module path to the load_predict_fn. The third takes a Docker image that contains an entrypoint that starts a process listening for requests at port 5005 using HTTP and exposes POST /predict and GET /healthz endpoints.
     
    Each of these modes of creating a model bundle is called a \"Flavor\".
     
    Info
     Creating From CallablesCreating From DirectoriesCreating From a Runnable Image 
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\ndef my_load_predict_fn(model):\ndef returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn():\ndef my_model(x):\nreturn x * 2\nreturn my_model\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"load_model_fn\": my_load_model_fn,\n\"load_predict_fn\": my_load_predict_fn,\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],  # list your requirements here\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_callable_v2(**BUNDLE_PARAMS)\n
     
    import os\nimport tempfile\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\ndirectory = tempfile.mkdtemp()\nmodel_filename = os.path.join(directory, \"model.py\")\nwith open(model_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_model_fn():\n    def my_model(x):\n        return x * 2\n    return my_model\n\"\"\")\npredict_filename = os.path.join(directory, \"predict.py\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\ndef my_load_predict_fn(model):\n    def returns_model_of_x_plus_len_of_y(x: int, y: str) -> int:\n        assert isinstance(x, int) and isinstance(y, str)\n        return model(x) + len(y)\n    return returns_model_of_x_plus_len_of_y\n\"\"\")\nrequirements_filename = os.path.join(directory, \"requirements.txt\")\nwith open(predict_filename, \"w\") as f:\nf.write(\"\"\"\npytest==7.2.1\nnumpy\n\"\"\"\n)\n\"\"\"\nThe directory structure should now look like\ndirectory/\n    model.py\n    predict.py\n    requirements.txt\n\"\"\"\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"base_paths\": [directory],\n\"load_predict_fn_module_path\": \"predict.my_load_predict_fn\",\n\"load_model_fn_module_path\": \"model.my_load_model_fn\",\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"requirements_path\": \"requirements.txt\",\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_dirs_v2(**BUNDLE_PARAMS)\n# Clean up files from demo\nos.remove(model_filename)\nos.remove(predict_filename)\nos.rmdir(directory)\n
     
    import os\nfrom pydantic import BaseModel\nfrom launch import LaunchClient\nclass MyRequestSchema(BaseModel):\nx: int\ny: str\nclass MyResponseSchema(BaseModel):\n__root__: int\nBUNDLE_PARAMS = {\n\"model_bundle_name\": \"test-bundle\",\n\"load_model_fn\": my_load_model_fn,\n\"load_predict_fn\": my_load_predict_fn,\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"repository\": \"launch_rearch\",\n\"tag\": \"12b9131c5a1489c76592cddd186962cce965f0f6-cpu\",\n\"command\": [\n\"dumb-init\",\n\"--\",\n\"ddtrace-run\",\n\"run-service\",\n\"--config\",\n\"/install/launch_rearch/config/service--user_defined_code.yaml\",\n\"--concurrency\",\n\"1\",\n\"--http\",\n\"production\",\n\"--port\",\n\"5005\",\n],\n\"env\": {\n\"TEST_KEY\": \"test_value\",\n},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.create_model_bundle_from_runnable_image_v2(**BUNDLE_PARAMS)\n
    "},{"location":"concepts/model_bundles/#choosing-the-right-model-bundle-flavor","title":"Choosing the right model bundle flavor","text":"
    Here are some tips for how to choose between the different flavors of ModelBundle:
     
    A CloudpickleArtifactFlavor (creating from callable) is good if:
     
       
    • You are creating the model bundle from a Jupyter notebook.
      •  
    • The model bundle is small without too many dependencies.
      •  
     
    A ZipArtifactFlavor (creating from directories) is good if:
     
       
    • You have a relatively constant set of dependencies.
      •  
    • You have a lot of custom code that you want to include in the model bundle.
      •  
    • You do not want to build a web server and Docker image to serve your model.
      •  
     
    A RunnableImageArtifactFlavor (creating from runnable image) is good if:
     
       
    • You have a lot of dependencies.
      •  
    • You have a lot of custom code that you want to include in the model bundle.
      •  
    • You are comfortable with building a web server and Docker image to serve your model.
      •  
    "},{"location":"concepts/model_bundles/#configuring-model-bundles","title":"Configuring Model Bundles","text":"
    The app_config field of a model bundle is a dictionary that can be used to configure the model bundle. If specified, the app_config is passed to the load_predict_fn when the model bundle is deployed, alongside the model. This can allow for more code reuse between multiple bundles that perform similar tasks.
     Creating Model Bundles with app_config
    import os\nfrom launch import LaunchClient\nfrom pydantic import BaseModel\nfrom typing import List, Union\nfrom typing_extensions import Literal\nclass MyRequestSchemaSingle(BaseModel):\nkind: Literal['single']\nx: int\ny: str\nclass MyRequestSchemaBatched(BaseModel):\nkind: Literal['batched']\nx: List[int]\ny: List[str]\nclass MyRequestSchema(BaseModel):\n__root__: Union[MyRequestSchemaSingle, MyRequestSchemaBatched]\nclass MyResponseSchema(BaseModel):\n__root__: Union[int, List[int]]\ndef my_load_predict_fn(app_config, model):\ndef returns_model_of_x_plus_len_of_y(x: Union[int, List[int]], y: Union[str, List[str]]) -> Union[int, List[int]]:\n\"\"\"MyRequestSchema -> MyResponseSchema\"\"\"\nif app_config[\"mode\"] == \"single\":\nassert isinstance(x, int) and isinstance(y, str)\nreturn model(x) + len(y)\nresult = []\nfor x_i, y_i in zip(x, y):\nresult.append(model(x_i) + len(y_i))\nreturn result\nreturn returns_model_of_x_plus_len_of_y\ndef my_load_model_fn(app_config):\ndef my_model_single(x: int):\nreturn x * 2\ndef my_model_batched(x: List[int]):\nreturn [my_model_single(x_i) for x_i in x]\nif app_config[\"mode\"] == \"single\":\nreturn my_model_single\nreturn my_model_batched\nBUNDLE_PARAMS_SINGLE = {\n\"model_bundle_name\": \"test-bundle-single\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n\"app_config\": {\"mode\": \"single\"},\n}\nBUNDLE_PARAMS_BATCHED = {\n\"model_bundle_name\": \"test-bundle-batched\",\n\"load_predict_fn\": my_load_predict_fn,\n\"load_model_fn\": my_load_model_fn,\n\"requirements\": [\"pytest==7.2.1\", \"numpy\"],\n\"request_schema\": MyRequestSchema,\n\"response_schema\": MyResponseSchema,\n\"pytorch_image_tag\": \"1.7.1-cuda11.0-cudnn8-runtime\",\n\"app_config\": {\"mode\": \"batched\"},\n}\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nbundle_single = client.create_model_bundle_from_callable_v2(**BUNDLE_PARAMS_SINGLE)\nbundle_batch = client.create_model_bundle_from_callable_v2(**BUNDLE_PARAMS_BATCHED)\n
    "},{"location":"concepts/model_bundles/#updating-model-bundles","title":"Updating Model Bundles","text":"
    Model Bundles are immutable, meaning they cannot be edited once created. However, it is possible to clone an existing model bundle with a new app_config using clone_model_bundle_with_changes_v2.
    "},{"location":"concepts/model_bundles/#listing-model-bundles","title":"Listing Model Bundles","text":"
    To list all the model bundles you own, use list_model_bundles_v2.
    "},{"location":"concepts/model_endpoints/","title":"Model Endpoints","text":"
    Model Endpoints are deployments of models that can receive requests and return predictions containing the results of the model's inference. Each model endpoint is associated with a model bundle, which contains the model's code. An endpoint specifies deployment parameters, such as the minimum and maximum number of workers, as well as the requested resources for each worker, such as the number of CPUs, amount of memory, GPU count, and type of GPU.
     
    Endpoints can be asynchronous or synchronous. Asynchronous endpoints return a future immediately after receiving a request, and the future can be used to retrieve the prediction once it is ready. Synchronous endpoints return the prediction directly after receiving a request.
     
    Info
    "},{"location":"concepts/model_endpoints/#choosing-the-right-inference-mode","title":"Choosing the right inference mode","text":"
    Here are some tips for how to choose between SyncEndpoint, AsyncEndpoint, and BatchJob for deploying your ModelBundle:
     
    A SyncEndpoint is good if:
     
       
    • You have strict latency requirements (e.g. on the order of seconds or less).
      •  
    • You are willing to have resources continually allocated.
      •  
     
    An AsyncEndpoint is good if:
     
       
    • You want to save on compute costs.
      •  
    • Your inference code takes a long time to run.
      •  
    • Your latency requirements are on the order of minutes.
      •  
     
    A BatchJob is good if:
     
       
    • You know there is a large batch of inputs ahead of time.
      •  
    • You want to optimize for throughput instead of latency.
      •  
    "},{"location":"concepts/model_endpoints/#creating-async-model-endpoints","title":"Creating Async Model Endpoints","text":"
    Async model endpoints are the most cost-efficient way to perform inference on tasks that are less latency-sensitive.
     Creating an Async Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-async\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#creating-sync-model-endpoints","title":"Creating Sync Model Endpoints","text":"
    Sync model endpoints are useful for latency-sensitive tasks, such as real-time inference. Sync endpoints are more expensive than async endpoints.
     
    Note
     
    Sync model endpoints require at least 1 min_worker.
     Creating a Sync Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-sync\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=1,\nendpoint_type=\"sync\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\n
    "},{"location":"concepts/model_endpoints/#managing-model-endpoints","title":"Managing Model Endpoints","text":"
    Model endpoints can be listed, updated, and deleted using the Launch API.
     Listing Model Endpoints
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoints = client.list_model_endpoints()\n
     Updating a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nclient.edit_model_endpoint(\nmodel_endpoint=\"demo-endpoint\",\nmax_workers=2,\n)\n
     Deleting a Model Endpoint
    import os\nfrom launch import LaunchClient\nclient = LaunchClient(api_key=os.getenv(\"LAUNCH_API_KEY\"))\nendpoint = client.create_model_endpoint(\nendpoint_name=\"demo-endpoint-tmp\",\nmodel_bundle=\"test-bundle\",\ncpus=1,\nmin_workers=0,\nendpoint_type=\"async\",\nupdate_if_exists=True,\nlabels={\n\"team\": \"MY_TEAM\",\n\"product\": \"MY_PRODUCT\",\n},\n)\nclient.delete_model_endpoint(model_endpoint=\"demo-endpoint-tmp\")\n
    "},{"location":"concepts/overview/","title":"Overview","text":"
    Creating deployments on Launch generally involves three steps:
     
       
    1.  
      Create and upload a ModelBundle. Pass your trained model    as well as pre-/post-processing code to the Scale Launch Python client, and     we\u2019ll create a model bundle based on the code and store it in our Bundle Store.
       
      2.  
    2.  
      Create a ModelEndpoint. Pass a ModelBundle as well as    infrastructure settings such as the desired number of GPUs to our client.    This provisions resources on Scale\u2019s cluster dedicated to your ModelEndpoint.
       
      3.  
    3.  
      Make requests to the ModelEndpoint. You can make requests through the Python    client, or make HTTP requests directly to Scale.
       
      4.  
    "}]}
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 9da3509e425d56c786b081cabe4da8f90624e755..615741f3103acb0103eec480e0b80e10cc0f3761 100644
GIT binary patch
delta 16
XcmX@WbbyInzMF&NcdqhAc1uP8E6oJM

delta 16
XcmX@WbbyInzMF%i*k5TQyCow4CtL&^


From a538983f7c7c3a1f1bee395e50e307819db7248c Mon Sep 17 00:00:00 2001
From: Phil Chen 
Date: Tue, 28 Mar 2023 22:37:19 -0700
Subject: [PATCH 13/28] Deployed 527ff16 with MkDocs version: 1.4.2

---
 concepts/model_bundles/index.html |   6 +++---
 search/search_index.json          |   2 +-
 sitemap.xml                       |  26 +++++++++++++-------------
 sitemap.xml.gz                    | Bin 320 -> 320 bytes
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/concepts/model_bundles/index.html b/concepts/model_bundles/index.html
index 7640dfdb..e76ca68b 100644
--- a/concepts/model_bundles/index.html
+++ b/concepts/model_bundles/index.html
@@ -590,8 +590,8 @@ 
    Creating Model Bundlespredict_fn, a function which takes in a request.
 The second takes in directories containing a load_predict_fn and the
 module path to the load_predict_fn.
-The third takes a Docker image that contains an entrypoint that starts a process
-listening for requests at port 5005 using HTTP and exposes POST /predict and
+The third takes a Docker image and a command that starts a process listening for
+requests at port 5005 using HTTP and exposes POST /predict and
 GET /healthz endpoints.
    
 
    Each of these modes of creating a model bundle is called a "Flavor".
    
 
    
@@ -609,7 +609,7 @@ 
    Choosing the right model bundle
 
  • You have a lot of custom code that you want to include in the model bundle.
    • 
 
  • You do not want to build a web server and Docker image to serve your model.
    • 
 
-
    A RunnableImageArtifactFlavor (creating from runnable image) is good if:
    
+
    A RunnableImageFlavor (creating from runnable image) is good if:

    @@ -2226,14 +2242,16 @@

    repository: str, tag: str, command: List[str], - env: Dict[str, str] -) -> CreateModelBundleV2Response + env: Dict[str, str], + readiness_initial_delay_seconds: int +) -> CreateModelBundleV2Response

    Create a model bundle from a runnable image. The specified command must start a process that will listen for requests on port 5005 using HTTP.

    +

    Inference requests must be served at the POST /predict route while the GET /readyz route is a healthcheck.

    Parameters:

    A dictionary containing the following keys:

    +

    An object containing the following keys:

    • model_bundle_id: The ID of the cloned model bundle.

    A dictionary containing the following keys:

    +

    An object containing the following keys:

    • model_bundle_id: The ID of the created model bundle.

    A dictionary containing the following keys:

    +

    An object containing the following keys:

    • model_bundle_id: The ID of the created model bundle.
    - Dict[str, Any] + CreateModelBundleV2Response

    A dictionary containing the following keys:

    +

    An object containing the following keys:

    • model_bundle_id: The ID of the created model bundle.

    A dictionary containing the following keys:

    +

    An object containing the following keys:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
      • @@ -3255,7 +3259,9 @@

      -
      get_model_bundle_v2(model_bundle_id: str) -> Dict[str, Any]
+
      get_model_bundle_v2(
+    model_bundle_id: str,
+) -> ModelBundleV2Response
 
      
 
   
      
@@ -3297,13 +3303,12 @@
    - Dict[str, Any] + ModelBundleV2Response

    A dictionary containing the following keys:

    +

    An object containing the following fields:

    • id: The ID of the model bundle.
    • name: The name of the model bundle.
      • -
    • schema_location: The location of the schema for the model bundle.
    • flavor: The flavor of the model bundle. Either RunnableImage, CloudpickleArtifact, or ZipArtifact.
    • created_at: The time the model bundle was created.
      • @@ -3412,7 +3417,7 @@

      -
      list_model_bundles_v2() -> List[Dict[str, Any]]
+
      list_model_bundles_v2() -> ListModelBundlesV2Response
 
      
 
   
      
@@ -3430,19 +3435,11 @@ 
      List[Dict[str, Any]]
+                ListModelBundlesV2Response

    A list of dictionaries containing the following keys:

    +

    An object containing the following keys:

      -
    • id: The ID of the model bundle.
      • -
    • name: The name of the model bundle.
      • -
    • schema_location: The location of the schema for the model bundle.
      • -
    • flavor: The flavor of the model bundle. Either RunnableImage, - CloudpickleArtifact, or ZipArtifact.
      • -
    • created_at: The time the model bundle was created.
      • -
    • metadata: A dictionary of metadata associated with the model bundle.
      • -
    • model_artifact_ids: A list of IDs of model artifacts associated with the - bundle.
      • +
    • model_bundles: A list of model bundles. Each model bundle is an object.

    Path to a requirements.txt file that will be used to install dependencies for the bundle. This file must be relative to the current working directory.

    		 - required + None
    urls - List[str] + Optional[List[str]]

    A list of urls, each pointing to a file containing model input. Must be accessible by Scale Launch, hence urls need to either be public or signedURLs.

    @@ -2317,6 +2335,245 @@

    required +

    + + + + + + +
    readiness_initial_delay_seconds + int +

    The number of seconds to wait for the HTTP server to become ready and +successfully respond on its healthcheck.

    		 + required +
    + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + CreateModelBundleV2Response +

    An object containing the following keys:

    +
      +
    • model_bundle_id: The ID of the created model bundle.
      • +
    + +

    + + + +
    + + + +

    + create_model_bundle_from_triton_enhanced_runnable_image_v2 + + +

    +
    create_model_bundle_from_triton_enhanced_runnable_image_v2(
+    *,
+    model_bundle_name: str,
+    request_schema: Type[BaseModel],
+    response_schema: Type[BaseModel],
+    repository: str,
+    tag: str,
+    command: List[str],
+    env: Dict[str, str],
+    readiness_initial_delay_seconds: int,
+    triton_model_repository: str,
+    triton_model_replicas: Optional[Dict[str, str]],
+    triton_num_cpu: float,
+    triton_commit_tag: str,
+    triton_storage: Optional[str],
+    triton_memory: Optional[str],
+    triton_readiness_initial_delay_seconds: int
+) -> CreateModelBundleV2Response
+
    + +
    + +

    Create a model bundle from a runnable image and a tritonserver image.

    +

    Same requirements as :param:create_model_bundle_from_runnable_image_v2 with additional constraints necessary +for configuring tritonserver's execution.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    model_bundle_name + str +

    The name of the model bundle you want to create.

    		 + required +
    request_schema + Type[BaseModel] +

    A Pydantic model that defines the request schema for the bundle.

    		 + required +
    response_schema + Type[BaseModel] +

    A Pydantic model that defines the response schema for the bundle.

    		 + required +
    repository + str +

    The name of the Docker repository for the runnable image.

    		 + required +
    tag + str +

    The tag for the runnable image.

    		 + required +
    command + List[str] +

    The command that will be used to start the process that listens for requests.

    		 + required +
    env + Dict[str, str] +

    A dictionary of environment variables that will be passed to the bundle when it +is run.

    		 + required +
    readiness_initial_delay_seconds + int +

    The number of seconds to wait for the HTTP server to become ready and +successfully respond on its healthcheck.

    		 + required +
    triton_model_repository + str +

    The S3 prefix that contains the contents of the model repository, formatted +according to https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md

    		 + required +
    triton_model_replicas + Optional[Dict[str, str]] +

    If supplied, the name and number of replicas to make for each model.

    		 + required +
    triton_num_cpu + float +

    Number of CPUs, fractional, to allocate to tritonserver.

    		 + required +
    triton_commit_tag + str +

    The image tag of the specific trionserver version.

    		 + required +
    triton_storage + Optional[str] +

    Amount of storage space to allocate for the tritonserver container.

    		 + required +
    triton_memory + Optional[str] +

    Amount of memory to allocate for the tritonserver container.

    		 + required +
    triton_readiness_initial_delay_seconds + int +

    Like readiness_initial_delay_seconds, but for tritonserver's +own healthcheck.

    		 + required +
    @@ -3172,7 +3429,7 @@

  • id: The ID of the model bundle.
  • name: The name of the model bundle.
  • flavor: The flavor of the model bundle. Either RunnableImage, - CloudpickleArtifact, or ZipArtifact.
    • + CloudpickleArtifact, ZipArtifact, or TritonEnhancedRunnableImageFlavor.

  • created_at: The time the model bundle was created.
  • metadata: A dictionary of metadata associated with the model bundle.
  • model_artifact_ids: A list of IDs of model artifacts associated with the @@ -3792,7 +4049,7 @@

    {"base": "../..", "features": ["search.suggest", "search.highlight", "content.tabs.link", "content.code.annotate", "content.code.copy"], "search": "../../assets/javascripts/workers/search.208ed371.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}} - + diff --git a/api/endpoint_predictions/index.html b/api/endpoint_predictions/index.html index 2bc159d0..5b6387dc 100644 --- a/api/endpoint_predictions/index.html +++ b/api/endpoint_predictions/index.html @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -89,7 +89,9 @@ -
    + + +

  • @@ -1180,40 +1148,6 @@ RunnableImageFlavor - -
  • @@ -1964,19 +1898,15 @@

    -
    flavor: Union[
-    CloudpickleArtifactFlavor,
-    ZipArtifactFlavor,
-    RunnableImageFlavor,
-] = Field(Ellipsis, discriminator="flavor")
+
    flavor: ModelBundleFlavors = Field(
+    Ellipsis, discriminator="flavor"
+)
 
    
 
   
    
   
-      
    Flavor of the Model Bundle, representing how the model bundle was packaged. Either
-CloudpickleArtifactFlavor,
-ZipArtifactFlavor, or
-RunnableImageFlavor.
    
+      
    Flavor of the Model Bundle, representing how the model bundle was packaged.
    
+
    See ModelBundleFlavors for details.
    @@ -2157,112 +2087,22 @@

    - Bases: BaseModel

    - - - - - - -
    - - - + Bases: RunnableImageLike

    - - - -
    - - - -

    - command - - - - class-attribute - - -

    -
    command: List[str]
-
    - -
    -

    Command to run the image.

    -
    - -
    - -
    - - - -

    - env +

    Model bundles that use custom docker images that expose an HTTP server for inference.

    - - - class-attribute - -

    -
    env: Optional[Dict[str, str]]
-
    -
    -

    Environment variables to set when running the image.

    -
    -
    - -
    - - - -

    - repository - - - - class-attribute - - -

    -
    repository: str
-
    - -
    - -

    Docker repository of the image.

    -
    - -
    +
    -
    -

    - tag - - - class-attribute - -

    -
    tag: str
-
    - -
    - -

    Docker tag of the image.

    -
    - -
    @@ -2534,7 +2374,7 @@

    {"base": "../..", "features": ["search.suggest", "search.highlight", "content.tabs.link", "content.code.annotate", "content.code.copy"], "search": "../../assets/javascripts/workers/search.208ed371.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}} - + diff --git a/api/model_endpoints/index.html b/api/model_endpoints/index.html index 0b8144b3..c80d323d 100644 --- a/api/model_endpoints/index.html +++ b/api/model_endpoints/index.html @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -89,7 +89,9 @@ -
    + + +

    • + +
  • + + create_docker_image_batch_job() + + +
    • + +
  • + + create_docker_image_batch_job_bundle() + +
  • @@ -550,6 +564,27 @@ get_batch_async_response() +
    • + +
  • + + get_docker_image_batch_job() + + +
    • + +
  • + + get_docker_image_batch_job_bundle() + + +
    • + +
  • + + get_latest_docker_image_batch_job_bundle() + +
  • @@ -578,6 +613,13 @@ get_model_endpoint() +
    • + +
  • + + list_docker_image_batch_job_bundles() + +
  • @@ -634,6 +676,13 @@ register_upload_bundle_fn() +
    • + +
  • + + update_docker_image_batch_job() + +
    • @@ -712,6 +761,55 @@ + + + + + + + + +
  • + + + + + + + + + + + +
    • + + + @@ -765,6 +863,20 @@ clone_model_bundle_with_changes_v2() + + +
  • + + create_docker_image_batch_job() + + +
    • + +
  • + + create_docker_image_batch_job_bundle() + +
  • @@ -835,6 +947,27 @@ get_batch_async_response() +
    • + +
  • + + get_docker_image_batch_job() + + +
    • + +
  • + + get_docker_image_batch_job_bundle() + + +
    • + +
  • + + get_latest_docker_image_batch_job_bundle() + +
  • @@ -863,6 +996,13 @@ get_model_endpoint() +
    • + +
  • + + list_docker_image_batch_job_bundles() + +
  • @@ -919,6 +1059,13 @@ register_upload_bundle_fn() +
    • + +
  • + + update_docker_image_batch_job() + +
    • @@ -1048,8 +1195,9 @@

    gpu_type: Optional[str] = None, storage: Optional[str] = None, max_workers: Optional[int] = None, - per_worker: Optional[int] = None -) -> Dict[str, Any] + per_worker: Optional[int] = None, + timeout_seconds: Optional[float] = None +) -> Dict[str, Any]
    @@ -1222,6 +1370,18 @@

    None + + timeout_seconds + + Optional[float] + +

    The maximum amount of time (in seconds) that the batch job can take. +If not specified, the server defaults to 12 hours. This includes the time required +to build the endpoint and the total time required for all the individual tasks.

    + + None + + @@ -1409,6 +1569,324 @@

    + create_docker_image_batch_job + + +

    +
    create_docker_image_batch_job(
+    *,
+    labels: Dict[str, str],
+    docker_image_batch_job_bundle: Optional[
+        Union[str, DockerImageBatchJobBundleResponse]
+    ] = None,
+    docker_image_batch_job_bundle_name: Optional[
+        str
+    ] = None,
+    job_config: Optional[Dict[str, Any]] = None,
+    cpus: Optional[int] = None,
+    memory: Optional[str] = None,
+    gpus: Optional[int] = None,
+    gpu_type: Optional[str] = None,
+    storage: Optional[str] = None
+)
+
    + +
    + +

    For self hosted mode only.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    docker_image_batch_job_bundle + Optional[Union[str, DockerImageBatchJobBundleResponse]] +

    Specifies the docker image bundle to use for the batch job. +Either the string id of a docker image bundle, or a +DockerImageBatchJobBundleResponse object. +Only one of docker_image_batch_job_bundle and docker_image_batch_job_bundle_name +can be specified.

    		 + None +
    docker_image_batch_job_bundle_name + Optional[str] +

    The name of a batch job bundle. If specified, +Launch will use the most recent bundle with that name owned by the current user. +Only one of docker_image_batch_job_bundle and docker_image_batch_job_bundle_name +can be specified.

    		 + None +
    labels + Dict[str, str] +

    Kubernetes labels that are present on the batch job.

    		 + required +
    job_config + Optional[Dict[str, Any]] +

    A JSON-serializable python object that will get passed to the batch job, +specifically as the contents of a file mounted at mount_location inside the bundle. +You can call python's json.load() on the file to retrieve the contents.

    		 + None +
    cpus + Optional[int] +

    Optional override for the number of cpus to give to your job. Either the default +must be specified in the bundle, or this must be specified.

    		 + None +
    memory + Optional[str] +

    Optional override for the amount of memory to give to your job. Either the default +must be specified in the bundle, or this must be specified.

    		 + None +
    gpus + Optional[int] +

    Optional number of gpus to give to the bundle. If not specified in the bundle or +here, will be interpreted as 0 gpus.

    		 + None +
    gpu_type + Optional[str] +

    Optional type of gpu. If the final number of gpus is positive, must be specified +either in the bundle or here.

    		 + None +
    storage + Optional[str] +

    Optional reserved amount of disk to give to your batch job. If not specified, +your job may be evicted if it is using too much disk.

    		 + None +
    + +
    + +
    + +
    + + + +

    + create_docker_image_batch_job_bundle + + +

    +
    create_docker_image_batch_job_bundle(
+    *,
+    name: str,
+    image_repository: str,
+    image_tag: str,
+    command: List[str],
+    env: Optional[Dict[str, str]],
+    mount_location: Optional[str],
+    cpus: Optional[int] = None,
+    memory: Optional[str] = None,
+    gpus: Optional[int] = None,
+    gpu_type: Optional[str] = None,
+    storage: Optional[str] = None
+) -> CreateDockerImageBatchJobBundleResponse
+
    + +
    + +

    For self hosted mode only.

    +

    Creates a Docker Image Batch Job Bundle.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    name + str +

    A user-defined name for the bundle. Does not need to be unique.

    		 + required +
    image_repository + str +

    The (short) repository of your image. For example, if your image is located at +123456789012.dkr.ecr.us-west-2.amazonaws.com/repo:tag, and your version of Launch +is configured to look at 123456789012.dkr.ecr.us-west-2.amazonaws.com for Docker Images, +you would pass the value repo for the image_repository parameter.

    		 + required +
    image_tag + str +

    The tag of your image inside of the repo. In the example above, you would pass +the value tag for the image_tag parameter.

    		 + required +
    command + List[str] +

    The command to run inside the docker image.

    		 + required +
    env + Optional[Dict[str, str]] +

    A dictionary of environment variables to inject into your docker image.

    		 + required +
    mount_location + Optional[str] +

    A location in the filesystem where you would like a json-formatted file, controllable +on runtime, to be mounted. This allows behavior to be specified on runtime. +(Specifically, the contents of this file can be read via json.load() inside of the +user-defined code.)

    		 + required +
    cpus + Optional[int] +

    Optional default value for the number of cpus to give the job.

    		 + None +
    memory + Optional[str] +

    Optional default value for the amount of memory to give the job.

    		 + None +
    gpus + Optional[int] +

    Optional default value for the number of gpus to give the job.

    		 + None +
    gpu_type + Optional[str] +

    Optional default value for the type of gpu to give the job.

    		 + None +
    storage + Optional[str] +

    Optional default value for the amount of disk to give the job.

    		 + None +
    + +
    + +
    + +
    + + +

    create_model_bundle @@ -1639,8 +2117,9 @@

    tensorflow_version: Optional[str] = None, custom_base_image_repository: Optional[str] = None, custom_base_image_tag: Optional[str] = None, - app_config: Optional[Union[Dict[str, Any], str]] = None -) -> CreateModelBundleV2Response + app_config: Optional[Union[Dict[str, Any], str]] = None, + metadata: Optional[Dict[str, Any]] = None +) -> CreateModelBundleV2Response

    @@ -1799,6 +2278,16 @@

    Optional[Dict[str, Any]] + +

    Metadata to record with the bundle.

    + + None + + @@ -2014,8 +2503,9 @@

    tensorflow_version: Optional[str] = None, custom_base_image_repository: Optional[str] = None, custom_base_image_tag: Optional[str] = None, - app_config: Optional[Dict[str, Any]] = None -) -> CreateModelBundleV2Response + app_config: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None +) -> CreateModelBundleV2Response

    @@ -2197,6 +2687,16 @@

    Optional[Dict[str, Any]] + +

    Metadata to record with the bundle.

    + + None + + @@ -2243,8 +2743,9 @@

    tag: str, command: List[str], env: Dict[str, str], - readiness_initial_delay_seconds: int -) -> CreateModelBundleV2Response + readiness_initial_delay_seconds: int, + metadata: Optional[Dict[str, Any]] = None +) -> CreateModelBundleV2Response

    @@ -2346,6 +2847,16 @@

    required + + metadata + + Optional[Dict[str, Any]] + +

    Metadata to record with the bundle.

    + + None + + @@ -2399,8 +2910,9 @@

    triton_commit_tag: str, triton_storage: Optional[str], triton_memory: Optional[str], - triton_readiness_initial_delay_seconds: int -) -> CreateModelBundleV2Response + triton_readiness_initial_delay_seconds: int, + metadata: Optional[Dict[str, Any]] = None +) -> CreateModelBundleV2Response

    @@ -2496,8 +3008,8 @@

    str -

    The S3 prefix that contains the contents of the model repository, formatted -according to https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md

    +

    The S3 prefix that contains the contents of the model +repository, formatted according to +https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_repository.md

    required @@ -2518,7 +3031,8 @@

    Optional[Dict[str, str]] -

    If supplied, the name and number of replicas to make for each model.

    +

    If supplied, the name and number of replicas to make for each +model.

    required @@ -2568,12 +3082,22 @@

    Optional[Dict[str, Any]] + +

    Metadata to record with the bundle.

    + + None + + @@ -3372,6 +3896,70 @@

    + get_docker_image_batch_job + + +

    +
    get_docker_image_batch_job(batch_job_id: str)
+
    + +
    + +

    For self hosted mode only. Gets information about a batch job given a batch job id.

    + +
    + +
    + +
    + + + +

    + get_docker_image_batch_job_bundle + + +

    +
    get_docker_image_batch_job_bundle(
+    docker_image_batch_job_bundle_id: str,
+) -> DockerImageBatchJobBundleResponse
+
    + +
    + +

    For self hosted mode only. Gets information for a single batch job bundle with a given id.

    + +
    + +
    + +
    + + + +

    + get_latest_docker_image_batch_job_bundle + + +

    +
    get_latest_docker_image_batch_job_bundle(
+    bundle_name: str,
+) -> DockerImageBatchJobBundleResponse
+
    + +
    + +

    For self hosted mode only. Gets information for the latest batch job bundle with a given name.

    + +
    + +
    + +
    + + +

    get_latest_model_bundle_v2 @@ -3631,6 +4219,63 @@

    +

    + list_docker_image_batch_job_bundles + + +

    +
    list_docker_image_batch_job_bundles(
+    bundle_name: Optional[str] = None,
+    order_by: Optional[Literal[newest, oldest]] = None,
+) -> ListDockerImageBatchJobBundleResponse
+
    + +
    + +

    For self hosted mode only. Gets information for multiple bundles.

    + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    bundle_name + Optional[str] +

    The name of the bundles to retrieve. If not specified, this will retrieve all

    		 + None +
    order_by + Optional[Literal[newest, oldest]] +

    Either "newest", "oldest", or not specified. Specify to sort by newest/oldest.

    		 + None +
    + +
    + +
    + +
    + + +

    list_model_bundles @@ -4000,6 +4645,29 @@

    + + + +

    + update_docker_image_batch_job + + +

    +
    update_docker_image_batch_job(
+    batch_job_id: str, cancel: bool
+)
+
    + +
    + +

    For self hosted mode only. Updates a batch job by id. +Use this if you want to cancel/delete a batch job.

    + +
    + +
    + @@ -4049,7 +4717,7 @@

    {"base": "../..", "features": ["search.suggest", "search.highlight", "content.tabs.link", "content.code.annotate", "content.code.copy"], "search": "../../assets/javascripts/workers/search.208ed371.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}} - + diff --git a/api/endpoint_predictions/index.html b/api/endpoint_predictions/index.html index 5b6387dc..6ae20c51 100644 --- a/api/endpoint_predictions/index.html +++ b/api/endpoint_predictions/index.html @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -558,6 +558,55 @@ + + + + + + + + +
  • + + + + + + + + + + + +
    • + + + @@ -1111,7 +1160,7 @@

    {"base": "../..", "features": ["search.suggest", "search.highlight", "content.tabs.link", "content.code.annotate", "content.code.copy"], "search": "../../assets/javascripts/workers/search.208ed371.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}} - + diff --git a/api/hooks/index.html b/api/hooks/index.html index 0d7acd79..ea6df9d7 100644 --- a/api/hooks/index.html +++ b/api/hooks/index.html @@ -16,8 +16,10 @@ + + - + @@ -25,7 +27,7 @@ - + @@ -525,6 +527,55 @@ + + + + + + + + +
  • + + + + + + + + + + + +
    • + + + @@ -674,7 +725,7 @@

    - + diff --git a/api/model_bundles/index.html b/api/model_bundles/index.html index a0b5c8ba..87700ef4 100644 --- a/api/model_bundles/index.html +++ b/api/model_bundles/index.html @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -858,6 +858,55 @@ + + + + + + + + +
  • + + + + + + + + + + + +
    • + + + @@ -2374,7 +2423,7 @@

    {"base": "../..", "features": ["search.suggest", "search.highlight", "content.tabs.link", "content.code.annotate", "content.code.copy"], "search": "../../assets/javascripts/workers/search.208ed371.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}} - + diff --git a/api/model_endpoints/index.html b/api/model_endpoints/index.html index c80d323d..a4ee2d9f 100644 --- a/api/model_endpoints/index.html +++ b/api/model_endpoints/index.html @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -571,6 +571,55 @@ + + + + + + + + +
  • + + + + + + + + + + + +
    • + + + @@ -1042,7 +1091,7 @@

    - + diff --git a/assets/javascripts/bundle.19047be9.min.js.map b/assets/javascripts/bundle.19047be9.min.js.map deleted file mode 100644 index 3ce61e96..00000000 --- a/assets/javascripts/bundle.19047be9.min.js.map +++ /dev/null @@ -1,8 +0,0 @@ -{ - "version": 3, - "sources": ["node_modules/focus-visible/dist/focus-visible.js", "node_modules/url-polyfill/url-polyfill.js", "node_modules/clipboard/dist/clipboard.js", "node_modules/escape-html/index.js", "node_modules/array-flat-polyfill/index.mjs", "src/assets/javascripts/bundle.ts", "node_modules/unfetch/polyfill/index.js", "node_modules/rxjs/node_modules/tslib/tslib.es6.js", "node_modules/rxjs/src/internal/util/isFunction.ts", "node_modules/rxjs/src/internal/util/createErrorClass.ts", "node_modules/rxjs/src/internal/util/UnsubscriptionError.ts", "node_modules/rxjs/src/internal/util/arrRemove.ts", "node_modules/rxjs/src/internal/Subscription.ts", "node_modules/rxjs/src/internal/config.ts", "node_modules/rxjs/src/internal/scheduler/timeoutProvider.ts", "node_modules/rxjs/src/internal/util/reportUnhandledError.ts", "node_modules/rxjs/src/internal/util/noop.ts", "node_modules/rxjs/src/internal/NotificationFactories.ts", "node_modules/rxjs/src/internal/util/errorContext.ts", "node_modules/rxjs/src/internal/Subscriber.ts", "node_modules/rxjs/src/internal/symbol/observable.ts", "node_modules/rxjs/src/internal/util/identity.ts", "node_modules/rxjs/src/internal/util/pipe.ts", "node_modules/rxjs/src/internal/Observable.ts", "node_modules/rxjs/src/internal/util/lift.ts", "node_modules/rxjs/src/internal/operators/OperatorSubscriber.ts", "node_modules/rxjs/src/internal/scheduler/animationFrameProvider.ts", "node_modules/rxjs/src/internal/util/ObjectUnsubscribedError.ts", "node_modules/rxjs/src/internal/Subject.ts", "node_modules/rxjs/src/internal/scheduler/dateTimestampProvider.ts", "node_modules/rxjs/src/internal/ReplaySubject.ts", "node_modules/rxjs/src/internal/scheduler/Action.ts", "node_modules/rxjs/src/internal/scheduler/intervalProvider.ts", "node_modules/rxjs/src/internal/scheduler/AsyncAction.ts", "node_modules/rxjs/src/internal/Scheduler.ts", "node_modules/rxjs/src/internal/scheduler/AsyncScheduler.ts", "node_modules/rxjs/src/internal/scheduler/async.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameAction.ts", "node_modules/rxjs/src/internal/scheduler/AnimationFrameScheduler.ts", "node_modules/rxjs/src/internal/scheduler/animationFrame.ts", "node_modules/rxjs/src/internal/observable/empty.ts", "node_modules/rxjs/src/internal/util/isScheduler.ts", "node_modules/rxjs/src/internal/util/args.ts", "node_modules/rxjs/src/internal/util/isArrayLike.ts", "node_modules/rxjs/src/internal/util/isPromise.ts", "node_modules/rxjs/src/internal/util/isInteropObservable.ts", "node_modules/rxjs/src/internal/util/isAsyncIterable.ts", "node_modules/rxjs/src/internal/util/throwUnobservableError.ts", "node_modules/rxjs/src/internal/symbol/iterator.ts", "node_modules/rxjs/src/internal/util/isIterable.ts", "node_modules/rxjs/src/internal/util/isReadableStreamLike.ts", "node_modules/rxjs/src/internal/observable/innerFrom.ts", "node_modules/rxjs/src/internal/util/executeSchedule.ts", "node_modules/rxjs/src/internal/operators/observeOn.ts", "node_modules/rxjs/src/internal/operators/subscribeOn.ts", "node_modules/rxjs/src/internal/scheduled/scheduleObservable.ts", "node_modules/rxjs/src/internal/scheduled/schedulePromise.ts", "node_modules/rxjs/src/internal/scheduled/scheduleArray.ts", "node_modules/rxjs/src/internal/scheduled/scheduleIterable.ts", "node_modules/rxjs/src/internal/scheduled/scheduleAsyncIterable.ts", "node_modules/rxjs/src/internal/scheduled/scheduleReadableStreamLike.ts", "node_modules/rxjs/src/internal/scheduled/scheduled.ts", "node_modules/rxjs/src/internal/observable/from.ts", "node_modules/rxjs/src/internal/observable/of.ts", "node_modules/rxjs/src/internal/observable/throwError.ts", "node_modules/rxjs/src/internal/util/EmptyError.ts", "node_modules/rxjs/src/internal/util/isDate.ts", "node_modules/rxjs/src/internal/operators/map.ts", "node_modules/rxjs/src/internal/util/mapOneOrManyArgs.ts", "node_modules/rxjs/src/internal/util/argsArgArrayOrObject.ts", "node_modules/rxjs/src/internal/util/createObject.ts", "node_modules/rxjs/src/internal/observable/combineLatest.ts", "node_modules/rxjs/src/internal/operators/mergeInternals.ts", "node_modules/rxjs/src/internal/operators/mergeMap.ts", "node_modules/rxjs/src/internal/operators/mergeAll.ts", "node_modules/rxjs/src/internal/operators/concatAll.ts", "node_modules/rxjs/src/internal/observable/concat.ts", "node_modules/rxjs/src/internal/observable/defer.ts", "node_modules/rxjs/src/internal/observable/fromEvent.ts", "node_modules/rxjs/src/internal/observable/fromEventPattern.ts", "node_modules/rxjs/src/internal/observable/timer.ts", "node_modules/rxjs/src/internal/observable/merge.ts", "node_modules/rxjs/src/internal/observable/never.ts", "node_modules/rxjs/src/internal/util/argsOrArgArray.ts", "node_modules/rxjs/src/internal/operators/filter.ts", "node_modules/rxjs/src/internal/observable/zip.ts", "node_modules/rxjs/src/internal/operators/audit.ts", "node_modules/rxjs/src/internal/operators/auditTime.ts", "node_modules/rxjs/src/internal/operators/bufferCount.ts", "node_modules/rxjs/src/internal/operators/catchError.ts", "node_modules/rxjs/src/internal/operators/scanInternals.ts", "node_modules/rxjs/src/internal/operators/combineLatest.ts", "node_modules/rxjs/src/internal/operators/combineLatestWith.ts", "node_modules/rxjs/src/internal/operators/debounceTime.ts", "node_modules/rxjs/src/internal/operators/defaultIfEmpty.ts", "node_modules/rxjs/src/internal/operators/take.ts", "node_modules/rxjs/src/internal/operators/ignoreElements.ts", "node_modules/rxjs/src/internal/operators/mapTo.ts", "node_modules/rxjs/src/internal/operators/delayWhen.ts", "node_modules/rxjs/src/internal/operators/delay.ts", "node_modules/rxjs/src/internal/operators/distinctUntilChanged.ts", "node_modules/rxjs/src/internal/operators/distinctUntilKeyChanged.ts", "node_modules/rxjs/src/internal/operators/throwIfEmpty.ts", "node_modules/rxjs/src/internal/operators/endWith.ts", "node_modules/rxjs/src/internal/operators/finalize.ts", "node_modules/rxjs/src/internal/operators/first.ts", "node_modules/rxjs/src/internal/operators/merge.ts", "node_modules/rxjs/src/internal/operators/mergeWith.ts", "node_modules/rxjs/src/internal/operators/repeat.ts", "node_modules/rxjs/src/internal/operators/scan.ts", "node_modules/rxjs/src/internal/operators/share.ts", "node_modules/rxjs/src/internal/operators/shareReplay.ts", "node_modules/rxjs/src/internal/operators/skip.ts", "node_modules/rxjs/src/internal/operators/skipUntil.ts", "node_modules/rxjs/src/internal/operators/startWith.ts", "node_modules/rxjs/src/internal/operators/switchMap.ts", "node_modules/rxjs/src/internal/operators/takeUntil.ts", "node_modules/rxjs/src/internal/operators/takeWhile.ts", "node_modules/rxjs/src/internal/operators/tap.ts", "node_modules/rxjs/src/internal/operators/throttle.ts", "node_modules/rxjs/src/internal/operators/throttleTime.ts", "node_modules/rxjs/src/internal/operators/withLatestFrom.ts", "node_modules/rxjs/src/internal/operators/zip.ts", "node_modules/rxjs/src/internal/operators/zipWith.ts", "src/assets/javascripts/browser/document/index.ts", "src/assets/javascripts/browser/element/_/index.ts", "src/assets/javascripts/browser/element/focus/index.ts", "src/assets/javascripts/browser/element/offset/_/index.ts", "src/assets/javascripts/browser/element/offset/content/index.ts", "node_modules/resize-observer-polyfill/dist/ResizeObserver.es.js", "src/assets/javascripts/browser/element/size/_/index.ts", "src/assets/javascripts/browser/element/size/content/index.ts", "src/assets/javascripts/browser/element/visibility/index.ts", "src/assets/javascripts/browser/toggle/index.ts", "src/assets/javascripts/browser/keyboard/index.ts", "src/assets/javascripts/browser/location/_/index.ts", "src/assets/javascripts/utilities/h/index.ts", "src/assets/javascripts/utilities/round/index.ts", "src/assets/javascripts/browser/location/hash/index.ts", "src/assets/javascripts/browser/media/index.ts", "src/assets/javascripts/browser/request/index.ts", "src/assets/javascripts/browser/script/index.ts", "src/assets/javascripts/browser/viewport/offset/index.ts", "src/assets/javascripts/browser/viewport/size/index.ts", "src/assets/javascripts/browser/viewport/_/index.ts", "src/assets/javascripts/browser/viewport/at/index.ts", "node_modules/iframe-worker/shim/index.js", "src/assets/javascripts/browser/worker/index.ts", "src/assets/javascripts/_/index.ts", "src/assets/javascripts/components/_/index.ts", "src/assets/javascripts/components/announce/index.ts", "src/assets/javascripts/components/consent/index.ts", "src/assets/javascripts/components/content/code/_/index.ts", "src/assets/javascripts/templates/tooltip/index.tsx", "src/assets/javascripts/templates/annotation/index.tsx", "src/assets/javascripts/templates/clipboard/index.tsx", "src/assets/javascripts/templates/search/index.tsx", "src/assets/javascripts/templates/source/index.tsx", "src/assets/javascripts/templates/tabbed/index.tsx", "src/assets/javascripts/templates/table/index.tsx", "src/assets/javascripts/templates/version/index.tsx", "src/assets/javascripts/components/content/annotation/_/index.ts", "src/assets/javascripts/components/content/annotation/list/index.ts", "src/assets/javascripts/components/content/code/mermaid/index.ts", "src/assets/javascripts/components/content/details/index.ts", "src/assets/javascripts/components/content/table/index.ts", "src/assets/javascripts/components/content/tabs/index.ts", "src/assets/javascripts/components/content/_/index.ts", "src/assets/javascripts/components/dialog/index.ts", "src/assets/javascripts/components/header/_/index.ts", "src/assets/javascripts/components/header/title/index.ts", "src/assets/javascripts/components/main/index.ts", "src/assets/javascripts/components/palette/index.ts", "src/assets/javascripts/integrations/clipboard/index.ts", "src/assets/javascripts/integrations/sitemap/index.ts", "src/assets/javascripts/integrations/instant/index.ts", "src/assets/javascripts/integrations/search/highlighter/index.ts", "src/assets/javascripts/integrations/search/worker/message/index.ts", "src/assets/javascripts/integrations/search/worker/_/index.ts", "src/assets/javascripts/integrations/version/index.ts", "src/assets/javascripts/components/search/query/index.ts", "src/assets/javascripts/components/search/result/index.ts", "src/assets/javascripts/components/search/share/index.ts", "src/assets/javascripts/components/search/suggest/index.ts", "src/assets/javascripts/components/search/_/index.ts", "src/assets/javascripts/components/search/highlight/index.ts", "src/assets/javascripts/components/sidebar/index.ts", "src/assets/javascripts/components/source/facts/github/index.ts", "src/assets/javascripts/components/source/facts/gitlab/index.ts", "src/assets/javascripts/components/source/facts/_/index.ts", "src/assets/javascripts/components/source/_/index.ts", "src/assets/javascripts/components/tabs/index.ts", "src/assets/javascripts/components/toc/index.ts", "src/assets/javascripts/components/top/index.ts", "src/assets/javascripts/patches/indeterminate/index.ts", "src/assets/javascripts/patches/scrollfix/index.ts", "src/assets/javascripts/patches/scrolllock/index.ts", "src/assets/javascripts/polyfills/index.ts"], - "sourceRoot": "../../..", - "sourcesContent": ["(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory() :\n typeof define === 'function' && define.amd ? define(factory) :\n (factory());\n}(this, (function () { 'use strict';\n\n /**\n * Applies the :focus-visible polyfill at the given scope.\n * A scope in this case is either the top-level Document or a Shadow Root.\n *\n * @param {(Document|ShadowRoot)} scope\n * @see https://github.com/WICG/focus-visible\n */\n function applyFocusVisiblePolyfill(scope) {\n var hadKeyboardEvent = true;\n var hadFocusVisibleRecently = false;\n var hadFocusVisibleRecentlyTimeout = null;\n\n var inputTypesAllowlist = {\n text: true,\n search: true,\n url: true,\n tel: true,\n email: true,\n password: true,\n number: true,\n date: true,\n month: true,\n week: true,\n time: true,\n datetime: true,\n 'datetime-local': true\n };\n\n /**\n * Helper function for legacy browsers and iframes which sometimes focus\n * elements like document, body, and non-interactive SVG.\n * @param {Element} el\n */\n function isValidFocusTarget(el) {\n if (\n el &&\n el !== document &&\n el.nodeName !== 'HTML' &&\n el.nodeName !== 'BODY' &&\n 'classList' in el &&\n 'contains' in el.classList\n ) {\n return true;\n }\n return false;\n }\n\n /**\n * Computes whether the given element should automatically trigger the\n * `focus-visible` class being added, i.e. whether it should always match\n * `:focus-visible` when focused.\n * @param {Element} el\n * @return {boolean}\n */\n function focusTriggersKeyboardModality(el) {\n var type = el.type;\n var tagName = el.tagName;\n\n if (tagName === 'INPUT' && inputTypesAllowlist[type] && !el.readOnly) {\n return true;\n }\n\n if (tagName === 'TEXTAREA' && !el.readOnly) {\n return true;\n }\n\n if (el.isContentEditable) {\n return true;\n }\n\n return false;\n }\n\n /**\n * Add the `focus-visible` class to the given element if it was not added by\n * the author.\n * @param {Element} el\n */\n function addFocusVisibleClass(el) {\n if (el.classList.contains('focus-visible')) {\n return;\n }\n el.classList.add('focus-visible');\n el.setAttribute('data-focus-visible-added', '');\n }\n\n /**\n * Remove the `focus-visible` class from the given element if it was not\n * originally added by the author.\n * @param {Element} el\n */\n function removeFocusVisibleClass(el) {\n if (!el.hasAttribute('data-focus-visible-added')) {\n return;\n }\n el.classList.remove('focus-visible');\n el.removeAttribute('data-focus-visible-added');\n }\n\n /**\n * If the most recent user interaction was via the keyboard;\n * and the key press did not include a meta, alt/option, or control key;\n * then the modality is keyboard. Otherwise, the modality is not keyboard.\n * Apply `focus-visible` to any current active element and keep track\n * of our keyboard modality state with `hadKeyboardEvent`.\n * @param {KeyboardEvent} e\n */\n function onKeyDown(e) {\n if (e.metaKey || e.altKey || e.ctrlKey) {\n return;\n }\n\n if (isValidFocusTarget(scope.activeElement)) {\n addFocusVisibleClass(scope.activeElement);\n }\n\n hadKeyboardEvent = true;\n }\n\n /**\n * If at any point a user clicks with a pointing device, ensure that we change\n * the modality away from keyboard.\n * This avoids the situation where a user presses a key on an already focused\n * element, and then clicks on a different element, focusing it with a\n * pointing device, while we still think we're in keyboard modality.\n * @param {Event} e\n */\n function onPointerDown(e) {\n hadKeyboardEvent = false;\n }\n\n /**\n * On `focus`, add the `focus-visible` class to the target if:\n * - the target received focus as a result of keyboard navigation, or\n * - the event target is an element that will likely require interaction\n * via the keyboard (e.g. a text box)\n * @param {Event} e\n */\n function onFocus(e) {\n // Prevent IE from focusing the document or HTML element.\n if (!isValidFocusTarget(e.target)) {\n return;\n }\n\n if (hadKeyboardEvent || focusTriggersKeyboardModality(e.target)) {\n addFocusVisibleClass(e.target);\n }\n }\n\n /**\n * On `blur`, remove the `focus-visible` class from the target.\n * @param {Event} e\n */\n function onBlur(e) {\n if (!isValidFocusTarget(e.target)) {\n return;\n }\n\n if (\n e.target.classList.contains('focus-visible') ||\n e.target.hasAttribute('data-focus-visible-added')\n ) {\n // To detect a tab/window switch, we look for a blur event followed\n // rapidly by a visibility change.\n // If we don't see a visibility change within 100ms, it's probably a\n // regular focus change.\n hadFocusVisibleRecently = true;\n window.clearTimeout(hadFocusVisibleRecentlyTimeout);\n hadFocusVisibleRecentlyTimeout = window.setTimeout(function() {\n hadFocusVisibleRecently = false;\n }, 100);\n removeFocusVisibleClass(e.target);\n }\n }\n\n /**\n * If the user changes tabs, keep track of whether or not the previously\n * focused element had .focus-visible.\n * @param {Event} e\n */\n function onVisibilityChange(e) {\n if (document.visibilityState === 'hidden') {\n // If the tab becomes active again, the browser will handle calling focus\n // on the element (Safari actually calls it twice).\n // If this tab change caused a blur on an element with focus-visible,\n // re-apply the class when the user switches back to the tab.\n if (hadFocusVisibleRecently) {\n hadKeyboardEvent = true;\n }\n addInitialPointerMoveListeners();\n }\n }\n\n /**\n * Add a group of listeners to detect usage of any pointing devices.\n * These listeners will be added when the polyfill first loads, and anytime\n * the window is blurred, so that they are active when the window regains\n * focus.\n */\n function addInitialPointerMoveListeners() {\n document.addEventListener('mousemove', onInitialPointerMove);\n document.addEventListener('mousedown', onInitialPointerMove);\n document.addEventListener('mouseup', onInitialPointerMove);\n document.addEventListener('pointermove', onInitialPointerMove);\n document.addEventListener('pointerdown', onInitialPointerMove);\n document.addEventListener('pointerup', onInitialPointerMove);\n document.addEventListener('touchmove', onInitialPointerMove);\n document.addEventListener('touchstart', onInitialPointerMove);\n document.addEventListener('touchend', onInitialPointerMove);\n }\n\n function removeInitialPointerMoveListeners() {\n document.removeEventListener('mousemove', onInitialPointerMove);\n document.removeEventListener('mousedown', onInitialPointerMove);\n document.removeEventListener('mouseup', onInitialPointerMove);\n document.removeEventListener('pointermove', onInitialPointerMove);\n document.removeEventListener('pointerdown', onInitialPointerMove);\n document.removeEventListener('pointerup', onInitialPointerMove);\n document.removeEventListener('touchmove', onInitialPointerMove);\n document.removeEventListener('touchstart', onInitialPointerMove);\n document.removeEventListener('touchend', onInitialPointerMove);\n }\n\n /**\n * When the polfyill first loads, assume the user is in keyboard modality.\n * If any event is received from a pointing device (e.g. mouse, pointer,\n * touch), turn off keyboard modality.\n * This accounts for situations where focus enters the page from the URL bar.\n * @param {Event} e\n */\n function onInitialPointerMove(e) {\n // Work around a Safari quirk that fires a mousemove on whenever the\n // window blurs, even if you're tabbing out of the page. \u00AF\\_(\u30C4)_/\u00AF\n if (e.target.nodeName && e.target.nodeName.toLowerCase() === 'html') {\n return;\n }\n\n hadKeyboardEvent = false;\n removeInitialPointerMoveListeners();\n }\n\n // For some kinds of state, we are interested in changes at the global scope\n // only. For example, global pointer input, global key presses and global\n // visibility change should affect the state at every scope:\n document.addEventListener('keydown', onKeyDown, true);\n document.addEventListener('mousedown', onPointerDown, true);\n document.addEventListener('pointerdown', onPointerDown, true);\n document.addEventListener('touchstart', onPointerDown, true);\n document.addEventListener('visibilitychange', onVisibilityChange, true);\n\n addInitialPointerMoveListeners();\n\n // For focus and blur, we specifically care about state changes in the local\n // scope. This is because focus / blur events that originate from within a\n // shadow root are not re-dispatched from the host element if it was already\n // the active element in its own scope:\n scope.addEventListener('focus', onFocus, true);\n scope.addEventListener('blur', onBlur, true);\n\n // We detect that a node is a ShadowRoot by ensuring that it is a\n // DocumentFragment and also has a host property. This check covers native\n // implementation and polyfill implementation transparently. If we only cared\n // about the native implementation, we could just check if the scope was\n // an instance of a ShadowRoot.\n if (scope.nodeType === Node.DOCUMENT_FRAGMENT_NODE && scope.host) {\n // Since a ShadowRoot is a special kind of DocumentFragment, it does not\n // have a root element to add a class to. So, we add this attribute to the\n // host element instead:\n scope.host.setAttribute('data-js-focus-visible', '');\n } else if (scope.nodeType === Node.DOCUMENT_NODE) {\n document.documentElement.classList.add('js-focus-visible');\n document.documentElement.setAttribute('data-js-focus-visible', '');\n }\n }\n\n // It is important to wrap all references to global window and document in\n // these checks to support server-side rendering use cases\n // @see https://github.com/WICG/focus-visible/issues/199\n if (typeof window !== 'undefined' && typeof document !== 'undefined') {\n // Make the polyfill helper globally available. This can be used as a signal\n // to interested libraries that wish to coordinate with the polyfill for e.g.,\n // applying the polyfill to a shadow root:\n window.applyFocusVisiblePolyfill = applyFocusVisiblePolyfill;\n\n // Notify interested libraries of the polyfill's presence, in case the\n // polyfill was loaded lazily:\n var event;\n\n try {\n event = new CustomEvent('focus-visible-polyfill-ready');\n } catch (error) {\n // IE11 does not support using CustomEvent as a constructor directly:\n event = document.createEvent('CustomEvent');\n event.initCustomEvent('focus-visible-polyfill-ready', false, false, {});\n }\n\n window.dispatchEvent(event);\n }\n\n if (typeof document !== 'undefined') {\n // Apply the polyfill to the global document, so that no JavaScript\n // coordination is required to use the polyfill in the top-level document:\n applyFocusVisiblePolyfill(document);\n }\n\n})));\n", "(function(global) {\r\n /**\r\n * Polyfill URLSearchParams\r\n *\r\n * Inspired from : https://github.com/WebReflection/url-search-params/blob/master/src/url-search-params.js\r\n */\r\n\r\n var checkIfIteratorIsSupported = function() {\r\n try {\r\n return !!Symbol.iterator;\r\n } catch (error) {\r\n return false;\r\n }\r\n };\r\n\r\n\r\n var iteratorSupported = checkIfIteratorIsSupported();\r\n\r\n var createIterator = function(items) {\r\n var iterator = {\r\n next: function() {\r\n var value = items.shift();\r\n return { done: value === void 0, value: value };\r\n }\r\n };\r\n\r\n if (iteratorSupported) {\r\n iterator[Symbol.iterator] = function() {\r\n return iterator;\r\n };\r\n }\r\n\r\n return iterator;\r\n };\r\n\r\n /**\r\n * Search param name and values should be encoded according to https://url.spec.whatwg.org/#urlencoded-serializing\r\n * encodeURIComponent() produces the same result except encoding spaces as `%20` instead of `+`.\r\n */\r\n var serializeParam = function(value) {\r\n return encodeURIComponent(value).replace(/%20/g, '+');\r\n };\r\n\r\n var deserializeParam = function(value) {\r\n return decodeURIComponent(String(value).replace(/\\+/g, ' '));\r\n };\r\n\r\n var polyfillURLSearchParams = function() {\r\n\r\n var URLSearchParams = function(searchString) {\r\n Object.defineProperty(this, '_entries', { writable: true, value: {} });\r\n var typeofSearchString = typeof searchString;\r\n\r\n if (typeofSearchString === 'undefined') {\r\n // do nothing\r\n } else if (typeofSearchString === 'string') {\r\n if (searchString !== '') {\r\n this._fromString(searchString);\r\n }\r\n } else if (searchString instanceof URLSearchParams) {\r\n var _this = this;\r\n searchString.forEach(function(value, name) {\r\n _this.append(name, value);\r\n });\r\n } else if ((searchString !== null) && (typeofSearchString === 'object')) {\r\n if (Object.prototype.toString.call(searchString) === '[object Array]') {\r\n for (var i = 0; i < searchString.length; i++) {\r\n var entry = searchString[i];\r\n if ((Object.prototype.toString.call(entry) === '[object Array]') || (entry.length !== 2)) {\r\n this.append(entry[0], entry[1]);\r\n } else {\r\n throw new TypeError('Expected [string, any] as entry at index ' + i + ' of URLSearchParams\\'s input');\r\n }\r\n }\r\n } else {\r\n for (var key in searchString) {\r\n if (searchString.hasOwnProperty(key)) {\r\n this.append(key, searchString[key]);\r\n }\r\n }\r\n }\r\n } else {\r\n throw new TypeError('Unsupported input\\'s type for URLSearchParams');\r\n }\r\n };\r\n\r\n var proto = URLSearchParams.prototype;\r\n\r\n proto.append = function(name, value) {\r\n if (name in this._entries) {\r\n this._entries[name].push(String(value));\r\n } else {\r\n this._entries[name] = [String(value)];\r\n }\r\n };\r\n\r\n proto.delete = function(name) {\r\n delete this._entries[name];\r\n };\r\n\r\n proto.get = function(name) {\r\n return (name in this._entries) ? this._entries[name][0] : null;\r\n };\r\n\r\n proto.getAll = function(name) {\r\n return (name in this._entries) ? this._entries[name].slice(0) : [];\r\n };\r\n\r\n proto.has = function(name) {\r\n return (name in this._entries);\r\n };\r\n\r\n proto.set = function(name, value) {\r\n this._entries[name] = [String(value)];\r\n };\r\n\r\n proto.forEach = function(callback, thisArg) {\r\n var entries;\r\n for (var name in this._entries) {\r\n if (this._entries.hasOwnProperty(name)) {\r\n entries = this._entries[name];\r\n for (var i = 0; i < entries.length; i++) {\r\n callback.call(thisArg, entries[i], name, this);\r\n }\r\n }\r\n }\r\n };\r\n\r\n proto.keys = function() {\r\n var items = [];\r\n this.forEach(function(value, name) {\r\n items.push(name);\r\n });\r\n return createIterator(items);\r\n };\r\n\r\n proto.values = function() {\r\n var items = [];\r\n this.forEach(function(value) {\r\n items.push(value);\r\n });\r\n return createIterator(items);\r\n };\r\n\r\n proto.entries = function() {\r\n var items = [];\r\n this.forEach(function(value, name) {\r\n items.push([name, value]);\r\n });\r\n return createIterator(items);\r\n };\r\n\r\n if (iteratorSupported) {\r\n proto[Symbol.iterator] = proto.entries;\r\n }\r\n\r\n proto.toString = function() {\r\n var searchArray = [];\r\n this.forEach(function(value, name) {\r\n searchArray.push(serializeParam(name) + '=' + serializeParam(value));\r\n });\r\n return searchArray.join('&');\r\n };\r\n\r\n\r\n global.URLSearchParams = URLSearchParams;\r\n };\r\n\r\n var checkIfURLSearchParamsSupported = function() {\r\n try {\r\n var URLSearchParams = global.URLSearchParams;\r\n\r\n return (\r\n (new URLSearchParams('?a=1').toString() === 'a=1') &&\r\n (typeof URLSearchParams.prototype.set === 'function') &&\r\n (typeof URLSearchParams.prototype.entries === 'function')\r\n );\r\n } catch (e) {\r\n return false;\r\n }\r\n };\r\n\r\n if (!checkIfURLSearchParamsSupported()) {\r\n polyfillURLSearchParams();\r\n }\r\n\r\n var proto = global.URLSearchParams.prototype;\r\n\r\n if (typeof proto.sort !== 'function') {\r\n proto.sort = function() {\r\n var _this = this;\r\n var items = [];\r\n this.forEach(function(value, name) {\r\n items.push([name, value]);\r\n if (!_this._entries) {\r\n _this.delete(name);\r\n }\r\n });\r\n items.sort(function(a, b) {\r\n if (a[0] < b[0]) {\r\n return -1;\r\n } else if (a[0] > b[0]) {\r\n return +1;\r\n } else {\r\n return 0;\r\n }\r\n });\r\n if (_this._entries) { // force reset because IE keeps keys index\r\n _this._entries = {};\r\n }\r\n for (var i = 0; i < items.length; i++) {\r\n this.append(items[i][0], items[i][1]);\r\n }\r\n };\r\n }\r\n\r\n if (typeof proto._fromString !== 'function') {\r\n Object.defineProperty(proto, '_fromString', {\r\n enumerable: false,\r\n configurable: false,\r\n writable: false,\r\n value: function(searchString) {\r\n if (this._entries) {\r\n this._entries = {};\r\n } else {\r\n var keys = [];\r\n this.forEach(function(value, name) {\r\n keys.push(name);\r\n });\r\n for (var i = 0; i < keys.length; i++) {\r\n this.delete(keys[i]);\r\n }\r\n }\r\n\r\n searchString = searchString.replace(/^\\?/, '');\r\n var attributes = searchString.split('&');\r\n var attribute;\r\n for (var i = 0; i < attributes.length; i++) {\r\n attribute = attributes[i].split('=');\r\n this.append(\r\n deserializeParam(attribute[0]),\r\n (attribute.length > 1) ? deserializeParam(attribute[1]) : ''\r\n );\r\n }\r\n }\r\n });\r\n }\r\n\r\n // HTMLAnchorElement\r\n\r\n})(\r\n (typeof global !== 'undefined') ? global\r\n : ((typeof window !== 'undefined') ? window\r\n : ((typeof self !== 'undefined') ? self : this))\r\n);\r\n\r\n(function(global) {\r\n /**\r\n * Polyfill URL\r\n *\r\n * Inspired from : https://github.com/arv/DOM-URL-Polyfill/blob/master/src/url.js\r\n */\r\n\r\n var checkIfURLIsSupported = function() {\r\n try {\r\n var u = new global.URL('b', 'http://a');\r\n u.pathname = 'c d';\r\n return (u.href === 'http://a/c%20d') && u.searchParams;\r\n } catch (e) {\r\n return false;\r\n }\r\n };\r\n\r\n\r\n var polyfillURL = function() {\r\n var _URL = global.URL;\r\n\r\n var URL = function(url, base) {\r\n if (typeof url !== 'string') url = String(url);\r\n if (base && typeof base !== 'string') base = String(base);\r\n\r\n // Only create another document if the base is different from current location.\r\n var doc = document, baseElement;\r\n if (base && (global.location === void 0 || base !== global.location.href)) {\r\n base = base.toLowerCase();\r\n doc = document.implementation.createHTMLDocument('');\r\n baseElement = doc.createElement('base');\r\n baseElement.href = base;\r\n doc.head.appendChild(baseElement);\r\n try {\r\n if (baseElement.href.indexOf(base) !== 0) throw new Error(baseElement.href);\r\n } catch (err) {\r\n throw new Error('URL unable to set base ' + base + ' due to ' + err);\r\n }\r\n }\r\n\r\n var anchorElement = doc.createElement('a');\r\n anchorElement.href = url;\r\n if (baseElement) {\r\n doc.body.appendChild(anchorElement);\r\n anchorElement.href = anchorElement.href; // force href to refresh\r\n }\r\n\r\n var inputElement = doc.createElement('input');\r\n inputElement.type = 'url';\r\n inputElement.value = url;\r\n\r\n if (anchorElement.protocol === ':' || !/:/.test(anchorElement.href) || (!inputElement.checkValidity() && !base)) {\r\n throw new TypeError('Invalid URL');\r\n }\r\n\r\n Object.defineProperty(this, '_anchorElement', {\r\n value: anchorElement\r\n });\r\n\r\n\r\n // create a linked searchParams which reflect its changes on URL\r\n var searchParams = new global.URLSearchParams(this.search);\r\n var enableSearchUpdate = true;\r\n var enableSearchParamsUpdate = true;\r\n var _this = this;\r\n ['append', 'delete', 'set'].forEach(function(methodName) {\r\n var method = searchParams[methodName];\r\n searchParams[methodName] = function() {\r\n method.apply(searchParams, arguments);\r\n if (enableSearchUpdate) {\r\n enableSearchParamsUpdate = false;\r\n _this.search = searchParams.toString();\r\n enableSearchParamsUpdate = true;\r\n }\r\n };\r\n });\r\n\r\n Object.defineProperty(this, 'searchParams', {\r\n value: searchParams,\r\n enumerable: true\r\n });\r\n\r\n var search = void 0;\r\n Object.defineProperty(this, '_updateSearchParams', {\r\n enumerable: false,\r\n configurable: false,\r\n writable: false,\r\n value: function() {\r\n if (this.search !== search) {\r\n search = this.search;\r\n if (enableSearchParamsUpdate) {\r\n enableSearchUpdate = false;\r\n this.searchParams._fromString(this.search);\r\n enableSearchUpdate = true;\r\n }\r\n }\r\n }\r\n });\r\n };\r\n\r\n var proto = URL.prototype;\r\n\r\n var linkURLWithAnchorAttribute = function(attributeName) {\r\n Object.defineProperty(proto, attributeName, {\r\n get: function() {\r\n return this._anchorElement[attributeName];\r\n },\r\n set: function(value) {\r\n this._anchorElement[attributeName] = value;\r\n },\r\n enumerable: true\r\n });\r\n };\r\n\r\n ['hash', 'host', 'hostname', 'port', 'protocol']\r\n .forEach(function(attributeName) {\r\n linkURLWithAnchorAttribute(attributeName);\r\n });\r\n\r\n Object.defineProperty(proto, 'search', {\r\n get: function() {\r\n return this._anchorElement['search'];\r\n },\r\n set: function(value) {\r\n this._anchorElement['search'] = value;\r\n this._updateSearchParams();\r\n },\r\n enumerable: true\r\n });\r\n\r\n Object.defineProperties(proto, {\r\n\r\n 'toString': {\r\n get: function() {\r\n var _this = this;\r\n return function() {\r\n return _this.href;\r\n };\r\n }\r\n },\r\n\r\n 'href': {\r\n get: function() {\r\n return this._anchorElement.href.replace(/\\?$/, '');\r\n },\r\n set: function(value) {\r\n this._anchorElement.href = value;\r\n this._updateSearchParams();\r\n },\r\n enumerable: true\r\n },\r\n\r\n 'pathname': {\r\n get: function() {\r\n return this._anchorElement.pathname.replace(/(^\\/?)/, '/');\r\n },\r\n set: function(value) {\r\n this._anchorElement.pathname = value;\r\n },\r\n enumerable: true\r\n },\r\n\r\n 'origin': {\r\n get: function() {\r\n // get expected port from protocol\r\n var expectedPort = { 'http:': 80, 'https:': 443, 'ftp:': 21 }[this._anchorElement.protocol];\r\n // add port to origin if, expected port is different than actual port\r\n // and it is not empty f.e http://foo:8080\r\n // 8080 != 80 && 8080 != ''\r\n var addPortToOrigin = this._anchorElement.port != expectedPort &&\r\n this._anchorElement.port !== '';\r\n\r\n return this._anchorElement.protocol +\r\n '//' +\r\n this._anchorElement.hostname +\r\n (addPortToOrigin ? (':' + this._anchorElement.port) : '');\r\n },\r\n enumerable: true\r\n },\r\n\r\n 'password': { // TODO\r\n get: function() {\r\n return '';\r\n },\r\n set: function(value) {\r\n },\r\n enumerable: true\r\n },\r\n\r\n 'username': { // TODO\r\n get: function() {\r\n return '';\r\n },\r\n set: function(value) {\r\n },\r\n enumerable: true\r\n },\r\n });\r\n\r\n URL.createObjectURL = function(blob) {\r\n return _URL.createObjectURL.apply(_URL, arguments);\r\n };\r\n\r\n URL.revokeObjectURL = function(url) {\r\n return _URL.revokeObjectURL.apply(_URL, arguments);\r\n };\r\n\r\n global.URL = URL;\r\n\r\n };\r\n\r\n if (!checkIfURLIsSupported()) {\r\n polyfillURL();\r\n }\r\n\r\n if ((global.location !== void 0) && !('origin' in global.location)) {\r\n var getOrigin = function() {\r\n return global.location.protocol + '//' + global.location.hostname + (global.location.port ? (':' + global.location.port) : '');\r\n };\r\n\r\n try {\r\n Object.defineProperty(global.location, 'origin', {\r\n get: getOrigin,\r\n enumerable: true\r\n });\r\n } catch (e) {\r\n setInterval(function() {\r\n global.location.origin = getOrigin();\r\n }, 100);\r\n }\r\n }\r\n\r\n})(\r\n (typeof global !== 'undefined') ? global\r\n : ((typeof window !== 'undefined') ? window\r\n : ((typeof self !== 'undefined') ? self : this))\r\n);\r\n", "/*!\n * clipboard.js v2.0.11\n * https://clipboardjs.com/\n *\n * Licensed MIT \u00A9 Zeno Rocha\n */\n(function webpackUniversalModuleDefinition(root, factory) {\n\tif(typeof exports === 'object' && typeof module === 'object')\n\t\tmodule.exports = factory();\n\telse if(typeof define === 'function' && define.amd)\n\t\tdefine([], factory);\n\telse if(typeof exports === 'object')\n\t\texports[\"ClipboardJS\"] = factory();\n\telse\n\t\troot[\"ClipboardJS\"] = factory();\n})(this, function() {\nreturn /******/ (function() { // webpackBootstrap\n/******/ \tvar __webpack_modules__ = ({\n\n/***/ 686:\n/***/ (function(__unused_webpack_module, __webpack_exports__, __webpack_require__) {\n\n\"use strict\";\n\n// EXPORTS\n__webpack_require__.d(__webpack_exports__, {\n \"default\": function() { return /* binding */ clipboard; }\n});\n\n// EXTERNAL MODULE: ./node_modules/tiny-emitter/index.js\nvar tiny_emitter = __webpack_require__(279);\nvar tiny_emitter_default = /*#__PURE__*/__webpack_require__.n(tiny_emitter);\n// EXTERNAL MODULE: ./node_modules/good-listener/src/listen.js\nvar listen = __webpack_require__(370);\nvar listen_default = /*#__PURE__*/__webpack_require__.n(listen);\n// EXTERNAL MODULE: ./node_modules/select/src/select.js\nvar src_select = __webpack_require__(817);\nvar select_default = /*#__PURE__*/__webpack_require__.n(src_select);\n;// CONCATENATED MODULE: ./src/common/command.js\n/**\n * Executes a given operation type.\n * @param {String} type\n * @return {Boolean}\n */\nfunction command(type) {\n try {\n return document.execCommand(type);\n } catch (err) {\n return false;\n }\n}\n;// CONCATENATED MODULE: ./src/actions/cut.js\n\n\n/**\n * Cut action wrapper.\n * @param {String|HTMLElement} target\n * @return {String}\n */\n\nvar ClipboardActionCut = function ClipboardActionCut(target) {\n var selectedText = select_default()(target);\n command('cut');\n return selectedText;\n};\n\n/* harmony default export */ var actions_cut = (ClipboardActionCut);\n;// CONCATENATED MODULE: ./src/common/create-fake-element.js\n/**\n * Creates a fake textarea element with a value.\n * @param {String} value\n * @return {HTMLElement}\n */\nfunction createFakeElement(value) {\n var isRTL = document.documentElement.getAttribute('dir') === 'rtl';\n var fakeElement = document.createElement('textarea'); // Prevent zooming on iOS\n\n fakeElement.style.fontSize = '12pt'; // Reset box model\n\n fakeElement.style.border = '0';\n fakeElement.style.padding = '0';\n fakeElement.style.margin = '0'; // Move element out of screen horizontally\n\n fakeElement.style.position = 'absolute';\n fakeElement.style[isRTL ? 'right' : 'left'] = '-9999px'; // Move element to the same position vertically\n\n var yPosition = window.pageYOffset || document.documentElement.scrollTop;\n fakeElement.style.top = \"\".concat(yPosition, \"px\");\n fakeElement.setAttribute('readonly', '');\n fakeElement.value = value;\n return fakeElement;\n}\n;// CONCATENATED MODULE: ./src/actions/copy.js\n\n\n\n/**\n * Create fake copy action wrapper using a fake element.\n * @param {String} target\n * @param {Object} options\n * @return {String}\n */\n\nvar fakeCopyAction = function fakeCopyAction(value, options) {\n var fakeElement = createFakeElement(value);\n options.container.appendChild(fakeElement);\n var selectedText = select_default()(fakeElement);\n command('copy');\n fakeElement.remove();\n return selectedText;\n};\n/**\n * Copy action wrapper.\n * @param {String|HTMLElement} target\n * @param {Object} options\n * @return {String}\n */\n\n\nvar ClipboardActionCopy = function ClipboardActionCopy(target) {\n var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {\n container: document.body\n };\n var selectedText = '';\n\n if (typeof target === 'string') {\n selectedText = fakeCopyAction(target, options);\n } else if (target instanceof HTMLInputElement && !['text', 'search', 'url', 'tel', 'password'].includes(target === null || target === void 0 ? void 0 : target.type)) {\n // If input type doesn't support `setSelectionRange`. Simulate it. https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement/setSelectionRange\n selectedText = fakeCopyAction(target.value, options);\n } else {\n selectedText = select_default()(target);\n command('copy');\n }\n\n return selectedText;\n};\n\n/* harmony default export */ var actions_copy = (ClipboardActionCopy);\n;// CONCATENATED MODULE: ./src/actions/default.js\nfunction _typeof(obj) { \"@babel/helpers - typeof\"; if (typeof Symbol === \"function\" && typeof Symbol.iterator === \"symbol\") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === \"function\" && obj.constructor === Symbol && obj !== Symbol.prototype ? \"symbol\" : typeof obj; }; } return _typeof(obj); }\n\n\n\n/**\n * Inner function which performs selection from either `text` or `target`\n * properties and then executes copy or cut operations.\n * @param {Object} options\n */\n\nvar ClipboardActionDefault = function ClipboardActionDefault() {\n var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};\n // Defines base properties passed from constructor.\n var _options$action = options.action,\n action = _options$action === void 0 ? 'copy' : _options$action,\n container = options.container,\n target = options.target,\n text = options.text; // Sets the `action` to be performed which can be either 'copy' or 'cut'.\n\n if (action !== 'copy' && action !== 'cut') {\n throw new Error('Invalid \"action\" value, use either \"copy\" or \"cut\"');\n } // Sets the `target` property using an element that will be have its content copied.\n\n\n if (target !== undefined) {\n if (target && _typeof(target) === 'object' && target.nodeType === 1) {\n if (action === 'copy' && target.hasAttribute('disabled')) {\n throw new Error('Invalid \"target\" attribute. Please use \"readonly\" instead of \"disabled\" attribute');\n }\n\n if (action === 'cut' && (target.hasAttribute('readonly') || target.hasAttribute('disabled'))) {\n throw new Error('Invalid \"target\" attribute. You can\\'t cut text from elements with \"readonly\" or \"disabled\" attributes');\n }\n } else {\n throw new Error('Invalid \"target\" value, use a valid Element');\n }\n } // Define selection strategy based on `text` property.\n\n\n if (text) {\n return actions_copy(text, {\n container: container\n });\n } // Defines which selection strategy based on `target` property.\n\n\n if (target) {\n return action === 'cut' ? actions_cut(target) : actions_copy(target, {\n container: container\n });\n }\n};\n\n/* harmony default export */ var actions_default = (ClipboardActionDefault);\n;// CONCATENATED MODULE: ./src/clipboard.js\nfunction clipboard_typeof(obj) { \"@babel/helpers - typeof\"; if (typeof Symbol === \"function\" && typeof Symbol.iterator === \"symbol\") { clipboard_typeof = function _typeof(obj) { return typeof obj; }; } else { clipboard_typeof = function _typeof(obj) { return obj && typeof Symbol === \"function\" && obj.constructor === Symbol && obj !== Symbol.prototype ? \"symbol\" : typeof obj; }; } return clipboard_typeof(obj); }\n\nfunction _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError(\"Cannot call a class as a function\"); } }\n\nfunction _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if (\"value\" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }\n\nfunction _createClass(Constructor, protoProps, staticProps) { if (protoProps) _defineProperties(Constructor.prototype, protoProps); if (staticProps) _defineProperties(Constructor, staticProps); return Constructor; }\n\nfunction _inherits(subClass, superClass) { if (typeof superClass !== \"function\" && superClass !== null) { throw new TypeError(\"Super expression must either be null or a function\"); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, writable: true, configurable: true } }); if (superClass) _setPrototypeOf(subClass, superClass); }\n\nfunction _setPrototypeOf(o, p) { _setPrototypeOf = Object.setPrototypeOf || function _setPrototypeOf(o, p) { o.__proto__ = p; return o; }; return _setPrototypeOf(o, p); }\n\nfunction _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = _getPrototypeOf(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = _getPrototypeOf(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return _possibleConstructorReturn(this, result); }; }\n\nfunction _possibleConstructorReturn(self, call) { if (call && (clipboard_typeof(call) === \"object\" || typeof call === \"function\")) { return call; } return _assertThisInitialized(self); }\n\nfunction _assertThisInitialized(self) { if (self === void 0) { throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\"); } return self; }\n\nfunction _isNativeReflectConstruct() { if (typeof Reflect === \"undefined\" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === \"function\") return true; try { Date.prototype.toString.call(Reflect.construct(Date, [], function () {})); return true; } catch (e) { return false; } }\n\nfunction _getPrototypeOf(o) { _getPrototypeOf = Object.setPrototypeOf ? Object.getPrototypeOf : function _getPrototypeOf(o) { return o.__proto__ || Object.getPrototypeOf(o); }; return _getPrototypeOf(o); }\n\n\n\n\n\n\n/**\n * Helper function to retrieve attribute value.\n * @param {String} suffix\n * @param {Element} element\n */\n\nfunction getAttributeValue(suffix, element) {\n var attribute = \"data-clipboard-\".concat(suffix);\n\n if (!element.hasAttribute(attribute)) {\n return;\n }\n\n return element.getAttribute(attribute);\n}\n/**\n * Base class which takes one or more elements, adds event listeners to them,\n * and instantiates a new `ClipboardAction` on each click.\n */\n\n\nvar Clipboard = /*#__PURE__*/function (_Emitter) {\n _inherits(Clipboard, _Emitter);\n\n var _super = _createSuper(Clipboard);\n\n /**\n * @param {String|HTMLElement|HTMLCollection|NodeList} trigger\n * @param {Object} options\n */\n function Clipboard(trigger, options) {\n var _this;\n\n _classCallCheck(this, Clipboard);\n\n _this = _super.call(this);\n\n _this.resolveOptions(options);\n\n _this.listenClick(trigger);\n\n return _this;\n }\n /**\n * Defines if attributes would be resolved using internal setter functions\n * or custom functions that were passed in the constructor.\n * @param {Object} options\n */\n\n\n _createClass(Clipboard, [{\n key: \"resolveOptions\",\n value: function resolveOptions() {\n var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};\n this.action = typeof options.action === 'function' ? options.action : this.defaultAction;\n this.target = typeof options.target === 'function' ? options.target : this.defaultTarget;\n this.text = typeof options.text === 'function' ? options.text : this.defaultText;\n this.container = clipboard_typeof(options.container) === 'object' ? options.container : document.body;\n }\n /**\n * Adds a click event listener to the passed trigger.\n * @param {String|HTMLElement|HTMLCollection|NodeList} trigger\n */\n\n }, {\n key: \"listenClick\",\n value: function listenClick(trigger) {\n var _this2 = this;\n\n this.listener = listen_default()(trigger, 'click', function (e) {\n return _this2.onClick(e);\n });\n }\n /**\n * Defines a new `ClipboardAction` on each click event.\n * @param {Event} e\n */\n\n }, {\n key: \"onClick\",\n value: function onClick(e) {\n var trigger = e.delegateTarget || e.currentTarget;\n var action = this.action(trigger) || 'copy';\n var text = actions_default({\n action: action,\n container: this.container,\n target: this.target(trigger),\n text: this.text(trigger)\n }); // Fires an event based on the copy operation result.\n\n this.emit(text ? 'success' : 'error', {\n action: action,\n text: text,\n trigger: trigger,\n clearSelection: function clearSelection() {\n if (trigger) {\n trigger.focus();\n }\n\n window.getSelection().removeAllRanges();\n }\n });\n }\n /**\n * Default `action` lookup function.\n * @param {Element} trigger\n */\n\n }, {\n key: \"defaultAction\",\n value: function defaultAction(trigger) {\n return getAttributeValue('action', trigger);\n }\n /**\n * Default `target` lookup function.\n * @param {Element} trigger\n */\n\n }, {\n key: \"defaultTarget\",\n value: function defaultTarget(trigger) {\n var selector = getAttributeValue('target', trigger);\n\n if (selector) {\n return document.querySelector(selector);\n }\n }\n /**\n * Allow fire programmatically a copy action\n * @param {String|HTMLElement} target\n * @param {Object} options\n * @returns Text copied.\n */\n\n }, {\n key: \"defaultText\",\n\n /**\n * Default `text` lookup function.\n * @param {Element} trigger\n */\n value: function defaultText(trigger) {\n return getAttributeValue('text', trigger);\n }\n /**\n * Destroy lifecycle.\n */\n\n }, {\n key: \"destroy\",\n value: function destroy() {\n this.listener.destroy();\n }\n }], [{\n key: \"copy\",\n value: function copy(target) {\n var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {\n container: document.body\n };\n return actions_copy(target, options);\n }\n /**\n * Allow fire programmatically a cut action\n * @param {String|HTMLElement} target\n * @returns Text cutted.\n */\n\n }, {\n key: \"cut\",\n value: function cut(target) {\n return actions_cut(target);\n }\n /**\n * Returns the support of the given action, or all actions if no action is\n * given.\n * @param {String} [action]\n */\n\n }, {\n key: \"isSupported\",\n value: function isSupported() {\n var action = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : ['copy', 'cut'];\n var actions = typeof action === 'string' ? [action] : action;\n var support = !!document.queryCommandSupported;\n actions.forEach(function (action) {\n support = support && !!document.queryCommandSupported(action);\n });\n return support;\n }\n }]);\n\n return Clipboard;\n}((tiny_emitter_default()));\n\n/* harmony default export */ var clipboard = (Clipboard);\n\n/***/ }),\n\n/***/ 828:\n/***/ (function(module) {\n\nvar DOCUMENT_NODE_TYPE = 9;\n\n/**\n * A polyfill for Element.matches()\n */\nif (typeof Element !== 'undefined' && !Element.prototype.matches) {\n var proto = Element.prototype;\n\n proto.matches = proto.matchesSelector ||\n proto.mozMatchesSelector ||\n proto.msMatchesSelector ||\n proto.oMatchesSelector ||\n proto.webkitMatchesSelector;\n}\n\n/**\n * Finds the closest parent that matches a selector.\n *\n * @param {Element} element\n * @param {String} selector\n * @return {Function}\n */\nfunction closest (element, selector) {\n while (element && element.nodeType !== DOCUMENT_NODE_TYPE) {\n if (typeof element.matches === 'function' &&\n element.matches(selector)) {\n return element;\n }\n element = element.parentNode;\n }\n}\n\nmodule.exports = closest;\n\n\n/***/ }),\n\n/***/ 438:\n/***/ (function(module, __unused_webpack_exports, __webpack_require__) {\n\nvar closest = __webpack_require__(828);\n\n/**\n * Delegates event to a selector.\n *\n * @param {Element} element\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @param {Boolean} useCapture\n * @return {Object}\n */\nfunction _delegate(element, selector, type, callback, useCapture) {\n var listenerFn = listener.apply(this, arguments);\n\n element.addEventListener(type, listenerFn, useCapture);\n\n return {\n destroy: function() {\n element.removeEventListener(type, listenerFn, useCapture);\n }\n }\n}\n\n/**\n * Delegates event to a selector.\n *\n * @param {Element|String|Array} [elements]\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @param {Boolean} useCapture\n * @return {Object}\n */\nfunction delegate(elements, selector, type, callback, useCapture) {\n // Handle the regular Element usage\n if (typeof elements.addEventListener === 'function') {\n return _delegate.apply(null, arguments);\n }\n\n // Handle Element-less usage, it defaults to global delegation\n if (typeof type === 'function') {\n // Use `document` as the first parameter, then apply arguments\n // This is a short way to .unshift `arguments` without running into deoptimizations\n return _delegate.bind(null, document).apply(null, arguments);\n }\n\n // Handle Selector-based usage\n if (typeof elements === 'string') {\n elements = document.querySelectorAll(elements);\n }\n\n // Handle Array-like based usage\n return Array.prototype.map.call(elements, function (element) {\n return _delegate(element, selector, type, callback, useCapture);\n });\n}\n\n/**\n * Finds closest match and invokes callback.\n *\n * @param {Element} element\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @return {Function}\n */\nfunction listener(element, selector, type, callback) {\n return function(e) {\n e.delegateTarget = closest(e.target, selector);\n\n if (e.delegateTarget) {\n callback.call(element, e);\n }\n }\n}\n\nmodule.exports = delegate;\n\n\n/***/ }),\n\n/***/ 879:\n/***/ (function(__unused_webpack_module, exports) {\n\n/**\n * Check if argument is a HTML element.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.node = function(value) {\n return value !== undefined\n && value instanceof HTMLElement\n && value.nodeType === 1;\n};\n\n/**\n * Check if argument is a list of HTML elements.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.nodeList = function(value) {\n var type = Object.prototype.toString.call(value);\n\n return value !== undefined\n && (type === '[object NodeList]' || type === '[object HTMLCollection]')\n && ('length' in value)\n && (value.length === 0 || exports.node(value[0]));\n};\n\n/**\n * Check if argument is a string.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.string = function(value) {\n return typeof value === 'string'\n || value instanceof String;\n};\n\n/**\n * Check if argument is a function.\n *\n * @param {Object} value\n * @return {Boolean}\n */\nexports.fn = function(value) {\n var type = Object.prototype.toString.call(value);\n\n return type === '[object Function]';\n};\n\n\n/***/ }),\n\n/***/ 370:\n/***/ (function(module, __unused_webpack_exports, __webpack_require__) {\n\nvar is = __webpack_require__(879);\nvar delegate = __webpack_require__(438);\n\n/**\n * Validates all params and calls the right\n * listener function based on its target type.\n *\n * @param {String|HTMLElement|HTMLCollection|NodeList} target\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listen(target, type, callback) {\n if (!target && !type && !callback) {\n throw new Error('Missing required arguments');\n }\n\n if (!is.string(type)) {\n throw new TypeError('Second argument must be a String');\n }\n\n if (!is.fn(callback)) {\n throw new TypeError('Third argument must be a Function');\n }\n\n if (is.node(target)) {\n return listenNode(target, type, callback);\n }\n else if (is.nodeList(target)) {\n return listenNodeList(target, type, callback);\n }\n else if (is.string(target)) {\n return listenSelector(target, type, callback);\n }\n else {\n throw new TypeError('First argument must be a String, HTMLElement, HTMLCollection, or NodeList');\n }\n}\n\n/**\n * Adds an event listener to a HTML element\n * and returns a remove listener function.\n *\n * @param {HTMLElement} node\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenNode(node, type, callback) {\n node.addEventListener(type, callback);\n\n return {\n destroy: function() {\n node.removeEventListener(type, callback);\n }\n }\n}\n\n/**\n * Add an event listener to a list of HTML elements\n * and returns a remove listener function.\n *\n * @param {NodeList|HTMLCollection} nodeList\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenNodeList(nodeList, type, callback) {\n Array.prototype.forEach.call(nodeList, function(node) {\n node.addEventListener(type, callback);\n });\n\n return {\n destroy: function() {\n Array.prototype.forEach.call(nodeList, function(node) {\n node.removeEventListener(type, callback);\n });\n }\n }\n}\n\n/**\n * Add an event listener to a selector\n * and returns a remove listener function.\n *\n * @param {String} selector\n * @param {String} type\n * @param {Function} callback\n * @return {Object}\n */\nfunction listenSelector(selector, type, callback) {\n return delegate(document.body, selector, type, callback);\n}\n\nmodule.exports = listen;\n\n\n/***/ }),\n\n/***/ 817:\n/***/ (function(module) {\n\nfunction select(element) {\n var selectedText;\n\n if (element.nodeName === 'SELECT') {\n element.focus();\n\n selectedText = element.value;\n }\n else if (element.nodeName === 'INPUT' || element.nodeName === 'TEXTAREA') {\n var isReadOnly = element.hasAttribute('readonly');\n\n if (!isReadOnly) {\n element.setAttribute('readonly', '');\n }\n\n element.select();\n element.setSelectionRange(0, element.value.length);\n\n if (!isReadOnly) {\n element.removeAttribute('readonly');\n }\n\n selectedText = element.value;\n }\n else {\n if (element.hasAttribute('contenteditable')) {\n element.focus();\n }\n\n var selection = window.getSelection();\n var range = document.createRange();\n\n range.selectNodeContents(element);\n selection.removeAllRanges();\n selection.addRange(range);\n\n selectedText = selection.toString();\n }\n\n return selectedText;\n}\n\nmodule.exports = select;\n\n\n/***/ }),\n\n/***/ 279:\n/***/ (function(module) {\n\nfunction E () {\n // Keep this empty so it's easier to inherit from\n // (via https://github.com/lipsmack from https://github.com/scottcorgan/tiny-emitter/issues/3)\n}\n\nE.prototype = {\n on: function (name, callback, ctx) {\n var e = this.e || (this.e = {});\n\n (e[name] || (e[name] = [])).push({\n fn: callback,\n ctx: ctx\n });\n\n return this;\n },\n\n once: function (name, callback, ctx) {\n var self = this;\n function listener () {\n self.off(name, listener);\n callback.apply(ctx, arguments);\n };\n\n listener._ = callback\n return this.on(name, listener, ctx);\n },\n\n emit: function (name) {\n var data = [].slice.call(arguments, 1);\n var evtArr = ((this.e || (this.e = {}))[name] || []).slice();\n var i = 0;\n var len = evtArr.length;\n\n for (i; i < len; i++) {\n evtArr[i].fn.apply(evtArr[i].ctx, data);\n }\n\n return this;\n },\n\n off: function (name, callback) {\n var e = this.e || (this.e = {});\n var evts = e[name];\n var liveEvents = [];\n\n if (evts && callback) {\n for (var i = 0, len = evts.length; i < len; i++) {\n if (evts[i].fn !== callback && evts[i].fn._ !== callback)\n liveEvents.push(evts[i]);\n }\n }\n\n // Remove event from queue to prevent memory leak\n // Suggested by https://github.com/lazd\n // Ref: https://github.com/scottcorgan/tiny-emitter/commit/c6ebfaa9bc973b33d110a84a307742b7cf94c953#commitcomment-5024910\n\n (liveEvents.length)\n ? e[name] = liveEvents\n : delete e[name];\n\n return this;\n }\n};\n\nmodule.exports = E;\nmodule.exports.TinyEmitter = E;\n\n\n/***/ })\n\n/******/ \t});\n/************************************************************************/\n/******/ \t// The module cache\n/******/ \tvar __webpack_module_cache__ = {};\n/******/ \t\n/******/ \t// The require function\n/******/ \tfunction __webpack_require__(moduleId) {\n/******/ \t\t// Check if module is in cache\n/******/ \t\tif(__webpack_module_cache__[moduleId]) {\n/******/ \t\t\treturn __webpack_module_cache__[moduleId].exports;\n/******/ \t\t}\n/******/ \t\t// Create a new module (and put it into the cache)\n/******/ \t\tvar module = __webpack_module_cache__[moduleId] = {\n/******/ \t\t\t// no module.id needed\n/******/ \t\t\t// no module.loaded needed\n/******/ \t\t\texports: {}\n/******/ \t\t};\n/******/ \t\n/******/ \t\t// Execute the module function\n/******/ \t\t__webpack_modules__[moduleId](module, module.exports, __webpack_require__);\n/******/ \t\n/******/ \t\t// Return the exports of the module\n/******/ \t\treturn module.exports;\n/******/ \t}\n/******/ \t\n/************************************************************************/\n/******/ \t/* webpack/runtime/compat get default export */\n/******/ \t!function() {\n/******/ \t\t// getDefaultExport function for compatibility with non-harmony modules\n/******/ \t\t__webpack_require__.n = function(module) {\n/******/ \t\t\tvar getter = module && module.__esModule ?\n/******/ \t\t\t\tfunction() { return module['default']; } :\n/******/ \t\t\t\tfunction() { return module; };\n/******/ \t\t\t__webpack_require__.d(getter, { a: getter });\n/******/ \t\t\treturn getter;\n/******/ \t\t};\n/******/ \t}();\n/******/ \t\n/******/ \t/* webpack/runtime/define property getters */\n/******/ \t!function() {\n/******/ \t\t// define getter functions for harmony exports\n/******/ \t\t__webpack_require__.d = function(exports, definition) {\n/******/ \t\t\tfor(var key in definition) {\n/******/ \t\t\t\tif(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n/******/ \t\t\t\t\tObject.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n/******/ \t\t\t\t}\n/******/ \t\t\t}\n/******/ \t\t};\n/******/ \t}();\n/******/ \t\n/******/ \t/* webpack/runtime/hasOwnProperty shorthand */\n/******/ \t!function() {\n/******/ \t\t__webpack_require__.o = function(obj, prop) { return Object.prototype.hasOwnProperty.call(obj, prop); }\n/******/ \t}();\n/******/ \t\n/************************************************************************/\n/******/ \t// module exports must be returned from runtime so entry inlining is disabled\n/******/ \t// startup\n/******/ \t// Load entry module and return exports\n/******/ \treturn __webpack_require__(686);\n/******/ })()\n.default;\n});", "/*!\n * escape-html\n * Copyright(c) 2012-2013 TJ Holowaychuk\n * Copyright(c) 2015 Andreas Lubbe\n * Copyright(c) 2015 Tiancheng \"Timothy\" Gu\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module variables.\n * @private\n */\n\nvar matchHtmlRegExp = /[\"'&<>]/;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = escapeHtml;\n\n/**\n * Escape special characters in the given string of html.\n *\n * @param {string} string The string to escape for inserting into HTML\n * @return {string}\n * @public\n */\n\nfunction escapeHtml(string) {\n var str = '' + string;\n var match = matchHtmlRegExp.exec(str);\n\n if (!match) {\n return str;\n }\n\n var escape;\n var html = '';\n var index = 0;\n var lastIndex = 0;\n\n for (index = match.index; index < str.length; index++) {\n switch (str.charCodeAt(index)) {\n case 34: // \"\n escape = '"';\n break;\n case 38: // &\n escape = '&';\n break;\n case 39: // '\n escape = ''';\n break;\n case 60: // <\n escape = '<';\n break;\n case 62: // >\n escape = '>';\n break;\n default:\n continue;\n }\n\n if (lastIndex !== index) {\n html += str.substring(lastIndex, index);\n }\n\n lastIndex = index + 1;\n html += escape;\n }\n\n return lastIndex !== index\n ? html + str.substring(lastIndex, index)\n : html;\n}\n", "Array.prototype.flat||Object.defineProperty(Array.prototype,\"flat\",{configurable:!0,value:function r(){var t=isNaN(arguments[0])?1:Number(arguments[0]);return t?Array.prototype.reduce.call(this,function(a,e){return Array.isArray(e)?a.push.apply(a,r.call(e,t-1)):a.push(e),a},[]):Array.prototype.slice.call(this)},writable:!0}),Array.prototype.flatMap||Object.defineProperty(Array.prototype,\"flatMap\",{configurable:!0,value:function(r){return Array.prototype.map.apply(this,arguments).flat()},writable:!0})\n", "/*\n * Copyright (c) 2016-2023 Martin Donath \n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\nimport \"array-flat-polyfill\"\nimport \"focus-visible\"\nimport \"unfetch/polyfill\"\nimport \"url-polyfill\"\n\nimport {\n EMPTY,\n NEVER,\n Observable,\n Subject,\n defer,\n delay,\n filter,\n map,\n merge,\n mergeWith,\n shareReplay,\n switchMap\n} from \"rxjs\"\n\nimport { configuration, feature } from \"./_\"\nimport {\n at,\n getActiveElement,\n getOptionalElement,\n requestJSON,\n setLocation,\n setToggle,\n watchDocument,\n watchKeyboard,\n watchLocation,\n watchLocationTarget,\n watchMedia,\n watchPrint,\n watchScript,\n watchViewport\n} from \"./browser\"\nimport {\n getComponentElement,\n getComponentElements,\n mountAnnounce,\n mountBackToTop,\n mountConsent,\n mountContent,\n mountDialog,\n mountHeader,\n mountHeaderTitle,\n mountPalette,\n mountSearch,\n mountSearchHiglight,\n mountSidebar,\n mountSource,\n mountTableOfContents,\n mountTabs,\n watchHeader,\n watchMain\n} from \"./components\"\nimport {\n SearchIndex,\n setupClipboardJS,\n setupInstantLoading,\n setupVersionSelector\n} from \"./integrations\"\nimport {\n patchIndeterminate,\n patchScrollfix,\n patchScrolllock\n} from \"./patches\"\nimport \"./polyfills\"\n\n/* ----------------------------------------------------------------------------\n * Functions - @todo refactor\n * ------------------------------------------------------------------------- */\n\n/**\n * Fetch search index\n *\n * @returns Search index observable\n */\nfunction fetchSearchIndex(): Observable {\n if (location.protocol === \"file:\") {\n return watchScript(\n `${new URL(\"search/search_index.js\", config.base)}`\n )\n .pipe(\n // @ts-ignore - @todo fix typings\n map(() => __index),\n shareReplay(1)\n )\n } else {\n return requestJSON(\n new URL(\"search/search_index.json\", config.base)\n )\n }\n}\n\n/* ----------------------------------------------------------------------------\n * Application\n * ------------------------------------------------------------------------- */\n\n/* Yay, JavaScript is available */\ndocument.documentElement.classList.remove(\"no-js\")\ndocument.documentElement.classList.add(\"js\")\n\n/* Set up navigation observables and subjects */\nconst document$ = watchDocument()\nconst location$ = watchLocation()\nconst target$ = watchLocationTarget(location$)\nconst keyboard$ = watchKeyboard()\n\n/* Set up media observables */\nconst viewport$ = watchViewport()\nconst tablet$ = watchMedia(\"(min-width: 960px)\")\nconst screen$ = watchMedia(\"(min-width: 1220px)\")\nconst print$ = watchPrint()\n\n/* Retrieve search index, if search is enabled */\nconst config = configuration()\nconst index$ = document.forms.namedItem(\"search\")\n ? fetchSearchIndex()\n : NEVER\n\n/* Set up Clipboard.js integration */\nconst alert$ = new Subject()\nsetupClipboardJS({ alert$ })\n\n/* Set up instant loading, if enabled */\nif (feature(\"navigation.instant\"))\n setupInstantLoading({ location$, viewport$ })\n .subscribe(document$)\n\n/* Set up version selector */\nif (config.version?.provider === \"mike\")\n setupVersionSelector({ document$ })\n\n/* Always close drawer and search on navigation */\nmerge(location$, target$)\n .pipe(\n delay(125)\n )\n .subscribe(() => {\n setToggle(\"drawer\", false)\n setToggle(\"search\", false)\n })\n\n/* Set up global keyboard handlers */\nkeyboard$\n .pipe(\n filter(({ mode }) => mode === \"global\")\n )\n .subscribe(key => {\n switch (key.type) {\n\n /* Go to previous page */\n case \"p\":\n case \",\":\n const prev = getOptionalElement(\"link[rel=prev]\")\n if (typeof prev !== \"undefined\")\n setLocation(prev)\n break\n\n /* Go to next page */\n case \"n\":\n case \".\":\n const next = getOptionalElement(\"link[rel=next]\")\n if (typeof next !== \"undefined\")\n setLocation(next)\n break\n\n /* Expand navigation, see https://bit.ly/3ZjG5io */\n case \"Enter\":\n const active = getActiveElement()\n if (active instanceof HTMLLabelElement)\n active.click()\n }\n })\n\n/* Set up patches */\npatchIndeterminate({ document$, tablet$ })\npatchScrollfix({ document$ })\npatchScrolllock({ viewport$, tablet$ })\n\n/* Set up header and main area observable */\nconst header$ = watchHeader(getComponentElement(\"header\"), { viewport$ })\nconst main$ = document$\n .pipe(\n map(() => getComponentElement(\"main\")),\n switchMap(el => watchMain(el, { viewport$, header$ })),\n shareReplay(1)\n )\n\n/* Set up control component observables */\nconst control$ = merge(\n\n /* Consent */\n ...getComponentElements(\"consent\")\n .map(el => mountConsent(el, { target$ })),\n\n /* Dialog */\n ...getComponentElements(\"dialog\")\n .map(el => mountDialog(el, { alert$ })),\n\n /* Header */\n ...getComponentElements(\"header\")\n .map(el => mountHeader(el, { viewport$, header$, main$ })),\n\n /* Color palette */\n ...getComponentElements(\"palette\")\n .map(el => mountPalette(el)),\n\n /* Search */\n ...getComponentElements(\"search\")\n .map(el => mountSearch(el, { index$, keyboard$ })),\n\n /* Repository information */\n ...getComponentElements(\"source\")\n .map(el => mountSource(el))\n)\n\n/* Set up content component observables */\nconst content$ = defer(() => merge(\n\n /* Announcement bar */\n ...getComponentElements(\"announce\")\n .map(el => mountAnnounce(el)),\n\n /* Content */\n ...getComponentElements(\"content\")\n .map(el => mountContent(el, { viewport$, target$, print$ })),\n\n /* Search highlighting */\n ...getComponentElements(\"content\")\n .map(el => feature(\"search.highlight\")\n ? mountSearchHiglight(el, { index$, location$ })\n : EMPTY\n ),\n\n /* Header title */\n ...getComponentElements(\"header-title\")\n .map(el => mountHeaderTitle(el, { viewport$, header$ })),\n\n /* Sidebar */\n ...getComponentElements(\"sidebar\")\n .map(el => el.getAttribute(\"data-md-type\") === \"navigation\"\n ? at(screen$, () => mountSidebar(el, { viewport$, header$, main$ }))\n : at(tablet$, () => mountSidebar(el, { viewport$, header$, main$ }))\n ),\n\n /* Navigation tabs */\n ...getComponentElements(\"tabs\")\n .map(el => mountTabs(el, { viewport$, header$ })),\n\n /* Table of contents */\n ...getComponentElements(\"toc\")\n .map(el => mountTableOfContents(el, {\n viewport$, header$, main$, target$\n })),\n\n /* Back-to-top button */\n ...getComponentElements(\"top\")\n .map(el => mountBackToTop(el, { viewport$, header$, main$, target$ }))\n))\n\n/* Set up component observables */\nconst component$ = document$\n .pipe(\n switchMap(() => content$),\n mergeWith(control$),\n shareReplay(1)\n )\n\n/* Subscribe to all components */\ncomponent$.subscribe()\n\n/* ----------------------------------------------------------------------------\n * Exports\n * ------------------------------------------------------------------------- */\n\nwindow.document$ = document$ /* Document observable */\nwindow.location$ = location$ /* Location subject */\nwindow.target$ = target$ /* Location target observable */\nwindow.keyboard$ = keyboard$ /* Keyboard observable */\nwindow.viewport$ = viewport$ /* Viewport observable */\nwindow.tablet$ = tablet$ /* Media tablet observable */\nwindow.screen$ = screen$ /* Media screen observable */\nwindow.print$ = print$ /* Media print observable */\nwindow.alert$ = alert$ /* Alert subject */\nwindow.component$ = component$ /* Component observable */\n", "self.fetch||(self.fetch=function(e,n){return n=n||{},new Promise(function(t,r){var s=new XMLHttpRequest,o=[],u={},a=function e(){return{ok:2==(s.status/100|0),statusText:s.statusText,status:s.status,url:s.responseURL,text:function(){return Promise.resolve(s.responseText)},json:function(){return Promise.resolve(s.responseText).then(JSON.parse)},blob:function(){return Promise.resolve(new Blob([s.response]))},clone:e,headers:{keys:function(){return o},entries:function(){return o.map(function(e){return[e,s.getResponseHeader(e)]})},get:function(e){return s.getResponseHeader(e)},has:function(e){return null!=s.getResponseHeader(e)}}}};for(var i in s.open(n.method||\"get\",e,!0),s.onload=function(){s.getAllResponseHeaders().toLowerCase().replace(/^(.+?):/gm,function(e,n){u[n]||o.push(u[n]=n)}),t(a())},s.onerror=r,s.withCredentials=\"include\"==n.credentials,n.headers)s.setRequestHeader(i,n.headers[i]);s.send(n.body||null)})});\n", "/*! *****************************************************************************\r\nCopyright (c) Microsoft Corporation.\r\n\r\nPermission to use, copy, modify, and/or distribute this software for any\r\npurpose with or without fee is hereby granted.\r\n\r\nTHE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH\r\nREGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY\r\nAND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,\r\nINDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM\r\nLOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR\r\nOTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR\r\nPERFORMANCE OF THIS SOFTWARE.\r\n***************************************************************************** */\r\n/* global Reflect, Promise */\r\n\r\nvar extendStatics = function(d, b) {\r\n extendStatics = Object.setPrototypeOf ||\r\n ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||\r\n function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };\r\n return extendStatics(d, b);\r\n};\r\n\r\nexport function __extends(d, b) {\r\n if (typeof b !== \"function\" && b !== null)\r\n throw new TypeError(\"Class extends value \" + String(b) + \" is not a constructor or null\");\r\n extendStatics(d, b);\r\n function __() { this.constructor = d; }\r\n d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());\r\n}\r\n\r\nexport var __assign = function() {\r\n __assign = Object.assign || function __assign(t) {\r\n for (var s, i = 1, n = arguments.length; i < n; i++) {\r\n s = arguments[i];\r\n for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];\r\n }\r\n return t;\r\n }\r\n return __assign.apply(this, arguments);\r\n}\r\n\r\nexport function __rest(s, e) {\r\n var t = {};\r\n for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)\r\n t[p] = s[p];\r\n if (s != null && typeof Object.getOwnPropertySymbols === \"function\")\r\n for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {\r\n if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))\r\n t[p[i]] = s[p[i]];\r\n }\r\n return t;\r\n}\r\n\r\nexport function __decorate(decorators, target, key, desc) {\r\n var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;\r\n if (typeof Reflect === \"object\" && typeof Reflect.decorate === \"function\") r = Reflect.decorate(decorators, target, key, desc);\r\n else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;\r\n return c > 3 && r && Object.defineProperty(target, key, r), r;\r\n}\r\n\r\nexport function __param(paramIndex, decorator) {\r\n return function (target, key) { decorator(target, key, paramIndex); }\r\n}\r\n\r\nexport function __metadata(metadataKey, metadataValue) {\r\n if (typeof Reflect === \"object\" && typeof Reflect.metadata === \"function\") return Reflect.metadata(metadataKey, metadataValue);\r\n}\r\n\r\nexport function __awaiter(thisArg, _arguments, P, generator) {\r\n function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }\r\n return new (P || (P = Promise))(function (resolve, reject) {\r\n function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }\r\n function rejected(value) { try { step(generator[\"throw\"](value)); } catch (e) { reject(e); } }\r\n function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }\r\n step((generator = generator.apply(thisArg, _arguments || [])).next());\r\n });\r\n}\r\n\r\nexport function __generator(thisArg, body) {\r\n var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;\r\n return g = { next: verb(0), \"throw\": verb(1), \"return\": verb(2) }, typeof Symbol === \"function\" && (g[Symbol.iterator] = function() { return this; }), g;\r\n function verb(n) { return function (v) { return step([n, v]); }; }\r\n function step(op) {\r\n if (f) throw new TypeError(\"Generator is already executing.\");\r\n while (_) try {\r\n if (f = 1, y && (t = op[0] & 2 ? y[\"return\"] : op[0] ? y[\"throw\"] || ((t = y[\"return\"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;\r\n if (y = 0, t) op = [op[0] & 2, t.value];\r\n switch (op[0]) {\r\n case 0: case 1: t = op; break;\r\n case 4: _.label++; return { value: op[1], done: false };\r\n case 5: _.label++; y = op[1]; op = [0]; continue;\r\n case 7: op = _.ops.pop(); _.trys.pop(); continue;\r\n default:\r\n if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }\r\n if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }\r\n if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }\r\n if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }\r\n if (t[2]) _.ops.pop();\r\n _.trys.pop(); continue;\r\n }\r\n op = body.call(thisArg, _);\r\n } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }\r\n if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };\r\n }\r\n}\r\n\r\nexport var __createBinding = Object.create ? (function(o, m, k, k2) {\r\n if (k2 === undefined) k2 = k;\r\n Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });\r\n}) : (function(o, m, k, k2) {\r\n if (k2 === undefined) k2 = k;\r\n o[k2] = m[k];\r\n});\r\n\r\nexport function __exportStar(m, o) {\r\n for (var p in m) if (p !== \"default\" && !Object.prototype.hasOwnProperty.call(o, p)) __createBinding(o, m, p);\r\n}\r\n\r\nexport function __values(o) {\r\n var s = typeof Symbol === \"function\" && Symbol.iterator, m = s && o[s], i = 0;\r\n if (m) return m.call(o);\r\n if (o && typeof o.length === \"number\") return {\r\n next: function () {\r\n if (o && i >= o.length) o = void 0;\r\n return { value: o && o[i++], done: !o };\r\n }\r\n };\r\n throw new TypeError(s ? \"Object is not iterable.\" : \"Symbol.iterator is not defined.\");\r\n}\r\n\r\nexport function __read(o, n) {\r\n var m = typeof Symbol === \"function\" && o[Symbol.iterator];\r\n if (!m) return o;\r\n var i = m.call(o), r, ar = [], e;\r\n try {\r\n while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);\r\n }\r\n catch (error) { e = { error: error }; }\r\n finally {\r\n try {\r\n if (r && !r.done && (m = i[\"return\"])) m.call(i);\r\n }\r\n finally { if (e) throw e.error; }\r\n }\r\n return ar;\r\n}\r\n\r\n/** @deprecated */\r\nexport function __spread() {\r\n for (var ar = [], i = 0; i < arguments.length; i++)\r\n ar = ar.concat(__read(arguments[i]));\r\n return ar;\r\n}\r\n\r\n/** @deprecated */\r\nexport function __spreadArrays() {\r\n for (var s = 0, i = 0, il = arguments.length; i < il; i++) s += arguments[i].length;\r\n for (var r = Array(s), k = 0, i = 0; i < il; i++)\r\n for (var a = arguments[i], j = 0, jl = a.length; j < jl; j++, k++)\r\n r[k] = a[j];\r\n return r;\r\n}\r\n\r\nexport function __spreadArray(to, from, pack) {\r\n if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {\r\n if (ar || !(i in from)) {\r\n if (!ar) ar = Array.prototype.slice.call(from, 0, i);\r\n ar[i] = from[i];\r\n }\r\n }\r\n return to.concat(ar || Array.prototype.slice.call(from));\r\n}\r\n\r\nexport function __await(v) {\r\n return this instanceof __await ? (this.v = v, this) : new __await(v);\r\n}\r\n\r\nexport function __asyncGenerator(thisArg, _arguments, generator) {\r\n if (!Symbol.asyncIterator) throw new TypeError(\"Symbol.asyncIterator is not defined.\");\r\n var g = generator.apply(thisArg, _arguments || []), i, q = [];\r\n return i = {}, verb(\"next\"), verb(\"throw\"), verb(\"return\"), i[Symbol.asyncIterator] = function () { return this; }, i;\r\n function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }\r\n function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }\r\n function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }\r\n function fulfill(value) { resume(\"next\", value); }\r\n function reject(value) { resume(\"throw\", value); }\r\n function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }\r\n}\r\n\r\nexport function __asyncDelegator(o) {\r\n var i, p;\r\n return i = {}, verb(\"next\"), verb(\"throw\", function (e) { throw e; }), verb(\"return\"), i[Symbol.iterator] = function () { return this; }, i;\r\n function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === \"return\" } : f ? f(v) : v; } : f; }\r\n}\r\n\r\nexport function __asyncValues(o) {\r\n if (!Symbol.asyncIterator) throw new TypeError(\"Symbol.asyncIterator is not defined.\");\r\n var m = o[Symbol.asyncIterator], i;\r\n return m ? m.call(o) : (o = typeof __values === \"function\" ? __values(o) : o[Symbol.iterator](), i = {}, verb(\"next\"), verb(\"throw\"), verb(\"return\"), i[Symbol.asyncIterator] = function () { return this; }, i);\r\n function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }\r\n function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }\r\n}\r\n\r\nexport function __makeTemplateObject(cooked, raw) {\r\n if (Object.defineProperty) { Object.defineProperty(cooked, \"raw\", { value: raw }); } else { cooked.raw = raw; }\r\n return cooked;\r\n};\r\n\r\nvar __setModuleDefault = Object.create ? (function(o, v) {\r\n Object.defineProperty(o, \"default\", { enumerable: true, value: v });\r\n}) : function(o, v) {\r\n o[\"default\"] = v;\r\n};\r\n\r\nexport function __importStar(mod) {\r\n if (mod && mod.__esModule) return mod;\r\n var result = {};\r\n if (mod != null) for (var k in mod) if (k !== \"default\" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);\r\n __setModuleDefault(result, mod);\r\n return result;\r\n}\r\n\r\nexport function __importDefault(mod) {\r\n return (mod && mod.__esModule) ? mod : { default: mod };\r\n}\r\n\r\nexport function __classPrivateFieldGet(receiver, state, kind, f) {\r\n if (kind === \"a\" && !f) throw new TypeError(\"Private accessor was defined without a getter\");\r\n if (typeof state === \"function\" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError(\"Cannot read private member from an object whose class did not declare it\");\r\n return kind === \"m\" ? f : kind === \"a\" ? f.call(receiver) : f ? f.value : state.get(receiver);\r\n}\r\n\r\nexport function __classPrivateFieldSet(receiver, state, value, kind, f) {\r\n if (kind === \"m\") throw new TypeError(\"Private method is not writable\");\r\n if (kind === \"a\" && !f) throw new TypeError(\"Private accessor was defined without a setter\");\r\n if (typeof state === \"function\" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError(\"Cannot write private member to an object whose class did not declare it\");\r\n return (kind === \"a\" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;\r\n}\r\n", "/**\n * Returns true if the object is a function.\n * @param value The value to check\n */\nexport function isFunction(value: any): value is (...args: any[]) => any {\n return typeof value === 'function';\n}\n", "/**\n * Used to create Error subclasses until the community moves away from ES5.\n *\n * This is because compiling from TypeScript down to ES5 has issues with subclassing Errors\n * as well as other built-in types: https://github.com/Microsoft/TypeScript/issues/12123\n *\n * @param createImpl A factory function to create the actual constructor implementation. The returned\n * function should be a named function that calls `_super` internally.\n */\nexport function createErrorClass(createImpl: (_super: any) => any): T {\n const _super = (instance: any) => {\n Error.call(instance);\n instance.stack = new Error().stack;\n };\n\n const ctorFunc = createImpl(_super);\n ctorFunc.prototype = Object.create(Error.prototype);\n ctorFunc.prototype.constructor = ctorFunc;\n return ctorFunc;\n}\n", "import { createErrorClass } from './createErrorClass';\n\nexport interface UnsubscriptionError extends Error {\n readonly errors: any[];\n}\n\nexport interface UnsubscriptionErrorCtor {\n /**\n * @deprecated Internal implementation detail. Do not construct error instances.\n * Cannot be tagged as internal: https://github.com/ReactiveX/rxjs/issues/6269\n */\n new (errors: any[]): UnsubscriptionError;\n}\n\n/**\n * An error thrown when one or more errors have occurred during the\n * `unsubscribe` of a {@link Subscription}.\n */\nexport const UnsubscriptionError: UnsubscriptionErrorCtor = createErrorClass(\n (_super) =>\n function UnsubscriptionErrorImpl(this: any, errors: (Error | string)[]) {\n _super(this);\n this.message = errors\n ? `${errors.length} errors occurred during unsubscription:\n${errors.map((err, i) => `${i + 1}) ${err.toString()}`).join('\\n ')}`\n : '';\n this.name = 'UnsubscriptionError';\n this.errors = errors;\n }\n);\n", "/**\n * Removes an item from an array, mutating it.\n * @param arr The array to remove the item from\n * @param item The item to remove\n */\nexport function arrRemove(arr: T[] | undefined | null, item: T) {\n if (arr) {\n const index = arr.indexOf(item);\n 0 <= index && arr.splice(index, 1);\n }\n}\n", "import { isFunction } from './util/isFunction';\nimport { UnsubscriptionError } from './util/UnsubscriptionError';\nimport { SubscriptionLike, TeardownLogic, Unsubscribable } from './types';\nimport { arrRemove } from './util/arrRemove';\n\n/**\n * Represents a disposable resource, such as the execution of an Observable. A\n * Subscription has one important method, `unsubscribe`, that takes no argument\n * and just disposes the resource held by the subscription.\n *\n * Additionally, subscriptions may be grouped together through the `add()`\n * method, which will attach a child Subscription to the current Subscription.\n * When a Subscription is unsubscribed, all its children (and its grandchildren)\n * will be unsubscribed as well.\n *\n * @class Subscription\n */\nexport class Subscription implements SubscriptionLike {\n /** @nocollapse */\n public static EMPTY = (() => {\n const empty = new Subscription();\n empty.closed = true;\n return empty;\n })();\n\n /**\n * A flag to indicate whether this Subscription has already been unsubscribed.\n */\n public closed = false;\n\n private _parentage: Subscription[] | Subscription | null = null;\n\n /**\n * The list of registered finalizers to execute upon unsubscription. Adding and removing from this\n * list occurs in the {@link #add} and {@link #remove} methods.\n */\n private _finalizers: Exclude[] | null = null;\n\n /**\n * @param initialTeardown A function executed first as part of the finalization\n * process that is kicked off when {@link #unsubscribe} is called.\n */\n constructor(private initialTeardown?: () => void) {}\n\n /**\n * Disposes the resources held by the subscription. May, for instance, cancel\n * an ongoing Observable execution or cancel any other type of work that\n * started when the Subscription was created.\n * @return {void}\n */\n unsubscribe(): void {\n let errors: any[] | undefined;\n\n if (!this.closed) {\n this.closed = true;\n\n // Remove this from it's parents.\n const { _parentage } = this;\n if (_parentage) {\n this._parentage = null;\n if (Array.isArray(_parentage)) {\n for (const parent of _parentage) {\n parent.remove(this);\n }\n } else {\n _parentage.remove(this);\n }\n }\n\n const { initialTeardown: initialFinalizer } = this;\n if (isFunction(initialFinalizer)) {\n try {\n initialFinalizer();\n } catch (e) {\n errors = e instanceof UnsubscriptionError ? e.errors : [e];\n }\n }\n\n const { _finalizers } = this;\n if (_finalizers) {\n this._finalizers = null;\n for (const finalizer of _finalizers) {\n try {\n execFinalizer(finalizer);\n } catch (err) {\n errors = errors ?? [];\n if (err instanceof UnsubscriptionError) {\n errors = [...errors, ...err.errors];\n } else {\n errors.push(err);\n }\n }\n }\n }\n\n if (errors) {\n throw new UnsubscriptionError(errors);\n }\n }\n }\n\n /**\n * Adds a finalizer to this subscription, so that finalization will be unsubscribed/called\n * when this subscription is unsubscribed. If this subscription is already {@link #closed},\n * because it has already been unsubscribed, then whatever finalizer is passed to it\n * will automatically be executed (unless the finalizer itself is also a closed subscription).\n *\n * Closed Subscriptions cannot be added as finalizers to any subscription. Adding a closed\n * subscription to a any subscription will result in no operation. (A noop).\n *\n * Adding a subscription to itself, or adding `null` or `undefined` will not perform any\n * operation at all. (A noop).\n *\n * `Subscription` instances that are added to this instance will automatically remove themselves\n * if they are unsubscribed. Functions and {@link Unsubscribable} objects that you wish to remove\n * will need to be removed manually with {@link #remove}\n *\n * @param teardown The finalization logic to add to this subscription.\n */\n add(teardown: TeardownLogic): void {\n // Only add the finalizer if it's not undefined\n // and don't add a subscription to itself.\n if (teardown && teardown !== this) {\n if (this.closed) {\n // If this subscription is already closed,\n // execute whatever finalizer is handed to it automatically.\n execFinalizer(teardown);\n } else {\n if (teardown instanceof Subscription) {\n // We don't add closed subscriptions, and we don't add the same subscription\n // twice. Subscription unsubscribe is idempotent.\n if (teardown.closed || teardown._hasParent(this)) {\n return;\n }\n teardown._addParent(this);\n }\n (this._finalizers = this._finalizers ?? []).push(teardown);\n }\n }\n }\n\n /**\n * Checks to see if a this subscription already has a particular parent.\n * This will signal that this subscription has already been added to the parent in question.\n * @param parent the parent to check for\n */\n private _hasParent(parent: Subscription) {\n const { _parentage } = this;\n return _parentage === parent || (Array.isArray(_parentage) && _parentage.includes(parent));\n }\n\n /**\n * Adds a parent to this subscription so it can be removed from the parent if it\n * unsubscribes on it's own.\n *\n * NOTE: THIS ASSUMES THAT {@link _hasParent} HAS ALREADY BEEN CHECKED.\n * @param parent The parent subscription to add\n */\n private _addParent(parent: Subscription) {\n const { _parentage } = this;\n this._parentage = Array.isArray(_parentage) ? (_parentage.push(parent), _parentage) : _parentage ? [_parentage, parent] : parent;\n }\n\n /**\n * Called on a child when it is removed via {@link #remove}.\n * @param parent The parent to remove\n */\n private _removeParent(parent: Subscription) {\n const { _parentage } = this;\n if (_parentage === parent) {\n this._parentage = null;\n } else if (Array.isArray(_parentage)) {\n arrRemove(_parentage, parent);\n }\n }\n\n /**\n * Removes a finalizer from this subscription that was previously added with the {@link #add} method.\n *\n * Note that `Subscription` instances, when unsubscribed, will automatically remove themselves\n * from every other `Subscription` they have been added to. This means that using the `remove` method\n * is not a common thing and should be used thoughtfully.\n *\n * If you add the same finalizer instance of a function or an unsubscribable object to a `Subscription` instance\n * more than once, you will need to call `remove` the same number of times to remove all instances.\n *\n * All finalizer instances are removed to free up memory upon unsubscription.\n *\n * @param teardown The finalizer to remove from this subscription\n */\n remove(teardown: Exclude): void {\n const { _finalizers } = this;\n _finalizers && arrRemove(_finalizers, teardown);\n\n if (teardown instanceof Subscription) {\n teardown._removeParent(this);\n }\n }\n}\n\nexport const EMPTY_SUBSCRIPTION = Subscription.EMPTY;\n\nexport function isSubscription(value: any): value is Subscription {\n return (\n value instanceof Subscription ||\n (value && 'closed' in value && isFunction(value.remove) && isFunction(value.add) && isFunction(value.unsubscribe))\n );\n}\n\nfunction execFinalizer(finalizer: Unsubscribable | (() => void)) {\n if (isFunction(finalizer)) {\n finalizer();\n } else {\n finalizer.unsubscribe();\n }\n}\n", "import { Subscriber } from './Subscriber';\nimport { ObservableNotification } from './types';\n\n/**\n * The {@link GlobalConfig} object for RxJS. It is used to configure things\n * like how to react on unhandled errors.\n */\nexport const config: GlobalConfig = {\n onUnhandledError: null,\n onStoppedNotification: null,\n Promise: undefined,\n useDeprecatedSynchronousErrorHandling: false,\n useDeprecatedNextContext: false,\n};\n\n/**\n * The global configuration object for RxJS, used to configure things\n * like how to react on unhandled errors. Accessible via {@link config}\n * object.\n */\nexport interface GlobalConfig {\n /**\n * A registration point for unhandled errors from RxJS. These are errors that\n * cannot were not handled by consuming code in the usual subscription path. For\n * example, if you have this configured, and you subscribe to an observable without\n * providing an error handler, errors from that subscription will end up here. This\n * will _always_ be called asynchronously on another job in the runtime. This is because\n * we do not want errors thrown in this user-configured handler to interfere with the\n * behavior of the library.\n */\n onUnhandledError: ((err: any) => void) | null;\n\n /**\n * A registration point for notifications that cannot be sent to subscribers because they\n * have completed, errored or have been explicitly unsubscribed. By default, next, complete\n * and error notifications sent to stopped subscribers are noops. However, sometimes callers\n * might want a different behavior. For example, with sources that attempt to report errors\n * to stopped subscribers, a caller can configure RxJS to throw an unhandled error instead.\n * This will _always_ be called asynchronously on another job in the runtime. This is because\n * we do not want errors thrown in this user-configured handler to interfere with the\n * behavior of the library.\n */\n onStoppedNotification: ((notification: ObservableNotification, subscriber: Subscriber) => void) | null;\n\n /**\n * The promise constructor used by default for {@link Observable#toPromise toPromise} and {@link Observable#forEach forEach}\n * methods.\n *\n * @deprecated As of version 8, RxJS will no longer support this sort of injection of a\n * Promise constructor. If you need a Promise implementation other than native promises,\n * please polyfill/patch Promise as you see appropriate. Will be removed in v8.\n */\n Promise?: PromiseConstructorLike;\n\n /**\n * If true, turns on synchronous error rethrowing, which is a deprecated behavior\n * in v6 and higher. This behavior enables bad patterns like wrapping a subscribe\n * call in a try/catch block. It also enables producer interference, a nasty bug\n * where a multicast can be broken for all observers by a downstream consumer with\n * an unhandled error. DO NOT USE THIS FLAG UNLESS IT'S NEEDED TO BUY TIME\n * FOR MIGRATION REASONS.\n *\n * @deprecated As of version 8, RxJS will no longer support synchronous throwing\n * of unhandled errors. All errors will be thrown on a separate call stack to prevent bad\n * behaviors described above. Will be removed in v8.\n */\n useDeprecatedSynchronousErrorHandling: boolean;\n\n /**\n * If true, enables an as-of-yet undocumented feature from v5: The ability to access\n * `unsubscribe()` via `this` context in `next` functions created in observers passed\n * to `subscribe`.\n *\n * This is being removed because the performance was severely problematic, and it could also cause\n * issues when types other than POJOs are passed to subscribe as subscribers, as they will likely have\n * their `this` context overwritten.\n *\n * @deprecated As of version 8, RxJS will no longer support altering the\n * context of next functions provided as part of an observer to Subscribe. Instead,\n * you will have access to a subscription or a signal or token that will allow you to do things like\n * unsubscribe and test closed status. Will be removed in v8.\n */\n useDeprecatedNextContext: boolean;\n}\n", "import type { TimerHandle } from './timerHandle';\ntype SetTimeoutFunction = (handler: () => void, timeout?: number, ...args: any[]) => TimerHandle;\ntype ClearTimeoutFunction = (handle: TimerHandle) => void;\n\ninterface TimeoutProvider {\n setTimeout: SetTimeoutFunction;\n clearTimeout: ClearTimeoutFunction;\n delegate:\n | {\n setTimeout: SetTimeoutFunction;\n clearTimeout: ClearTimeoutFunction;\n }\n | undefined;\n}\n\nexport const timeoutProvider: TimeoutProvider = {\n // When accessing the delegate, use the variable rather than `this` so that\n // the functions can be called without being bound to the provider.\n setTimeout(handler: () => void, timeout?: number, ...args) {\n const { delegate } = timeoutProvider;\n if (delegate?.setTimeout) {\n return delegate.setTimeout(handler, timeout, ...args);\n }\n return setTimeout(handler, timeout, ...args);\n },\n clearTimeout(handle) {\n const { delegate } = timeoutProvider;\n return (delegate?.clearTimeout || clearTimeout)(handle as any);\n },\n delegate: undefined,\n};\n", "import { config } from '../config';\nimport { timeoutProvider } from '../scheduler/timeoutProvider';\n\n/**\n * Handles an error on another job either with the user-configured {@link onUnhandledError},\n * or by throwing it on that new job so it can be picked up by `window.onerror`, `process.on('error')`, etc.\n *\n * This should be called whenever there is an error that is out-of-band with the subscription\n * or when an error hits a terminal boundary of the subscription and no error handler was provided.\n *\n * @param err the error to report\n */\nexport function reportUnhandledError(err: any) {\n timeoutProvider.setTimeout(() => {\n const { onUnhandledError } = config;\n if (onUnhandledError) {\n // Execute the user-configured error handler.\n onUnhandledError(err);\n } else {\n // Throw so it is picked up by the runtime's uncaught error mechanism.\n throw err;\n }\n });\n}\n", "/* tslint:disable:no-empty */\nexport function noop() { }\n", "import { CompleteNotification, NextNotification, ErrorNotification } from './types';\n\n/**\n * A completion object optimized for memory use and created to be the\n * same \"shape\" as other notifications in v8.\n * @internal\n */\nexport const COMPLETE_NOTIFICATION = (() => createNotification('C', undefined, undefined) as CompleteNotification)();\n\n/**\n * Internal use only. Creates an optimized error notification that is the same \"shape\"\n * as other notifications.\n * @internal\n */\nexport function errorNotification(error: any): ErrorNotification {\n return createNotification('E', undefined, error) as any;\n}\n\n/**\n * Internal use only. Creates an optimized next notification that is the same \"shape\"\n * as other notifications.\n * @internal\n */\nexport function nextNotification(value: T) {\n return createNotification('N', value, undefined) as NextNotification;\n}\n\n/**\n * Ensures that all notifications created internally have the same \"shape\" in v8.\n *\n * TODO: This is only exported to support a crazy legacy test in `groupBy`.\n * @internal\n */\nexport function createNotification(kind: 'N' | 'E' | 'C', value: any, error: any) {\n return {\n kind,\n value,\n error,\n };\n}\n", "import { config } from '../config';\n\nlet context: { errorThrown: boolean; error: any } | null = null;\n\n/**\n * Handles dealing with errors for super-gross mode. Creates a context, in which\n * any synchronously thrown errors will be passed to {@link captureError}. Which\n * will record the error such that it will be rethrown after the call back is complete.\n * TODO: Remove in v8\n * @param cb An immediately executed function.\n */\nexport function errorContext(cb: () => void) {\n if (config.useDeprecatedSynchronousErrorHandling) {\n const isRoot = !context;\n if (isRoot) {\n context = { errorThrown: false, error: null };\n }\n cb();\n if (isRoot) {\n const { errorThrown, error } = context!;\n context = null;\n if (errorThrown) {\n throw error;\n }\n }\n } else {\n // This is the general non-deprecated path for everyone that\n // isn't crazy enough to use super-gross mode (useDeprecatedSynchronousErrorHandling)\n cb();\n }\n}\n\n/**\n * Captures errors only in super-gross mode.\n * @param err the error to capture\n */\nexport function captureError(err: any) {\n if (config.useDeprecatedSynchronousErrorHandling && context) {\n context.errorThrown = true;\n context.error = err;\n }\n}\n", "import { isFunction } from './util/isFunction';\nimport { Observer, ObservableNotification } from './types';\nimport { isSubscription, Subscription } from './Subscription';\nimport { config } from './config';\nimport { reportUnhandledError } from './util/reportUnhandledError';\nimport { noop } from './util/noop';\nimport { nextNotification, errorNotification, COMPLETE_NOTIFICATION } from './NotificationFactories';\nimport { timeoutProvider } from './scheduler/timeoutProvider';\nimport { captureError } from './util/errorContext';\n\n/**\n * Implements the {@link Observer} interface and extends the\n * {@link Subscription} class. While the {@link Observer} is the public API for\n * consuming the values of an {@link Observable}, all Observers get converted to\n * a Subscriber, in order to provide Subscription-like capabilities such as\n * `unsubscribe`. Subscriber is a common type in RxJS, and crucial for\n * implementing operators, but it is rarely used as a public API.\n *\n * @class Subscriber\n */\nexport class Subscriber extends Subscription implements Observer {\n /**\n * A static factory for a Subscriber, given a (potentially partial) definition\n * of an Observer.\n * @param next The `next` callback of an Observer.\n * @param error The `error` callback of an\n * Observer.\n * @param complete The `complete` callback of an\n * Observer.\n * @return A Subscriber wrapping the (partially defined)\n * Observer represented by the given arguments.\n * @nocollapse\n * @deprecated Do not use. Will be removed in v8. There is no replacement for this\n * method, and there is no reason to be creating instances of `Subscriber` directly.\n * If you have a specific use case, please file an issue.\n */\n static create(next?: (x?: T) => void, error?: (e?: any) => void, complete?: () => void): Subscriber {\n return new SafeSubscriber(next, error, complete);\n }\n\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n protected isStopped: boolean = false;\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n protected destination: Subscriber | Observer; // this `any` is the escape hatch to erase extra type param (e.g. R)\n\n /**\n * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n * There is no reason to directly create an instance of Subscriber. This type is exported for typings reasons.\n */\n constructor(destination?: Subscriber | Observer) {\n super();\n if (destination) {\n this.destination = destination;\n // Automatically chain subscriptions together here.\n // if destination is a Subscription, then it is a Subscriber.\n if (isSubscription(destination)) {\n destination.add(this);\n }\n } else {\n this.destination = EMPTY_OBSERVER;\n }\n }\n\n /**\n * The {@link Observer} callback to receive notifications of type `next` from\n * the Observable, with a value. The Observable may call this method 0 or more\n * times.\n * @param {T} [value] The `next` value.\n * @return {void}\n */\n next(value?: T): void {\n if (this.isStopped) {\n handleStoppedNotification(nextNotification(value), this);\n } else {\n this._next(value!);\n }\n }\n\n /**\n * The {@link Observer} callback to receive notifications of type `error` from\n * the Observable, with an attached `Error`. Notifies the Observer that\n * the Observable has experienced an error condition.\n * @param {any} [err] The `error` exception.\n * @return {void}\n */\n error(err?: any): void {\n if (this.isStopped) {\n handleStoppedNotification(errorNotification(err), this);\n } else {\n this.isStopped = true;\n this._error(err);\n }\n }\n\n /**\n * The {@link Observer} callback to receive a valueless notification of type\n * `complete` from the Observable. Notifies the Observer that the Observable\n * has finished sending push-based notifications.\n * @return {void}\n */\n complete(): void {\n if (this.isStopped) {\n handleStoppedNotification(COMPLETE_NOTIFICATION, this);\n } else {\n this.isStopped = true;\n this._complete();\n }\n }\n\n unsubscribe(): void {\n if (!this.closed) {\n this.isStopped = true;\n super.unsubscribe();\n this.destination = null!;\n }\n }\n\n protected _next(value: T): void {\n this.destination.next(value);\n }\n\n protected _error(err: any): void {\n try {\n this.destination.error(err);\n } finally {\n this.unsubscribe();\n }\n }\n\n protected _complete(): void {\n try {\n this.destination.complete();\n } finally {\n this.unsubscribe();\n }\n }\n}\n\n/**\n * This bind is captured here because we want to be able to have\n * compatibility with monoid libraries that tend to use a method named\n * `bind`. In particular, a library called Monio requires this.\n */\nconst _bind = Function.prototype.bind;\n\nfunction bind any>(fn: Fn, thisArg: any): Fn {\n return _bind.call(fn, thisArg);\n}\n\n/**\n * Internal optimization only, DO NOT EXPOSE.\n * @internal\n */\nclass ConsumerObserver implements Observer {\n constructor(private partialObserver: Partial>) {}\n\n next(value: T): void {\n const { partialObserver } = this;\n if (partialObserver.next) {\n try {\n partialObserver.next(value);\n } catch (error) {\n handleUnhandledError(error);\n }\n }\n }\n\n error(err: any): void {\n const { partialObserver } = this;\n if (partialObserver.error) {\n try {\n partialObserver.error(err);\n } catch (error) {\n handleUnhandledError(error);\n }\n } else {\n handleUnhandledError(err);\n }\n }\n\n complete(): void {\n const { partialObserver } = this;\n if (partialObserver.complete) {\n try {\n partialObserver.complete();\n } catch (error) {\n handleUnhandledError(error);\n }\n }\n }\n}\n\nexport class SafeSubscriber extends Subscriber {\n constructor(\n observerOrNext?: Partial> | ((value: T) => void) | null,\n error?: ((e?: any) => void) | null,\n complete?: (() => void) | null\n ) {\n super();\n\n let partialObserver: Partial>;\n if (isFunction(observerOrNext) || !observerOrNext) {\n // The first argument is a function, not an observer. The next\n // two arguments *could* be observers, or they could be empty.\n partialObserver = {\n next: (observerOrNext ?? undefined) as (((value: T) => void) | undefined),\n error: error ?? undefined,\n complete: complete ?? undefined,\n };\n } else {\n // The first argument is a partial observer.\n let context: any;\n if (this && config.useDeprecatedNextContext) {\n // This is a deprecated path that made `this.unsubscribe()` available in\n // next handler functions passed to subscribe. This only exists behind a flag\n // now, as it is *very* slow.\n context = Object.create(observerOrNext);\n context.unsubscribe = () => this.unsubscribe();\n partialObserver = {\n next: observerOrNext.next && bind(observerOrNext.next, context),\n error: observerOrNext.error && bind(observerOrNext.error, context),\n complete: observerOrNext.complete && bind(observerOrNext.complete, context),\n };\n } else {\n // The \"normal\" path. Just use the partial observer directly.\n partialObserver = observerOrNext;\n }\n }\n\n // Wrap the partial observer to ensure it's a full observer, and\n // make sure proper error handling is accounted for.\n this.destination = new ConsumerObserver(partialObserver);\n }\n}\n\nfunction handleUnhandledError(error: any) {\n if (config.useDeprecatedSynchronousErrorHandling) {\n captureError(error);\n } else {\n // Ideal path, we report this as an unhandled error,\n // which is thrown on a new call stack.\n reportUnhandledError(error);\n }\n}\n\n/**\n * An error handler used when no error handler was supplied\n * to the SafeSubscriber -- meaning no error handler was supplied\n * do the `subscribe` call on our observable.\n * @param err The error to handle\n */\nfunction defaultErrorHandler(err: any) {\n throw err;\n}\n\n/**\n * A handler for notifications that cannot be sent to a stopped subscriber.\n * @param notification The notification being sent\n * @param subscriber The stopped subscriber\n */\nfunction handleStoppedNotification(notification: ObservableNotification, subscriber: Subscriber) {\n const { onStoppedNotification } = config;\n onStoppedNotification && timeoutProvider.setTimeout(() => onStoppedNotification(notification, subscriber));\n}\n\n/**\n * The observer used as a stub for subscriptions where the user did not\n * pass any arguments to `subscribe`. Comes with the default error handling\n * behavior.\n */\nexport const EMPTY_OBSERVER: Readonly> & { closed: true } = {\n closed: true,\n next: noop,\n error: defaultErrorHandler,\n complete: noop,\n};\n", "/**\n * Symbol.observable or a string \"@@observable\". Used for interop\n *\n * @deprecated We will no longer be exporting this symbol in upcoming versions of RxJS.\n * Instead polyfill and use Symbol.observable directly *or* use https://www.npmjs.com/package/symbol-observable\n */\nexport const observable: string | symbol = (() => (typeof Symbol === 'function' && Symbol.observable) || '@@observable')();\n", "/**\n * This function takes one parameter and just returns it. Simply put,\n * this is like `(x: T): T => x`.\n *\n * ## Examples\n *\n * This is useful in some cases when using things like `mergeMap`\n *\n * ```ts\n * import { interval, take, map, range, mergeMap, identity } from 'rxjs';\n *\n * const source$ = interval(1000).pipe(take(5));\n *\n * const result$ = source$.pipe(\n * map(i => range(i)),\n * mergeMap(identity) // same as mergeMap(x => x)\n * );\n *\n * result$.subscribe({\n * next: console.log\n * });\n * ```\n *\n * Or when you want to selectively apply an operator\n *\n * ```ts\n * import { interval, take, identity } from 'rxjs';\n *\n * const shouldLimit = () => Math.random() < 0.5;\n *\n * const source$ = interval(1000);\n *\n * const result$ = source$.pipe(shouldLimit() ? take(5) : identity);\n *\n * result$.subscribe({\n * next: console.log\n * });\n * ```\n *\n * @param x Any value that is returned by this function\n * @returns The value passed as the first parameter to this function\n */\nexport function identity(x: T): T {\n return x;\n}\n", "import { identity } from './identity';\nimport { UnaryFunction } from '../types';\n\nexport function pipe(): typeof identity;\nexport function pipe(fn1: UnaryFunction): UnaryFunction;\nexport function pipe(fn1: UnaryFunction, fn2: UnaryFunction): UnaryFunction;\nexport function pipe(fn1: UnaryFunction, fn2: UnaryFunction, fn3: UnaryFunction): UnaryFunction;\nexport function pipe(\n fn1: UnaryFunction,\n fn2: UnaryFunction,\n fn3: UnaryFunction,\n fn4: UnaryFunction\n): UnaryFunction;\nexport function pipe(\n fn1: UnaryFunction,\n fn2: UnaryFunction,\n fn3: UnaryFunction,\n fn4: UnaryFunction,\n fn5: UnaryFunction\n): UnaryFunction;\nexport function pipe(\n fn1: UnaryFunction,\n fn2: UnaryFunction,\n fn3: UnaryFunction,\n fn4: UnaryFunction,\n fn5: UnaryFunction,\n fn6: UnaryFunction\n): UnaryFunction;\nexport function pipe(\n fn1: UnaryFunction,\n fn2: UnaryFunction,\n fn3: UnaryFunction,\n fn4: UnaryFunction,\n fn5: UnaryFunction,\n fn6: UnaryFunction,\n fn7: UnaryFunction\n): UnaryFunction;\nexport function pipe(\n fn1: UnaryFunction,\n fn2: UnaryFunction,\n fn3: UnaryFunction,\n fn4: UnaryFunction,\n fn5: UnaryFunction,\n fn6: UnaryFunction,\n fn7: UnaryFunction,\n fn8: UnaryFunction\n): UnaryFunction;\nexport function pipe(\n fn1: UnaryFunction,\n fn2: UnaryFunction,\n fn3: UnaryFunction,\n fn4: UnaryFunction,\n fn5: UnaryFunction,\n fn6: UnaryFunction,\n fn7: UnaryFunction,\n fn8: UnaryFunction,\n fn9: UnaryFunction\n): UnaryFunction;\nexport function pipe(\n fn1: UnaryFunction,\n fn2: UnaryFunction,\n fn3: UnaryFunction,\n fn4: UnaryFunction,\n fn5: UnaryFunction,\n fn6: UnaryFunction,\n fn7: UnaryFunction,\n fn8: UnaryFunction,\n fn9: UnaryFunction,\n ...fns: UnaryFunction[]\n): UnaryFunction;\n\n/**\n * pipe() can be called on one or more functions, each of which can take one argument (\"UnaryFunction\")\n * and uses it to return a value.\n * It returns a function that takes one argument, passes it to the first UnaryFunction, and then\n * passes the result to the next one, passes that result to the next one, and so on. \n */\nexport function pipe(...fns: Array>): UnaryFunction {\n return pipeFromArray(fns);\n}\n\n/** @internal */\nexport function pipeFromArray(fns: Array>): UnaryFunction {\n if (fns.length === 0) {\n return identity as UnaryFunction;\n }\n\n if (fns.length === 1) {\n return fns[0];\n }\n\n return function piped(input: T): R {\n return fns.reduce((prev: any, fn: UnaryFunction) => fn(prev), input as any);\n };\n}\n", "import { Operator } from './Operator';\nimport { SafeSubscriber, Subscriber } from './Subscriber';\nimport { isSubscription, Subscription } from './Subscription';\nimport { TeardownLogic, OperatorFunction, Subscribable, Observer } from './types';\nimport { observable as Symbol_observable } from './symbol/observable';\nimport { pipeFromArray } from './util/pipe';\nimport { config } from './config';\nimport { isFunction } from './util/isFunction';\nimport { errorContext } from './util/errorContext';\n\n/**\n * A representation of any set of values over any amount of time. This is the most basic building block\n * of RxJS.\n *\n * @class Observable\n */\nexport class Observable implements Subscribable {\n /**\n * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n */\n source: Observable | undefined;\n\n /**\n * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n */\n operator: Operator | undefined;\n\n /**\n * @constructor\n * @param {Function} subscribe the function that is called when the Observable is\n * initially subscribed to. This function is given a Subscriber, to which new values\n * can be `next`ed, or an `error` method can be called to raise an error, or\n * `complete` can be called to notify of a successful completion.\n */\n constructor(subscribe?: (this: Observable, subscriber: Subscriber) => TeardownLogic) {\n if (subscribe) {\n this._subscribe = subscribe;\n }\n }\n\n // HACK: Since TypeScript inherits static properties too, we have to\n // fight against TypeScript here so Subject can have a different static create signature\n /**\n * Creates a new Observable by calling the Observable constructor\n * @owner Observable\n * @method create\n * @param {Function} subscribe? the subscriber function to be passed to the Observable constructor\n * @return {Observable} a new observable\n * @nocollapse\n * @deprecated Use `new Observable()` instead. Will be removed in v8.\n */\n static create: (...args: any[]) => any = (subscribe?: (subscriber: Subscriber) => TeardownLogic) => {\n return new Observable(subscribe);\n };\n\n /**\n * Creates a new Observable, with this Observable instance as the source, and the passed\n * operator defined as the new observable's operator.\n * @method lift\n * @param operator the operator defining the operation to take on the observable\n * @return a new observable with the Operator applied\n * @deprecated Internal implementation detail, do not use directly. Will be made internal in v8.\n * If you have implemented an operator using `lift`, it is recommended that you create an\n * operator by simply returning `new Observable()` directly. See \"Creating new operators from\n * scratch\" section here: https://rxjs.dev/guide/operators\n */\n lift(operator?: Operator): Observable {\n const observable = new Observable();\n observable.source = this;\n observable.operator = operator;\n return observable;\n }\n\n subscribe(observerOrNext?: Partial> | ((value: T) => void)): Subscription;\n /** @deprecated Instead of passing separate callback arguments, use an observer argument. Signatures taking separate callback arguments will be removed in v8. Details: https://rxjs.dev/deprecations/subscribe-arguments */\n subscribe(next?: ((value: T) => void) | null, error?: ((error: any) => void) | null, complete?: (() => void) | null): Subscription;\n /**\n * Invokes an execution of an Observable and registers Observer handlers for notifications it will emit.\n *\n * Use it when you have all these Observables, but still nothing is happening.\n *\n * `subscribe` is not a regular operator, but a method that calls Observable's internal `subscribe` function. It\n * might be for example a function that you passed to Observable's constructor, but most of the time it is\n * a library implementation, which defines what will be emitted by an Observable, and when it be will emitted. This means\n * that calling `subscribe` is actually the moment when Observable starts its work, not when it is created, as it is often\n * the thought.\n *\n * Apart from starting the execution of an Observable, this method allows you to listen for values\n * that an Observable emits, as well as for when it completes or errors. You can achieve this in two\n * of the following ways.\n *\n * The first way is creating an object that implements {@link Observer} interface. It should have methods\n * defined by that interface, but note that it should be just a regular JavaScript object, which you can create\n * yourself in any way you want (ES6 class, classic function constructor, object literal etc.). In particular, do\n * not attempt to use any RxJS implementation details to create Observers - you don't need them. Remember also\n * that your object does not have to implement all methods. If you find yourself creating a method that doesn't\n * do anything, you can simply omit it. Note however, if the `error` method is not provided and an error happens,\n * it will be thrown asynchronously. Errors thrown asynchronously cannot be caught using `try`/`catch`. Instead,\n * use the {@link onUnhandledError} configuration option or use a runtime handler (like `window.onerror` or\n * `process.on('error)`) to be notified of unhandled errors. Because of this, it's recommended that you provide\n * an `error` method to avoid missing thrown errors.\n *\n * The second way is to give up on Observer object altogether and simply provide callback functions in place of its methods.\n * This means you can provide three functions as arguments to `subscribe`, where the first function is equivalent\n * of a `next` method, the second of an `error` method and the third of a `complete` method. Just as in case of an Observer,\n * if you do not need to listen for something, you can omit a function by passing `undefined` or `null`,\n * since `subscribe` recognizes these functions by where they were placed in function call. When it comes\n * to the `error` function, as with an Observer, if not provided, errors emitted by an Observable will be thrown asynchronously.\n *\n * You can, however, subscribe with no parameters at all. This may be the case where you're not interested in terminal events\n * and you also handled emissions internally by using operators (e.g. using `tap`).\n *\n * Whichever style of calling `subscribe` you use, in both cases it returns a Subscription object.\n * This object allows you to call `unsubscribe` on it, which in turn will stop the work that an Observable does and will clean\n * up all resources that an Observable used. Note that cancelling a subscription will not call `complete` callback\n * provided to `subscribe` function, which is reserved for a regular completion signal that comes from an Observable.\n *\n * Remember that callbacks provided to `subscribe` are not guaranteed to be called asynchronously.\n * It is an Observable itself that decides when these functions will be called. For example {@link of}\n * by default emits all its values synchronously. Always check documentation for how given Observable\n * will behave when subscribed and if its default behavior can be modified with a `scheduler`.\n *\n * #### Examples\n *\n * Subscribe with an {@link guide/observer Observer}\n *\n * ```ts\n * import { of } from 'rxjs';\n *\n * const sumObserver = {\n * sum: 0,\n * next(value) {\n * console.log('Adding: ' + value);\n * this.sum = this.sum + value;\n * },\n * error() {\n * // We actually could just remove this method,\n * // since we do not really care about errors right now.\n * },\n * complete() {\n * console.log('Sum equals: ' + this.sum);\n * }\n * };\n *\n * of(1, 2, 3) // Synchronously emits 1, 2, 3 and then completes.\n * .subscribe(sumObserver);\n *\n * // Logs:\n * // 'Adding: 1'\n * // 'Adding: 2'\n * // 'Adding: 3'\n * // 'Sum equals: 6'\n * ```\n *\n * Subscribe with functions ({@link deprecations/subscribe-arguments deprecated})\n *\n * ```ts\n * import { of } from 'rxjs'\n *\n * let sum = 0;\n *\n * of(1, 2, 3).subscribe(\n * value => {\n * console.log('Adding: ' + value);\n * sum = sum + value;\n * },\n * undefined,\n * () => console.log('Sum equals: ' + sum)\n * );\n *\n * // Logs:\n * // 'Adding: 1'\n * // 'Adding: 2'\n * // 'Adding: 3'\n * // 'Sum equals: 6'\n * ```\n *\n * Cancel a subscription\n *\n * ```ts\n * import { interval } from 'rxjs';\n *\n * const subscription = interval(1000).subscribe({\n * next(num) {\n * console.log(num)\n * },\n * complete() {\n * // Will not be called, even when cancelling subscription.\n * console.log('completed!');\n * }\n * });\n *\n * setTimeout(() => {\n * subscription.unsubscribe();\n * console.log('unsubscribed!');\n * }, 2500);\n *\n * // Logs:\n * // 0 after 1s\n * // 1 after 2s\n * // 'unsubscribed!' after 2.5s\n * ```\n *\n * @param {Observer|Function} observerOrNext (optional) Either an observer with methods to be called,\n * or the first of three possible handlers, which is the handler for each value emitted from the subscribed\n * Observable.\n * @param {Function} error (optional) A handler for a terminal event resulting from an error. If no error handler is provided,\n * the error will be thrown asynchronously as unhandled.\n * @param {Function} complete (optional) A handler for a terminal event resulting from successful completion.\n * @return {Subscription} a subscription reference to the registered handlers\n * @method subscribe\n */\n subscribe(\n observerOrNext?: Partial> | ((value: T) => void) | null,\n error?: ((error: any) => void) | null,\n complete?: (() => void) | null\n ): Subscription {\n const subscriber = isSubscriber(observerOrNext) ? observerOrNext : new SafeSubscriber(observerOrNext, error, complete);\n\n errorContext(() => {\n const { operator, source } = this;\n subscriber.add(\n operator\n ? // We're dealing with a subscription in the\n // operator chain to one of our lifted operators.\n operator.call(subscriber, source)\n : source\n ? // If `source` has a value, but `operator` does not, something that\n // had intimate knowledge of our API, like our `Subject`, must have\n // set it. We're going to just call `_subscribe` directly.\n this._subscribe(subscriber)\n : // In all other cases, we're likely wrapping a user-provided initializer\n // function, so we need to catch errors and handle them appropriately.\n this._trySubscribe(subscriber)\n );\n });\n\n return subscriber;\n }\n\n /** @internal */\n protected _trySubscribe(sink: Subscriber): TeardownLogic {\n try {\n return this._subscribe(sink);\n } catch (err) {\n // We don't need to return anything in this case,\n // because it's just going to try to `add()` to a subscription\n // above.\n sink.error(err);\n }\n }\n\n /**\n * Used as a NON-CANCELLABLE means of subscribing to an observable, for use with\n * APIs that expect promises, like `async/await`. You cannot unsubscribe from this.\n *\n * **WARNING**: Only use this with observables you *know* will complete. If the source\n * observable does not complete, you will end up with a promise that is hung up, and\n * potentially all of the state of an async function hanging out in memory. To avoid\n * this situation, look into adding something like {@link timeout}, {@link take},\n * {@link takeWhile}, or {@link takeUntil} amongst others.\n *\n * #### Example\n *\n * ```ts\n * import { interval, take } from 'rxjs';\n *\n * const source$ = interval(1000).pipe(take(4));\n *\n * async function getTotal() {\n * let total = 0;\n *\n * await source$.forEach(value => {\n * total += value;\n * console.log('observable -> ' + value);\n * });\n *\n * return total;\n * }\n *\n * getTotal().then(\n * total => console.log('Total: ' + total)\n * );\n *\n * // Expected:\n * // 'observable -> 0'\n * // 'observable -> 1'\n * // 'observable -> 2'\n * // 'observable -> 3'\n * // 'Total: 6'\n * ```\n *\n * @param next a handler for each value emitted by the observable\n * @return a promise that either resolves on observable completion or\n * rejects with the handled error\n */\n forEach(next: (value: T) => void): Promise;\n\n /**\n * @param next a handler for each value emitted by the observable\n * @param promiseCtor a constructor function used to instantiate the Promise\n * @return a promise that either resolves on observable completion or\n * rejects with the handled error\n * @deprecated Passing a Promise constructor will no longer be available\n * in upcoming versions of RxJS. This is because it adds weight to the library, for very\n * little benefit. If you need this functionality, it is recommended that you either\n * polyfill Promise, or you create an adapter to convert the returned native promise\n * to whatever promise implementation you wanted. Will be removed in v8.\n */\n forEach(next: (value: T) => void, promiseCtor: PromiseConstructorLike): Promise;\n\n forEach(next: (value: T) => void, promiseCtor?: PromiseConstructorLike): Promise {\n promiseCtor = getPromiseCtor(promiseCtor);\n\n return new promiseCtor((resolve, reject) => {\n const subscriber = new SafeSubscriber({\n next: (value) => {\n try {\n next(value);\n } catch (err) {\n reject(err);\n subscriber.unsubscribe();\n }\n },\n error: reject,\n complete: resolve,\n });\n this.subscribe(subscriber);\n }) as Promise;\n }\n\n /** @internal */\n protected _subscribe(subscriber: Subscriber): TeardownLogic {\n return this.source?.subscribe(subscriber);\n }\n\n /**\n * An interop point defined by the es7-observable spec https://github.com/zenparsing/es-observable\n * @method Symbol.observable\n * @return {Observable} this instance of the observable\n */\n [Symbol_observable]() {\n return this;\n }\n\n /* tslint:disable:max-line-length */\n pipe(): Observable;\n pipe(op1: OperatorFunction): Observable;\n pipe(op1: OperatorFunction, op2: OperatorFunction): Observable;\n pipe(op1: OperatorFunction, op2: OperatorFunction, op3: OperatorFunction): Observable;\n pipe(\n op1: OperatorFunction,\n op2: OperatorFunction,\n op3: OperatorFunction,\n op4: OperatorFunction\n ): Observable;\n pipe(\n op1: OperatorFunction,\n op2: OperatorFunction,\n op3: OperatorFunction,\n op4: OperatorFunction,\n op5: OperatorFunction\n ): Observable;\n pipe(\n op1: OperatorFunction,\n op2: OperatorFunction,\n op3: OperatorFunction,\n op4: OperatorFunction,\n op5: OperatorFunction,\n op6: OperatorFunction\n ): Observable;\n pipe(\n op1: OperatorFunction,\n op2: OperatorFunction,\n op3: OperatorFunction,\n op4: OperatorFunction,\n op5: OperatorFunction,\n op6: OperatorFunction,\n op7: OperatorFunction\n ): Observable;\n pipe(\n op1: OperatorFunction,\n op2: OperatorFunction,\n op3: OperatorFunction,\n op4: OperatorFunction,\n op5: OperatorFunction,\n op6: OperatorFunction,\n op7: OperatorFunction,\n op8: OperatorFunction\n ): Observable;\n pipe(\n op1: OperatorFunction,\n op2: OperatorFunction,\n op3: OperatorFunction,\n op4: OperatorFunction,\n op5: OperatorFunction,\n op6: OperatorFunction,\n op7: OperatorFunction,\n op8: OperatorFunction,\n op9: OperatorFunction\n ): Observable;\n pipe(\n op1: OperatorFunction,\n op2: OperatorFunction,\n op3: OperatorFunction,\n op4: OperatorFunction,\n op5: OperatorFunction,\n op6: OperatorFunction,\n op7: OperatorFunction,\n op8: OperatorFunction,\n op9: OperatorFunction,\n ...operations: OperatorFunction[]\n ): Observable;\n /* tslint:enable:max-line-length */\n\n /**\n * Used to stitch together functional operators into a chain.\n * @method pipe\n * @return {Observable} the Observable result of all of the operators having\n * been called in the order they were passed in.\n *\n * ## Example\n *\n * ```ts\n * import { interval, filter, map, scan } from 'rxjs';\n *\n * interval(1000)\n * .pipe(\n * filter(x => x % 2 === 0),\n * map(x => x + x),\n * scan((acc, x) => acc + x)\n * )\n * .subscribe(x => console.log(x));\n * ```\n */\n pipe(...operations: OperatorFunction[]): Observable {\n return pipeFromArray(operations)(this);\n }\n\n /* tslint:disable:max-line-length */\n /** @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise */\n toPromise(): Promise;\n /** @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise */\n toPromise(PromiseCtor: typeof Promise): Promise;\n /** @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise */\n toPromise(PromiseCtor: PromiseConstructorLike): Promise;\n /* tslint:enable:max-line-length */\n\n /**\n * Subscribe to this Observable and get a Promise resolving on\n * `complete` with the last emission (if any).\n *\n * **WARNING**: Only use this with observables you *know* will complete. If the source\n * observable does not complete, you will end up with a promise that is hung up, and\n * potentially all of the state of an async function hanging out in memory. To avoid\n * this situation, look into adding something like {@link timeout}, {@link take},\n * {@link takeWhile}, or {@link takeUntil} amongst others.\n *\n * @method toPromise\n * @param [promiseCtor] a constructor function used to instantiate\n * the Promise\n * @return A Promise that resolves with the last value emit, or\n * rejects on an error. If there were no emissions, Promise\n * resolves with undefined.\n * @deprecated Replaced with {@link firstValueFrom} and {@link lastValueFrom}. Will be removed in v8. Details: https://rxjs.dev/deprecations/to-promise\n */\n toPromise(promiseCtor?: PromiseConstructorLike): Promise {\n promiseCtor = getPromiseCtor(promiseCtor);\n\n return new promiseCtor((resolve, reject) => {\n let value: T | undefined;\n this.subscribe(\n (x: T) => (value = x),\n (err: any) => reject(err),\n () => resolve(value)\n );\n }) as Promise;\n }\n}\n\n/**\n * Decides between a passed promise constructor from consuming code,\n * A default configured promise constructor, and the native promise\n * constructor and returns it. If nothing can be found, it will throw\n * an error.\n * @param promiseCtor The optional promise constructor to passed by consuming code\n */\nfunction getPromiseCtor(promiseCtor: PromiseConstructorLike | undefined) {\n return promiseCtor ?? config.Promise ?? Promise;\n}\n\nfunction isObserver(value: any): value is Observer {\n return value && isFunction(value.next) && isFunction(value.error) && isFunction(value.complete);\n}\n\nfunction isSubscriber(value: any): value is Subscriber {\n return (value && value instanceof Subscriber) || (isObserver(value) && isSubscription(value));\n}\n", "import { Observable } from '../Observable';\nimport { Subscriber } from '../Subscriber';\nimport { OperatorFunction } from '../types';\nimport { isFunction } from './isFunction';\n\n/**\n * Used to determine if an object is an Observable with a lift function.\n */\nexport function hasLift(source: any): source is { lift: InstanceType['lift'] } {\n return isFunction(source?.lift);\n}\n\n/**\n * Creates an `OperatorFunction`. Used to define operators throughout the library in a concise way.\n * @param init The logic to connect the liftedSource to the subscriber at the moment of subscription.\n */\nexport function operate(\n init: (liftedSource: Observable, subscriber: Subscriber) => (() => void) | void\n): OperatorFunction {\n return (source: Observable) => {\n if (hasLift(source)) {\n return source.lift(function (this: Subscriber, liftedSource: Observable) {\n try {\n return init(liftedSource, this);\n } catch (err) {\n this.error(err);\n }\n });\n }\n throw new TypeError('Unable to lift unknown Observable type');\n };\n}\n", "import { Subscriber } from '../Subscriber';\n\n/**\n * Creates an instance of an `OperatorSubscriber`.\n * @param destination The downstream subscriber.\n * @param onNext Handles next values, only called if this subscriber is not stopped or closed. Any\n * error that occurs in this function is caught and sent to the `error` method of this subscriber.\n * @param onError Handles errors from the subscription, any errors that occur in this handler are caught\n * and send to the `destination` error handler.\n * @param onComplete Handles completion notification from the subscription. Any errors that occur in\n * this handler are sent to the `destination` error handler.\n * @param onFinalize Additional teardown logic here. This will only be called on teardown if the\n * subscriber itself is not already closed. This is called after all other teardown logic is executed.\n */\nexport function createOperatorSubscriber(\n destination: Subscriber,\n onNext?: (value: T) => void,\n onComplete?: () => void,\n onError?: (err: any) => void,\n onFinalize?: () => void\n): Subscriber {\n return new OperatorSubscriber(destination, onNext, onComplete, onError, onFinalize);\n}\n\n/**\n * A generic helper for allowing operators to be created with a Subscriber and\n * use closures to capture necessary state from the operator function itself.\n */\nexport class OperatorSubscriber extends Subscriber {\n /**\n * Creates an instance of an `OperatorSubscriber`.\n * @param destination The downstream subscriber.\n * @param onNext Handles next values, only called if this subscriber is not stopped or closed. Any\n * error that occurs in this function is caught and sent to the `error` method of this subscriber.\n * @param onError Handles errors from the subscription, any errors that occur in this handler are caught\n * and send to the `destination` error handler.\n * @param onComplete Handles completion notification from the subscription. Any errors that occur in\n * this handler are sent to the `destination` error handler.\n * @param onFinalize Additional finalization logic here. This will only be called on finalization if the\n * subscriber itself is not already closed. This is called after all other finalization logic is executed.\n * @param shouldUnsubscribe An optional check to see if an unsubscribe call should truly unsubscribe.\n * NOTE: This currently **ONLY** exists to support the strange behavior of {@link groupBy}, where unsubscription\n * to the resulting observable does not actually disconnect from the source if there are active subscriptions\n * to any grouped observable. (DO NOT EXPOSE OR USE EXTERNALLY!!!)\n */\n constructor(\n destination: Subscriber,\n onNext?: (value: T) => void,\n onComplete?: () => void,\n onError?: (err: any) => void,\n private onFinalize?: () => void,\n private shouldUnsubscribe?: () => boolean\n ) {\n // It's important - for performance reasons - that all of this class's\n // members are initialized and that they are always initialized in the same\n // order. This will ensure that all OperatorSubscriber instances have the\n // same hidden class in V8. This, in turn, will help keep the number of\n // hidden classes involved in property accesses within the base class as\n // low as possible. If the number of hidden classes involved exceeds four,\n // the property accesses will become megamorphic and performance penalties\n // will be incurred - i.e. inline caches won't be used.\n //\n // The reasons for ensuring all instances have the same hidden class are\n // further discussed in this blog post from Benedikt Meurer:\n // https://benediktmeurer.de/2018/03/23/impact-of-polymorphism-on-component-based-frameworks-like-react/\n super(destination);\n this._next = onNext\n ? function (this: OperatorSubscriber, value: T) {\n try {\n onNext(value);\n } catch (err) {\n destination.error(err);\n }\n }\n : super._next;\n this._error = onError\n ? function (this: OperatorSubscriber, err: any) {\n try {\n onError(err);\n } catch (err) {\n // Send any errors that occur down stream.\n destination.error(err);\n } finally {\n // Ensure finalization.\n this.unsubscribe();\n }\n }\n : super._error;\n this._complete = onComplete\n ? function (this: OperatorSubscriber) {\n try {\n onComplete();\n } catch (err) {\n // Send any errors that occur down stream.\n destination.error(err);\n } finally {\n // Ensure finalization.\n this.unsubscribe();\n }\n }\n : super._complete;\n }\n\n unsubscribe() {\n if (!this.shouldUnsubscribe || this.shouldUnsubscribe()) {\n const { closed } = this;\n super.unsubscribe();\n // Execute additional teardown if we have any and we didn't already do so.\n !closed && this.onFinalize?.();\n }\n }\n}\n", "import { Subscription } from '../Subscription';\n\ninterface AnimationFrameProvider {\n schedule(callback: FrameRequestCallback): Subscription;\n requestAnimationFrame: typeof requestAnimationFrame;\n cancelAnimationFrame: typeof cancelAnimationFrame;\n delegate:\n | {\n requestAnimationFrame: typeof requestAnimationFrame;\n cancelAnimationFrame: typeof cancelAnimationFrame;\n }\n | undefined;\n}\n\nexport const animationFrameProvider: AnimationFrameProvider = {\n // When accessing the delegate, use the variable rather than `this` so that\n // the functions can be called without being bound to the provider.\n schedule(callback) {\n let request = requestAnimationFrame;\n let cancel: typeof cancelAnimationFrame | undefined = cancelAnimationFrame;\n const { delegate } = animationFrameProvider;\n if (delegate) {\n request = delegate.requestAnimationFrame;\n cancel = delegate.cancelAnimationFrame;\n }\n const handle = request((timestamp) => {\n // Clear the cancel function. The request has been fulfilled, so\n // attempting to cancel the request upon unsubscription would be\n // pointless.\n cancel = undefined;\n callback(timestamp);\n });\n return new Subscription(() => cancel?.(handle));\n },\n requestAnimationFrame(...args) {\n const { delegate } = animationFrameProvider;\n return (delegate?.requestAnimationFrame || requestAnimationFrame)(...args);\n },\n cancelAnimationFrame(...args) {\n const { delegate } = animationFrameProvider;\n return (delegate?.cancelAnimationFrame || cancelAnimationFrame)(...args);\n },\n delegate: undefined,\n};\n", "import { createErrorClass } from './createErrorClass';\n\nexport interface ObjectUnsubscribedError extends Error {}\n\nexport interface ObjectUnsubscribedErrorCtor {\n /**\n * @deprecated Internal implementation detail. Do not construct error instances.\n * Cannot be tagged as internal: https://github.com/ReactiveX/rxjs/issues/6269\n */\n new (): ObjectUnsubscribedError;\n}\n\n/**\n * An error thrown when an action is invalid because the object has been\n * unsubscribed.\n *\n * @see {@link Subject}\n * @see {@link BehaviorSubject}\n *\n * @class ObjectUnsubscribedError\n */\nexport const ObjectUnsubscribedError: ObjectUnsubscribedErrorCtor = createErrorClass(\n (_super) =>\n function ObjectUnsubscribedErrorImpl(this: any) {\n _super(this);\n this.name = 'ObjectUnsubscribedError';\n this.message = 'object unsubscribed';\n }\n);\n", "import { Operator } from './Operator';\nimport { Observable } from './Observable';\nimport { Subscriber } from './Subscriber';\nimport { Subscription, EMPTY_SUBSCRIPTION } from './Subscription';\nimport { Observer, SubscriptionLike, TeardownLogic } from './types';\nimport { ObjectUnsubscribedError } from './util/ObjectUnsubscribedError';\nimport { arrRemove } from './util/arrRemove';\nimport { errorContext } from './util/errorContext';\n\n/**\n * A Subject is a special type of Observable that allows values to be\n * multicasted to many Observers. Subjects are like EventEmitters.\n *\n * Every Subject is an Observable and an Observer. You can subscribe to a\n * Subject, and you can call next to feed values as well as error and complete.\n */\nexport class Subject extends Observable implements SubscriptionLike {\n closed = false;\n\n private currentObservers: Observer[] | null = null;\n\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n observers: Observer[] = [];\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n isStopped = false;\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n hasError = false;\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n thrownError: any = null;\n\n /**\n * Creates a \"subject\" by basically gluing an observer to an observable.\n *\n * @nocollapse\n * @deprecated Recommended you do not use. Will be removed at some point in the future. Plans for replacement still under discussion.\n */\n static create: (...args: any[]) => any = (destination: Observer, source: Observable): AnonymousSubject => {\n return new AnonymousSubject(destination, source);\n };\n\n constructor() {\n // NOTE: This must be here to obscure Observable's constructor.\n super();\n }\n\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n lift(operator: Operator): Observable {\n const subject = new AnonymousSubject(this, this);\n subject.operator = operator as any;\n return subject as any;\n }\n\n /** @internal */\n protected _throwIfClosed() {\n if (this.closed) {\n throw new ObjectUnsubscribedError();\n }\n }\n\n next(value: T) {\n errorContext(() => {\n this._throwIfClosed();\n if (!this.isStopped) {\n if (!this.currentObservers) {\n this.currentObservers = Array.from(this.observers);\n }\n for (const observer of this.currentObservers) {\n observer.next(value);\n }\n }\n });\n }\n\n error(err: any) {\n errorContext(() => {\n this._throwIfClosed();\n if (!this.isStopped) {\n this.hasError = this.isStopped = true;\n this.thrownError = err;\n const { observers } = this;\n while (observers.length) {\n observers.shift()!.error(err);\n }\n }\n });\n }\n\n complete() {\n errorContext(() => {\n this._throwIfClosed();\n if (!this.isStopped) {\n this.isStopped = true;\n const { observers } = this;\n while (observers.length) {\n observers.shift()!.complete();\n }\n }\n });\n }\n\n unsubscribe() {\n this.isStopped = this.closed = true;\n this.observers = this.currentObservers = null!;\n }\n\n get observed() {\n return this.observers?.length > 0;\n }\n\n /** @internal */\n protected _trySubscribe(subscriber: Subscriber): TeardownLogic {\n this._throwIfClosed();\n return super._trySubscribe(subscriber);\n }\n\n /** @internal */\n protected _subscribe(subscriber: Subscriber): Subscription {\n this._throwIfClosed();\n this._checkFinalizedStatuses(subscriber);\n return this._innerSubscribe(subscriber);\n }\n\n /** @internal */\n protected _innerSubscribe(subscriber: Subscriber) {\n const { hasError, isStopped, observers } = this;\n if (hasError || isStopped) {\n return EMPTY_SUBSCRIPTION;\n }\n this.currentObservers = null;\n observers.push(subscriber);\n return new Subscription(() => {\n this.currentObservers = null;\n arrRemove(observers, subscriber);\n });\n }\n\n /** @internal */\n protected _checkFinalizedStatuses(subscriber: Subscriber) {\n const { hasError, thrownError, isStopped } = this;\n if (hasError) {\n subscriber.error(thrownError);\n } else if (isStopped) {\n subscriber.complete();\n }\n }\n\n /**\n * Creates a new Observable with this Subject as the source. You can do this\n * to create custom Observer-side logic of the Subject and conceal it from\n * code that uses the Observable.\n * @return {Observable} Observable that the Subject casts to\n */\n asObservable(): Observable {\n const observable: any = new Observable();\n observable.source = this;\n return observable;\n }\n}\n\n/**\n * @class AnonymousSubject\n */\nexport class AnonymousSubject extends Subject {\n constructor(\n /** @deprecated Internal implementation detail, do not use directly. Will be made internal in v8. */\n public destination?: Observer,\n source?: Observable\n ) {\n super();\n this.source = source;\n }\n\n next(value: T) {\n this.destination?.next?.(value);\n }\n\n error(err: any) {\n this.destination?.error?.(err);\n }\n\n complete() {\n this.destination?.complete?.();\n }\n\n /** @internal */\n protected _subscribe(subscriber: Subscriber): Subscription {\n return this.source?.subscribe(subscriber) ?? EMPTY_SUBSCRIPTION;\n }\n}\n", "import { TimestampProvider } from '../types';\n\ninterface DateTimestampProvider extends TimestampProvider {\n delegate: TimestampProvider | undefined;\n}\n\nexport const dateTimestampProvider: DateTimestampProvider = {\n now() {\n // Use the variable rather than `this` so that the function can be called\n // without being bound to the provider.\n return (dateTimestampProvider.delegate || Date).now();\n },\n delegate: undefined,\n};\n", "import { Subject } from './Subject';\nimport { TimestampProvider } from './types';\nimport { Subscriber } from './Subscriber';\nimport { Subscription } from './Subscription';\nimport { dateTimestampProvider } from './scheduler/dateTimestampProvider';\n\n/**\n * A variant of {@link Subject} that \"replays\" old values to new subscribers by emitting them when they first subscribe.\n *\n * `ReplaySubject` has an internal buffer that will store a specified number of values that it has observed. Like `Subject`,\n * `ReplaySubject` \"observes\" values by having them passed to its `next` method. When it observes a value, it will store that\n * value for a time determined by the configuration of the `ReplaySubject`, as passed to its constructor.\n *\n * When a new subscriber subscribes to the `ReplaySubject` instance, it will synchronously emit all values in its buffer in\n * a First-In-First-Out (FIFO) manner. The `ReplaySubject` will also complete, if it has observed completion; and it will\n * error if it has observed an error.\n *\n * There are two main configuration items to be concerned with:\n *\n * 1. `bufferSize` - This will determine how many items are stored in the buffer, defaults to infinite.\n * 2. `windowTime` - The amount of time to hold a value in the buffer before removing it from the buffer.\n *\n * Both configurations may exist simultaneously. So if you would like to buffer a maximum of 3 values, as long as the values\n * are less than 2 seconds old, you could do so with a `new ReplaySubject(3, 2000)`.\n *\n * ### Differences with BehaviorSubject\n *\n * `BehaviorSubject` is similar to `new ReplaySubject(1)`, with a couple of exceptions:\n *\n * 1. `BehaviorSubject` comes \"primed\" with a single value upon construction.\n * 2. `ReplaySubject` will replay values, even after observing an error, where `BehaviorSubject` will not.\n *\n * @see {@link Subject}\n * @see {@link BehaviorSubject}\n * @see {@link shareReplay}\n */\nexport class ReplaySubject extends Subject {\n private _buffer: (T | number)[] = [];\n private _infiniteTimeWindow = true;\n\n /**\n * @param bufferSize The size of the buffer to replay on subscription\n * @param windowTime The amount of time the buffered items will stay buffered\n * @param timestampProvider An object with a `now()` method that provides the current timestamp. This is used to\n * calculate the amount of time something has been buffered.\n */\n constructor(\n private _bufferSize = Infinity,\n private _windowTime = Infinity,\n private _timestampProvider: TimestampProvider = dateTimestampProvider\n ) {\n super();\n this._infiniteTimeWindow = _windowTime === Infinity;\n this._bufferSize = Math.max(1, _bufferSize);\n this._windowTime = Math.max(1, _windowTime);\n }\n\n next(value: T): void {\n const { isStopped, _buffer, _infiniteTimeWindow, _timestampProvider, _windowTime } = this;\n if (!isStopped) {\n _buffer.push(value);\n !_infiniteTimeWindow && _buffer.push(_timestampProvider.now() + _windowTime);\n }\n this._trimBuffer();\n super.next(value);\n }\n\n /** @internal */\n protected _subscribe(subscriber: Subscriber): Subscription {\n this._throwIfClosed();\n this._trimBuffer();\n\n const subscription = this._innerSubscribe(subscriber);\n\n const { _infiniteTimeWindow, _buffer } = this;\n // We use a copy here, so reentrant code does not mutate our array while we're\n // emitting it to a new subscriber.\n const copy = _buffer.slice();\n for (let i = 0; i < copy.length && !subscriber.closed; i += _infiniteTimeWindow ? 1 : 2) {\n subscriber.next(copy[i] as T);\n }\n\n this._checkFinalizedStatuses(subscriber);\n\n return subscription;\n }\n\n private _trimBuffer() {\n const { _bufferSize, _timestampProvider, _buffer, _infiniteTimeWindow } = this;\n // If we don't have an infinite buffer size, and we're over the length,\n // use splice to truncate the old buffer values off. Note that we have to\n // double the size for instances where we're not using an infinite time window\n // because we're storing the values and the timestamps in the same array.\n const adjustedBufferSize = (_infiniteTimeWindow ? 1 : 2) * _bufferSize;\n _bufferSize < Infinity && adjustedBufferSize < _buffer.length && _buffer.splice(0, _buffer.length - adjustedBufferSize);\n\n // Now, if we're not in an infinite time window, remove all values where the time is\n // older than what is allowed.\n if (!_infiniteTimeWindow) {\n const now = _timestampProvider.now();\n let last = 0;\n // Search the array for the first timestamp that isn't expired and\n // truncate the buffer up to that point.\n for (let i = 1; i < _buffer.length && (_buffer[i] as number) <= now; i += 2) {\n last = i;\n }\n last && _buffer.splice(0, last + 1);\n }\n }\n}\n", "import { Scheduler } from '../Scheduler';\nimport { Subscription } from '../Subscription';\nimport { SchedulerAction } from '../types';\n\n/**\n * A unit of work to be executed in a `scheduler`. An action is typically\n * created from within a {@link SchedulerLike} and an RxJS user does not need to concern\n * themselves about creating and manipulating an Action.\n *\n * ```ts\n * class Action extends Subscription {\n * new (scheduler: Scheduler, work: (state?: T) => void);\n * schedule(state?: T, delay: number = 0): Subscription;\n * }\n * ```\n *\n * @class Action\n */\nexport class Action extends Subscription {\n constructor(scheduler: Scheduler, work: (this: SchedulerAction, state?: T) => void) {\n super();\n }\n /**\n * Schedules this action on its parent {@link SchedulerLike} for execution. May be passed\n * some context object, `state`. May happen at some point in the future,\n * according to the `delay` parameter, if specified.\n * @param {T} [state] Some contextual data that the `work` function uses when\n * called by the Scheduler.\n * @param {number} [delay] Time to wait before executing the work, where the\n * time unit is implicit and defined by the Scheduler.\n * @return {void}\n */\n public schedule(state?: T, delay: number = 0): Subscription {\n return this;\n }\n}\n", "import type { TimerHandle } from './timerHandle';\ntype SetIntervalFunction = (handler: () => void, timeout?: number, ...args: any[]) => TimerHandle;\ntype ClearIntervalFunction = (handle: TimerHandle) => void;\n\ninterface IntervalProvider {\n setInterval: SetIntervalFunction;\n clearInterval: ClearIntervalFunction;\n delegate:\n | {\n setInterval: SetIntervalFunction;\n clearInterval: ClearIntervalFunction;\n }\n | undefined;\n}\n\nexport const intervalProvider: IntervalProvider = {\n // When accessing the delegate, use the variable rather than `this` so that\n // the functions can be called without being bound to the provider.\n setInterval(handler: () => void, timeout?: number, ...args) {\n const { delegate } = intervalProvider;\n if (delegate?.setInterval) {\n return delegate.setInterval(handler, timeout, ...args);\n }\n return setInterval(handler, timeout, ...args);\n },\n clearInterval(handle) {\n const { delegate } = intervalProvider;\n return (delegate?.clearInterval || clearInterval)(handle as any);\n },\n delegate: undefined,\n};\n", "import { Action } from './Action';\nimport { SchedulerAction } from '../types';\nimport { Subscription } from '../Subscription';\nimport { AsyncScheduler } from './AsyncScheduler';\nimport { intervalProvider } from './intervalProvider';\nimport { arrRemove } from '../util/arrRemove';\nimport { TimerHandle } from './timerHandle';\n\nexport class AsyncAction extends Action {\n public id: TimerHandle | undefined;\n public state?: T;\n // @ts-ignore: Property has no initializer and is not definitely assigned\n public delay: number;\n protected pending: boolean = false;\n\n constructor(protected scheduler: AsyncScheduler, protected work: (this: SchedulerAction, state?: T) => void) {\n super(scheduler, work);\n }\n\n public schedule(state?: T, delay: number = 0): Subscription {\n if (this.closed) {\n return this;\n }\n\n // Always replace the current state with the new state.\n this.state = state;\n\n const id = this.id;\n const scheduler = this.scheduler;\n\n //\n // Important implementation note:\n //\n // Actions only execute once by default, unless rescheduled from within the\n // scheduled callback. This allows us to implement single and repeat\n // actions via the same code path, without adding API surface area, as well\n // as mimic traditional recursion but across asynchronous boundaries.\n //\n // However, JS runtimes and timers distinguish between intervals achieved by\n // serial `setTimeout` calls vs. a single `setInterval` call. An interval of\n // serial `setTimeout` calls can be individually delayed, which delays\n // scheduling the next `setTimeout`, and so on. `setInterval` attempts to\n // guarantee the interval callback will be invoked more precisely to the\n // interval period, regardless of load.\n //\n // Therefore, we use `setInterval` to schedule single and repeat actions.\n // If the action reschedules itself with the same delay, the interval is not\n // canceled. If the action doesn't reschedule, or reschedules with a\n // different delay, the interval will be canceled after scheduled callback\n // execution.\n //\n if (id != null) {\n this.id = this.recycleAsyncId(scheduler, id, delay);\n }\n\n // Set the pending flag indicating that this action has been scheduled, or\n // has recursively rescheduled itself.\n this.pending = true;\n\n this.delay = delay;\n // If this action has already an async Id, don't request a new one.\n this.id = this.id ?? this.requestAsyncId(scheduler, this.id, delay);\n\n return this;\n }\n\n protected requestAsyncId(scheduler: AsyncScheduler, _id?: TimerHandle, delay: number = 0): TimerHandle {\n return intervalProvider.setInterval(scheduler.flush.bind(scheduler, this), delay);\n }\n\n protected recycleAsyncId(_scheduler: AsyncScheduler, id?: TimerHandle, delay: number | null = 0): TimerHandle | undefined {\n // If this action is rescheduled with the same delay time, don't clear the interval id.\n if (delay != null && this.delay === delay && this.pending === false) {\n return id;\n }\n // Otherwise, if the action's delay time is different from the current delay,\n // or the action has been rescheduled before it's executed, clear the interval id\n if (id != null) {\n intervalProvider.clearInterval(id);\n }\n\n return undefined;\n }\n\n /**\n * Immediately executes this action and the `work` it contains.\n * @return {any}\n */\n public execute(state: T, delay: number): any {\n if (this.closed) {\n return new Error('executing a cancelled action');\n }\n\n this.pending = false;\n const error = this._execute(state, delay);\n if (error) {\n return error;\n } else if (this.pending === false && this.id != null) {\n // Dequeue if the action didn't reschedule itself. Don't call\n // unsubscribe(), because the action could reschedule later.\n // For example:\n // ```\n // scheduler.schedule(function doWork(counter) {\n // /* ... I'm a busy worker bee ... */\n // var originalAction = this;\n // /* wait 100ms before rescheduling the action */\n // setTimeout(function () {\n // originalAction.schedule(counter + 1);\n // }, 100);\n // }, 1000);\n // ```\n this.id = this.recycleAsyncId(this.scheduler, this.id, null);\n }\n }\n\n protected _execute(state: T, _delay: number): any {\n let errored: boolean = false;\n let errorValue: any;\n try {\n this.work(state);\n } catch (e) {\n errored = true;\n // HACK: Since code elsewhere is relying on the \"truthiness\" of the\n // return here, we can't have it return \"\" or 0 or false.\n // TODO: Clean this up when we refactor schedulers mid-version-8 or so.\n errorValue = e ? e : new Error('Scheduled action threw falsy error');\n }\n if (errored) {\n this.unsubscribe();\n return errorValue;\n }\n }\n\n unsubscribe() {\n if (!this.closed) {\n const { id, scheduler } = this;\n const { actions } = scheduler;\n\n this.work = this.state = this.scheduler = null!;\n this.pending = false;\n\n arrRemove(actions, this);\n if (id != null) {\n this.id = this.recycleAsyncId(scheduler, id, null);\n }\n\n this.delay = null!;\n super.unsubscribe();\n }\n }\n}\n", "import { Action } from './scheduler/Action';\nimport { Subscription } from './Subscription';\nimport { SchedulerLike, SchedulerAction } from './types';\nimport { dateTimestampProvider } from './scheduler/dateTimestampProvider';\n\n/**\n * An execution context and a data structure to order tasks and schedule their\n * execution. Provides a notion of (potentially virtual) time, through the\n * `now()` getter method.\n *\n * Each unit of work in a Scheduler is called an `Action`.\n *\n * ```ts\n * class Scheduler {\n * now(): number;\n * schedule(work, delay?, state?): Subscription;\n * }\n * ```\n *\n * @class Scheduler\n * @deprecated Scheduler is an internal implementation detail of RxJS, and\n * should not be used directly. Rather, create your own class and implement\n * {@link SchedulerLike}. Will be made internal in v8.\n */\nexport class Scheduler implements SchedulerLike {\n public static now: () => number = dateTimestampProvider.now;\n\n constructor(private schedulerActionCtor: typeof Action, now: () => number = Scheduler.now) {\n this.now = now;\n }\n\n /**\n * A getter method that returns a number representing the current time\n * (at the time this function was called) according to the scheduler's own\n * internal clock.\n * @return {number} A number that represents the current time. May or may not\n * have a relation to wall-clock time. May or may not refer to a time unit\n * (e.g. milliseconds).\n */\n public now: () => number;\n\n /**\n * Schedules a function, `work`, for execution. May happen at some point in\n * the future, according to the `delay` parameter, if specified. May be passed\n * some context object, `state`, which will be passed to the `work` function.\n *\n * The given arguments will be processed an stored as an Action object in a\n * queue of actions.\n *\n * @param {function(state: ?T): ?Subscription} work A function representing a\n * task, or some unit of work to be executed by the Scheduler.\n * @param {number} [delay] Time to wait before executing the work, where the\n * time unit is implicit and defined by the Scheduler itself.\n * @param {T} [state] Some contextual data that the `work` function uses when\n * called by the Scheduler.\n * @return {Subscription} A subscription in order to be able to unsubscribe\n * the scheduled work.\n */\n public schedule(work: (this: SchedulerAction, state?: T) => void, delay: number = 0, state?: T): Subscription {\n return new this.schedulerActionCtor(this, work).schedule(state, delay);\n }\n}\n", "import { Scheduler } from '../Scheduler';\nimport { Action } from './Action';\nimport { AsyncAction } from './AsyncAction';\nimport { TimerHandle } from './timerHandle';\n\nexport class AsyncScheduler extends Scheduler {\n public actions: Array> = [];\n /**\n * A flag to indicate whether the Scheduler is currently executing a batch of\n * queued actions.\n * @type {boolean}\n * @internal\n */\n public _active: boolean = false;\n /**\n * An internal ID used to track the latest asynchronous task such as those\n * coming from `setTimeout`, `setInterval`, `requestAnimationFrame`, and\n * others.\n * @type {any}\n * @internal\n */\n public _scheduled: TimerHandle | undefined;\n\n constructor(SchedulerAction: typeof Action, now: () => number = Scheduler.now) {\n super(SchedulerAction, now);\n }\n\n public flush(action: AsyncAction): void {\n const { actions } = this;\n\n if (this._active) {\n actions.push(action);\n return;\n }\n\n let error: any;\n this._active = true;\n\n do {\n if ((error = action.execute(action.state, action.delay))) {\n break;\n }\n } while ((action = actions.shift()!)); // exhaust the scheduler queue\n\n this._active = false;\n\n if (error) {\n while ((action = actions.shift()!)) {\n action.unsubscribe();\n }\n throw error;\n }\n }\n}\n", "import { AsyncAction } from './AsyncAction';\nimport { AsyncScheduler } from './AsyncScheduler';\n\n/**\n *\n * Async Scheduler\n *\n * Schedule task as if you used setTimeout(task, duration)\n *\n * `async` scheduler schedules tasks asynchronously, by putting them on the JavaScript\n * event loop queue. It is best used to delay tasks in time or to schedule tasks repeating\n * in intervals.\n *\n * If you just want to \"defer\" task, that is to perform it right after currently\n * executing synchronous code ends (commonly achieved by `setTimeout(deferredTask, 0)`),\n * better choice will be the {@link asapScheduler} scheduler.\n *\n * ## Examples\n * Use async scheduler to delay task\n * ```ts\n * import { asyncScheduler } from 'rxjs';\n *\n * const task = () => console.log('it works!');\n *\n * asyncScheduler.schedule(task, 2000);\n *\n * // After 2 seconds logs:\n * // \"it works!\"\n * ```\n *\n * Use async scheduler to repeat task in intervals\n * ```ts\n * import { asyncScheduler } from 'rxjs';\n *\n * function task(state) {\n * console.log(state);\n * this.schedule(state + 1, 1000); // `this` references currently executing Action,\n * // which we reschedule with new state and delay\n * }\n *\n * asyncScheduler.schedule(task, 3000, 0);\n *\n * // Logs:\n * // 0 after 3s\n * // 1 after 4s\n * // 2 after 5s\n * // 3 after 6s\n * ```\n */\n\nexport const asyncScheduler = new AsyncScheduler(AsyncAction);\n\n/**\n * @deprecated Renamed to {@link asyncScheduler}. Will be removed in v8.\n */\nexport const async = asyncScheduler;\n", "import { AsyncAction } from './AsyncAction';\nimport { AnimationFrameScheduler } from './AnimationFrameScheduler';\nimport { SchedulerAction } from '../types';\nimport { animationFrameProvider } from './animationFrameProvider';\nimport { TimerHandle } from './timerHandle';\n\nexport class AnimationFrameAction extends AsyncAction {\n constructor(protected scheduler: AnimationFrameScheduler, protected work: (this: SchedulerAction, state?: T) => void) {\n super(scheduler, work);\n }\n\n protected requestAsyncId(scheduler: AnimationFrameScheduler, id?: TimerHandle, delay: number = 0): TimerHandle {\n // If delay is greater than 0, request as an async action.\n if (delay !== null && delay > 0) {\n return super.requestAsyncId(scheduler, id, delay);\n }\n // Push the action to the end of the scheduler queue.\n scheduler.actions.push(this);\n // If an animation frame has already been requested, don't request another\n // one. If an animation frame hasn't been requested yet, request one. Return\n // the current animation frame request id.\n return scheduler._scheduled || (scheduler._scheduled = animationFrameProvider.requestAnimationFrame(() => scheduler.flush(undefined)));\n }\n\n protected recycleAsyncId(scheduler: AnimationFrameScheduler, id?: TimerHandle, delay: number = 0): TimerHandle | undefined {\n // If delay exists and is greater than 0, or if the delay is null (the\n // action wasn't rescheduled) but was originally scheduled as an async\n // action, then recycle as an async action.\n if (delay != null ? delay > 0 : this.delay > 0) {\n return super.recycleAsyncId(scheduler, id, delay);\n }\n // If the scheduler queue has no remaining actions with the same async id,\n // cancel the requested animation frame and set the scheduled flag to\n // undefined so the next AnimationFrameAction will request its own.\n const { actions } = scheduler;\n if (id != null && actions[actions.length - 1]?.id !== id) {\n animationFrameProvider.cancelAnimationFrame(id as number);\n scheduler._scheduled = undefined;\n }\n // Return undefined so the action knows to request a new async id if it's rescheduled.\n return undefined;\n }\n}\n", "import { AsyncAction } from './AsyncAction';\nimport { AsyncScheduler } from './AsyncScheduler';\n\nexport class AnimationFrameScheduler extends AsyncScheduler {\n public flush(action?: AsyncAction): void {\n this._active = true;\n // The async id that effects a call to flush is stored in _scheduled.\n // Before executing an action, it's necessary to check the action's async\n // id to determine whether it's supposed to be executed in the current\n // flush.\n // Previous implementations of this method used a count to determine this,\n // but that was unsound, as actions that are unsubscribed - i.e. cancelled -\n // are removed from the actions array and that can shift actions that are\n // scheduled to be executed in a subsequent flush into positions at which\n // they are executed within the current flush.\n const flushId = this._scheduled;\n this._scheduled = undefined;\n\n const { actions } = this;\n let error: any;\n action = action || actions.shift()!;\n\n do {\n if ((error = action.execute(action.state, action.delay))) {\n break;\n }\n } while ((action = actions[0]) && action.id === flushId && actions.shift());\n\n this._active = false;\n\n if (error) {\n while ((action = actions[0]) && action.id === flushId && actions.shift()) {\n action.unsubscribe();\n }\n throw error;\n }\n }\n}\n", "import { AnimationFrameAction } from './AnimationFrameAction';\nimport { AnimationFrameScheduler } from './AnimationFrameScheduler';\n\n/**\n *\n * Animation Frame Scheduler\n *\n * Perform task when `window.requestAnimationFrame` would fire\n *\n * When `animationFrame` scheduler is used with delay, it will fall back to {@link asyncScheduler} scheduler\n * behaviour.\n *\n * Without delay, `animationFrame` scheduler can be used to create smooth browser animations.\n * It makes sure scheduled task will happen just before next browser content repaint,\n * thus performing animations as efficiently as possible.\n *\n * ## Example\n * Schedule div height animation\n * ```ts\n * // html:
    \n * import { animationFrameScheduler } from 'rxjs';\n *\n * const div = document.querySelector('div');\n *\n * animationFrameScheduler.schedule(function(height) {\n * div.style.height = height + \"px\";\n *\n * this.schedule(height + 1); // `this` references currently executing Action,\n * // which we reschedule with new state\n * }, 0, 0);\n *\n * // You will see a div element growing in height\n * ```\n */\n\nexport const animationFrameScheduler = new AnimationFrameScheduler(AnimationFrameAction);\n\n/**\n * @deprecated Renamed to {@link animationFrameScheduler}. Will be removed in v8.\n */\nexport const animationFrame = animationFrameScheduler;\n", "import { Observable } from '../Observable';\nimport { SchedulerLike } from '../types';\n\n/**\n * A simple Observable that emits no items to the Observer and immediately\n * emits a complete notification.\n *\n * Just emits 'complete', and nothing else.\n *\n * ![](empty.png)\n *\n * A simple Observable that only emits the complete notification. It can be used\n * for composing with other Observables, such as in a {@link mergeMap}.\n *\n * ## Examples\n *\n * Log complete notification\n *\n * ```ts\n * import { EMPTY } from 'rxjs';\n *\n * EMPTY.subscribe({\n * next: () => console.log('Next'),\n * complete: () => console.log('Complete!')\n * });\n *\n * // Outputs\n * // Complete!\n * ```\n *\n * Emit the number 7, then complete\n *\n * ```ts\n * import { EMPTY, startWith } from 'rxjs';\n *\n * const result = EMPTY.pipe(startWith(7));\n * result.subscribe(x => console.log(x));\n *\n * // Outputs\n * // 7\n * ```\n *\n * Map and flatten only odd numbers to the sequence `'a'`, `'b'`, `'c'`\n *\n * ```ts\n * import { interval, mergeMap, of, EMPTY } from 'rxjs';\n *\n * const interval$ = interval(1000);\n * const result = interval$.pipe(\n * mergeMap(x => x % 2 === 1 ? of('a', 'b', 'c') : EMPTY),\n * );\n * result.subscribe(x => console.log(x));\n *\n * // Results in the following to the console:\n * // x is equal to the count on the interval, e.g. (0, 1, 2, 3, ...)\n * // x will occur every 1000ms\n * // if x % 2 is equal to 1, print a, b, c (each on its own)\n * // if x % 2 is not equal to 1, nothing will be output\n * ```\n *\n * @see {@link Observable}\n * @see {@link NEVER}\n * @see {@link of}\n * @see {@link throwError}\n */\nexport const EMPTY = new Observable((subscriber) => subscriber.complete());\n\n/**\n * @param scheduler A {@link SchedulerLike} to use for scheduling\n * the emission of the complete notification.\n * @deprecated Replaced with the {@link EMPTY} constant or {@link scheduled} (e.g. `scheduled([], scheduler)`). Will be removed in v8.\n */\nexport function empty(scheduler?: SchedulerLike) {\n return scheduler ? emptyScheduled(scheduler) : EMPTY;\n}\n\nfunction emptyScheduled(scheduler: SchedulerLike) {\n return new Observable((subscriber) => scheduler.schedule(() => subscriber.complete()));\n}\n", "import { SchedulerLike } from '../types';\nimport { isFunction } from './isFunction';\n\nexport function isScheduler(value: any): value is SchedulerLike {\n return value && isFunction(value.schedule);\n}\n", "import { SchedulerLike } from '../types';\nimport { isFunction } from './isFunction';\nimport { isScheduler } from './isScheduler';\n\nfunction last(arr: T[]): T | undefined {\n return arr[arr.length - 1];\n}\n\nexport function popResultSelector(args: any[]): ((...args: unknown[]) => unknown) | undefined {\n return isFunction(last(args)) ? args.pop() : undefined;\n}\n\nexport function popScheduler(args: any[]): SchedulerLike | undefined {\n return isScheduler(last(args)) ? args.pop() : undefined;\n}\n\nexport function popNumber(args: any[], defaultValue: number): number {\n return typeof last(args) === 'number' ? args.pop()! : defaultValue;\n}\n", "export const isArrayLike = ((x: any): x is ArrayLike => x && typeof x.length === 'number' && typeof x !== 'function');", "import { isFunction } from \"./isFunction\";\n\n/**\n * Tests to see if the object is \"thennable\".\n * @param value the object to test\n */\nexport function isPromise(value: any): value is PromiseLike {\n return isFunction(value?.then);\n}\n", "import { InteropObservable } from '../types';\nimport { observable as Symbol_observable } from '../symbol/observable';\nimport { isFunction } from './isFunction';\n\n/** Identifies an input as being Observable (but not necessary an Rx Observable) */\nexport function isInteropObservable(input: any): input is InteropObservable {\n return isFunction(input[Symbol_observable]);\n}\n", "import { isFunction } from './isFunction';\n\nexport function isAsyncIterable(obj: any): obj is AsyncIterable {\n return Symbol.asyncIterator && isFunction(obj?.[Symbol.asyncIterator]);\n}\n", "/**\n * Creates the TypeError to throw if an invalid object is passed to `from` or `scheduled`.\n * @param input The object that was passed.\n */\nexport function createInvalidObservableTypeError(input: any) {\n // TODO: We should create error codes that can be looked up, so this can be less verbose.\n return new TypeError(\n `You provided ${\n input !== null && typeof input === 'object' ? 'an invalid object' : `'${input}'`\n } where a stream was expected. You can provide an Observable, Promise, ReadableStream, Array, AsyncIterable, or Iterable.`\n );\n}\n", "export function getSymbolIterator(): symbol {\n if (typeof Symbol !== 'function' || !Symbol.iterator) {\n return '@@iterator' as any;\n }\n\n return Symbol.iterator;\n}\n\nexport const iterator = getSymbolIterator();\n", "import { iterator as Symbol_iterator } from '../symbol/iterator';\nimport { isFunction } from './isFunction';\n\n/** Identifies an input as being an Iterable */\nexport function isIterable(input: any): input is Iterable {\n return isFunction(input?.[Symbol_iterator]);\n}\n", "import { ReadableStreamLike } from '../types';\nimport { isFunction } from './isFunction';\n\nexport async function* readableStreamLikeToAsyncGenerator(readableStream: ReadableStreamLike): AsyncGenerator {\n const reader = readableStream.getReader();\n try {\n while (true) {\n const { value, done } = await reader.read();\n if (done) {\n return;\n }\n yield value!;\n }\n } finally {\n reader.releaseLock();\n }\n}\n\nexport function isReadableStreamLike(obj: any): obj is ReadableStreamLike {\n // We don't want to use instanceof checks because they would return\n // false for instances from another Realm, like an