From 072f442c1f17d73cf093e2e0243beea82aa40abe Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Wed, 2 Nov 2016 00:39:35 -0700 Subject: [PATCH] Update worker.py and services.py to use plasma and the local scheduler. (#19) * Update worker code and services code to use plasma and the local scheduler. * Cleanups. * Fix bug in which threads were started before the worker mode was set. This caused remote functions to be defined on workers before the worker knew it was in WORKER_MODE. * Fix bug in install-dependencies.sh. * Lengthen timeout in failure_test.py. * Cleanups. * Cleanup services.start_ray_local. * Clean up random name generation. * Cleanups. --- .travis.yml | 5 + CMakeLists.txt | 147 ----- data/README.md | 5 - data/mini.tar | Bin 51200 -> 0 bytes install-dependencies.sh | 4 +- lib/python/ray/__init__.py | 6 +- lib/python/ray/default_worker.py | 25 + lib/python/ray/graph.py | 34 -- lib/python/ray/internal/__init__.py | 0 lib/python/ray/serialization.py | 84 +-- lib/python/ray/services.py | 191 +++--- lib/python/ray/worker.py | 886 ++++++++++++++-------------- scripts/default_worker.py | 16 - setup-env.sh | 19 - src/photon/photon_algorithm.c | 2 +- src/photon/photon_scheduler.c | 2 +- src/plasma/lib/python/plasma.py | 30 +- test/array_test.py | 2 +- test/failure_test.py | 66 +-- test/runtest.py | 307 +--------- 20 files changed, 623 insertions(+), 1208 deletions(-) delete mode 100644 CMakeLists.txt delete mode 100644 data/README.md delete mode 100644 data/mini.tar create mode 100644 lib/python/ray/default_worker.py delete mode 100644 lib/python/ray/graph.py delete mode 100644 lib/python/ray/internal/__init__.py delete mode 100644 scripts/default_worker.py delete mode 100644 setup-env.sh diff --git a/.travis.yml b/.travis.yml index e4c3d0d24f04..9bf0c215c3db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,3 +69,8 @@ script: - python src/common/test/test.py - python src/plasma/test/test.py - python src/photon/test/test.py + + - python test/runtest.py + - python test/array_test.py + - python test/failure_test.py + - python test/microbenchmarks.py diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 5ffc42cb133e..000000000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,147 +0,0 @@ -cmake_minimum_required(VERSION 2.8) - -project(ray) - -set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") - -list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) - -set(CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/" ${CMAKE_PREFIX_PATH}) - -if(NOT APPLE) - find_package(PythonInterp REQUIRED) - find_package(PythonLibs REQUIRED) - set(CUSTOM_PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}) -else() - find_program(CUSTOM_PYTHON_EXECUTABLE python) - message("-- Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") - execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c - "import sys; print 'python' + sys.version[0:3]" - OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c - "import sys; print sys.exec_prefix" - OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) - FIND_LIBRARY(PYTHON_LIBRARIES - NAMES ${PYTHON_LIBRARY_NAME} - HINTS "${PYTHON_PREFIX}" - PATH_SUFFIXES "lib" "libs" - NO_DEFAULT_PATH) - execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c - "from distutils.sysconfig import *; print get_python_inc()" - OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) - if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS) - SET(PYTHONLIBS_FOUND TRUE) - message("-- Found PythonLibs: " ${PYTHON_LIBRARIES}) - message("-- -- Used custom search path") - else() - find_package(PythonLibs REQUIRED) - message("-- -- Used find_package(PythonLibs)") - endif() -endif() - -find_package(NumPy REQUIRED) -find_package(Boost REQUIRED) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") - -include_directories("${CMAKE_SOURCE_DIR}/include") -include_directories("${CMAKE_SOURCE_DIR}/thirdparty/grpc/include/") -include_directories("${CMAKE_SOURCE_DIR}/thirdparty/grpc/third_party/protobuf/src") -include_directories("${PYTHON_INCLUDE_DIRS}") -include_directories("${NUMPY_INCLUDE_DIR}") -include_directories("/usr/local/include") -include_directories("${Boost_INCLUDE_DIRS}") - -set(PROTO_PATH "${CMAKE_SOURCE_DIR}/protos") - -set(GRAPH_PROTO "${PROTO_PATH}/graph.proto") -set(RAY_PROTO "${PROTO_PATH}/ray.proto") -set(TYPES_PROTO "${PROTO_PATH}/types.proto") -set(GENERATED_PROTOBUF_PATH "${CMAKE_BINARY_DIR}/generated") -file(MAKE_DIRECTORY ${GENERATED_PROTOBUF_PATH}) - -set(GRAPH_PB_CPP_FILE "${GENERATED_PROTOBUF_PATH}/graph.pb.cc") -set(GRAPH_PB_H_FILE "${GENERATED_PROTOBUF_PATH}/graph.pb.h") - -set(RAY_PB_CPP_FILE "${GENERATED_PROTOBUF_PATH}/ray.pb.cc") -set(RAY_PB_H_FILE "${GENERATED_PROTOBUF_PATH}/ray.pb.h") -set(RAY_GRPC_PB_CPP_FILE "${GENERATED_PROTOBUF_PATH}/ray.grpc.pb.cc") -set(RAY_GRPC_PB_H_FILE "${GENERATED_PROTOBUF_PATH}/ray.grpc.pb.h") - -set(TYPES_PB_CPP_FILE "${GENERATED_PROTOBUF_PATH}/types.pb.cc") -set(TYPES_PB_H_FILE "${GENERATED_PROTOBUF_PATH}/types.pb.h") - -add_custom_command( - OUTPUT "${GRAPH_PB_H_FILE}" - "${GRAPH_PB_CPP_FILE}" - COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc - ARGS "--proto_path=${PROTO_PATH}" - "--cpp_out=${GENERATED_PROTOBUF_PATH}" - "${GRAPH_PROTO}" - ) - -add_custom_command( - OUTPUT "${RAY_PB_H_FILE}" - "${RAY_PB_CPP_FILE}" - "${RAY_GRPC_PB_H_FILE}" - "${RAY_GRPC_PB_CPP_FILE}" - COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc - ARGS "--proto_path=${PROTO_PATH}" - "--cpp_out=${GENERATED_PROTOBUF_PATH}" - "${RAY_PROTO}" - COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc - ARGS "--proto_path=${PROTO_PATH}" - "--grpc_out=${GENERATED_PROTOBUF_PATH}" - "--plugin=protoc-gen-grpc=${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/grpc_cpp_plugin" - "${RAY_PROTO}" - ) - -add_custom_command( - OUTPUT "${TYPES_PB_H_FILE}" - "${TYPES_PB_CPP_FILE}" - COMMAND ${CMAKE_SOURCE_DIR}/thirdparty/grpc/bins/opt/protobuf/protoc - ARGS "--proto_path=${PROTO_PATH}" - "--cpp_out=${GENERATED_PROTOBUF_PATH}" - "${TYPES_PROTO}" - ) - -set(GENERATED_PROTOBUF_FILES - ${GRAPH_PB_H_FILE} ${GRAPH_PB_CPP_FILE} - ${RAY_PB_H_FILE} ${RAY_PB_CPP_FILE} - ${RAY_GRPC_PB_H_FILE} ${RAY_GRPC_PB_CPP_FILE} - ${TYPES_PB_H_FILE} ${TYPES_PB_CPP_FILE}) - -include_directories(${GENERATED_PROTOBUF_PATH}) - -link_libraries(${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/libgrpc++_unsecure.a - ${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/libgrpc++.a - ${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/libgrpc.a - ${CMAKE_SOURCE_DIR}/thirdparty/grpc/libs/opt/protobuf/libprotobuf.a - ${CMAKE_SOURCE_DIR}/thirdparty/hiredis/libhiredis.a - pthread) - -if(UNIX AND NOT APPLE) - link_libraries(rt) -endif() - -if(APPLE) - SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif(APPLE) - -set(ARROW_LIB ${CMAKE_SOURCE_DIR}/thirdparty/arrow-old/cpp/build/release/libarrow.a) - -add_definitions(-fPIC) - -add_executable(objstore src/objstore.cc src/ipc.cc src/utils.cc ${GENERATED_PROTOBUF_FILES}) -add_executable(scheduler src/scheduler.cc src/computation_graph.cc src/utils.cc ${GENERATED_PROTOBUF_FILES}) -add_library(raylib SHARED src/raylib.cc src/worker.cc src/ipc.cc src/utils.cc ${GENERATED_PROTOBUF_FILES}) -target_link_libraries(raylib ${PYTHON_LIBRARIES}) - -get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME) -if(APPLE) - add_custom_command(TARGET raylib - POST_BUILD COMMAND - ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libraylib.so) -endif(APPLE) - -install(TARGETS objstore scheduler raylib DESTINATION ${CMAKE_SOURCE_DIR}/lib/python/ray) diff --git a/data/README.md b/data/README.md deleted file mode 100644 index 9f4765b5a220..000000000000 --- a/data/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Data for Ray - -This folder contains data neccessary to run tests, etc. Only very small amounts -of data should be stored here and if a loader for a large dataset is tested, a -miniature version of this dataset should be created. diff --git a/data/mini.tar b/data/mini.tar deleted file mode 100644 index 73098dfe53e0ff7aac7f3d6f86f1b1d904ab385a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51200 zcmd42bx<79^Dnx%1PN}71PMWd!=eF#TX1)0!v+?23z|iC(V!u?yKL|TcXxLuc#y!& z_jl{Q_v+Ps^`!0}ucxNY)ajm6{h3oU)t~Mj2roa6pn#wdkGZgb5Qtm(gPa23f3@)Z z6CnWsz`uc4Sdi!cIQ~b&E5QF>bN?~>j{tah`Gkap01Q0;s~!2Dt>@w9ZV6>z0C-q= zK-@k4XBPi`?*C5ze|_iwuQku}&+CAf%JNF`02CAe0Oj8ScwPd?0~nZTWK?tvA~7jBB{l79`uDv2g2JNWlG56``i91)X5^2b zzq)&R`}zk4C#R-oX6NP?7B@Dxws&^-_74uvFD|dHZ*KqG-Tw#Ie{lW}`R~a7Pq>Kw z;X*}6M?=T@4=xl`uYV^R5jw^zUQA+XO)N`S5=OodY*HC`ZcP^s6TjAPGArl=E;+Nn z2Fv+>p#2xJ|7XBL|34x7-+=w^xE29;Xej?S4~+;Q1*mheX`A%_BOSN;4k+Rxi+4W@ z$+kDO0(&mk9BPu{`QY$v7iu*3KgzkW6@Wz{-hUjUi2eKqcImP&Yjkh5t@=!j0@E`FhFC@LT*l8UlV(9^Spw1E7IL~UyGhj!_5 zaZMZ$hgkwzgdrbzk-cW3XID_7v8iHSeej7pr|9xVE7)#l3Q@7*>%u;Ac(iyH!xT5V zO66`qMu$D*tRFr7a{I=DFz_>V%B{}0I0>*_=Sx6Iiw}(A^wo8r>V10tkFQP33*#2k zSdk1l-O`-?@CfG6Np9?I-^|_GopWn*C%8BGmsXcmhNNH0M}!1c8?^V#^jxwym7-4V zV?jv0t=jn>2qpOD?8sEiVfW1cJ6rAyvGl|ID}4;3Z#u0FQyla8X0vEjI@l5*=lZvx$9IIwOQxc5~CeBF{oBm(O-at9JtG2si{TABMSb_dY ztd8kq&H|g)wi?;8qZA>%TYb};NTk1h(MW0E_n!Ra#RvdP@4t&N24314glPIsWrLcR zUJ8!Z#vtU9pQ_iuz4Nf!V=rbN%%M^S*3x@A(?nP(Wgf4(22*+h9;UXH{JIO5>lcU{ zGB~xZMs_`1(@fZPu^b1_2;#vbP*q*t|+E0 zx6x$NJUX!z#N2x$z)^|n^>FDivF&}rDNv6lVox)8$lK=aO>^q;ho)g`sAQmo=+=?y zltlx}ECt_tmnBM@Xf&I!qy7TY&8|gPr2I2LVcLDDO>QHqJtcB| z4rtMpQ##buS7(!>jeKMl`n}p??N03nqIl)2^up^+P?4`v{hdIISW9E`!+U&STq;v8 z4Na(2csnFH(qj|fWU=$M@r4*agdmD4vYEj~ZzsFM(S!=IDy9a{cw?8XOE04r2p0{$emrnGu07 z{h~!*O7~7&?_HEZM=0wWB3|x-*QvcM`(ggW##7COs|%e!vU+y5j;otuX~XDSH=yzo zpo1fWgb|<8{M&v162aHxvo)Ddr<{)`cX)yB0_m)Fx+(6hZAiZ~OgoychkR7;t`ism z*iEU`_=}=cV`F(^H!}q>=G{yP?yxV>l0(f;n0OHJG4^}B3mySL z$^@q4TaZb|ToY?<vlVyi#WmJjf=C6!uY5DlisZ=8>R5+?);nxxi?xxFbVsJt;pJR{KmE^DPa2r8 za_vrix7{Y|IR&xZTlVXhV&4h2Jt72aL*nZOMZ+z-(D&#fkBsC^^>($xo92fQmpao| zvW5FKN_Li(RBO$V=9_z!(kEvzru1Fp_f|vPEgpD@qdF-WJD{=mXuF_Cj{KIi9v9mt zwM2!SrXPU(Lf%%t$IhGrOLb=fjse@oM&yj3=H%wz3@fx1b)I);Fg=q=eHB@K$`&Da zPab1K`QFI^=aYoTTYc}Cmm1&W4E{PbpQGR2!y0|bnsNjT4c~Gn+Gx6FG7=a(=y_)4 zuXtDW2PkOejE1y!4msC~XNU4>XdcB(#ykVme;=O|Dio17!^AVlROr;vYWgJiB=B)X zSE7gV)u~@_{0+SL)M!qBl4iRkwU{7+R!rm;ENY{-Ed*Ec5&V2EO_h1_HCKyV3-m#H z?-N$MxzRXcbxM0`_2=rHTu<*z_Vzp7CX(h_7#>vQURM)iHnAYAYVa+dMh&E*?`%tR z(B>IH5j2?LWkkTf$S3>F#F1xt%yDO>t{XQrSGndXIo6V5aW{Kr2fpXQe))D8ujti> zn=UHWHjkUPSXph{UsAE@z30~nbp=jT)FU=AoyGIz*kCB#w+E* zo{E;uut+4Us~hae1L|Wo_uipmLpZ)@u>AwBk)DinRHbO?_xC-QRvVjj3-LKWJ-=rO_Ki(97?&99H@o#iHBy{qrtC`wH z82fodD-C|9!|5}BCvmTBlsGum+Q1URY&gf&OpAJWQ`OGEyE(4wvqyRTQ^A z^!3y8dSoV|{JB)Syh^OxNAZI4n(MFctZrXcsY#zBXXyY5X4XCG_V1iz@;R+0xxZ{S zZ3##GrJv1HC_R6FnUYeUsdF%5d+hn}%WH60go&2_&kVk{cFpf!VFIreKcJJJ3l9$} zXkH17(0AYJ&uP(o!9F=&=$Pn&xGe>|&ba+SC|9J_*ua%sF;jO~Bi`T9c7Xs!6ROr= zHL_=CRV*GtTZI;F5{&!j^}n{?N&v1Zr*xZ}nqydmVkZ0fa-#t2?r77S)mJZ;Bp{BA zNE|fdIsuz++kqz5WZw}rz$!O1-HZpJPt|=`Uq88g##+@Ml3)+Q>&#imX-&&h;E70#dDIAc@|EWJOURx8kY%P_gRa;9g%AJRcrSVz_TJ9s|KJhxec9|S8 z?LJ5Bcqa7S*@#OXB#xYYy)Js~L6_Fm=i2}#OZ@_IuB_BD2B+ClB{2}KcrjG{&K8Yh zUxnL#+B7F@``y_ZE8g+7u9bi^$KUmjt_1sUq3=Y?Yz-m#HfXuM0Ljdp-n!G--MO|z zwm&H}ZPteAqld!vQ_jl+93qSM67p-heSyhuBX|6kcGJ~L9Kj?3S*{mxJc>7)DTjB& zFES*og?(K%ndrz3+C1r=0jJscCNUEXla;oi-uX4n;aWIT!O}6w?Vf$?zfeBigi4WU#Kos-=(rd7}S6`gQzPC_6G!hFDEU1 zgntg#vElUoi*uycs#(&9UvK2mNM=Km6|x;N_4uG86>w1znJ?(usM}oM$fwMd_GaVM zR!@qTmPv0+ditI|FfEXAx&aJkBT<{&OJ6!|E#(8;ocdl@;4A<9xk(9uG`QD~@8}BE zy!tiN>zOS^Clp_D^ufgYYyA+;l=9T#uCdR1@3iAL;n4h$0jwgk=?&U?B){MHawAAZ zrMEAa=;o#|NGDRxgQqGrwwWEo1cTjTNjR;9r+g_~j75DB)!?jzw%kSY61AV=JAC{+ z0Y}hG!cBWxYxH7IKCM58*IPsGiL=OZtrG0CI=ShH-|aOYf7V0%9-a*9??PkEW*bx& zm3YZQ#SJNLu`mAHc#kE&bo0f6N^yXF9?Rt{?(>6Q+h^ifhJ{sTD?d*rc$=+yPY^*z z1(f28&ocmrtYW1hv%}_%b(WU}6TkobJ+b1=h;uB&+h56%cyaVxg@!E)6q_ZllQ>~q zdfA1Qoh;m*=ilVucc>cr9wFRFK5>fM!LAZ7-_`ff7H~wb-|`UfIH$K5tYbs|J7LT5 z)SOmbgC?LBT%GNwQXyg{P3+bmqtj`{evUuKcR-cIK5wPrP@mCvYDi;m^*ipng(q&3H>Z_Q0fptq7Y ziPJApSQnE_=uPGKDkHsrgolqCbF}FDtkE!fxA}$52Zb~LaBfy=EmN{INJ3bpJ?3Q+ z7d1_m8GRjDT`Gv+$Qmxr?cLMZaBkC&KeH+Z8$APtEt}f-YBQw}`QNVZ;`jo?=yr1} zI3@iCC!PVpM8Ev-yHrwPhR8MIZ!F(M(0K=F4gO9nrAN$OZ*X&SSvTNPVc+G9P%$KU zTkt8Ng&1fZ3KQ1d?u6D@HhY1NeRG0_r#VmOIrmTd@)5wqtL?;b7xt{XkM-;VZD)Qu z?$!@nCl)n6XHkvp)|ZTCUl78+A=wdSe=BF0gvP&jw!oqrLY(jGk|Ey;YOewQqb7y_&RN1PjZ39i@Epf z6WPHt;IKk0C40ZC@Zd*mIb5|M-9z8UU@t+IFzM>(kG_zw65ZA-=lKtv!3LL8VL5Zl z(=MaEdR7!J5CRE_GF?``H1%{pYN*_&Jfr@b!XCMgqL_nGbHCWB9K!55a198jn%0koDt=O5+v)mJGxGEzmLvS3rXC&A(&B2t?7 z9}1S4D{SW@wiGcirji}KV!I-|Ph$37BhxQ+QhIv@_{*|3#=|wISm)wq_K8uLIoHvs z%v(VxhI-{OXWTP*t@PSDEhK?O0zTlZ~mw4MV;dJrS|ErJoIG#gSPC)m@!jVnY=zh8CsHF6LTa=#@pF{Fgq>Ao68F^Pjp@Tbq5W2Qlz>rZ5c zZYjEimKzDrgaBnZ-%yj0wsyB{4->mFOR|jTucxD94rG>%MV#|sRZYb`&IhRz1`C3j zcAx^Thd)4DL&A3ofAlbR8vaf<&Jq70Q@tCZlZDTr&6auReyS&{cC?BghRZ4le3aIP zB}q2--_Mqc56JKOKgJzjMRQwIqZQ^r4FA|gdJUKL$sU*_{-n`br5T}kUyaiAQs<=e zv1E}nRj)BUpgmV4x{_H~Me1v8ci2VZBeQGB6dAwSLDPsG!d&iCAlgg+4lZWq?Y(O2 zIi@%b<;tiV3-$s%E=^J~5i z)~8BET8{MgS#j+*exNFDn{-`kq8Lr8PAcY*XTWUv0@IA(3g@D^p$W~BjWu&I;Tcn} z^wOhRHoxZaLmG6&SmhaDL|gEym8ah2klG)Ad_epO{Pt7wOKT}DFrQvsxa3g4<^r|! zC4aY20-^JQ1P46jmu@TS&Fbt9nW7=%18JyCUhFcA_}z#KVPwNAFS^PUKF1*%NZhK! z{3+VQFRFplAmt)CWBorb5p7j>wBmtG6XR|5Bxa`URgeDSA==?j=i6aNan}j{8tCW4 z7Io8})emsKQoM6Sr|(vJ9W%L=Ba12MRIOVxZ8OW27-ihE>w2sJVqo&Cc%Nhmn^=BP zQ;(Yhw9g~z789;cjw~-RQa>JuqxD_w;VySSsG@!^1!-!|x%DUc!T0$Tj|bbrO`~xl zcIP<05UhgHtOKt&eL2ASs#;q7@jUbQGav*oyOVYGLvW-!_2L(hv*ymV&~zr5N5#T5 zK`e7y2SUDyv(u;~Q2mS7w6|=p6y6-t48+^oG0`?DF;Y|?U5?q|a3;SP`rx-T5IfFr zS4}yTY7O)Gmo|F_bjFf6&8EA)1s6(NdM*%bVQkW`uy~_C18TjmM~p)6ESkVHO4QDZ z3JruAA!1t?2{nCbIQtq>KQ)xM&0nPrFv@>OUOY>oTz_oM3_Ci-w;x`dvi?p=^5^&|ZK9 zffJ=W1$qgJ^3rZi3@$kIesOE6x;L;hy9lg>P~l!4#lArPeGT)$Bw5mrKImjLiVw#$ zxnpxhbBn1GF*|%Qz^@cEzr}mwtM_@{i9dPw;tM$+HK#(6Puw~d+juti2k})B0(Q|` z&K>Lmk%)Jl7TyQbblH1}tI^hQu1w*9@Rh9X9MzhA$*e=sU9`XRVZ`l7+lueKM{$VJj^`}6x}!1YmC0Qt(M0z!dfJXmqeb*khUAU^BSn{xw$ z$>$9MULr>c-R5&XIUN-laZ!$bLDp<&kc~iU4(1TOjf3Sa+gk5HTkx^1IDS*^EQaKK zSGuhh8mOeXdVs{y@ayX4#vnnaSDLpjdGTXag<^AR8*Q%6tS{VyL(6%C=>glo_*TUc zlF&cjZA))$JSdvZ``l+IbK`WyWUb?Jipw`3wWb3+Ci2prJj`R3wygKxba`a`gDY`l5W%YrJ`JF+*=0`($#-m4k8-X3Sibd+%QA zmGZH+ONg4EG`CltA9j8}*;h=(RrDZ?2j?|UxbP*=*fU`2P7&HUIo;qtkB7Te$F*r- zBA{TII~@~3!MPM}~aF2n?jaMXL;&%UR=SpIo# z*x6tHE=V943n@NL{3n#g*+|S0k|#p*-i3`}FEd5*DOtSslT7CK?_YWaDBw}k;g>wp zz>=#He}(+U`tJgZlk$;{t_jF3GG~BaB$6GC2T;7vK5QlH$6mZlff0ZE4!GO!rM}Wq z3^?0hbs_yX%b6n0QpJyyH35eq<^J3#{a6SLCQAr0<6w^E>9oO=al4DD$(PFT3h?3# z;X2VR?BItTi`~rD1B3g9jqfC-!Fe}y&F`CVNKbU*8Hom$k26jI((?h zmE|Ww*I-neb()P<;^5=kx4Ri5O|8=f@gVz@vs>^pATDO_%zem|iZNS_U7#fv=i=Kd zE0Y^YLEZJeb%g@U6-By4#iYsM8^wRn&QYjEp2*F<#cLznGAH_fQ1s_FP9L1MsxLa| zWoT#H`WQQ)n4*}KwLqXgw_6pWQpE4pTI0Z&_`$kSv_1orqa0%%xn%|6Z8JSPi}P%T zMso@*$w|3-59jx!NTKeKsSQL_KS z-3~T{fjA?J~sPC!MA>v#6w;CT*-n?jA z-g49fkX)hf<^E%U;f|?(;X18wt&_oWFTC|nR2y~@nI{{6226m!blZGe@tI2lXY))< z9V_gAc@rr=i9WhSIg;}EfQ$Ekh7=DCr(tLvc_mw*vtZwJlbfmB>G6|lnOX%GQyaM# zjIE!QakeDCxvk|4rr0#bMmL)C4>5V`tYr@e_)s0p(bZS%mVoWNo8GOO2=j&F5&&Y} zse4|awyR^)A_*KICp*so#b_1U9vRrng#k{=cq`Y;5Un|#-mgS)hTE}RU*t%N2#_>U zKum2Ifpq85pNq;9Qeufw+!GeQ?Fhi*3Xd;Bj3_-bwYdpZ4;R6`6*Yivoq*|TJ6a@opZCAfc`ODV4-;ITG*pM+8_{B|KJ?bC`=K=jA$$c6Pha{73-Zip*X?46-%-_d{E`rgs=8D(fY*-ZmvQhpJ-Q%4ib!();VsF^}7*RB%6N)M30< zkZ0?}%LnGD<9h}&w+P*M4%*vmTwU4I_E=v-pueQ%C<6l^+kA zDzhKveN1UDIg^JMsI@j+>UlQg46NbweL}6Bf;Ob)5>$~=e+sj^*uoLK3a5-~@#TFC z3-mBMuzLo4PQQ6=@h^wj+q@oH%sblK|{It}r}C+${@t{x1yn$}}USXH@4hE6W9>x zT;lyxh|kyoT5iAVund5;uixHQWCVELYQx^m+R+}${_@}^qp69}9RC+)E^E1}j2Acl zRUK-H=XQL5v+FoX%%UDL4va8Al#tV}Y0S2V)>X7;cn+RB2a=okxBn3jyDqUEzZNr9 z+e0Sr)h#ufp+s{TSB-w%32^NK#uzff-C)jX2c+8XvmF+9CeAUa@DUG z!hc?itaVxV+!M_S6l}ubOs%{&#M@aVudouEI!xew(({vjwN%}?dmc$ZVp9yTM zWQ$3{TVNaa3t84cst7(Ba_iG?jy(o=zn| z^X;MvRcwANfX2j~HVw~x8)lzQ8lkaGC?HEJ-8G1TPRu*15wH@(9(G(oE%JGYA<2!Y znzKma-AU)z++t=_8MKTe9|2=bSh$quBaJ)8l6`zYD2OV&8I<}UxfyhvJPVKBOxEQit?5EB+N%3~x`jC<)+-_DPy zuffauyUcIai^mG(#q4Jb2XmDuhBXpu^WRm=@QyVzXX6~G#ch9BV)hbaiPMd=bfhA2 zKnp{(wP%Uy^)}tSkF}~*Fw__pwqQG^m|A|KEv4aMO>3QGtHBuxo8B7fOO zx;2=YyBKQ|GqXM6V|Vv6rl^w!d(9!6Z|P><)Ow5_2l(yQBlLbaT7;m>i8#)qs!K0R zU;zWs+H5t8m4^>CRPuyIOz{B%R6o<@EwNT`7ur}9%lNns^7PuZ|y+XRjVOzVlHW=vG%UZ z8^Pc<%lg0aJ`61p0U4=(#&5Mx^IeU#V^Rf2eN<9r1=kcJ#>^DUw`ua;b};=RSSg~t z-;%(tETy%2v*qU%lF&y!2FJ9u@L{DD4HT&onmxIurRi@rK45;}IBw$1;p^Kso~puZ ztdI5G)2t(@(PXj{UgVFfrnc&5aMXbQMR(F{r7PKxFHYlC@cQ0)iT3L!bz7_UEefC(Ic;&VG$#cysl!7+qQ~nCU|UV$b$(zF(y! zLM(0EMTH>mP{Aqc#yb7SI705Fnnc~QPwHz`Bohz~MrdhD*7BXJtCPW$u{K-G{(e>x zi6`4b|FKtc&i>|$D-4(oq#yq)_fdILrJ#2RC1x^7R>#7I$lK5Ld?`{YDc zxq@ko?^dG%YkEKRM}E#*^6^$ZBGFuwc1IcjVcPAz-waE@4QW<=v>BI%_RJ?@hB{{B z_;g(t=9BoML#HcY-}>8H2lCrL-ToorIAY`v}~k-%@MPvFNGBzVu~dqS1Jso01FW*LUy#3Sc~!7Ab+ABTbi9OKh(Bzfz2lKdQ^WZt zR_5tsJE4>jtCOi?TPn*iLI`}2f#pBa{Men!+BEeDR-<0rGx?%wrRrI-gAi+Cc?M{P zTvd=Ft=@0=I2qJEs72&*<-Z0oB)bsxZq&c!7^~VDk@YEB7|Scm1v@0r*Og~BAjQ)z zSNZQsZYslU$mja^2X@03sI!CDp8`0Tv{U9xes9F_Hbn7ZN;jc<-j3p1(wsW%jCY5x zTD5d8H{<@~>r(2Ih?ZQC06S5n9-o>q+bw$4V=GzCuV=#J@~GDD^;pVIV}$W3Rz=Vu zA8*aP_d;W(T&HKUczwyWcrJ?4ZxF$y-(J0g zbtj@6a+Fx0cwe zDevmeZX-wfuOHc1m*sqvUwDScpGG~@RLW!{8SvIUmL+1D$CxKtwMp7&Zu=u?B79tr zmYDYC2q28Hr(7~+1O5Ah`Ed&7s_^(csr3c#Bn>YJa15t}$PZDKvkX&f` zD=my%GxNxV6fxK3Eq30exyiyN9Eh|}0izJHf0tXZ2RFFM#5{M3(zm?JU9WJtgRFcQTJ~QBrxZg|ZYJ*~qV=b9XF!by+Rt+OS?5K9$9iB_3%9a=VV`?7DRFeQ9vX#YINvDsAxf70{D3_uGQ}qVDBa z60FRR_?b2?Qg4rR{7pCZx2iL`>rCnS6xR(bbq_sDF9Jg=>jED>I&I)Aexu8~gWc=R z>|AG$OsPUD9*aYbe*_*-QBa|k@q*S}O{Z55q}+Y}K? zx5ZtIR2tp2*DrnY+tj%#C%Rd#GT{XmDq)+jP@i5gUmJa+d$OD_^<^AwGUO`F7hfV4 zl_Pcx0CLN*j!mit>JTOmkknPv1fzd+-4^j? zdyzEpX99Eqtu)7o7I#gsnSrDI31<*SGS?xQ-H^QTWxgScjFWpN+gnkVl9?Y6@B?U{ z-e6CbX~t~&N|35bli5zX-j)>R%nXDa-03NE3FXM@Gf#p`s$9_yE9^$tISjh{flX`p zM+Y(b9g4?{)swpv+IWQu_Z@IaE8}mP_cIyM77n&`jjvjFFYv;0_jd1tMYFdbbl-5Z zOq(AQ!Eto?+Gl8-r&#g&zKQH}2u%sX%w@6)(<@LYl}d zFL&5wpKKA;WvC>D{D*xK0Zsvs9UBCV8rRp2CX}D@d`m6;|H>zB2R%@)^w|3QK?8%I zuj}JfN0^uICh3IfE!;vT>tVpG;UYmY8;?dWI^Vk(GE-)3{)?u$@pWAaA95eY#NA+$ zMpabiy+ru*ZAF&R+HnYarFj25pU4M3!l~kQH^ZiJuA$x?MG%V?F9N5N((KQP!Ot

M@-pdSPpY$zbBwb7%Od0PHoE!ti zftMU&I5>!3go7+PqNoHDE(3}&2}XpZy~prrJkf)U7V0Osj_V&v0>|wT?iZ^v56atd z09wfgYo*_J32Mc|rJu+_#2`xh>5|(+P{ELG^K|7Xi%y^W`Rj!^cWmDgCY(;@(EQ8F>bjK=u9?@1U!d4m@S;X4$ytO>jNROr+V?PpwJmP7v> zjoT~q{l)@Q)}zngz0#iC5jsB%PtxwSmeHU_jmP$l{i?ohZw+0S!ln`X1k z_k<*!H^9!11Lea%WR}De@bT$~Vqae!%oe3!xXT!RZV@U=eg+sOZxw0eJ~aqW+d8ts z`mGh25gPz)>VS#IcL5?I5^)No*%ppr?+w@qQ%9o`mw5(&0X|qF;%>^;6yyW$m-0u{ z7tNF{JJgEeRuJuAnx}b-)K=VEr&`sw!fwAnDD~W=aGxgU9eaKcr zLgul^$=HlKzG)Oq+QK?4t)|JfcR%V7hV$?%GbfkKYLVf7p{F0HYl-rxb@(r+|cboibDvt_S9? zdx%_>*S&zdLMgGC)Fu^&p&;~-rK+hr-M{LatGkPc=*ALVlt!n>)Bj3&6uo!h9dVb}noy(4Z*YBW*A$8((JkZ(1C)R+Det`=y|E_0 zAywRMQj807ocxJ#evjnXSE5&dLP+QiGLvLg;vE>}5` zA($GI&Q6nPbMiJ4iYNadQyq3$jgkFMG}v{H)+z5|jKNR{LGU3Q-MVnnHNL0~o9Ij6 z5Rhc}+fu!2UCBqAbs&^quN6e9<;b&Q7CQv_sxRI`e;O0vfcV$TN2gfuv$l{wY#!kK z$o|w+;IuirIHo})<2VHvltf5DBEK-@R;>`tV|Mk(L zcBV3sc$6e+Yiu}1fu>VDM_qJ|mK40c#QgPV&dU{rw1mr>H~kNNPw7PCp1?WNjRTC? zX#C}bIwi(C-jbbs<~^sqIog2e@*@^-->2cl*rlD6X8?lJcxlAT4U)nOLZKE@q^)i@ z%A_*6w`R>QsJT)Z*fLgWq2-NVV0512uU@@%%h|ZqoRSc!pixO%Z=M!7*p5G`pae&R z$B$8BVSl~?ocGPOSssY0?G^<|{?#qN5)&CfB6-745W#;%a}L38SYMkOGM6`Egc?hr z=}a>>`Rr!-w>bR4X%SiW8CXKNPDy|LR+|$G2A@13%s}(C--wU&PW) zXBg;FHc&0k*8)ds6hBa|dyJk+^w>KIkuR1vrYr`m@rqEOZ|`)p<(P!!d`w>zbr|*W zGR|`QT~+!SCe{15QL(Ik!>#um z*_CU}b)DUmo~P><%lY}$-YvkPm&<@%`W5dI5%>OJBT8>jO9SOii$fbXE!Yb)&ktdIW@?*hLWJkVZqJ}{8yNMLt zEBvsWZuiP{<9zbZ&HHH#zSadwpMWmCM4Fm_jdz`W4Ny8z%QQE)`(G&)gxuE`Cd$?U z9Sog-ze?hMdVHSxc5r=be~pdi>k~STGS|inO0VwDEhef(@Cdh60ISx5;Lq}jQiP>d zZrxH`j>ce{z2OW~%_O zaeX4YHRapn&j5!8=&K4bl?SC;W22;IX}b6Fx_JEabx?UQs7#D; zwlK3!w#m8t=Aub)CZXmZv-GpR(+&5+dqsW*8GHwg@=QL&JkOShDn&ObCjR_khleS- zNipEJI!d1lmf2On_c!w#CKQ+NIBE6x2ut39?$dv+#JvB)C`#0~6IsFH?n=16a@;liTfJLe1<3I~wn4fg0@A zUNo@WE3L0bNDZmcZ_{s8R(ixIMSnF}^;3Sb!D$eh2!-=q>kcEEN3qo9tRO{fa5E+<}}n5_LuEufH+E56TL^4OiO+ju;{$<y)RN{LElzV4F`tAEUMenzOGcRDAOFO)w zkHiGBKzo=w=zV_zb{3%O!9hRld-4+WUU^BOv^`Y-#Nq zVpaE*P~)U=G1WCKI3Wo4HQZIGj$lPvC(KE_CMv&=i(y}8D_+ZCveno>#Y^7{F4H); zj@p?ccz=vmv^B4JQ6n>DeM zOe)?Zx4@Aj-=NTw>Lw#qxLRvWyD+`DEl^u3bU*Oa{Al*H#6F5vS zAx3jIK$Vr14cgtd^`hd(-oofw{en?JD@P`V7shhPAs^Oktw2-{j8ey{vc*^&GG^v` z^H`eQRw?~9dKa$!D&Ic&>6}yD`|-$F^{XJN5ji~rMuLk%yA&>~cXz30Qe5U?bzh;F z@^|5I9EYW9V950t>vT1lzItskW0JzI0?jj^kP|Aj8!z&fPAsltd-AXyZ_a*NZl(=Z zyJRA&#;L}LMz{Ljes}wH8{XAf@#5igV+_quz{WZ%A2!YBuo~KdtJ^KXFPe=H$rQs% z;5?;b1;>2Dz|)b9%c3MVggQ^qx2kuO0vu}e>O!;oAnmsEOHiK{IOcb6s z9M%g)ZXwB_R4Xx}1GCR8n0#0!z};HNJq>s|H;CP6b5{ws^wpRMW#4P0n!uRn6xd5q z6GF-C3hxhx{%aNMtku{u3S^Bq{A0Ax`!&?;YhO27IcfTKNpYt}< z&&+O4$o$+#OEM$tb5Up%xhbt|yoLU9`=OMiSojVXf2%}GgBfi^Lnyt83%`6QK|_Q& zp|6BqdAJTK>aqO8<02}OS)eA8nNaqcYOBEgLCP(i`-Jkc5-eGy?ZgE`P};<9(zh;4 zUZSi4VIGIRFeAs}{_cD!qKY{1<0fnGt}ulGm(D z46P1cDm22&wvHFEe+Zh_-PR<)@yW}J`~DQ_rzK#9OR|`1|L|Vk;8xeUY@ZS`S(Vh+ zZ1O=-JH29_duWME)gwwDZw8>R&0Cf5nQTZIP@3_s_Chplt&mMH=Vg`m+6*Z3@p}D_ zJrvWzuWM-_MT*IM*7Uq4^li)Z>fVAtRs2y)hZuCe@+Mpvwl09nJDRLWbSPE>--~#D zeIoY${JfOaylmICDvcd&1>&XrM2>+sY@dmC0H zQjTx>Aec%&v)Y=i*!LTVU!)}t^9qS~b9)E6UX4oO&9clTlg7Hl{Q~FJuqJe7w<0t^ zBc9aIKA%q#lc9qq31i7C{dYC<=kXWd!nQm$0k^JpJ6jx^rhvxt(l=tB#3p<6Ra9fo zfXbxk??}FX1=!ew@Hdzu%wAacGi7Pcp+43Y51=8A_}2EY9N+kUSFtlW*h?T_zVD-u z=-&WWlR^0&5R(N3Z-nVHKoaw1T#oz0O)(3Dm^Bf^`Vu7$Yh|p!G+EcU->L=fm1(ml z5k!|`G?I?R{}Eco?*91mb@( zYmAweMgO4CxNF0N(Y3b}mwk0`TE(>&H#vH!2w}-m&7h%4D6@~dTMS}vDi=KLZKRg_ zn$D*DuU2BeA5`J1Q$<BVm7Kw;FLhd!*gq}+JN<@|Ih8a6~Uo+6&8@A0e9 z_)YaxRecM%A)$WNf3&>gIFvMg_+0zYi{IN%kyqjE*J9;n#6}b1Y-({H)>>?_4xK+g)5lDvoeZmu%#0$Oa^3f6BFXZLPefG*kE^)K4rzWnhKXD70}s**{>d zbHXorZ*~ni?bC4RK>Rzwm^xJ-0wf!(?lH*L?fpH~+<0daTi$k&i4R5dNeoTpB**7%QhG@v_G8W)ZHO@g+Hi5EuK`twGGo(f0cz1t1}QZ z)Ph_FUb+qSAmtJtS7+*S0$Y2^7oWI02gX>u;=E6?gN$~4xd^h2>n#>E@_IY0n%gpm zSc%kFzQMM}tD%&B*$S##F^BaXq<_<1_!U>0CT?H8j#9{$NDI5htfg0Kn5OoY?6rl3 zgWg{({aw%BV?vt5n>CDR6s_(PxBlEXb#E|VC2XtlBbS}FZ~n`H!Rp8V{ey;%uiEL4 zY$pYvGqCWr4JThECO^!+P{F7GW13DURZEeU@s3Cu%hP9|PpvQK8`;3YV)znioWNJS ziPLv6tTII%|61d_MueY;#cb0>ScS@o{A<{4J*}I7C3NdL>I8B!6NjdA*BJ0E?KjR# zuv)aKDSGK1-6^i^6hU}WF+@hVkXeDr_?#*1PfQJ1uzZ5k(+jSy zX^=J)&woK1L^xQE9O3rKdh9 zC>y7Sf+s}_`^W20)io%Ot}7fCdpYR2X;k7_qF)t8T}zxl1Ln%KT#Sh~hHrV@caZ5I zCJF~4od`q5fB6Z@Bco{h?S=34PVI&mq2JK&BkuRA8Z4Du&JX>#k_+BbalXTI9k>es zL)Nq@QdEh>(c&}LA7JnSvT$AI(?;8_qC-Q3$PTW5=;61QTWFL|jJ|Zd$6&byql1lT z%+5S$3dr_UiMHz`GsJL=vxx_TxYQ=av?TnaXV-d@x&(#_#^3;C!O$u;4Wih&((HQ= zm6u{Nxll{_%3J|7+IG;)#is(i?~Bh6w=a4Il$H#XEcUz@t!qi`8-VPgxq!H_dkFyI z+d5V|&J7PiX{aCWh8MUp!}^&Rw&L28a|A0!vW}|_PdxxXFd_M{IFW6Sft%}!Ts2`w zPl1Ty@I49&<<49zN6=QN!k+1y%!6%(S{E0cHv>8!_t3;&2Jw!#b`@l35=UtDF&6^Z z#u(z3V~$cai93(BQzE6r`e^?1$93}-V-*=@l4cTfhVx4w4L7Jv1oFSJ_f}tR{o> zEfm^9q0r(E!QI`97l+_df|+HR^G#QFL%$^54EPi-mPv+;ky{w>$oJH z+A}=sF0iDT^9aCo361d0RT;#%~6t)cWY*Ezt9gy{07NH6HCvy~bv|11pAb_+H^D!)x zrgD@Q!-!~LI>jHB{H3yo@+4QSD(P@HVsc;WqVPaS?Zvf#w?CyW1D5kEs@MG7UeBnC zY^{;Se0CeaRgitI92||FuugUp&v%-*eBeLx`$E7JVb7G5u5@ZdU@8Dz3_n{Kwbgkc z7e?~scY#wSJu-VA4At7b51bimdxu8NgQSxgTJPaIILfBz#L#O|;MmD)i}tkr*&fGJ zW70Ze!ZX%EkuMra9R>hmMFGiubM(%&@kFlM%?M*ouq;AYQw`3}G!1r(jv80Id{IP>SE`Q`?Sli2x<^j z`r;UPGX0p%25d0doqQ5mubXr{#Z|0qZWERGJ?LN9oILWD5Yg*|tGm|@pz#MC~$sV?V z&(oKgiOMK2J;uA~v1%pCu+h3*TE_rWv>qToVuO`W z4SRXCJvt^?;Z##mPqLkmAT^*~nFG>$P|I~C+m>9`_sv!HyW&*X;`68$bOy~*{xC3h zUqZjV`iA`1l*7INhnRX7#@BNjCbtZ{m39famtqS{A)A1P#LaiI29aHXG_b-(!@yvsT zYxFx;8!8PHtL$bJoM@vxW9c$Hz1FL+DkV$e=IXOT>gh7o)rNR+|k~OU?CSonfvb`E9hvIOR~Ue&r~7rGTpb zbTeW%_UaA6tfu#3=JnZz;Uk00(c^{z@zY26aYRmRcTdf(9IJ?3DSEqvsBY(bmjLuC zVW{3D*zCy>aU_d|6JzYZo;t$*`e2s&IaNBpYG8#&>c{xjF!sKm_&tc$-@T0_mGZiRds$m$Igp3ppgT0Z2qz;RaywX)P@Kv2 zIsAbCR_r1cAAoK*tgRt)RXuNH)%fn_gB)co=;bCiA2UTt=~>w0Nv;pnOg!vW9fIxi z$%UY^z0|U1@Y&vTzvh@(*Lh+^jh{h33IR@l4goTE$V-tX>sF{Ic8_r6i^VzB)`kWy zwhX_gqbw6aiKQg;gHfvRoKA#XYXI{0?V_fyy{*W;F~uGN4JdxynIb1y;a&!wV7hTD z^ortxgC&$t9LwLwO*?_u9*ZBhg;EuFRXw^WUdI4%;xA8o_k&m?hogVAuU|UltMf{K_2xC);NSgZKVk3K6?_woOUSC6v)8wm%95uV%MBPX|Nlz&^V#PNdV${YYi;x#qoN|rpj}H%4vh{d|`*ZOvBtM!o+&$o8xtf zz}4lAP6yYu9n7_3lp;Ukxl$d(OKAvb%zb5?hEMl6hf$|=vqf5xE>J=&(8h1Wfj(Zq z{cg#OXUIP;2-@WEd3=ZEOMI(Xz#S0!^B3<3)PpkjZwC5JL#UNxXt>CKXk;lD=8aKf zLUJzRjcw!a+Oh0Jdbj=n0~3y6)v0+Y{y!tDoN4~r6kIzRdu1B^T&A01E&UJega50` z6E#Kq;wd|8g?2v>ILSwZrg@5K;o*`n_2(_e*OYN=v1CKbHoaxBN&rP}a7ztkYsQ`J zGN7lE_@y2&TtN0-8J$-4Wx(Ziv3T2b)|dcY&$j0(b+@UpQusctrhELGZQc~T^U7|y zYlTRT6U!FgzR{GO@ao+E(C`B9wyx^@H)oIafqzCZ8iqx8g6BL5!z5lHdL`Z*WA(TU zm+a5rt*jb0#p8Ecc~8wX=`jVpJxTkTmdiA?18{ zpf!d}zO^UD4>A$`O0kMQy>`g&isp2ju|ameb3;=EVx zANO7EiE?A^lF$4PO%@|uvc0DDt~HCISg9pInKvGapG+8deNF znvEjMdQLOLRjO7CP_91AH>Bp8EO);at-8aEE{(;Q#uycN8hX)ODDfzV`73Ipv8+27 zllgm|fi$1V+@9a@h?J0DHpYZT^S5SNuL^JuW{Q5o>nebLz}FD(+>ZT^_y;M$Ul)Wn z#>JBG{#C=yQD5Rl`#1F-H^ckUU-_;->CO8tUnqk`SHLC4M}MYT=#z!gYPq*7g-6lR zrOX}FIgipPYGI3hjeiSFXGQJ|0q&Nu~{nC!@=uKf=8Yl$gxIs?!xVL z)E(7BkYjF3xP2jwQH&);tGe$|L_xRcENZ74ZhkKcZyeKDi4a2|L~b=^nUGq?h)li! zGL?!ho5X}d$cg1e79az3U%+=lgifEhmsn9@!CE!|F)VQqO&A!&A~#15z#S8dznfQS z8Tw}Zu4O_6)W%*#*-Hv)-*LxdlH6Jla9F)mqbI+8uBn;5(#7czfxRK20F{RfFY2V*rO&-9RW#+G__y73-DRCe*79U|D?M#iWKi}+h)s+_qsVUbTD=-oRk$aHUu>i#L{(G zEGTnL4x5&BrG!X3(yqR2G&1;1#yvnzDQ#_F`%l$X@sK4uz8=vt3b*p}RK^FAe_z0f zR9~484aYNwHF4ECuj194A3xW}LrMwkyEH(7 ztT&40(m}t-pwtnocaM9bX@7b8`{-J)_L9+cJ4SaqPaM>`V0zOBCE& zqj-KMR!EdgNpv?#pY-Je#lEa!G3X%Y0=X1$XMuJXT~luy-5hT+G})9};=YAJ+Cd_9 ziCeuj@4o-aHb6bW2;}1gFfug77L-=$XCfam!cW#vR)I+@l3Ffq4@^hdyW&dX`T!w5 zllpLWYi&VfBU%7Hg5vs4?_x|HL3%ITXI}-FU!`uF!Q{VpsEE#+$zWcy+;3IPW4rUg_inUYPKvedBE-c`J^KRpv74 z;a0I~w7mGqnI=9XMH||=bBl2{V%8$6mCr@_2U)A^phja7^y4;70-rsy6nmyYlocZ0io+wDD}q|I#YF z;H1-N)F&lrjl4nbW!=>COcFO^8^(ckm7BgIlM{qKN*wG^8kcfn=0lryb0E^c0q zA23zDoS0p-@r?GbxCnhgz=*N-rq2F#?!{ls6gU)TrWpR^nR`t^Wnf&DFb@YkjaDT> zYgDamm@_AT4k_W&7ly`~q$+gC-JAzOX4cjX@GQMOW6nrbsr+r?@UEsm`f~~ur2I|U z7|`C)68_27t~_(R|I@a$a^F7kVliBTtuAF|K`m#(*kHT)mt|zpH`j!-p_4`Wq{>Mv zGrQ|~eO~@HqBDf4;Irz3kN;7C7D|^48Wht=U;Z zKLzhke3DVFK}*kupwR=ilzi0oS7*HY>Y(Q(!2*}rZ~Yl)S;sl8n2EY(!iNwYr>v#YDZWw2iCe|*CGGkNILgjlcBy_d zw2{Mx-tm_JN_kKgy|`n_7(P;w%qsWU>Y{d1=EgzjWI=gxucUUb`h4q;~i} zg(S!YI!)Z0A=F#5fl@tBlS_bzLWxkUy9L#tTKnvRI1YB&p6)IUfzN_f?DmfCV=p!k z)Sg>Ad&2AbdXm{*ij6` z@0Zb$OZubOFEuOtklKQ7%aJZX>lie~srq$=)Uug~wsV;u6`K57M$ zCohEn%!0?FN|VVdhxsyEwpMbxTmskL6k~D006dk9>+3z$=9H(qUHD3@IoB`0s|*+DF-T#MYa6Z=`RJ+mZ}p#Iqnbq@J!;4Bgcs1VwJy25e3xg$4T};{kdL` z(@ypGWM$zn%n7M9w3{8{zhKzed^&gZN&qcfL9(QE4#_j`yN= zd&DbeeaIFY`@xyCLr+jqz-`ByyRf3Xb6C&{&2f94p@KBMNCOmBEPeq&Nf zQ|;CfG3s}iK5%Cqz1uDDrd3-sFnpmqy!s$>Rde0Ba+O9FS>k6@UO<9i)=DT$RF^`Z zDYCOt>aP~ZH{nElpYD{pfnN@5I;xXXv%lU<07Vw^bYZ^GrBc{>ekdH4v7LO}vMpjL zPYoy(GxfDO*mM_?$eYMX(aN=5oKOE1!w(H&48NXH^B?C*ccS9-Q-i8Fg;;-hcc(>%!PX|<_rWsXN3c0 zu&yH;j0LR|mkrheo$=7$9ZI~OG;3vhm8I}Ky}jKat%3<%L@LAK*VjNb4-06i%JF<3 zW&ZNE)w^nQwt`NDRb+s^2GKW=m-V~d*pAJF&q)rbP6O4O=fMzeR&ALVn?wSLgG^y4 zy>l^=MDQ%lTIBvY#J2|-t}N|}d|IV7Oi%i*QYg&o0n^wpv}XC>_~Sw{`nk&vbYzR0PSTj zbr<3lN#Om7s}LA%vr#Hv?L$}okb>E(gmk-6l+uB*k|(vS9r$oNS+$Iyv7n9d@`b@( zse;qTAm+eK{W#t)k*ljfJ--qL%CIel9+?y2-7v6i|`mphR~?~&J?bl z{LsO!PfCwPaw%D0ROUkFCwLywxclKc$2f-AvED|!aqq;6KKAKKglN=l2UafrY;Iy68`bLF*o_8RWT8b}35q9KaNzMZQ{9_A z@%Z=9$inLADkyr`;JX>mh9XB^>fNmOwxwC=*85% z9Kh!gQ%W>Tz_xI`z`gblVu}o6PttA{yh{R^a(VjcfXy`PqC4_r4IKgh<|m=g{cw4G zV2VR#RzS61sA&=JjHW)-Cca0yPOWtdbS^78AUL#92Jbq?e&j0=j{}&1n94(-DsB#g zl2#`Pj-wT#ZpydPnu?LUD5Qo}_qGuW zx|M2;bClZACvelh(68r69q!TM{8%c45ZF>O0sa9{wi`K;hAcmCOfon{Qg(bj;EB zE9qG63al)nX75+~S8>`YXa|(@B?C$KK1$ zZl$$9aP*}($)seZ*V83L-3)}x$UC3wsk8h!9cE1C0yH663L?)Hz}&PLw;=CaE-kK z!75$j7iXi-we|V`hPB1WvJX$@QS{*_v-VbQV&QKEkyiK%!plqOiFgiUgC>J;&M%EV zvC9HFG)l#0;{9Fd)GIoNj`{Pmdf5tb=fXtuj2D1JAzD4G&;Hx`%6{d>E6b)v6X;lK zhkc`gY>KgY*7$(BHkqwWL;VMqsacN|JCR$ZDW2A zk8V!vhRaW^o8ahEQ00L(`E|>J_}9iLV_)bTqv8l!2<{9X#vCP?J(vhv)tlCe6}^I7 z_lgM~7e2IlZ;*YsZ^B}mye^F0&ZRb&h zSCqQ|Vd$A3uPFRhI%%#+jwL^NDY(msZr_g+aq`YCDCc;Rv@?Uxi#Co)SGwADY%wsis)xvpqQ%ePk_z{cv;m3$du2OcD8=n@{5Z)KR-tF7b%W ze zcY}oKDke9vfFJZsYr-`uZAAO6$`yhb*u?n~3HcL@k%A-%l>;?8j1>H(7Bpuz_ zskqS|LRbeg&Q-@dc$`$ymm5&MXgIh(dqYyS1X^oc86&q+kVEd%BasHFcbbh!5`w7>(!LTqgv8cawhl27{d)iIF;LmdXS7RIU1dvSNQ1QJ(GYSYE5 z2iAVym|BH-xV#ISiC>H*5NGLF1SZktrg^wbG$M4uE=_Z_&+Fq; ztya#M$ywo1_Rkw@R_*?qAfQ1z@WDqo!_IGg$X^TVkB1D3d zHg0!l`Diaz?}H#JNqd-8QREb5n0oY%;VZB~Iz?ro1jQ-k8eWc zSA&tmgIZENe-3spr1lpvRmqF|eTDe~y!*}Qt?S~uTcz&1wv zE{8=vCRHk1Qx2uqR4nThBz3@|nNVxu6dPwsns5{p94ZZA2B%}TR2r-~s6PLVj@Z+`COmdf(#i}FvArUSyTq2fjfPik zCttgqDF`!Iq<5l}#57tJ#4Zj@#ibT61_i!y4};xFREWo^MJU4K1NK^QP3nGc@97IW zzVqJF?oeaxC84AJYH5SxHbp9)%;PuPRfX$?H32R{ct~#Nh%&7?NqP+&)}i^||B6_I3UHU`-G*e$ zzDpc#mR=e`1PJTqnHstbu5x;CP2||$?Jg7V>r?h9;Nm?md9C!`X}{rLQJ8mBaN%L# ztQW#&#gEc@niumKbegLE=dh(Q+Y+Qb^C>GeP#<4OuX~W1*WY&WuRLdIpwwpGD^B1o z%7O`hN*v(llD|!bYuA&vOkgsu5c1#B{}xO9|NZLxfB9ed1bBu2zwo~Za0>|X{4XcW z|Ni}-{uiGA@xT1Paqs<)|K)%DFaNhC+b_9RaPFO8#{RxaDGxk|k z5+0Vzp13z@Yx`xbN3xRIb>9O0QrS>HaR_B9XNfISrYdrAjpw6faw|Pq8QWG~zDwI` zBIB>^K&B}7GC|bN7>kW&05nmST|cvGR%!F+`!;ReIQm8t^`jgx7c(M9jbURBHWLlA^f2BQ}SD9P|n4xX)VQg*kR)M@v6HC^V-X6&uA z&M`O_DttRgRu~5W0y8?$G(hk&QnuY(Rw@*DF~EmDzlLvHdDn#7I$p=iE-}63k-xm* zIip)omU&qO|5>WzuCc;f*SyBL%0g0^qOB6wRJ^pJv?mOplD@1=SvD@y-u}z|bMDv1 z{wi#bfNKnCbvK_}72=hbUg;H!W8@+4dc=H-C!jJvr0m5A`2~*}xzpEUFK~oxBw(J% zNTz+O;ALLsRp8SfN#viq;@g>lrhLi3a^E8~^z5LQViq(0RS0#wCv0(?yv;)&dMRoC)@T)UWSi;8)N`1?q<7+x`CMg?r1?Vsye+l|2bLpm$sK zinNt7MRK&1k+X6Q(fW*lbBj>+Cwerhx!D8nHTziXy1DfQ`eJ?|vOtfC4oHqvq>I{* zL5EQEh(9mX-;`BFGkKJ?$IXcOp-B&LVRm2We2=j*K~c3mu)~Y z8G-9z>T1R^YThDUYVwvnd?Rb(;2SmXJ)`hFHNPzpi(EYaUHZQhHDV?cqYd=P=Y);i z=Qe^AkS1|^?db) ziOYkcU>VfpFlemJuuHgR$->TdZh^a?jBMkFKB!Vbyp zZJ%UPyFq8g?7!!21XfFYJb$k_zS5djH1f{Q^dU~}eX=ZRMP{S;IXFpdi1}#t$Rl)b z9O>3v5+{L_YqT{^U8*(AcFXdUxfkPX?<;$`+E0ycLb{WcvS`ry6Ip{8u@N2LOu&l) zxZ%wLaxa=(Smq3M>@`K_)dW94zHMGbNjNLd(iX+)quroGtp}kj@eBhpInD-&(GsjH z$GqkaAJ)`|$I~L3+Rf5cRPi5+#9fn8tjyWy3klU7?ozg_-P1Bt`l}i0)oOU3As#JP z{J;J~`=oT9hbFO7W4M zFjV-}D9TjfCxI_?GwgcsEm}p3a^(a>5rD^pfx^Ni#OtU><| zjl;OW(q26!EpVc^?pO}zz|GY*oJwQ-7X>aWsd&{c8`II+e4d+lCouo-?=T4Y?y$+c z)OB)~yzQ;B6SWD@E%GO$_AE#Z6lvJHPM3?W&!H3aI~T2*Z_0mL0K1Az35IV07IgUdwP5ZGV4l}-QX@#z?4A3f& z%H%5%QnV#cVw4Ns-djGWr`e9(#{e$X>}?8+;QlJYtSa#)q|w;d^i>5G%Q*fRWOCpj zmQN3v)07VmxL5C)@Rtp^YTvS|GY_i(Ztx8eQ?(Gnas*32jdl&RhjT3;3y|{9AXp)9 zD)6*Sa8WE+r|)B$z$GVCS3=u>!Nf$v6E^035qEGUY}FAH6NB8# zomZ9PNW%qas&nURO~4BT`v_SARHkF=U+2S+*tDqZ4pC>}KLje_*4mE}=I1L6nC< zU;=Bi5pvnMsv%nNZGPOC2y8tqv)lqlm2P(Uli_zMv$3^GqfF|`oIu^_X%Xatg7b3; z8y&Z47RrCJr@ET&1>ng88ho;&@b=@aL(713qX6 zHpxc%QqXudE=4(;#63rSw`v!>^c{wvt6nv(C5K0`2R46sNZQjXHjK2-S9FZ|T`BE0 z`Oo-jxY+u!`?`LLQ&=FoxB$Rp$`!^94QPk1JpO1-N)BTXc`LKBaKZ&j9n|3S< z7&TD=BLB?p`k`zg$wTichvCt z0ZH~!O*Q8G>m#&gD7W*o7XF)INzCn zid$v~lK~wWa^F6GDgu?3{^Xus68cZysMmB84U3eq^}Na0pH3N_7c_kHdOZJ?nk^x; zG4|#5aJV9rxKSmrJ;Pb>@z|#Q#+E9 z5!x5Ovsda>1)4Grw$X&l{Rg5I^t2oHEDJi--A;|LD~b*MzQA4}gAR3ChkUyJKs_O5 z$d7H732Tz+6*$0Ygo%s?Imt^5ZLC0DGPQ0g=ZRA8FMW63(9zFqA6P`ReJFrOdhXkI zB%)UD)x0Q(wkvf)$)FCwl%Mhq844!+FpqJ2^=#vOekE>B1R~SlisJeInQ=N{OxZlw zw_y<6c<#{F-^;1F-{xMmRChBz=;Eph6^6;)pSr$cJH~_C(B?N>qGq( zi=`D+e4gGP*B1AEE-K}yN@TNW6|^vZgx-z5c$>*dFYk3u>o=IBPCVoUPi2pji~7+@ zUh6F0w$8y(O(wCzQij%YnS8ZrSO24x%V9aTF-rzT1B|4=XnSjpe8SjW=(d7d#yERs zVnK}@M^XGcwb3{Ezbg~^;pViAm%J3rd-J))kLBTMTvh&JUId@OSwLoZy`~?{R(3Pw zH~zzLuP^}jKeR{xGIyV4iOKW^f;S)iN*~HJDt7dU3{O7pN8)^ju=6^%uB${cE=L4VuW*#LU|?rBfL!I-2X-=N4|FYjFaIs5Q^ zY`O!}#L>EBb&~K6>?`OCU*> zETb#t>Rj8#F7;Bq?T6Rj8t(}li=SuM0$&B-m*b?s%aH~3nw~Jt;hFB6AtJPY5inU- z1!U5mkqihInu(f1)J3C4T>M~94psuiRO!hVt>o-q9-;q7;- ze|P$&gC*pSZ5gi&rZ-tKxAzXTcWnJA36eyc)fED)E-+NXF%6ZU$ns~w+cm6TitKKR zD38jL`md8+{N#{=uUQAsf*MXw9F$5LXHm*ZpyqH|n6$jiLxTa#zN^H@bBCm+iz0X0 zRHL-9nSiafA|V>GI1h;L1Q#Pr`34qnT}=o_C>tFcQwLhL#{s}5Qh*kOjMFg9XSkwc z(3nx|Nbke)Fx^PU5-+Y~;oiqIq>nr-rSc?e`1vRz_y(XMRo;`R-d=W)A)6A`U+j+e zftJ`ilIX^~_I_8SQyff{Uf@ki?_2kpx&M?)pquH986sDJS%s`8-e(qWI_ieM+&&TN zZB*eU(bsNo=&dk-p|j{YXt!$0kqsI8`-1+y|G8~u`0}id*3!_2hh3W5yIk2^PSR9> z;{Z{l8fz$HrczJ&N(iOk7Z}s1o0Qk{N{@JZxmM}8L(EWV9e!Oh9==gRZ!5bZ3^jp+ z3I#lj)dstXc}rf<7jOhcLpAJX$iFy{yJW&nPxS=|??w*Zk6B$^)ufuolp8?FIZJSi z=Q}pU1jlumss&HL8(odsi=?+oR~4Lz8S}^cm_Ui5fDgL5Do?Uqm4GN3Z^BpCW!Hok z40){?i#3OA%4&5vS%$~#%Y;40OyfCLqc;eNHz>`;f8H`uQ{KY#IHGgDSLxZP8R7(5 zwm(;lEhqr;(=$o9j+@!_+X1LNU3kOV7|fIKUHsalrD0&h@{>o z-V)nO-c?;lWQ+bFdzhHFHG8Hsf?2&c*DW!)qN-wT0`P}6QjP~k8S{)&P5n2oo@dQi zIO&MZJNkyb{h3MsADXF0**LsSBcp()$JUQRnbCwV87K4x!}Ce+&$8MpSOLS9bVCGP zv7`u^SM9h2?mU5W9UyPp(?sFJ3l%gxi9=*+P^OwSPiS>780Lz+to8I~w5a7+PtqKa zSIj7^SUA!&IQVNUd+u~GPP?2mjinNWZSP(dD?PsZUZaQm(6~;#i4=cmO*G;G089-W zmtI<&Sn#a*mczWt5H1Xc=-W{=oEoXQ;1;a!)785#yvR^SP$oHyFI>5?`U(E|zJ(UL zwn}=W%de&kJ*{XC)Ni)$s1;7J>Mruob9TQ z;Up@$ul;e;Rzt(^LAyv4fht>nVghU@zS6QD=<>MmVl&ah9g}A(9u4&YYeB-Yhq^ip zv>tkY8pYSMLE_*%Bbf`lelseKsDVIBo_M#1!xmUJEMXVyv|XL=aYen?V+-F`Hs}qK zoZHf&tVJuxc|)R=q!X{#rQ>XgNx@r?hW`h2d!fHEeGGa>w^2cZwk1M(Nf|96rsst1 z&3;^4_=@DvmDwHJG)r4e>N-AIlo9i_#cB_)Xha-%A4uwf}dHwhoksOs9Z%!%Z&ds~II5iqZtE%!Q&fei3B~jnQjbx^xmJDSvORL-)Apd+TNr z)bo(BT$`-4h!qcszlTq@@v)iguPM!*uJ?Zmi-?aT0py1lSeF_cjPfL!bme_Z36?u# z6ZsMOJ!0I2m%V9Z{ak6EO=0W`L)9&meuSAiHEcm(2I^S6xjK@;Of#`+souA>XcGE{ zDK~rWn_Z+ELD0C}m^>$-q}9=0yy8V8CZ4H!8ec zFxJVzH~LMkqdVW6_kQB_587MJT#tYHpC}TLWo(RInPl$1-&7N4j6H0@do?3jq=6J$ zalHk<$;^ILeXd_ssdAtsVpy;7q=t&=H9g>aDaP)|vc8K;e=Xx%hG zYC5^C$;Ol#(rdSq>YEmaH`CwMXzJ+8kwPCs&TaDYREqpwye|(&K*x+kaWq~rpNE4s z?UQ5asfqqd^cJo50?$y&zSwd6IfZ~HJ=nIV!fRJNK+a;rF#Fewg}AyR8UhHQvAdB- z45p!8pwc{}xDzR>aB6iF(n z4)=h)cy_DI*+Z2w43~H1$Q1-@)&agGTfcVg&qfmP_kCMSk?In?BPAPgvOq(RDk%*wo{@^*iqlqWmQq8 z)jRemS&S~2@Dx`3b!kbJdqy^%aR&j*CkSLjs~tp`GQyAruL<0f;(jUl4YMxRe-=(@9c<;T?Fo?U(Ko zsU(nk{Na8c?lIu3_x&SQC?9-5jlZ33+G2_u?MOjrbp?)`X?*mRU+bkD9W)In;GHuyiPFW7zZvO!YEfs?Ax}HO zNqF=1#Oup_Zb8Ib`4|8V}orOLyUa2RW%s6%5kt0&VHEq?6$7|`MGr8MIzf@khQpO z`ybkz5%;(~TAWxc-pSK_$9iso(U&uZyxJcWTEwxEhXej`Oujw1V;}um0!Xez!B~#tyHdsW+ezKn=F)mjs@5q|*?W8!>~IE<)n~xRIXdxi_FRYAWhgyPjYVD0z3zdXe&AjF z(k8-hCzD3j(yi#@2ykNX!@VFO=M|&4S_cB59JTMz#i_xeV>&cDa+Uib zm~|=FZ z6z|o8(@=BiUIC=7PYiaR&`#6FVfL{va^c1zeImy^`uqr?wl)v320z^}5)sm?hbFCq zQ`|f+yE;hs#J$%0FPNeQuB-tyM+qR5+koykm^-%n`^ci?%{f1Z*JsZ8+8(OBQd9ek z-{-ek->Y~qaNI+pDV2YyarcAl!7eYI5=rLyrG+&y(S8Wm&onP`21*IrUVQ3{&ls9q8jQUJCUQevUo9zl{2}JX3}M>nRe(+PI8#1$ z6aa;zuqujQP+`_Wx~lwHEaU@z>H_J<-E`WRqH&Z~i+)?3+gcnrmd(M+>_<9_jU}P+ zPCshh=b^7cu2PZA&vz6f6B~iZe}D^}U<8}3OVi{lD8byOdaO;5idKqQ^K#vF9$_aA z&O%ktDu>2HTy5s1FLD$z8kLmK*_c87_RS>@Ijl4s3PD%pM^U`7M&Rw}e`xX->bM?c zd4+eWM~VomuLKz$TNFHI<>kj3nU;2mpykPty~N8D9$hBVD**~lLCn?XmRtJhp~kt;4Hw-nTjTvz=%)RslP|qLFpI%=Tt-y&gPy6)ZA)*8KY^y6wsRN0t zCT`p?<&_}xbfuSwlnipS0u{~Xm__7L{&Vg2_?ArzY7eR*)j&07ENH zy0wTfWYvS{g4jl$=qjZxDkZ{AocgDQ$FpYVhfcwc@f*6=rT11}Qn;x+=xhW0@5Az? z-cGlCOcYHhKk@$vY~{7Da-xZzDqtaRwBqyy;8l=HUMVcY?$z7oBH?z)gG7i=c%#pW zqj;aAq%WU=U@={~dkFvj*%%2I3{!>;HVsJ{UT!IYZ}w!9M~J0TFpyDz*LE!e*W_l` z&gxW4ZRT2d10}$IM5*FWfeARNC>yN{!eM&AR^`!2P&J2*AA8EwO$x^#@*8bJA8Y#) z5y=VPaEmhG{Q<-jV2K$^B03PKD){2L(`~^m6;nIvZ1c+zk<+@U_fV3tlDFoW6F`7$wDWV7b*=0edc$j9Y%yLCKm9O3U`F8ZUe z<$9;BY=nM$Q&;qazG*$tjycWx{T5`{A3n2Vy!d6YwyyH`A?3Xfn#vwi39b~+ff3-+ z`lp^FkpQ{zr=|qpP8oBQZW?7T0q+>v0VGj_-kH^y^7k8;!GlPf*0+!M&pjWaYn zv-EdSc{rWyHuHw1Udh$O^xp2u7`PQoRR3xOukjPuKebNZZVbdX*~Xeca;XaXXW?@I z2=Pw(;&@pG`N%eJRv|C+cgZ;jT$rdRH%p!1y#G_YH6Q=gSt5(71-Da8kOs))>Jn^2 zwg!u}iVcI@cw_&*{~^4Y+AoRS239y!tt!qZX>CMQ{U7bUbz4;58}>bvNQZ*N&>)R4 zgwzm9Hz*wfLrBNaC4wL^#0*`7NOyO4hcpa5G|~bh==YiX9Xx;C|J=V$Sr6~B?9*$_)(t+H+U znMDWs{FDy)DJmSpqBu4CyLOkl8fL=3Fr0Bs{h&sS$IuJBO#DQbM$aHPFnAW1H5Ao# zg_7HAy3qb@V7u{cOe0QEk%21RUAxlCVr_kdQP>ODDO;6pOXz46jK7`PSmJQ&W|`4E zfdXcjrLJKkNL9D~%JSgx*%*xIRv0S@{`n$5I!?iL^q0qGNo3h&VF$_0hVvPi6(2l& z|L6p3pJSsCG#YzuChXdu9lHF81;7-1W#SD5@wB^;el_N(P@j}3C0Sw92a?yEC!RlD z7?e*91rA`lS8~$_@AnoP9#k`(mR?Yn^i_Gv((P~6vA3r5Nes)! zwh~9DN*svy{N>44G@J5^Zt(W-+q$nvEE{TXiZHsIu5?>>TJmv)j{>C!>dvm8#}(Lf z&c$2}#G%tK8*1@i4QmxOR5iZH$m2U3WTSwy{5z*{(@mIdNap(`(fW|%bP}#dnbM%W zPR^bTgygz5s^6tiSNk%IE3jE2&Awr4KX6J637P$^oLGdD_*`Fnl6B62Mh0 zY@DYreDM?uF96`rc4Vx^OP<(|VBzdbuEJ=UABg-0k;VAxUw*&z$@g#z!qKW1 zyf>EHVF!!fpk?DcyMPPGrT1|hpsb+NMA9ux9GUs^CU`@duV%XwDAitKn&z?eXT$68 zu3Bz+5lKPgo#gh(FU;AY1&cv#8QqHqjgp>ORSbAzc0;9uD|n;S8$ z!OW{$B2x@Sbh^e8136Tq>u}Zd=N5sNm)4|d;nslCINT+%hD~}~1U$G$f0{}s8Qkz5 zG&T|DX~eaW@7lN!d-j6~wt}&4Dq8lPBCN6cF57|JBHI|#zj}&kX=UE-3;2c*P@^Ey zSPFtntm0L^9yt58WGIWy({Z6g z32E<_O%;2MGd@Zb1uwOQSa9%hx+ICSf$#MlLG!Dyf}IC)d4}mnjt>qQv-0+YrnH5L zXAoeZ>BoO;*;js4E^c(e^5_Jw0W3twC9~Hp<7TBkrLl!$k-DD=ccPGYqKij+_$EC2 zpD0JY`G~;AbC)@nrWK~k^F%yMrH4gwKkK=l5-+i$THt%-n7Ed^c`gy1qW>Px3k5?n z2qpcat!Te#L+%{?JgX6E`|w(vr={=nC@m+>%R2s7HZP}4v!Yf?lj)U396~L#`NEdJG>Y&Ze5aupeAWS0q_ z-W#F8o>CX`RPPMi=JsX_-@wv@Qpr`rR6G*LwDSO0!R5~emTut%zLbM(+L1Q4m;zx+ z7>3OFD_&)Xd3uCNMr)MRN+3ZTl9t_wKQ{jXUa95#>3Pl{TMV9yELZJ=o`@%Zdnjfr zIfq%-QeUa+yS%hXM-BMY9Q!j>3IE2?H9F4kpLAW)n)^y~<;zHsu<;+jK9N-F+8~3| z&OsmcM)`@zI~ejF9P#EiqG{ioWt^8zzLKTEP7v>H`i}))Nkq7t^%I{&t~pqR+u2j-*>OGncFDm~@lGt%YibRcS01b%yL%*~@V1gEyS8|`PF6{7d*ly2S z9MiyF!LM#~o-Xe2)jb_mqK|dk^Se*mWOLJMJ>)4Itl@W^<5stqN>4(4#4mj7Lqzl1 zV4q)>>x+o1GOoKY(y<-YHsmUxXVlT#CYu(He7vpXJ)~RT^T&Rk6N=?W1GEe@IFD+~ z;ARVQEersd1KmQlt9T}15;Z2seCmm(GJnEs%gKT%7jg%W#l7N$Yt(=wJi+x-p*&-ypDQ}lOogq* z=N%=NKJ!HZc1(~}@o!UwFeZ}Kl?+k5S1%9%1?@HZe~1Ehp{Cq?(@7Q|opI zG1?at-(acEFvd=0%yhU9{7acbUcME70hE7i=wbU|w~4086TgZ?|_rm%d+0&$2l z(mso6LY?v(^TBexp^$?c59*lkhsquOt+dOK)&^!^_{bz_O(9b$$923$D>T_6G&ddR zG_@kX1BT{R+i?o$tXT5EPZtrCqU~rjW6l18i*@JE3>}tfocDX%%UzV6eH>!kBu(1OWPt@ralhl9{v#aLmpD0C&<{zc<*OtqQlP!#_bwkZg);@6BCRi?P}0b|udut$@s2Bm2Ntp7;q;?Hb)ph7%E^EQSE7+`0jm^=8(-<71T1 z#WMSoGiVspP`1gyi8U64B0}_g5z=;LFyQ!iePY~yOZY17g)O8t z*ab@w;yP%loG8F9R>Yxpz_iL~)oDUHyPfjKX5I+%IDo5|`6?+IUJYNDGtoJkwlsqZIKDbX zGfG$jm(yz*n9GIy5+ScndkGn5k#NPg|cIiRFSe0mp-Z0d8Y)?V=Ya+R-GZ0&48o#+w5#mYrPs)pu z1qV>J7x_f>Pr!cTQQk5t*&kg9JJlsvpnkhjcXogP93_0KtYo$FE-KE1UGai6>Ej82 zMN(Tf_7_wc<*WCVcSxczQrv8SP!AJ)zbObzHCUBEHO^K!`NJNc(|rv^Qbo6OmEv|m*q+cAton=pV(>W9 z?}+DZkzi%<&qb2GizD60PS z+q_ME*vE+RfpR$Gbhh^z*iU9zG~CyP|BIahW^{6|q_-EuxE_Fm3@#UaEH6<~s=!9S0VYm(-b|wyaS?By~uQO&; z=YH|xk`H0~n%EuiixUIb^Ak04wv6A{07t)IwhtS)KGXjFF05Su_AVgtIMUSrMwRN_ zr*w8rBn<21hBMRqHtG2!wz~8W{x+%=!Z5Q?EL>!$b@@s~+La?E0(H>Z!-iDCXDX(u zxKj2AF`O94(%Qq}(NFOB9sb)u>~1&h<~t~NXwpgLOKKA$@)0UO-~7@7&Up+&7R39qErHnhrC=rAl=}3s#UE31G_z(=v+p{8$N}La z>xqF7H!Y`kI^B)&Jt_z)xE91#q6Zh^L!-EYAembf7npX3BKGD=bn5pUkLgl=Q z=uNAYo2#7oo%J__e|e^J?WntMg+P;tD^5FW+%QrOA9%SWj_$NoM49GmmUaUHPC z^CeXkzptOx>Vb$?Rre+p6gL;af)W;P%9DFF)TUx8F*W!4JpfNXw1B~QrCY_s5kybz$Rw)_q@$x zDNKvRRQ)9KH3i2Q8le0f!F8{Vm+aefa}=MFnOTvkRmfl1AHAsm(w8JD)Q_$9j*8Gz zw@ye^(pTUEkJ-^TaRI=Zd5tgG_K+kc{(+f)+owi($+3$sY2(Ws*=N9mCds>l6LXrD zRBCwY$A}K6Xr$7}syvS&!<80vw$_{E<#FKsoLbxu*4RL+`u4?Ss-IA0B~7OidiUwQ z>PePNU0B$dp!_X6(_$s2WUt2@G5I1`;wzJtUrc?5K7=%Y@>j%rz*>?IjzLQbo)+L@`5_1Sjla zu>`Tb6;F#zQ}~a=Ql07TG|mwmB`WvS8>q=9M+q{u38L<0(0&8{YO>L`;;xAu+zk$VnLjxh*qF;;s zs;+esP1o<;8z=sG(hL!DjSdB%YpMeE9L`;1*CjOrbT#1h7i8MQv{OHH2jgVlU2h|$ zn&aQloUYO1hY9~NR0h@7WwtJ_Pua%2;*Mp0GuHEU6A1!ekyuSwx2jSa648q>ytVv< z?AA3kpuRJ4i>`LFNe_-+eVQp|l?WXp1-*b>8ozF?RXXt{|9j}%hC3v@R$au)?dH8} z4`v@5OfmU5q2BUxfGeO>;?J`YDz`qpH46(12MtPf(_|mQ#|Qv+aL+J4RT|%wC@glk?qV%0nRP7c_Sm9Q?oMNIrp)KQfgxdaAi#8R5#ldnCE z=zuL7+6@T}(*Qp4e9N_(vv0Yi>XS~x*^gvCaD2Uf!0@N$G$9~{fV9}fA<#x~V3C=B zaR)Y5z5Ml9D)2DkeL*qAAN^R!# z_g-k)*){n=U|FH0f`l&kOo6ncM{_MzI+MHu#%p;xXo5!<_>@e>zu@w`W}9Hvek~=0 zUcsbS#3c>!3XisH_(^7lVr5{a+DS;E)L%v?uZ{mNW#a5e*pbNp9@d2>|%eyn`U|ZD57=*q_Y@cI4Gz@y z%vFYJIZ7W`KJcqP&AYsAg2MiUC>J?4PUS$Tdcz0`-mr1df8-bes~)VOQLBwrBatG3 zUt@!$mTTk&=XWh%-%~~N|MXd!8GYV#+uqNt2(uOiM{a!W*7`)}nA#jAwo!WsQdbo? z?H>uFu+3*m&%GgUgGHiz-DP4x4XD2Sx6_xgp7S7Za{?ZvnB-+BPKMv~YF z2r%{q9HzI2LdfLtj}?gln8S);l%$BG9-5JhNJ z;6anH>5I&=!`HKgefEKPP7!a)=yAjEcj22zUI`}c2YF$F7dTU}gWXZKl3~_%ut{KG zLF5nS#%>3~$u7lv-x6lZrzeYzU)Z>sIcwDl%$#j{WwJc%>>9lhp0Ni)fHb!Fkb;}a z4>C-wMhNN{9~sCg=Eutruq14iM?)b{S$O?2BVFaQ2P`SUwXq(rXJhF#MY#C8MxZ;v z*+O={)AE>ub6a-*D*vxpqc>fSlgt!cZW;8Yy^qsinH104#f?b=s&3^6P+2?3@>|a? zqqgp66Y9GkMntj6p;;%{L#CDwWl!WVT%X_iL{87U>?1`fGS~!&5^myFR1Zo|W>e1w zM9xd!{N+H5^h^E+$WO6|(a#4rh5C!5ky4lDIT+XEM?V{r)jZG;u?zJYwPgnY{2#z< zKS|@_@yb|}v>0fsqeOAx-t#z|E8m0OXm**E62Vsg>d(+$E`Ag*lyJS+uvb@e4jF#s zHCEW|oVi#WTHY3*a*(O#!x9j|;@Vgfa=e^qFUhAk7M6I?> z?)k%;uxp3>8Hd3R=X{`zFDKfOn4cNxzwQ)G_e&>9n`Yq>T`udvNcxg3rlHn2@eS4Ds`Lw*kZFpnNGz?zUWwy? ze~t5|{0;vYA~yq-dD~U@RYDFee*Zgn-1m*0k0{z#99Oj3J{6k=J*CoJRh>I9*u znA-?04E~nG71q)+bW4mTnb|_FYs;iq<(W=$u*zuX@X+iSFoA5)I5XF91FtB0_1O_3 zklHaSBXl~Io)+CYC}G0$SDCBiSPsdMxUMJy;!(@WNOIlPA*FrG2n~}=*r|Lm=VCXJ z$AH6y#R8{NlEA^e0eF(J*Z!(V?rd#sGZJvAmR*qUEij8r0ILA6J{1>2Gm(|j)Jh~r zdo80!H&emakL5DP?MgfKd-amRO6NbX?qfgZt7R&#=GC#=PN_=&-4PlF>IW7#`5Vtv z$Vd78$nuw}O%y-CmkF!kcRN30jUT&uV*s24@?WLc0nUbl68<%ssR+38Z(Gq`v$lYk ziS81^mYLqRggjRR*2#~T)y^n0bkvhfGkZ@7eZIdv)wlCS$a4R4 zN@KxJn%ZZ!{1j)kYNmEqS*!ve*7!L1o8jR#HKUw_^71#jUj^?tqvrReyHXRPnoCh-qH;wAhuZxcd`!Og!jm z7>Os|%lR{Fb1;LeClM$cd4Y@O8G zQCYBVYZUv=b^V;HP=$X)`i4Q~d#A8o^r`|yTLY9?Sqcjm0)Rc-FsTi9&OT|yuAJX6 z|7n$CW6SZvVVmHnVXkd$X=OueVpl-aC%Ip8hZovtvbKed?!PSJCuC3A#^~sDch93&EHZ+;Xt{mIqu4j5#TCM$9ghfq}uF zOc+v!+#}Xu_adWqwW$z3T|Uz=Zi^4`a%_+L75SP@_HKn)Q4@WjeXZTx4w-QH{X(u_ zMC#KOf~(u@PO)A1@9;^nK^MEvh`lefByrCqt)frj14tzpj(rqVJq*Fxcx+p^-jm;pBwkTU2@77J*^`rva4cPp-89g4 z0}uGXnU}ZJtbO?;%ialg#QYvoJo_h-LK?L;2LJsUXZ4U8yiio&Lf28{Gv#{Vt@om?Ecqa667 zHWZd^DI&~}T0%VsjpoRU%7TRb2))U~Cr@P>hkvK8^1qqZMvx$#6s|_hFOC81XTP@W zuWl+3o<3bsCFO@sO@;hTx$ZN(sqmW1GRl#sL(sFpK)R@niZI@BzX4A{*~4SPQayK% zn=hvSmD9KFvoJDMl(A*y#ATG4uQK1RpG<&}#3hDq=olaqwvEG3rJwNe` z&Nke&M$&&wnHYu+ncd(gmbz4)KAgM)e1b^I}nfK2|uRI}ms(z}Brt!Bj7gF#^PdsPkqE7`?OwWiXaOFbfFeHPxR|{HkglAOiJIyncUC_Z zC*KRU1KMBs5J()7cuVdz-v)O;9BH=VB8O?$Ji;_RJfhATf4jO_4;IAtLX_@(KM!wj z)N5xzUliCljt1O9M+WPkp?ThMGL{7f&IkA*F_6n=CeMH#Vn^&AN%7*e$6B()`e3!i z?fp0_KATl#{(1XO{(n_%wiQ09bwOh?p82IzZ{Tqen1FqZ&d^_nYU{*;mE$`#KZwM z#q}0s^INedJ5=ik_F$Oi^a}2J%~y}7`RIlIyhLU4M^SLx?Iop+%V~SfPq4P^Yy*3H zB-c*Iy~bz1D0lN{j#x+y&ErY#BBfI_^|t$Ler*bLSUMEnW438Ii-JkS^zFx5>GXSb z+4tuA=aL`Qr0$120h9E0kppk>vJ-J%7u80BW$4ipj=d63E*)=wyU$?OPIG z(F+c0Q6_GVgw~UDqn!l~zi)^SqPNfL$OWYU1jmAsOS(@Zs!n$@1blMg-@Qp`vQw|= z>D!=(1XOW5^wiDS>+(RK*$nJ{Vhya1_q{l^llGFqI#dUBB(+9-9%Xpk&8g?p)tE!W z`&F3)!WAX!*5pu)IyU!j@k(pg?Mo-829KOB9D4(=K@k};Hn7o0c3;_Qw1Jy9O@0@g z#Sbmtsi7r=@aMk~AGFzDYIB%=y24He{^{H0EIgsNzBJAfa-Gdc_Uo=#l{#JT`seNa zm*;*mE(0KRr0ju&XmsgXVOReC>J)9**L&qRflL7iqQm_rqawYTF@hF~O9TM`7HC~G z?WOTTxNT+*x$|1(U~W6Ey7Hn9SC$v&&e$>VrrlJWB5c&XiZ=Or*yDbYG?zPO=iG`} z#2CMOOi5PWwAPVw(w?N=?0YK76@1jv@6XM5wFwlJ76u;qTE4zD>3#f;UMSx63pYIf zJ?%YXMxLDFjt8p^2<6ierQs_1#|W>*n!n*HEp138Wq4CqAeot=j7`|SF?Rp6QW?`c zpla^Pp_7VnyUKpo67M0?s@_KMchJyEC6Oh3EI~Ucx6;wvFW`A={XC(Cf5nfHx#>G2 zQZ1~bVWtQv#W@pu#HN;J>fY6Ta$E)eI!?3bC*!c|2W+Lz)FU*?sCuf}y`=$r5_M_DkzSoz^qG!4(X$@J4aB#e>KY9~N!?g%QfII^acwa4VeKL(6Cd?t6S4Pg>mo1o z3T$1IosC9^=oy^gk>V!*7U@ed1DG{F2BZF9Rj}-zg(Gr@Mag#4X$L{2q{z_t_hfH3 zFUy;elylb2PM;}7wn6Nu#1^ml+c~bxLD_&pFHfQIRD7P`4NEA&y{`yJiIaJ~Bxo zUsK;MP8EFl&K<(2JHpHb#<$S^;5dSZkP5y0sh}-{#=%O5Panl(w3}0={EQMcy=q!~ z2(tM_v&NunoV*4jn9lVPZpA*v~L=VNe}^jz0Y@PK|f-8c}*r#M1d*= zJ!Cl#g6;#5msQ*r=bi=C-wzEtd4BNKk5Qzk4Yi_S|AI0_qSfma>im75hRAgft-F)F z??h1k==XeE_eMLtyZ0Yr z_Rq~{e>PRGN+?%Um)1q=W~h?#{I7C@w6=mn*876o+zj{E0tCZgkC=lS-YlnH_C-&1 zVCZ(o;KQ&q{5KML14lICAmjjZI12>(!-Y z@_xFTbe^O+cUA$3~3p!Gs3%-bky1n)6y`mKNxEIZzufxXsz$5Uqvkl*o!2v#xHU*d|uevNn@e)=C}HrT%&o{ z=Fvt<%J-!Am9$>;?cDskK6*#Yma+Grcu=)-q*0Vk0&8Mg>V(JJktm(|7Xa+x^FX?c z_T$*3BmQ^dZqL?Vt)Lr2Wc2*-tGEC8SIOXI2&gG_OLaXR<_QSLz}zY%TITHto~hE+ zn3enzP4dsz_Nwt@#X3Pqf zOk93p?+N&-6}_BtPp9QP${ebTuyK&reV#@8I=nCJQ;t55S0DZ42lg$RbhICSRWQ_q zbMmY^G5c4;2e*#+IqjJ10d*(aHt{dvcc_FBd;^(HwE5G>wdH=?GoON(<8!80*>R|L z7^QzjEPbrHN9dp{Q1tidun@zbl>9MXIeRD;iSn}Tgn%CJR++}+?aj2Tc-LM)C% zO(9Yd!8_O)_w~=Fi`z8$s|T&|vUM`s{>e$d55kS706qiv8EG zQxj^F5IaiIRrXOX+h42rz`F|elxE>|N6X+G)()N}JS)w1q|oT}yHia&9-|>sN8rmx z%3yB1RmC7z;w;_Jp&#vuepI!E4(`Y5zJb1MVx}zeyUnG8-lHf3S9HxJ%D|meX+r?O*}IGg=U7w`-+TQ{|11tle*jD6CMm zwP$FIjxnaKA0>H=urF+x%Ww16N!@i*e4BbOua8Q%Nf|G9fssv>d}wK==*8vd5u_y~ z^(FWF*XX|VPApEAtQSWlW(z;!sWQ-BI?2>B0s|7rBMO*$4qkRuw(lO0%>VGPmh8ha(MWI0ETbB|k>Tsx7h2 zv+yeC`@SX0ymiVdt1WDMdrr%e*h-m0!s+id*HQM z_Oo?Ul>d~v5ae2L2ur-uqU2rO^4#n}Q`#=qdCKnD9$6JH=ms-=na_T2p~ler92Vw* zl*-2VUGpIyZR&-`y>K%;xHO0%-%42L*t(r2wXRriXF}QH5;&rUWz{4QhBl?9@8XdT z0_PjMj}ckE44txctMyh65Qh29Zsfk1LRPf3aF{mQz}bu2yOi|-+%6(^?e9zPcP~7A zQ+(1!+|)MURDjZ8jm0xXc#@c{x+S{qbrXHqP~3^yGPQ##3{SIvD*vzpIkLz;0xsq{ zq^`$Kpe(2L6P6hNI=$`{gOlOpcRsjTbx8 zsqVsyyx-paN=jgW^!U>kk4b$gu}Pul5Zqj->#$NJ1E2hJmMYrtdcmRVxvL6)uRy09 z{_W-lGVlGymJMk9?q58w+VR^$kfJBx>H;kt8aNju*@PgVUJu##rChu6)O3VT+_lCQ z{3hg=@kQ+QHrh=5Xx0-uxah5Vr+4vF(!9@6?44?p7cbpG2?;`bmv^?xs-v^LEn?T1 zW(hu|^B>?(XgewA_VfQkQGNeM+ z#D5MjD7|uZT|VmC3b#i5qzpekxnGZ^wZB~s8tU8UMlyNBm^`yx-i`dlDt{sC*0Dg; z?^>hCbWWy&b@L~4Kj&qG!W94U|3kr||2M)b|F_2f6!@P4|5M<93j9xj|0(c41^%bN T{}lM20{>Ise+vBnR^a~wBl 1000: - # The argument is too big, so we will not pass it by value. - return None - # Return the serialized argument. - return serialized_value - -def deserialize_argument(serialized_value): - """This method deserializes arguments that are passed by value. - - The argument will have been serialized by serialize_argument. - """ - return eval(serialized_value) - def check_serializable(cls): """Throws an exception if Ray cannot serialize this class efficiently. diff --git a/lib/python/ray/services.py b/lib/python/ray/services.py index 95ed7927ecb6..75a9b7a6dd54 100644 --- a/lib/python/ray/services.py +++ b/lib/python/ray/services.py @@ -1,31 +1,29 @@ +from __future__ import print_function + import os import sys import time -import subprocess32 as subprocess +import subprocess import string import random # Ray modules import config -_services_env = os.environ.copy() -_services_env["PATH"] = os.pathsep.join([os.path.dirname(os.path.abspath(__file__)), _services_env["PATH"]]) -# Make GRPC only print error messages. -_services_env["GRPC_VERBOSITY"] = "ERROR" - # all_processes is a list of the scheduler, object store, and worker processes # that have been started by this services module if Ray is being used in local # mode. all_processes = [] -TIMEOUT_SECONDS = 5 - def address(host, port): return host + ":" + str(port) -def new_scheduler_port(): +def new_port(): return random.randint(10000, 65535) +def random_name(): + return str(random.randint(0, 99999999)) + def cleanup(): """When running in local mode, shutdown the Ray processes. @@ -36,7 +34,8 @@ def cleanup(): """ global all_processes successfully_shut_down = True - for p in all_processes: + # Terminate the processes in reverse order. + for p in all_processes[::-1]: if p.poll() is not None: # process has already terminated continue p.kill() @@ -49,146 +48,112 @@ def cleanup(): continue successfully_shut_down = False if successfully_shut_down: - print "Successfully shut down Ray." + print("Successfully shut down Ray.") else: - print "Ray did not shut down properly." + print("Ray did not shut down properly.") all_processes = [] -def start_scheduler(scheduler_address, cleanup): - """This method starts a scheduler process. +def start_redis(port): + redis_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../common/thirdparty/redis-3.2.3/src/redis-server") + p = subprocess.Popen([redis_filepath, "--port", str(port), "--loglevel", "warning"]) + if cleanup: + all_processes.append(p) - Args: - scheduler_address (str): The ip address and port to use for the scheduler. - cleanup (bool): True if using Ray in local mode. If cleanup is true, then - this process will be killed by serices.cleanup() when the Python process - that imported services exits. - """ - scheduler_port = scheduler_address.split(":")[1] - p = subprocess.Popen(["scheduler", scheduler_address, "--log-file-name", config.get_log_file_path("scheduler-" + scheduler_port + ".log")], env=_services_env) +def start_local_scheduler(redis_address, plasma_store_name): + local_scheduler_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../photon/build/photon_scheduler") + local_scheduler_name = "/tmp/scheduler{}".format(random_name()) + p = subprocess.Popen([local_scheduler_filepath, "-s", local_scheduler_name, "-r", redis_address, "-p", plasma_store_name]) if cleanup: all_processes.append(p) + return local_scheduler_name -def start_objstore(scheduler_address, node_ip_address, cleanup): +def start_objstore(node_ip_address, redis_address, cleanup): """This method starts an object store process. Args: - scheduler_address (str): The ip address and port of the scheduler to connect - to. node_ip_address (str): The ip address of the node running the object store. cleanup (bool): True if using Ray in local mode. If cleanup is true, then this process will be killed by serices.cleanup() when the Python process that imported services exits. """ - random_string = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) - p = subprocess.Popen(["objstore", scheduler_address, node_ip_address, "--log-file-name", config.get_log_file_path("-".join(["objstore", random_string]) + ".log")], env=_services_env) + plasma_store_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../plasma/build/plasma_store") + store_name = "/tmp/ray_plasma_store{}".format(random_name()) + p1 = subprocess.Popen([plasma_store_executable, "-s", store_name]) + + plasma_manager_executable = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../plasma/build/plasma_manager") + manager_name = "/tmp/ray_plasma_manager{}".format(random_name()) + manager_port = new_port() + p2 = subprocess.Popen([plasma_manager_executable, + "-s", store_name, + "-m", manager_name, + "-h", node_ip_address, + "-p", str(manager_port), + "-r", redis_address]) + if cleanup: - all_processes.append(p) + all_processes.append(p1) + all_processes.append(p2) -def start_worker(node_ip_address, worker_path, scheduler_address, objstore_address=None, cleanup=True): + return store_name, manager_name, manager_port + +def start_worker(address_info, worker_path, cleanup=True): """This method starts a worker process. Args: - node_ip_address (str): The IP address of the node that the worker runs on. + address_info (dict): This dictionary contains the node_ip_address, + redis_port, object_store_name, object_store_manager_name, and + local_scheduler_name. worker_path (str): The path of the source code which the worker process will run. - scheduler_address (str): The ip address and port of the scheduler to connect - to. - objstore_address (Optional[str]): The ip address and port of the object - store to connect to. - cleanup (Optional[bool]): True if using Ray in local mode. If cleanup is - true, then this process will be killed by serices.cleanup() when the - Python process that imported services exits. This is True by default. + cleanup (bool): True if using Ray in local mode. If cleanup is true, then + this process will be killed by services.cleanup() when the Python process + that imported services exits. This is True by default. """ command = ["python", worker_path, - "--node-ip-address=" + node_ip_address, - "--scheduler-address=" + scheduler_address] - if objstore_address is not None: - command.append("--objstore-address=" + objstore_address) + "--node-ip-address=" + address_info["node_ip_address"], + "--object-store-name=" + address_info["object_store_name"], + "--object-store-manager-name=" + address_info["object_store_manager_name"], + "--local-scheduler-name=" + address_info["local_scheduler_name"], + "--redis-port=" + str(address_info["redis_port"])] p = subprocess.Popen(command) if cleanup: all_processes.append(p) -def start_node(scheduler_address, node_ip_address, num_workers, worker_path=None, cleanup=False): - """Start an object store and associated workers in the cluster setting. - - This starts an object store and the associated workers when Ray is being used - in the cluster setting. This assumes the scheduler has already been started. - - Args: - scheduler_address (str): IP address and port of the scheduler (which may run - on a different node). - node_ip_address (str): IP address (without port) of the node this function - is run on. - num_workers (int): The number of workers to be started on this node. - worker_path (str): Path of the Python worker script that will be run on the - worker. - cleanup (bool): If cleanup is True, then the processes started by this - command will be killed when the process that imported services exits. - """ - start_objstore(scheduler_address, node_ip_address, cleanup=cleanup) - time.sleep(0.2) - if worker_path is None: - worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../scripts/default_worker.py") - for _ in range(num_workers): - start_worker(node_ip_address, worker_path, scheduler_address, cleanup=cleanup) - time.sleep(0.5) - -def start_workers(scheduler_address, objstore_address, num_workers, worker_path): - """Start a new set of workers on this node. - - Start a new set of workers on this node. This assumes that the scheduler is - already running and that the object store on this node is already running. The - intended use case is that a developer wants to update the code running on the - worker processes so first kills all of the workers and then runs this method. - - Args: - scheduler_address (str): ip address and port of the scheduler (which may run - on a different node) - objstore_address (str): ip address and port of the object store (which runs - on the same node) - num_workers (int): the number of workers to be started on this node - worker_path (str): path of the source code that will be run on the worker - """ - node_ip_address = objstore_address.split(":")[0] - for _ in range(num_workers): - start_worker(node_ip_address, worker_path, scheduler_address, cleanup=False) - -def start_ray_local(node_ip_address="127.0.0.1", num_objstores=1, num_workers=0, worker_path=None): +def start_ray_local(node_ip_address="127.0.0.1", num_workers=0, worker_path=None): """Start Ray in local mode. - This method starts Ray in local mode (as opposed to cluster mode, which is - handled by cluster.py). - Args: - num_objstores (int): The number of object stores to start. Aside from - testing, this should be one. num_workers (int): The number of workers to start. worker_path (str): The path of the source code that will be run by the worker. Returns: - The address of the scheduler and the addresses of all of the object stores. + This returns a tuple of three things. The first element is a tuple of the + Redis hostname and port. The second """ if worker_path is None: - worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../scripts/default_worker.py") - if num_objstores < 1: - raise Exception("`num_objstores` is {}, but should be at least 1.".format(num_objstores)) - scheduler_address = address(node_ip_address, new_scheduler_port()) - start_scheduler(scheduler_address, cleanup=True) + worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "default_worker.py") + # Start Redis. + redis_port = new_port() + redis_address = address(node_ip_address, redis_port) + start_redis(redis_port) + time.sleep(0.1) + # Start Plasma. + object_store_name, object_store_manager_name, object_store_manager_port = start_objstore(node_ip_address, redis_address, cleanup=True) + # Start the local scheduler. time.sleep(0.1) - # create objstores - for i in range(num_objstores): - start_objstore(scheduler_address, node_ip_address, cleanup=True) - time.sleep(0.2) - if i < num_objstores - 1: - num_workers_to_start = num_workers / num_objstores - else: - # In case num_workers is not divisible by num_objstores, start the correct - # remaining number of workers. - num_workers_to_start = num_workers - (num_objstores - 1) * (num_workers / num_objstores) - for _ in range(num_workers_to_start): - start_worker(node_ip_address, worker_path, scheduler_address, cleanup=True) - time.sleep(0.3) - - return scheduler_address + local_scheduler_name = start_local_scheduler(redis_address, object_store_name) + time.sleep(0.2) + # Aggregate the address information together. + address_info = {"node_ip_address": node_ip_address, + "redis_port": redis_port, + "object_store_name": object_store_name, + "object_store_manager_name": object_store_manager_name, + "local_scheduler_name": local_scheduler_name} + # Start the workers. + for _ in range(num_workers): + start_worker(address_info, worker_path, cleanup=True) + time.sleep(0.3) + # Return the addresses of the relevant processes. + return address_info diff --git a/lib/python/ray/worker.py b/lib/python/ray/worker.py index 4f9abadd0d6a..a3370cf9e801 100644 --- a/lib/python/ray/worker.py +++ b/lib/python/ray/worker.py @@ -1,26 +1,46 @@ +from __future__ import print_function + +import hashlib import os import sys import time import traceback import copy -import logging import funcsigs import numpy as np import colorama import atexit +import random +import redis import threading import string -import weakref # Ray modules import config import pickling import serialization -import internal.graph_pb2 -import graph import services import numbuf -import libraylib as raylib +import photon +import plasma + +SCRIPT_MODE = 0 +WORKER_MODE = 1 +PYTHON_MODE = 2 +SILENT_MODE = 3 + +def random_object_id(): + return photon.ObjectID("".join([chr(random.randint(0, 255)) for _ in range(20)])) + +def random_string(): + return "".join([chr(random.randint(0, 255)) for _ in range(20)]) + +class FunctionID(object): + def __init__(self, function_id): + self.function_id = function_id + + def id(self): + return self.function_id contained_objectids = [] def numbuf_serialize(value): @@ -93,7 +113,7 @@ def __init__(self, objectid, task_error): def __str__(self): """Format a RayGetError as a string.""" - return "Could not get objectid {}. It was created by remote function {}{}{} which failed with:\n\n{}".format(self.objectid, colorama.Fore.RED, self.task_error.function_name, colorama.Fore.RESET, self.task_error) + return "Could not get objectid {}. It was created by remote function {}{}{} which failed with:\n\n{}".format(self.objectid.id(), colorama.Fore.RED, self.task_error.function_name, colorama.Fore.RESET, self.task_error) class RayGetArgumentError(Exception): """An exception used when a task's argument was produced by a failed task. @@ -116,7 +136,7 @@ def __init__(self, function_name, argument_index, objectid, task_error): def __str__(self): """Format a RayGetArgumentError as a string.""" - return "Failed to get objectid {} as argument {} for remote function {}{}{}. It was created by remote function {}{}{} which failed with:\n{}".format(self.objectid, self.argument_index, colorama.Fore.RED, self.function_name, colorama.Fore.RESET, colorama.Fore.RED, self.task_error.function_name, colorama.Fore.RESET, self.task_error) + return "Failed to get objectid {} as argument {} for remote function {}{}{}. It was created by remote function {}{}{} which failed with:\n{}".format(self.objectid.id(), self.argument_index, colorama.Fore.RED, self.function_name, colorama.Fore.RESET, colorama.Fore.RED, self.task_error.function_name, colorama.Fore.RESET, self.task_error) class Reusable(object): @@ -216,7 +236,7 @@ def _create_and_export(self, name, reusable): # Export the reusable variable to the workers if we are on the driver. If # ray.init has not been called yet, then cache the reusable variable to # export later. - if _mode() in [raylib.SCRIPT_MODE, raylib.SILENT_MODE]: + if _mode() in [SCRIPT_MODE, SILENT_MODE]: _export_reusable_variable(name, reusable) elif _mode() is None: self._cached_reusables.append((name, reusable)) @@ -224,7 +244,7 @@ def _create_and_export(self, name, reusable): # We create a second copy of the reusable variable on the driver to use # inside of remote functions that run locally. This occurs when we start Ray # in PYTHON_MODE and when we call a remote function locally. - if _mode() in [raylib.SCRIPT_MODE, raylib.SILENT_MODE, raylib.PYTHON_MODE]: + if _mode() in [SCRIPT_MODE, SILENT_MODE, PYTHON_MODE]: self._local_mode_reusables[name] = reusable.initializer() def _reinitialize(self): @@ -234,7 +254,7 @@ def _reinitialize(self): new_value = self._reinitializers[name](current_value) # If we are on the driver, reset the copy of the reusable variable in the # _local_mode_reusables dictionary. - if _mode() in [raylib.SCRIPT_MODE, raylib.SILENT_MODE, raylib.PYTHON_MODE]: + if _mode() in [SCRIPT_MODE, SILENT_MODE, PYTHON_MODE]: assert self._running_remote_function_locally self._local_mode_reusables[name] = new_value else: @@ -308,30 +328,16 @@ def __delattr__(self, name): raise Exception("Attempted deletion of attribute {}. Attributes of a RayReusable object may not be deleted.".format(name)) class ObjectFixture(object): - """This is used to handle unmapping objects backed by the object store. - - The object referred to by objectid will get unmaped when the fixture is - deallocated. In addition, the ObjectFixture holds the objectid as a field, - which ensures that the corresponding object will not be deallocated from the - object store while the ObjectFixture is alive. ObjectFixture is used as the - base object for numpy arrays that are contained in the object referred to by - objectid and prevents memory that is used by them from getting unmapped by the - worker or deallocated by the object store. + """This is used to handle releasing objects backed by the object store. + + This keeps a PlasmaBuffer in scope as long as an object that is backed by that + PlasmaBuffer is in scope. This prevents memory in the object store from getting + released while it is still being used to back a Python object. """ - def __init__(self, objectid, segmentid, handle): + def __init__(self, plasma_buffer): """Initialize an ObjectFixture object.""" - self.objectid = objectid - self.segmentid = segmentid - self.handle = handle - - def __del__(self): - """Unmap the segment when the object goes out of scope.""" - # We probably shouldn't have this if statement, but if raylib gets set to - # None before this __del__ call happens, then an exception will be thrown - # at exit. - if raylib is not None: - raylib.unmap_object(self.handle, self.segmentid) + self.plasma_buffer = plasma_buffer class Worker(object): """A class used to define the control flow of a worker process. @@ -344,7 +350,7 @@ class Worker(object): functions (Dict[str, Callable]): A dictionary mapping the name of a remote function to the remote function itself. This is the set of remote functions that can be executed by this worker. - handle (worker capsule): A Python object wrapping a C++ Worker object. + connected (bool): True if Ray has been started and False otherwise. mode: The mode of the worker. One of SCRIPT_MODE, PYTHON_MODE, SILENT_MODE, and WORKER_MODE. cached_remote_functions (List[Tuple[str, str]]): A list of pairs @@ -356,15 +362,24 @@ class Worker(object): that connect has been called already. cached_functions_to_run (List): A list of functions to run on all of the workers that should be exported as soon as connect is called. + driver_export_counter (int): The number of exports that the driver has + exported. This is only used on the driver. + worker_import_counter (int): The number of exports that the worker has + imported so far. This is only used on the workers. """ def __init__(self): """Initialize a Worker object.""" self.functions = {} - self.handle = None + self.num_return_vals = {} + self.function_names = {} + self.function_export_counters = {} + self.connected = False self.mode = None self.cached_remote_functions = [] self.cached_functions_to_run = [] + self.driver_export_counter = 0 + self.worker_import_counter = 0 def set_mode(self, mode): """Set the mode of the worker. @@ -397,30 +412,21 @@ def put_object(self, objectid, value): local object store. Args: - objectid (raylib.ObjectID): The object ID of the value to be put. + objectid (object_id.ObjectID): The object ID of the value to be put. value (serializable object): The value to put in the object store. """ - # We put the value into a list here because in arrow the concept of - # "serializing a single object" does not exits. + # Serialize and put the object in the object store. schema, size, serialized = numbuf_serialize(value) + size = size + 4096 * 4 + 8 # The last 8 bytes are for the metadata offset. This is temporary. + buff = self.plasma_client.create(objectid.id(), size, buffer(schema)) + data = np.frombuffer(buff.buffer, dtype="byte")[8:] + metadata_offset = numbuf.write_to_buffer(serialized, memoryview(data)) + np.frombuffer(buff.buffer, dtype="int64", count=1)[0] = metadata_offset + self.plasma_client.seal(objectid.id()) + global contained_objectids - raylib.add_contained_objectids(self.handle, objectid, contained_objectids) + # Optionally do something with the contained_objectids here. contained_objectids = [] - # TODO(pcm): Right now, metadata is serialized twice, change that in the future - # in the following line, the "8" is for storing the metadata size, - # the len(schema) is for storing the metadata and the 8192 is for storing - # the metadata in the batch (see INITIAL_METADATA_SIZE in arrow) - size = size + 8 + len(schema) + 4096 * 4 - buff, segmentid = raylib.allocate_buffer(self.handle, objectid, size) - # write the metadata length - np.frombuffer(buff, dtype="int64", count=1)[0] = len(schema) - # metadata buffer - metadata = np.frombuffer(buff, dtype="byte", offset=8, count=len(schema)) - # write the metadata - metadata[:] = schema - data = np.frombuffer(buff, dtype="byte")[8 + len(schema):] - metadata_offset = numbuf.write_to_buffer(serialized, memoryview(data)) - raylib.finish_buffer(self.handle, objectid, segmentid, metadata_offset) def get_object(self, objectid): """Get the value in the local object store associated with objectid. @@ -429,34 +435,24 @@ def get_object(self, objectid): until the value for objectid has been written to the local object store. Args: - objectid (raylib.ObjectID): The object ID of the value to retrieve. + objectid (object_id.ObjectID): The object ID of the value to retrieve. """ - assert raylib.is_arrow(self.handle, objectid), "All objects should be serialized using Arrow." - buff, segmentid, metadata_offset = raylib.get_buffer(self.handle, objectid) - metadata_size = int(np.frombuffer(buff, dtype="int64", count=1)[0]) - metadata = np.frombuffer(buff, dtype="byte", offset=8, count=metadata_size) - data = np.frombuffer(buff, dtype="byte")[8 + metadata_size:] + buff = self.plasma_client.get(objectid.id()) + metadata = self.plasma_client.get_metadata(objectid.id()) + metadata_size = len(metadata) + data = np.frombuffer(buff.buffer, dtype="byte")[8:] + metadata_offset = int(np.frombuffer(buff.buffer, dtype="int64", count=1)[0]) serialized = numbuf.read_from_buffer(memoryview(data), bytearray(metadata), metadata_offset) - # If there is currently no ObjectFixture for this ObjectID, then create a - # new one. The object_fixtures object is a WeakValueDictionary, so entries - # will be discarded when there are no strong references to their values. - # We create object_fixture outside of the assignment because if we created - # it inside the assignement it would immediately go out of scope. - object_fixture = None - if objectid.id not in object_fixtures: - object_fixture = ObjectFixture(objectid, segmentid, self.handle) - object_fixtures[objectid.id] = object_fixture - deserialized = numbuf.deserialize_list(serialized, object_fixtures[objectid.id]) - # Unwrap the object from the list (it was wrapped put_object) + # Create an ObjectFixture. If the object we are getting is backed by the + # PlasmaBuffer, this ObjectFixture will keep the PlasmaBuffer in scope as + # long as the object is in scope. + object_fixture = ObjectFixture(buff) + deserialized = numbuf.deserialize_list(serialized, object_fixture) + # Unwrap the object from the list (it was wrapped put_object). assert len(deserialized) == 1 - result = deserialized[0] - return result - - def alias_objectids(self, alias_objectid, target_objectid): - """Make two object IDs refer to the same object.""" - raylib.alias_objectids(self.handle, alias_objectid, target_objectid) + return deserialized[0] - def submit_task(self, func_name, args): + def submit_task(self, function_id, func_name, args): """Submit a remote task to the scheduler. Tell the scheduler to schedule the execution of the function with name @@ -469,24 +465,22 @@ def submit_task(self, func_name, args): be object IDs or they can be values. If they are values, they must be serializable objecs. """ - # Convert all of the argumens to object IDs. It is a little strange that we - # are calling put, which is external to this class. - serialized_args = [] + # Put large or complex arguments that are passed by value in the object + # store first. + args_for_photon = [] for arg in args: - if isinstance(arg, raylib.ObjectID): - next_arg = arg + if isinstance(arg, photon.ObjectID): + args_for_photon.append(arg) + elif photon.check_simple_value(arg): + args_for_photon.append(arg) else: - serialized_arg = serialization.serialize_argument_if_possible(arg) - if serialized_arg is not None: - # Serialize the argument and pass it by value. - next_arg = serialized_arg - else: - # Put the objet in the object store under the hood. - next_arg = put(arg) - serialized_args.append(next_arg) - task_capsule = raylib.serialize_task(self.handle, func_name, serialized_args) - objectids = raylib.submit_task(self.handle, task_capsule) - return objectids + args_for_photon.append(put(arg)) + + # Submit the task to Photon. + task = photon.Task(photon.ObjectID(function_id.id()), args_for_photon, self.num_return_vals[function_id.id()]) + self.photon_client.submit(task) + + return task.returns() def export_function_to_run_on_all_workers(self, function): """Export this function and run it on all workers. @@ -496,11 +490,11 @@ def export_function_to_run_on_all_workers(self, function): not take any arguments. If it returns anything, its return values will not be used. """ - if self.mode not in [raylib.SCRIPT_MODE, raylib.SILENT_MODE, raylib.PYTHON_MODE]: + if self.mode not in [SCRIPT_MODE, SILENT_MODE, PYTHON_MODE]: raise Exception("run_function_on_all_workers can only be called on a driver.") # Run the function on all of the workers. - if self.mode in [raylib.SCRIPT_MODE, raylib.SILENT_MODE]: - raylib.run_function_on_all_workers(self.handle, pickling.dumps(function)) + if self.mode in [SCRIPT_MODE, SILENT_MODE]: + self.run_function_on_all_workers(function) def run_function_on_all_workers(self, function): @@ -516,7 +510,7 @@ def run_function_on_all_workers(self, function): not take any arguments. If it returns anything, its return values will not be used. """ - if self.mode not in [None, raylib.SCRIPT_MODE, raylib.SILENT_MODE, raylib.PYTHON_MODE]: + if self.mode not in [None, SCRIPT_MODE, SILENT_MODE, PYTHON_MODE]: raise Exception("run_function_on_all_workers can only be called on a driver.") # First run the function on the driver. function(self) @@ -525,7 +519,12 @@ def run_function_on_all_workers(self, function): if self.mode is None: self.cached_functions_to_run.append(function) else: - self.export_function_to_run_on_all_workers(function) + function_to_run_id = random_string() + key = "FunctionsToRun:{}".format(function_to_run_id) + self.redis_client.hmset(key, {"function_id": function_to_run_id, + "function": pickling.dumps(function)}) + self.redis_client.rpush("Exports", key) + self.driver_export_counter += 1 global_worker = Worker() """Worker: The global Worker object for this worker process. @@ -544,18 +543,6 @@ def run_function_on_all_workers(self, function): made by one task do not affect other tasks. """ -logger = logging.getLogger("ray") -"""Logger: The logging object for the Python worker code.""" - -object_fixtures = weakref.WeakValueDictionary() -"""WeakValueDictionary: The mapping from ObjectID to ObjectFixture object. - -This is to ensure that we have only one ObjectFixture per ObjectID. That way, if -we call get on an object twice, we do not unmap the segment before both of the -results go out of scope. It is a WeakValueDictionary instead of a regular -dictionary so that it does not keep the ObjectFixtures in scope forever. -""" - class RayConnectionError(Exception): pass @@ -565,8 +552,8 @@ def check_connected(worker=global_worker): Raises: Exception: An exception is raised if the worker is not connected. """ - if worker.handle is None and worker.mode != raylib.PYTHON_MODE: - raise RayConnectionError("This command cannot be called before a Ray cluster has been started. You can start one with 'ray.init(start_ray_local=True, num_workers=1)'.") + if not worker.connected: + raise RayConnectionError("This command cannot be called before Ray has been started. You can start Ray with 'ray.init(start_ray_local=True, num_workers=1)'.") def print_failed_task(task_status): """Print information about failed tasks. @@ -575,59 +562,29 @@ def print_failed_task(task_status): task_status (Dict): A dictionary containing the name, operationid, and error message for a failed task. """ - print """ + print(""" Error: Task failed Function Name: {} Task ID: {} Error Message: \n{} - """.format(task_status["function_name"], task_status["operationid"], task_status["error_message"]) - -def scheduler_info(worker=global_worker): - """Return information about the state of the scheduler.""" - check_connected(worker) - return raylib.scheduler_info(worker.handle) + """.format(task_status["function_name"], task_status["operationid"], task_status["error_message"])) -def visualize_computation_graph(file_path=None, view=False, worker=global_worker): - """Write the computation graph to a pdf file. - - Args: - file_path (str): The name of a pdf file that the rendered computation graph - will be written to. If this argument is None, a temporary path will be - used. - view (bool): If true, the result the python graphviz package will try to - open the result in a viewer. - - Examples: - Try the following code. - - >>> import ray.array.distributed as da - >>> x = da.zeros([20, 20]) - >>> y = da.zeros([20, 20]) - >>> z = da.dot(x, y) - >>> ray.visualize_computation_graph(view=True) - """ - check_connected(worker) - if file_path is None: - file_path = config.get_log_file_path("computation-graph.pdf") - - base_path, extension = os.path.splitext(file_path) - if extension != ".pdf": - raise Exception("File path must be a .pdf file") - proto_path = base_path + ".binaryproto" - - raylib.dump_computation_graph(worker.handle, proto_path) - g = internal.graph_pb2.CompGraph() - g.ParseFromString(open(proto_path).read()) - graph.graph_to_graphviz(g).render(base_path, view=view) - - print "Wrote graph dot description to file {}".format(base_path) - print "Wrote graph protocol buffer description to file {}".format(proto_path) - print "Wrote computation graph to file {}.pdf".format(base_path) - -def task_info(worker=global_worker): +def error_info(worker=global_worker): """Return information about failed tasks.""" check_connected(worker) - return raylib.task_info(worker.handle) + result = {"TaskError": [], + "RemoteFunctionImportError": [], + "ReusableVariableImportError": [], + "ReusableVariableReinitializeError": [], + "FunctionToRunError": [] + } + error_keys = worker.redis_client.lrange("ErrorKeys", 0, -1) + for error_key in error_keys: + error_type = error_key.split(":", 1)[0] + error_contents = worker.redis_client.hgetall(error_key) + result[error_type].append(error_contents) + + return result def initialize_numbuf(worker=global_worker): """Initialize the serialization library. @@ -639,19 +596,19 @@ def initialize_numbuf(worker=global_worker): def objectid_custom_serializer(obj): class_identifier = serialization.class_identifier(type(obj)) contained_objectids.append(obj) - return raylib.serialize_objectid(worker.handle, obj) + return obj.id() def objectid_custom_deserializer(serialized_obj): - return raylib.deserialize_objectid(worker.handle, serialized_obj) - serialization.add_class_to_whitelist(raylib.ObjectID, pickle=False, custom_serializer=objectid_custom_serializer, custom_deserializer=objectid_custom_deserializer) + return photon.ObjectID(serialized_obj) + serialization.add_class_to_whitelist(photon.ObjectID, pickle=False, custom_serializer=objectid_custom_serializer, custom_deserializer=objectid_custom_deserializer) - if worker.mode in [raylib.SCRIPT_MODE, raylib.SILENT_MODE]: + if worker.mode in [SCRIPT_MODE, SILENT_MODE]: # These should only be called on the driver because register_class will # export the class to all of the workers. register_class(RayTaskError) register_class(RayGetError) register_class(RayGetArgumentError) -def init(start_ray_local=False, num_workers=None, num_objstores=None, scheduler_address=None, node_ip_address=None, driver_mode=raylib.SCRIPT_MODE): +def init(start_ray_local=False, num_workers=None, driver_mode=SCRIPT_MODE): """Either connect to an existing Ray cluster or start one and connect to it. This method handles two cases. Either a Ray cluster already exists and we @@ -664,54 +621,37 @@ def init(start_ray_local=False, num_workers=None, num_objstores=None, scheduler_ existing Ray cluster. num_workers (Optional[int]): The number of workers to start if start_ray_local is True. - num_objstores (Optional[int]): The number of object stores to start if - start_ray_local is True. - scheduler_address (Optional[str]): The address of the scheduler to connect - to if start_ray_local is False. - node_ip_address (Optional[str]): The address of the node the worker is - running on. It is required if start_ray_local is False and it cannot be - provided otherwise. driver_mode (Optional[bool]): The mode in which to start the driver. This should be one of SCRIPT_MODE, PYTHON_MODE, and SILENT_MODE. Returns: - A string containing the address of the scheduler. + The address of the Redis server. Raises: Exception: An exception is raised if an inappropriate combination of arguments is passed in. """ - # Make GRPC only print error messages. - os.environ["GRPC_VERBOSITY"] = "ERROR" - if driver_mode == raylib.PYTHON_MODE: + if driver_mode == PYTHON_MODE: # If starting Ray in PYTHON_MODE, don't start any other processes. - pass + address_info = {} elif start_ray_local: # In this case, we launch a scheduler, a new object store, and some workers, # and we connect to them. - if (scheduler_address is not None) or (node_ip_address is not None): - raise Exception("If start_ray_local=True, then you cannot pass in a scheduler_address or a node_ip_address.") - if driver_mode not in [raylib.SCRIPT_MODE, raylib.PYTHON_MODE, raylib.SILENT_MODE]: + if driver_mode not in [SCRIPT_MODE, PYTHON_MODE, SILENT_MODE]: raise Exception("If start_ray_local=True, then driver_mode must be in [ray.SCRIPT_MODE, ray.PYTHON_MODE, ray.SILENT_MODE].") # Use the address 127.0.0.1 in local mode. - node_ip_address = "127.0.0.1" num_workers = 1 if num_workers is None else num_workers - num_objstores = 1 if num_objstores is None else num_objstores # Start the scheduler, object store, and some workers. These will be killed # by the call to cleanup(), which happens when the Python script exits. - scheduler_address = services.start_ray_local(num_objstores=num_objstores, num_workers=num_workers, worker_path=None) + address_info = services.start_ray_local(num_workers=num_workers) else: - # In this case, there is an existing scheduler and object store, and we do - # not need to start any processes. - if (num_workers is not None) or (num_objstores is not None): - raise Exception("The arguments num_workers and num_objstores must not be provided unless start_ray_local=True.") - if (node_ip_address is None) or (scheduler_address is None): - raise Exception("When start_ray_local=False, node_ip_address and scheduler_address must be provided.") - # Connect this driver to the scheduler and object store. The corresponing call - # to disconnect will happen in the call to cleanup() when the Python script - # exits. - connect(node_ip_address, scheduler_address, worker=global_worker, mode=driver_mode) - return scheduler_address + raise Exception("This mode is currently not enabled.") + # Connect this driver to Redis, the object store, and the local scheduler. The + # corresponing call to disconnect will happen in the call to cleanup() when + # the Python script exits. + connect(address_info, driver_mode, worker=global_worker) + if driver_mode != PYTHON_MODE: + return "{}:{}".format(address_info["node_ip_address"], address_info["redis_port"]) def cleanup(worker=global_worker): """Disconnect the driver, and terminate any processes started in init. @@ -723,99 +663,221 @@ def cleanup(worker=global_worker): """ disconnect(worker) worker.set_mode(None) + worker.driver_export_counter = 0 + worker.worker_import_counter = 0 + if hasattr(worker, "plasma_client"): + worker.plasma_client.shutdown() services.cleanup() atexit.register(cleanup) -def print_error_messages(worker=global_worker): - num_failed_tasks = 0 - num_failed_remote_function_imports = 0 - num_failed_reusable_variable_imports = 0 - num_failed_reusable_variable_reinitializations = 0 - num_failed_function_to_runs = 0 - while True: +def print_error_messages(worker): + """Print error messages in the background on the driver. + + This runs in a separate thread on the driver and prints error messages in the + background. + """ + worker.error_message_pubsub_client = worker.redis_client.pubsub() + # Exports that are published after the call to + # error_message_pubsub_client.psubscribe and before the call to + # error_message_pubsub_client.listen will still be processed in the loop. + worker.error_message_pubsub_client.psubscribe("__keyspace@0__:ErrorKeys") + num_errors_printed = 0 + + # Get the exports that occurred before the call to psubscribe. + try: + worker.lock.acquire() + error_keys = worker.redis_client.lrange("ErrorKeys", 0, -1) + for error_key in error_keys: + error_message = worker.redis_client.hget(error_key, "message") + print(error_message) + num_errors_printed += 1 + finally: + worker.lock.release() + + try: + for msg in worker.error_message_pubsub_client.listen(): + try: + worker.lock.acquire() + for error_key in worker.redis_client.lrange("ErrorKeys", num_errors_printed, -1): + error_message = worker.redis_client.hget(error_key, "message") + print(error_message) + num_errors_printed += 1 + finally: + worker.lock.release() + except redis.ConnectionError: + # When Redis terminates the listen call will throw a ConnectionError, which + # we catch here. + pass + +def fetch_and_register_remote_function(key, worker=global_worker): + """Import a remote function.""" + function_id_str, function_name, serialized_function, num_return_vals, module, function_export_counter = worker.redis_client.hmget(key, ["function_id", "name", "function", "num_return_vals", "module", "driver_export_counter"]) + function_id = photon.ObjectID(function_id_str) + num_return_vals = int(num_return_vals) + try: + function = pickling.loads(serialized_function) + except: + # If an exception was thrown when the remote function was imported, we + # record the traceback and notify the scheduler of the failure. + traceback_str = format_error_message(traceback.format_exc()) + # Log the error message. + error_key = "RemoteFunctionImportError:{}".format(function_id.id()) + worker.redis_client.hmset(error_key, {"function_id": function_id.id(), + "function_name": function_name, + "message": traceback_str}) + worker.redis_client.rpush("ErrorKeys", error_key) + else: + # TODO(rkn): Why is the below line necessary? + function.__module__ = module + function_name = "{}.{}".format(function.__module__, function.__name__) + worker.functions[function_id.id()] = remote(num_return_vals=num_return_vals, function_id=function_id)(function) + worker.function_names[function_id.id()] = function_name + worker.num_return_vals[function_id.id()] = num_return_vals + worker.function_export_counters[function_id.id()] = function_export_counter + # Add the function to the function table. + worker.redis_client.rpush("FunctionTable:{}".format(function_id.id()), worker.worker_id) + +def fetch_and_register_reusable_variable(key, worker=global_worker): + """Import a reusable variable.""" + reusable_variable_name, serialized_initializer, serialized_reinitializer = worker.redis_client.hmget(key, ["name", "initializer", "reinitializer"]) + try: + initializer = pickling.loads(serialized_initializer) + reinitializer = pickling.loads(serialized_reinitializer) + reusables.__setattr__(reusable_variable_name, Reusable(initializer, reinitializer)) + except: + # If an exception was thrown when the reusable variable was imported, we + # record the traceback and notify the scheduler of the failure. + traceback_str = format_error_message(traceback.format_exc()) + # Log the error message. + error_key = "ReusableVariableImportError:{}".format(random_string()) + worker.redis_client.hmset(error_key, {"name": reusable_variable_name, + "message": traceback_str}) + worker.redis_client.rpush("ErrorKeys", error_key) + +def fetch_and_execute_function_to_run(key, worker=global_worker): + """Run on arbitrary function on the worker.""" + serialized_function, = worker.redis_client.hmget(key, ["function"]) + try: + # Deserialize the function. + function = pickling.loads(serialized_function) + # Run the function. + function(worker) + except: + # If an exception was thrown when the function was run, we record the + # traceback and notify the scheduler of the failure. + traceback_str = traceback.format_exc() + # Log the error message. + name = function.__name__ if "function" in locals() and hasattr(function, "__name__") else "" + error_key = "FunctionToRunError:{}".format(random_string()) + worker.redis_client.hmset(error_key, {"name": name, + "message": traceback_str}) + worker.redis_client.rpush("ErrorKeys", error_key) + +def import_thread(worker): + worker.import_pubsub_client = worker.redis_client.pubsub() + # Exports that are published after the call to import_pubsub_client.psubscribe + # and before the call to import_pubsub_client.listen will still be processed + # in the loop. + worker.import_pubsub_client.psubscribe("__keyspace@0__:Exports") + worker_info_key = "WorkerInfo:{}".format(worker.worker_id) + worker.redis_client.hset(worker_info_key, "export_counter", 0) + worker.worker_import_counter = 0 + + # Get the exports that occurred before the call to psubscribe. + try: + worker.lock.acquire() + export_keys = worker.redis_client.lrange("Exports", 0, -1) + for key in export_keys: + if key.startswith("RemoteFunction"): + fetch_and_register_remote_function(key, worker=worker) + elif key.startswith("ReusableVariables"): + fetch_and_register_reusable_variable(key, worker=worker) + elif key.startswith("FunctionsToRun"): + fetch_and_execute_function_to_run(key, worker=worker) + else: + raise Exception("This code should be unreachable.") + worker.redis_client.hincrby(worker_info_key, "export_counter", 1) + worker.worker_import_counter += 1 + finally: + worker.lock.release() + + for msg in worker.import_pubsub_client.listen(): try: - info = task_info(worker=worker) - # Print failed task errors. - for error in info["failed_tasks"][num_failed_tasks:]: - print error["error_message"] - num_failed_tasks = len(info["failed_tasks"]) - # Print remote function import errors. - for error in info["failed_remote_function_imports"][num_failed_remote_function_imports:]: - print error["error_message"] - num_failed_remote_function_imports = len(info["failed_remote_function_imports"]) - # Print reusable variable import errors. - for error in info["failed_reusable_variable_imports"][num_failed_reusable_variable_imports:]: - print error["error_message"] - num_failed_reusable_variable_imports = len(info["failed_reusable_variable_imports"]) - # Print reusable variable reinitialization errors. - for error in info["failed_reinitialize_reusable_variables"][num_failed_reusable_variable_reinitializations:]: - print error["error_message"] - num_failed_reusable_variable_reinitializations = len(info["failed_reinitialize_reusable_variables"]) - for error in info["failed_function_to_runs"][num_failed_function_to_runs:]: - print error["error_message"] - num_failed_function_to_runs = len(info["failed_function_to_runs"]) - time.sleep(0.2) - except: - # When the driver is exiting, we set worker.handle to None, which will - # cause the check_connected call inside of task_info to raise an - # exception. We use the try block here to suppress that exception. In - # addition, when the script exits, the different names get set to None, - # for example, the time module and the task_info method get set to None, - # and so a TypeError will be thrown when we attempt to call time.sleep or - # task_info. - pass - -def connect(node_ip_address, scheduler_address, objstore_address=None, worker=global_worker, mode=raylib.WORKER_MODE): + worker.lock.acquire() + if msg["type"] == "psubscribe": + continue + assert msg["data"] == "rpush" + num_imports = worker.redis_client.llen("Exports") + assert num_imports >= worker.worker_import_counter + for i in range(worker.worker_import_counter, num_imports): + key = worker.redis_client.lindex("Exports", i) + if key.startswith("RemoteFunction"): + fetch_and_register_remote_function(key, worker=worker) + elif key.startswith("ReusableVariables"): + fetch_and_register_reusable_variable(key, worker=worker) + elif key.startswith("FunctionsToRun"): + fetch_and_execute_function_to_run(key, worker=worker) + else: + raise Exception("This code should be unreachable.") + worker.redis_client.hincrby(worker_info_key, "export_counter", 1) + worker.worker_import_counter += 1 + finally: + worker.lock.release() + +def connect(address_info, mode=WORKER_MODE, worker=global_worker): """Connect this worker to the scheduler and an object store. Args: - node_ip_address (str): The ip address of the node the worker runs on. - scheduler_address (str): The ip address and port of the scheduler. - objstore_address (Optional[str]): The ip address and port of the local - object store. Normally, this argument should be omitted and the scheduler - will tell the worker what object store to connect to. + address_info (dict): This contains the entries node_ip_address, + redis_address, object_store_name, object_store_manager_name, and + local_scheduler_name. mode: The mode of the worker. One of SCRIPT_MODE, WORKER_MODE, PYTHON_MODE, and SILENT_MODE. """ - assert worker.handle is None, "When connect is called, worker.handle should be None." + worker.worker_id = random_string() + worker.connected = True + worker.set_mode(mode) # If running Ray in PYTHON_MODE, there is no need to create call create_worker # or to start the worker service. - if mode == raylib.PYTHON_MODE: - worker.mode = raylib.PYTHON_MODE + if mode == PYTHON_MODE: return - - worker.scheduler_address = scheduler_address - random_string = "".join(np.random.choice(list(string.ascii_uppercase + string.digits)) for _ in range(10)) - cpp_log_file_name = config.get_log_file_path("-".join(["worker", random_string, "c++"]) + ".log") - python_log_file_name = config.get_log_file_path("-".join(["worker", random_string]) + ".log") - # Create a worker object. This also creates the worker service, which can - # receive commands from the scheduler. This call also sets up a queue between - # the worker and the worker service. - worker.handle, worker.worker_address = raylib.create_worker(node_ip_address, scheduler_address, objstore_address if objstore_address is not None else "", mode, cpp_log_file_name) + # Create a Redis client. + worker.redis_client = redis.StrictRedis(host=address_info["node_ip_address"], port=address_info["redis_port"]) + worker.redis_client.config_set("notify-keyspace-events", "AKE") + worker.lock = threading.Lock() + # Create an object store client. + worker.plasma_client = plasma.PlasmaClient(address_info["object_store_name"], address_info["object_store_manager_name"]) + # Create the local scheduler client. + worker.photon_client = photon.PhotonClient(address_info["local_scheduler_name"]) + # Register the worker with Redis. + if mode in [SCRIPT_MODE, SILENT_MODE]: + worker.redis_client.rpush("Drivers", worker.worker_id) + elif mode == WORKER_MODE: + worker.redis_client.rpush("Workers", worker.worker_id) + else: + raise Exception("This code should be unreachable.") + # If this is a worker, then start a thread to import exports from the driver. + if mode == WORKER_MODE: + t = threading.Thread(target=import_thread, args=(worker,)) + # Making the thread a daemon causes it to exit when the main thread exits. + t.daemon = True + t.start() # If this is a driver running in SCRIPT_MODE, start a thread to print error # messages asynchronously in the background. Ideally the scheduler would push # messages to the driver's worker service, but we ran into bugs when trying to # properly shutdown the driver's worker service, so we are temporarily using # this implementation which constantly queries the scheduler for new error # messages. - if mode == raylib.SCRIPT_MODE: + if mode == SCRIPT_MODE: t = threading.Thread(target=print_error_messages, args=(worker,)) # Making the thread a daemon causes it to exit when the main thread exits. t.daemon = True t.start() - worker.set_mode(mode) - FORMAT = "%(asctime)-15s %(message)s" - # Configure the Python logging module. Note that if we do not provide our own - # logger, then our logging will interfere with other Python modules that also - # use the logging module. - log_handler = logging.FileHandler(python_log_file_name) - log_handler.setLevel(logging.DEBUG) - log_handler.setFormatter(logging.Formatter(FORMAT)) - _logger().addHandler(log_handler) - _logger().setLevel(logging.DEBUG) - _logger().propagate = False - if mode in [raylib.SCRIPT_MODE, raylib.SILENT_MODE, raylib.PYTHON_MODE]: + # Initialize the serialization library. This registers some classes, and so + # it must be run before we export all of the cached remote functions. + initialize_numbuf() + if mode in [SCRIPT_MODE, SILENT_MODE]: # Add the directory containing the script that is running to the Python # paths of the workers. Also add the current directory. Note that this # assumes that the directory structures on the machines in the clusters are @@ -828,25 +890,21 @@ def connect(node_ip_address, scheduler_address, objstore_address=None, worker=gl for function in worker.cached_functions_to_run: worker.export_function_to_run_on_all_workers(function) # Export cached remote functions to the workers. - for function_name, function_to_export in worker.cached_remote_functions: - raylib.export_remote_function(worker.handle, function_name, function_to_export) - # Export the cached reusable variables. + for function_id, func_name, func, num_return_vals in worker.cached_remote_functions: + export_remote_function(function_id, func_name, func, num_return_vals, worker) + # Export cached reusable variables to the workers. for name, reusable_variable in reusables._cached_reusables: _export_reusable_variable(name, reusable_variable) - # Initialize the serialization library. - initialize_numbuf() worker.cached_functions_to_run = None worker.cached_remote_functions = None reusables._cached_reusables = None def disconnect(worker=global_worker): """Disconnect this worker from the scheduler and object store.""" - if worker.handle is not None: - raylib.disconnect(worker.handle) - worker.handle = None # Reset the list of cached remote functions so that if more remote functions # are defined and then connect is called again, the remote functions will be # exported. This is mostly relevant for the tests. + worker.connected = False worker.cached_functions_to_run = [] worker.cached_remote_functions = [] reusables._cached_reusables = [] @@ -871,7 +929,7 @@ def register_class(cls, pickle=False, worker=global_worker): # If the worker is not a driver, then return. We do this so that Python # modules can register classes and these modules can be imported on workers # without any trouble. - if worker.mode == raylib.WORKER_MODE: + if worker.mode == WORKER_MODE: return # Raise an exception if cls cannot be serialized efficiently by Ray. if not pickle: @@ -896,16 +954,14 @@ def get(objectid, worker=global_worker): A Python object or a list of Python objects. """ check_connected(worker) - if worker.mode == raylib.PYTHON_MODE: - return objectid # In raylib.PYTHON_MODE, ray.get is the identity operation (the input will actually be a value not an objectid) + if worker.mode == PYTHON_MODE: + return objectid # In PYTHON_MODE, ray.get is the identity operation (the input will actually be a value not an objectid) if isinstance(objectid, list): - [raylib.request_object(worker.handle, x) for x in objectid] values = [worker.get_object(x) for x in objectid] for i, value in enumerate(values): if isinstance(value, RayTaskError): raise RayGetError(objectid[i], value) return values - raylib.request_object(worker.handle, objectid) value = worker.get_object(objectid) if isinstance(value, RayTaskError): # If the result is a RayTaskError, then the task that created this object @@ -923,13 +979,13 @@ def put(value, worker=global_worker): The object ID assigned to this value. """ check_connected(worker) - if worker.mode == raylib.PYTHON_MODE: - return value # In raylib.PYTHON_MODE, ray.put is the identity operation - objectid = raylib.get_objectid(worker.handle) + if worker.mode == PYTHON_MODE: + return value # In PYTHON_MODE, ray.put is the identity operation + objectid = random_object_id() worker.put_object(objectid, value) return objectid -def wait(objectids, num_returns=1, timeout=None, worker=global_worker): +def wait(object_ids, num_returns=1, timeout=None, worker=global_worker): """Return a list of IDs that are ready and a list of IDs that are not ready. If timeout is set, the function returns either when the requested number of @@ -942,65 +998,20 @@ def wait(objectids, num_returns=1, timeout=None, worker=global_worker): corresponds to the rest of the object IDs (which may or may not be ready). Args: - objectids (List[raylib.ObjectID]): List of object IDs for objects that may + object_ids (List[ObjectID]): List of object IDs for objects that may or may not be ready. num_returns (int): The number of object IDs that should be returned. - timeout (float): The maximum amount of time in seconds that should be spent - polling the scheduler. + timeout (int): The maximum amount of time in milliseconds to wait before + returning. Returns: A list of object IDs that are ready and a list of the remaining object IDs. """ check_connected(worker) - if num_returns < 0: - raise Exception("num_returns cannot be less than 0.") - if num_returns > len(objectids): - raise Exception("num_returns cannot be greater than the length of the input list: num_objects is {}, and the length is {}.".format(num_returns, len(objectids))) - start_time = time.time() - ready_indices = raylib.wait(worker.handle, objectids) - # Polls scheduler until enough objects are ready. - while len(ready_indices) < num_returns and (time.time() - start_time < timeout or timeout is None): - ready_indices = raylib.wait(worker.handle, objectids) - time.sleep(0.1) - # Return indices for exactly the requested number of objects. - ready_ids = [objectids[i] for i in ready_indices[:num_returns]] - not_ready_ids = [objectids[i] for i in range(len(objectids)) if i not in ready_indices[:num_returns]] - return ready_ids, not_ready_ids - -def kill_workers(worker=global_worker): - """Kill all of the workers in the cluster. This does not kill drivers. - - Note: - Currently, we only support killing workers if all submitted tasks have been - run. If some workers are still running tasks or if the scheduler still has - tasks in its queue, then this method will not do anything. - - Returns: - True if workers were successfully killed. False otherwise. - """ - success = raylib.kill_workers(worker.handle) - if not success: - print "Could not kill all workers. We currently do not support killing workers when tasks are running." - return success - -def restart_workers_local(num_workers, worker_path, worker=global_worker): - """Restart workers locally. - - This method kills all of the workers and starts new workers locally on the - same node as the driver. This is intended for use in the case where Ray is - being used on a single node. - - Args: - num_workers (int): The number of workers to be started. - worker_path (str): The path of the source code that workers will run. - - Returns: - True if workers were successfully restarted. False otherwise. - """ - if not kill_workers(worker): - return False - services.start_workers(worker.scheduler_address, worker.objstore_address, num_workers, worker_path) - return True + object_id_strs = [object_id.id() for object_id in object_ids] + timeout = timeout if timeout is not None else 2 ** 36 + ready_ids, remaining_ids = worker.plasma_client.wait(object_id_strs, timeout, num_returns) + return ready_ids, remaining_ids def format_error_message(exception_message): """Improve the formatting of an exception thrown by a remote function. @@ -1030,10 +1041,6 @@ def main_loop(worker=global_worker): process. The worker executes the command, notifies the scheduler of any errors that occurred while executing the command, and waits for the next command. """ - if not raylib.connected(worker.handle): - raise Exception("Worker is attempting to enter main_loop but has not been connected yet.") - # Notify the scheduler that the worker is ready to start receiving tasks. - raylib.ready_for_new_task(worker.handle) def process_task(task): # wrapping these lines in a function should cause the local variables to go out of scope more quickly, which is useful for inspecting reference counts """Execute a task assigned to this worker. @@ -1046,27 +1053,30 @@ def process_task(task): # wrapping these lines in a function should cause the lo After the task executes, the worker resets any reusable variables that were accessed by the task. """ - function_name, serialized_args, return_objectids = task + function_id = task.function_id() + args = task.arguments() + return_object_ids = task.returns() + function_name = worker.function_names[function_id.id()] try: - arguments = get_arguments_for_execution(worker.functions[function_name], serialized_args, worker) # get args from objstore - outputs = worker.functions[function_name].executor(arguments) # execute the function - if len(return_objectids) == 1: + arguments = get_arguments_for_execution(worker.functions[function_id.id()], args, worker) # get args from objstore + outputs = worker.functions[function_id.id()].executor(arguments) # execute the function + if len(return_object_ids) == 1: outputs = (outputs,) - store_outputs_in_objstore(return_objectids, outputs, worker) # store output in local object store + store_outputs_in_objstore(return_object_ids, outputs, worker) # store output in local object store except Exception as e: # If the task threw an exception, then record the traceback. We determine # whether the exception was thrown in the task execution by whether the # variable "arguments" is defined. traceback_str = format_error_message(traceback.format_exc()) if "arguments" in locals() else None failure_object = RayTaskError(function_name, e, traceback_str) - failure_objects = [failure_object for _ in range(len(return_objectids))] - store_outputs_in_objstore(return_objectids, failure_objects, worker) - # Notify the scheduler that the task failed. - raylib.notify_failure(worker.handle, function_name, str(failure_object), raylib.FailedTask) - _logger().info("While running function {}, worker threw exception with message: \n\n{}\n".format(function_name, str(failure_object))) - # Notify the scheduler that the task is done. This happens regardless of - # whether the task succeeded or failed. - raylib.ready_for_new_task(worker.handle) + failure_objects = [failure_object for _ in range(len(return_object_ids))] + store_outputs_in_objstore(return_object_ids, failure_objects, worker) + # Log the error message. + error_key = "TaskError:{}".format(random_string()) + worker.redis_client.hmset(error_key, {"function_id": function_id.id(), + "function_name": function_name, + "message": traceback_str}) + worker.redis_client.rpush("ErrorKeys", error_key) try: # Reinitialize the values of reusable variables that were used in the task # above so that changes made to their state do not affect other tasks. @@ -1075,90 +1085,35 @@ def process_task(task): # wrapping these lines in a function should cause the lo # The attempt to reinitialize the reusable variables threw an exception. # We record the traceback and notify the scheduler. traceback_str = format_error_message(traceback.format_exc()) - raylib.notify_failure(worker.handle, function_name, traceback_str, raylib.FailedReinitializeReusableVariable) - _logger().info("While attempting to reinitialize the reusable variables after running function {}, the worker threw exception with message: \n\n{}\n".format(function_name, traceback_str)) - - def process_remote_function(function_name, serialized_function): - """Import a remote function.""" - try: - function, num_return_vals, module = pickling.loads(serialized_function) - except: - # If an exception was thrown when the remote function was imported, we - # record the traceback and notify the scheduler of the failure. - traceback_str = format_error_message(traceback.format_exc()) - _logger().info("Failed to import remote function {}. Failed with message: \n\n{}\n".format(function_name, traceback_str)) - # Notify the scheduler that the remote function failed to import. - raylib.notify_failure(worker.handle, function_name, traceback_str, raylib.FailedRemoteFunctionImport) - else: - # TODO(rkn): Why is the below line necessary? - function.__module__ = module - assert function_name == "{}.{}".format(function.__module__, function.__name__), "The remote function name does not match the name that was passed in." - worker.functions[function_name] = remote(num_return_vals=num_return_vals)(function) - _logger().info("Successfully imported remote function {}.".format(function_name)) - # Noify the scheduler that the remote function imported successfully. - # We pass an empty error message string because the import succeeded. - raylib.register_remote_function(worker.handle, function_name, num_return_vals) - - def process_reusable_variable(reusable_variable_name, initializer_str, reinitializer_str): - """Import a reusable variable.""" - try: - initializer = pickling.loads(initializer_str) - reinitializer = pickling.loads(reinitializer_str) - reusables.__setattr__(reusable_variable_name, Reusable(initializer, reinitializer)) - except: - # If an exception was thrown when the reusable variable was imported, we - # record the traceback and notify the scheduler of the failure. - traceback_str = format_error_message(traceback.format_exc()) - _logger().info("Failed to import reusable variable {}. Failed with message: \n\n{}\n".format(reusable_variable_name, traceback_str)) - # Notify the scheduler that the reusable variable failed to import. - raylib.notify_failure(worker.handle, reusable_variable_name, traceback_str, raylib.FailedReusableVariableImport) - else: - _logger().info("Successfully imported reusable variable {}.".format(reusable_variable_name)) - - def process_function_to_run(serialized_function): - """Run on arbitrary function on the worker.""" - try: - # Deserialize the function. - function = pickling.loads(serialized_function) - # Run the function. - function(worker) - except: - # If an exception was thrown when the function was run, we record the - # traceback and notify the scheduler of the failure. - traceback_str = traceback.format_exc() - _logger().info("Failed to run function on worker. Failed with message: \n\n{}\n".format(traceback_str)) - # Notify the scheduler that running the function failed. - name = function.__name__ if "function" in locals() and hasattr(function, "__name__") else "" - raylib.notify_failure(worker.handle, name, traceback_str, raylib.FailedFunctionToRun) - else: - _logger().info("Successfully ran function on worker.") + error_key = "ReusableVariableReinitializeError:{}".format(random_string()) + worker.redis_client.hmset(error_key, {"task_instance_id": "NOTIMPLEMENTED", + "task_id": "NOTIMPLEMENTED", + "function_id": function_id.id(), + "function_name": function_name, + "message": traceback_str}) + worker.redis_client.rpush("ErrorKeys", error_key) while True: - command, command_args = raylib.wait_for_next_message(worker.handle) + task = worker.photon_client.get_task() + function_id = task.function_id() + # Check that the number of imports we have is at least as great as the + # export counter for the task. If not, wait until we have imported enough. + while True: + try: + worker.lock.acquire() + if worker.functions.has_key(function_id.id()) and worker.function_export_counters[function_id.id()] <= worker.worker_import_counter: + break + time.sleep(0.001) + finally: + worker.lock.release() + # Execute the task. try: - if command == "die": - # We use this as a mechanism to allow the scheduler to kill workers. - _logger().info("Received a 'die' command, and will exit now.") - break - elif command == "task": - process_task(command_args) - elif command == "function": - function_name, serialized_function = command_args - process_remote_function(function_name, serialized_function) - elif command == "reusable_variable": - name, initializer_str, reinitializer_str = command_args - process_reusable_variable(name, initializer_str, reinitializer_str) - elif command == "function_to_run": - serialized_function = command_args - process_function_to_run(serialized_function) - else: - _logger().info("Reached the end of the if-else loop in the main loop. This should be unreachable.") - assert False, "This code should be unreachable." + worker.lock.acquire() + process_task(task) finally: - # Allow releasing the variables BEFORE we wait for the next message or exit the block - del command_args + worker.lock.release() -def _submit_task(func_name, args, worker=global_worker): +def _submit_task(function_id, func_name, args, worker=global_worker): """This is a wrapper around worker.submit_task. We use this wrapper so that in the remote decorator, we can call _submit_task @@ -1166,7 +1121,7 @@ def _submit_task(func_name, args, worker=global_worker): serialize remote functions, we don't attempt to serialize the worker object, which cannot be serialized. """ - return worker.submit_task(func_name, args) + return worker.submit_task(function_id, func_name, args) def _mode(worker=global_worker): """This is a wrapper around worker.mode. @@ -1178,15 +1133,6 @@ def _mode(worker=global_worker): """ return worker.mode -def _logger(): - """Return the logger object. - - We use this wrapper because so that functions which do logging can be pickled. - Normally a logger object is specific to a machine (it opens a local file), and - so cannot be pickled. - """ - return logger - def _reusables(): """Return the reusables object. @@ -1203,9 +1149,32 @@ def _export_reusable_variable(name, reusable, worker=global_worker): reusable (Reusable): The reusable object containing code for initializing and reinitializing the variable. """ - if _mode(worker) not in [raylib.SCRIPT_MODE, raylib.SILENT_MODE]: + if _mode(worker) not in [SCRIPT_MODE, SILENT_MODE]: raise Exception("_export_reusable_variable can only be called on a driver.") - raylib.export_reusable_variable(worker.handle, name, pickling.dumps(reusable.initializer), pickling.dumps(reusable.reinitializer)) + + reusable_variable_id = name + key = "ReusableVariables:{}".format(reusable_variable_id) + worker.redis_client.hmset(key, {"name": name, + "initializer": pickling.dumps(reusable.initializer), + "reinitializer": pickling.dumps(reusable.reinitializer)}) + worker.redis_client.rpush("Exports", key) + worker.driver_export_counter += 1 + +def export_remote_function(function_id, func_name, func, num_return_vals, worker=global_worker): + key = "RemoteFunction:{}".format(function_id.id()) + + worker.num_return_vals[function_id.id()] = num_return_vals + + + pickled_func = pickling.dumps(func) + worker.redis_client.hmset(key, {"function_id": function_id.id(), + "name": func_name, + "module": func.__module__, + "function": pickled_func, + "num_return_vals": num_return_vals, + "function_export_counter": worker.driver_export_counter}) + worker.redis_client.rpush("Exports", key) + worker.driver_export_counter += 1 def remote(*args, **kwargs): """This decorator is used to create remote functions. @@ -1215,8 +1184,14 @@ def remote(*args, **kwargs): should return. """ worker = global_worker - def make_remote_decorator(num_return_vals): + def make_remote_decorator(num_return_vals, func_id=None): def remote_decorator(func): + func_name = "{}.{}".format(func.__module__, func.__name__) + if func_id is None: + function_id = FunctionID((hashlib.sha256(func_name).digest())[:20]) + else: + function_id = func_id + def func_call(*args, **kwargs): """This gets run immediately when a worker calls a remote function.""" check_connected() @@ -1224,8 +1199,8 @@ def func_call(*args, **kwargs): args.extend([kwargs[keyword] if kwargs.has_key(keyword) else default for keyword, default in keyword_defaults[len(args):]]) # fill in the remaining arguments if any([arg is funcsigs._empty for arg in args]): raise Exception("Not enough arguments were provided to {}.".format(func_name)) - if _mode() == raylib.PYTHON_MODE: - # In raylib.PYTHON_MODE, remote calls simply execute the function. We copy the + if _mode() == PYTHON_MODE: + # In PYTHON_MODE, remote calls simply execute the function. We copy the # arguments to prevent the function call from mutating them and to match # the usual behavior of immutable remote objects. try: @@ -1235,18 +1210,16 @@ def func_call(*args, **kwargs): _reusables()._reinitialize() _reusables()._running_remote_function_locally = False return result - objectids = _submit_task(func_name, args) + objectids = _submit_task(function_id, func_name, args) if len(objectids) == 1: return objectids[0] elif len(objectids) > 1: return objectids def func_executor(arguments): """This gets run when the remote function is executed.""" - _logger().info("Calling function {}".format(func.__name__)) start_time = time.time() result = func(*arguments) end_time = time.time() - _logger().info("Finished executing function {}, it took {} seconds".format(func.__name__, end_time - start_time)) return result def func_invoker(*args, **kwargs): """This is returned by the decorator and used to invoke the function.""" @@ -1266,7 +1239,7 @@ def func_invoker(*args, **kwargs): check_signature_supported(has_kwargs_param, has_vararg_param, keyword_defaults, func_name) # Everything ready - export the function - if worker.mode in [None, raylib.SCRIPT_MODE, raylib.SILENT_MODE]: + if worker.mode in [None, SCRIPT_MODE, SILENT_MODE]: func_name_global_valid = func.__name__ in func.__globals__ func_name_global_value = func.__globals__.get(func.__name__) # Set the function globally to make it refer to itself @@ -1277,14 +1250,20 @@ def func_invoker(*args, **kwargs): # Undo our changes if func_name_global_valid: func.__globals__[func.__name__] = func_name_global_value else: del func.__globals__[func.__name__] - if worker.mode in [raylib.SCRIPT_MODE, raylib.SILENT_MODE]: - raylib.export_remote_function(worker.handle, func_name, to_export) + if worker.mode in [SCRIPT_MODE, SILENT_MODE]: + export_remote_function(function_id, func_name, func, num_return_vals) elif worker.mode is None: - worker.cached_remote_functions.append((func_name, to_export)) + worker.cached_remote_functions.append((function_id, func_name, func, num_return_vals)) return func_invoker return remote_decorator + if _mode() == WORKER_MODE: + if kwargs.has_key("function_id"): + num_return_vals = kwargs["num_return_vals"] + function_id = kwargs["function_id"] + return make_remote_decorator(num_return_vals, function_id) + if len(args) == 1 and len(kwargs) == 0 and callable(args[0]): # This is the case where the decorator is just @ray.remote. num_return_vals = 1 @@ -1293,8 +1272,9 @@ def func_invoker(*args, **kwargs): else: # This is the case where the decorator is something like # @ray.remote(num_return_vals=2). - assert len(args) == 0 and "num_return_vals" in kwargs.keys(), "The @ray.remote decorator must be applied either with no arguments and no parentheses, for example '@ray.remote', or it must be applied with only the argument num_return_vals, like '@ray.remote(num_return_vals=2)'." + assert len(args) == 0 and kwargs.has_key("num_return_vals"), "The @ray.remote decorator must be applied either with no arguments and no parentheses, for example '@ray.remote', or it must be applied with only the argument num_return_vals, like '@ray.remote(num_return_vals=2)'." num_return_vals = kwargs["num_return_vals"] + assert not kwargs.has_key("function_id") return make_remote_decorator(num_return_vals) def check_signature_supported(has_kwargs_param, has_vararg_param, keyword_defaults, name): @@ -1346,18 +1326,16 @@ def get_arguments_for_execution(function, serialized_args, worker=global_worker) """ arguments = [] for (i, arg) in enumerate(serialized_args): - if isinstance(arg, raylib.ObjectID): + if isinstance(arg, photon.ObjectID): # get the object from the local object store - _logger().info("Getting argument {} for function {}.".format(i, function.__name__)) argument = worker.get_object(arg) if isinstance(argument, RayTaskError): # If the result is a RayTaskError, then the task that created this # object failed, and we should propagate the error message here. raise RayGetArgumentError(function.__name__, i, arg, argument) - _logger().info("Successfully retrieved argument {} for function {}.".format(i, function.__name__)) else: # pass the argument by value - argument = serialization.deserialize_argument(arg) + argument = arg arguments.append(argument) return arguments @@ -1374,7 +1352,7 @@ def store_outputs_in_objstore(objectids, outputs, worker=global_worker): The arguments objectids and outputs should have the same length. Args: - objectids (List[raylib.ObjectID]): The object IDs that were assigned to the + objectids (List[ObjectID]): The object IDs that were assigned to the outputs of the remote function call. outputs (Tuple): The value returned by the remote function. If the remote function was supposed to only return one value, then its output was @@ -1382,7 +1360,7 @@ def store_outputs_in_objstore(objectids, outputs, worker=global_worker): function. """ for i in range(len(objectids)): - if isinstance(outputs[i], raylib.ObjectID): + if isinstance(outputs[i], photon.ObjectID): raise Exception("This remote function returned an ObjectID as its {}th return value. This is not allowed.".format(i)) for i in range(len(objectids)): worker.put_object(objectids[i], outputs[i]) diff --git a/scripts/default_worker.py b/scripts/default_worker.py deleted file mode 100644 index 54f31d7774ca..000000000000 --- a/scripts/default_worker.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys -import argparse -import numpy as np - -import ray - -parser = argparse.ArgumentParser(description="Parse addresses for the worker to connect to.") -parser.add_argument("--node-ip-address", required=True, type=str, help="the ip address of the worker's node") -parser.add_argument("--scheduler-address", required=True, type=str, help="the scheduler's address") -parser.add_argument("--objstore-address", type=str, help="the objstore's address") - -if __name__ == "__main__": - args = parser.parse_args() - ray.worker.connect(args.node_ip_address, args.scheduler_address) - - ray.worker.main_loop() diff --git a/setup-env.sh b/setup-env.sh deleted file mode 100644 index f86348d2e2b4..000000000000 --- a/setup-env.sh +++ /dev/null @@ -1,19 +0,0 @@ -# NO shebang! Force the user to run this using the 'source' command without spawning a new shell; otherwise, variable exports won't persist. - -echo "Adding Ray to PYTHONPATH" 1>&2 - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -export PYTHONPATH="$ROOT_DIR/lib/python/:$ROOT_DIR/thirdparty/numbuf/build:$PYTHONPATH" - -# Print instructions for adding Ray to your bashrc. -unamestr="$(uname)" -if [[ "$unamestr" == "Linux" ]]; then - BASH_RC="~/.bashrc" -elif [[ "$unamestr" == "Darwin" ]]; then - BASH_RC="~/.bash_profile" -fi -echo "To permanently add Ray to your Python path, run, - -echo 'export PYTHONPATH=$ROOT_DIR/lib/python/:$ROOT_DIR/thirdparty/numbuf/build:\$PYTHONPATH' >> $BASH_RC -" diff --git a/src/photon/photon_algorithm.c b/src/photon/photon_algorithm.c index 8f2d6c1b5b22..e0af933227b3 100644 --- a/src/photon/photon_algorithm.c +++ b/src/photon/photon_algorithm.c @@ -169,7 +169,7 @@ void handle_worker_available(scheduler_info *info, /* Add client_sock to a list of available workers. This struct will be freed * when a task is assigned to this worker. */ utarray_push_back(state->available_workers, &worker_index); - LOG_INFO("Adding worker_index %d to available workers.\n", worker_index); + LOG_DEBUG("Adding worker_index %d to available workers.\n", worker_index); } } diff --git a/src/photon/photon_scheduler.c b/src/photon/photon_scheduler.c index 1b3474e62514..7fbf4fede5eb 100644 --- a/src/photon/photon_scheduler.c +++ b/src/photon/photon_scheduler.c @@ -148,7 +148,7 @@ void new_client_connection(event_loop *loop, int listener_sock, void *context, local_scheduler_state *s = context; int new_socket = accept_client(listener_sock); event_loop_add_file(loop, new_socket, EVENT_LOOP_READ, process_message, s); - LOG_INFO("new connection with fd %d", new_socket); + LOG_DEBUG("new connection with fd %d", new_socket); /* Add worker to list of workers. */ /* TODO(pcm): Where shall we free this? */ worker_index *new_worker_index = malloc(sizeof(worker_index)); diff --git a/src/plasma/lib/python/plasma.py b/src/plasma/lib/python/plasma.py index af197df74b7c..a869d136a5a7 100644 --- a/src/plasma/lib/python/plasma.py +++ b/src/plasma/lib/python/plasma.py @@ -40,8 +40,12 @@ def __init__(self, buff, plasma_id, plasma_client): self.plasma_client = plasma_client def __del__(self): - """Notify Plasma that the object is no longer needed.""" - self.plasma_client.client.plasma_release(self.plasma_client.plasma_conn, self.plasma_id) + """Notify Plasma that the object is no longer needed. + + If the plasma client has been shut down, then don't do anything. + """ + if self.plasma_client.alive: + self.plasma_client.client.plasma_release(self.plasma_client.plasma_conn, self.plasma_id) def __getitem__(self, index): """Read from the PlasmaBuffer as if it were just a regular buffer.""" @@ -73,7 +77,7 @@ def __init__(self, store_socket_name, manager_socket_name=None): store_socket_name (str): Name of the socket the plasma store is listening at. manager_socket_name (str): Name of the socket the plasma manager is listening at. """ - + self.alive = True plasma_client_library = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../build/plasma_client.so") self.client = ctypes.cdll.LoadLibrary(plasma_client_library) @@ -85,6 +89,7 @@ def __init__(self, store_socket_name, manager_socket_name=None): self.client.plasma_seal.restype = None self.client.plasma_delete.restype = None self.client.plasma_subscribe.restype = ctypes.c_int + self.client.plasma_wait.restype = ctypes.c_int self.buffer_from_memory = ctypes.pythonapi.PyBuffer_FromMemory self.buffer_from_memory.argtypes = [ctypes.c_void_p, ctypes.c_int64] @@ -101,6 +106,15 @@ def __init__(self, store_socket_name, manager_socket_name=None): self.has_manager_conn = False self.plasma_conn = ctypes.c_void_p(self.client.plasma_connect(store_socket_name, None)) + def shutdown(self): + """Shutdown the client so that it does not send messages. + + If we kill the Plasma store and Plasma manager that this client is connected + to, then we can use this method to prevent the client from trying to send + messages to the killed processes. + """ + self.alive = False + def create(self, object_id, size, metadata=None): """Create a new buffer in the PlasmaStore for a particular object ID. @@ -233,6 +247,12 @@ def wait(self, object_ids, timeout, num_returns): """ if not self.has_manager_conn: raise Exception("Not connected to the plasma manager socket") + if num_returns < 0: + raise Exception("The argument num_returns cannot be less than one.") + if num_returns > len(object_ids): + raise Exception("The argument num_returns cannot be greater than len(object_ids): num_returns is {}, len(object_ids) is {}.".format(num_returns, len(object_ids))) + if timeout > 2 ** 36: + raise Exception("The method wait currently cannot be used with a timeout greater than 2 ** 36.") object_id_array = (len(object_ids) * PlasmaID)() for i, object_id in enumerate(object_ids): object_id_array[i] = make_plasma_id(object_id) @@ -240,7 +260,9 @@ def wait(self, object_ids, timeout, num_returns): num_return_objects = self.client.plasma_wait(self.plasma_conn, object_id_array._length_, object_id_array, - timeout, num_returns, return_id_array) + ctypes.c_int64(timeout), + num_returns, + return_id_array) ready_ids = map(plasma_id_to_str, return_id_array[num_returns-num_return_objects:]) return ready_ids, list(set(object_ids) - set(ready_ids)) diff --git a/test/array_test.py b/test/array_test.py index 5d00cbe61e81..c09f804b3bf0 100644 --- a/test/array_test.py +++ b/test/array_test.py @@ -58,7 +58,7 @@ def testAssemble(self): def testMethods(self): for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]: reload(module) - ray.init(start_ray_local=True, num_objstores=2, num_workers=10) + ray.init(start_ray_local=True, num_workers=10) x = da.zeros.remote([9, 25, 51], "float") assert_equal(ray.get(da.assemble.remote(x)), np.zeros([9, 25, 51])) diff --git a/test/failure_test.py b/test/failure_test.py index 78e6c6227749..2e251767d042 100644 --- a/test/failure_test.py +++ b/test/failure_test.py @@ -4,16 +4,24 @@ import test_functions +def wait_for_errors(error_type, num_errors, timeout=10): + start_time = time.time() + while time.time() - start_time < timeout: + error_info = ray.error_info() + if len(error_info[error_type]) >= num_errors: + return + time.sleep(0.1) + print("Timing out of wait.") + class FailureTest(unittest.TestCase): def testUnknownSerialization(self): reload(test_functions) ray.init(start_ray_local=True, num_workers=1, driver_mode=ray.SILENT_MODE) test_functions.test_unknown_type.remote() - time.sleep(0.2) - task_info = ray.task_info() - self.assertEqual(len(task_info["failed_tasks"]), 1) - self.assertEqual(len(task_info["running_tasks"]), 0) + wait_for_errors("TaskError", 1) + error_info = ray.error_info() + self.assertEqual(len(error_info["TaskError"]), 1) ray.worker.cleanup() @@ -45,19 +53,11 @@ def testFailedTask(self): test_functions.throw_exception_fct1.remote() test_functions.throw_exception_fct1.remote() - for _ in range(100): # Retry if we need to wait longer. - if len(ray.task_info()["failed_tasks"]) >= 2: - break - time.sleep(0.1) - result = ray.task_info() - self.assertEqual(len(result["failed_tasks"]), 2) - task_ids = set() - for task in result["failed_tasks"]: - self.assertTrue(task.has_key("worker_address")) - self.assertTrue(task.has_key("operationid")) - self.assertTrue("Test function 1 intentionally failed." in task.get("error_message")) - self.assertTrue(task["operationid"] not in task_ids) - task_ids.add(task["operationid"]) + wait_for_errors("TaskError", 2) + result = ray.error_info() + self.assertEqual(len(result["TaskError"]), 2) + for task in result["TaskError"]: + self.assertTrue("Test function 1 intentionally failed." in task.get("message")) x = test_functions.throw_exception_fct2.remote() try: @@ -96,11 +96,8 @@ def __reduce__(self): def __call__(self): return ray.remote(Foo()) - for _ in range(100): # Retry if we need to wait longer. - if len(ray.task_info()["failed_remote_function_imports"]) >= 1: - break - time.sleep(0.1) - self.assertTrue("There is a problem here." in ray.task_info()["failed_remote_function_imports"][0]["error_message"]) + wait_for_errors("RemoteFunctionImportError", 1) + self.assertTrue("There is a problem here." in ray.error_info()["RemoteFunctionImportError"][0]["message"]) ray.worker.cleanup() @@ -114,12 +111,9 @@ def initializer(): raise Exception("The initializer failed.") return 0 ray.reusables.foo = ray.Reusable(initializer) - for _ in range(100): # Retry if we need to wait longer. - if len(ray.task_info()["failed_reusable_variable_imports"]) >= 1: - break - time.sleep(0.1) + wait_for_errors("ReusableVariableImportError", 1) # Check that the error message is in the task info. - self.assertTrue("The initializer failed." in ray.task_info()["failed_reusable_variable_imports"][0]["error_message"]) + self.assertTrue("The initializer failed." in ray.error_info()["ReusableVariableImportError"][0]["message"]) ray.worker.cleanup() @@ -135,12 +129,9 @@ def reinitializer(foo): def use_foo(): ray.reusables.foo use_foo.remote() - for _ in range(100): # Retry if we need to wait longer. - if len(ray.task_info()["failed_reinitialize_reusable_variables"]) >= 1: - break - time.sleep(0.1) + wait_for_errors("ReusableVariableReinitializeError", 1) # Check that the error message is in the task info. - self.assertTrue("The reinitializer failed." in ray.task_info()["failed_reinitialize_reusable_variables"][0]["error_message"]) + self.assertTrue("The reinitializer failed." in ray.error_info()["ReusableVariableReinitializeError"][0]["message"]) ray.worker.cleanup() @@ -151,14 +142,11 @@ def f(worker): if ray.worker.global_worker.mode == ray.WORKER_MODE: raise Exception("Function to run failed.") ray.worker.global_worker.run_function_on_all_workers(f) - for _ in range(100): # Retry if we need to wait longer. - if len(ray.task_info()["failed_function_to_runs"]) >= 2: - break - time.sleep(0.1) + wait_for_errors("FunctionToRunError", 2) # Check that the error message is in the task info. - self.assertEqual(len(ray.task_info()["failed_function_to_runs"]), 2) - self.assertTrue("Function to run failed." in ray.task_info()["failed_function_to_runs"][0]["error_message"]) - self.assertTrue("Function to run failed." in ray.task_info()["failed_function_to_runs"][1]["error_message"]) + self.assertEqual(len(ray.error_info()["FunctionToRunError"]), 2) + self.assertTrue("Function to run failed." in ray.error_info()["FunctionToRunError"][0]["message"]) + self.assertTrue("Function to run failed." in ray.error_info()["FunctionToRunError"][1]["message"]) ray.worker.cleanup() diff --git a/test/runtest.py b/test/runtest.py index a02405a02433..445a398fa973 100644 --- a/test/runtest.py +++ b/test/runtest.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import unittest import ray import numpy as np @@ -142,64 +144,6 @@ class ClassA(object): ray.worker.cleanup() -class ObjStoreTest(unittest.TestCase): - - # Test setting up object stores, transfering data between them and retrieving data to a client - def testObjStore(self): - node_ip_address = "127.0.0.1" - scheduler_address = ray.services.start_ray_local(num_objstores=2, num_workers=0, worker_path=None) - ray.connect(node_ip_address, scheduler_address, mode=ray.SCRIPT_MODE) - objstore_addresses = [objstore_info["address"] for objstore_info in ray.scheduler_info()["objstores"]] - w1 = ray.worker.Worker() - w2 = ray.worker.Worker() - ray.reusables._cached_reusables = [] # This is a hack to make the test run. - ray.connect(node_ip_address, scheduler_address, objstore_address=objstore_addresses[0], mode=ray.SCRIPT_MODE, worker=w1) - ray.reusables._cached_reusables = [] # This is a hack to make the test run. - ray.connect(node_ip_address, scheduler_address, objstore_address=objstore_addresses[1], mode=ray.SCRIPT_MODE, worker=w2) - - for cls in [Foo, Bar, Baz, Qux, SubQux, Exception, CustomError, Point, NamedTupleExample]: - ray.register_class(cls) - - # putting and getting an object shouldn't change it - for data in RAY_TEST_OBJECTS: - objectid = ray.put(data, w1) - result = ray.get(objectid, w1) - assert_equal(result, data) - - # putting an object, shipping it to another worker, and getting it shouldn't change it - for data in RAY_TEST_OBJECTS: - objectid = ray.put(data, w1) - result = ray.get(objectid, w2) - assert_equal(result, data) - - # putting an object, shipping it to another worker, and getting it shouldn't change it - for data in RAY_TEST_OBJECTS: - objectid = ray.put(data, w2) - result = ray.get(objectid, w1) - assert_equal(result, data) - - # This test fails. See https://github.com/ray-project/ray/issues/159. - # getting multiple times shouldn't matter - # for data in [np.zeros([10, 20]), np.random.normal(size=[45, 25]), np.zeros([10, 20], dtype=np.dtype("float64")), np.zeros([10, 20], dtype=np.dtype("float32")), np.zeros([10, 20], dtype=np.dtype("int64")), np.zeros([10, 20], dtype=np.dtype("int32"))]: - # objectid = worker.put(data, w1) - # result = worker.get(objectid, w2) - # result = worker.get(objectid, w2) - # result = worker.get(objectid, w2) - # assert_equal(result, data) - - # Getting a buffer after modifying it before it finishes should return updated buffer - objectid = ray.libraylib.get_objectid(w1.handle) - buf = ray.libraylib.allocate_buffer(w1.handle, objectid, 100) - buf[0][0] = 1 - ray.libraylib.finish_buffer(w1.handle, objectid, buf[1], 0) - completedbuffer = ray.libraylib.get_buffer(w1.handle, objectid) - self.assertEqual(completedbuffer[0][0], 1) - - # We started multiple drivers manually, so we will disconnect them manually. - ray.disconnect(worker=w1) - ray.disconnect(worker=w2) - ray.worker.cleanup() - class WorkerTest(unittest.TestCase): def testPutGet(self): @@ -233,29 +177,6 @@ def testPutGet(self): class APITest(unittest.TestCase): - def testPassingArgumentsByValue(self): - ray.init(start_ray_local=True, num_workers=0) - - # The types that can be passed by value are defined by - # is_argument_serializable in serialization.py. - class Foo(object): - pass - CAN_PASS_BY_VALUE = [1, 1L, 1.0, True, False, None, [1L, 1.0, True, None], - ([1, 2, 3], {False: [1.0, u"hi", ()]}), 100 * ["a"]] - CANNOT_PASS_BY_VALUE = [int, np.int64(0), np.float64(0), Foo(), [Foo()], - (Foo()), {0: Foo()}, [[[int]]], 101 * [1], - np.zeros(10)] - - for obj in CAN_PASS_BY_VALUE: - self.assertTrue(ray.serialization.is_argument_serializable(obj)) - self.assertEqual(obj, ray.serialization.deserialize_argument(ray.serialization.serialize_argument_if_possible(obj))) - - for obj in CANNOT_PASS_BY_VALUE: - self.assertFalse(ray.serialization.is_argument_serializable(obj)) - self.assertEqual(None, ray.serialization.serialize_argument_if_possible(obj)) - - ray.worker.cleanup() - def testRegisterClass(self): ray.init(start_ray_local=True, num_workers=0) @@ -328,11 +249,7 @@ def testNoArgs(self): reload(test_functions) ray.init(start_ray_local=True, num_workers=1) - test_functions.no_op.remote() - time.sleep(0.2) - task_info = ray.task_info() - self.assertEqual(len(task_info["failed_tasks"]), 0) - self.assertEqual(len(task_info["running_tasks"]), 0) + ray.get(test_functions.no_op.remote()) ray.worker.cleanup() @@ -400,22 +317,22 @@ def f(delay): objectids = [f.remote(1.0), f.remote(0.5), f.remote(0.5), f.remote(0.5)] ready_ids, remaining_ids = ray.wait(objectids) - self.assertTrue(len(ready_ids) == 1) - self.assertTrue(len(remaining_ids) == 3) + self.assertEqual(len(ready_ids), 1) + self.assertEqual(len(remaining_ids), 3) ready_ids, remaining_ids = ray.wait(objectids, num_returns=4) - self.assertEqual(ready_ids, objectids) + self.assertEqual(set(ready_ids), set([object_id.id() for object_id in objectids])) self.assertEqual(remaining_ids, []) objectids = [f.remote(0.5), f.remote(0.5), f.remote(0.5), f.remote(0.5)] start_time = time.time() - ready_ids, remaining_ids = ray.wait(objectids, timeout=1.75, num_returns=4) - self.assertTrue(time.time() - start_time < 2) + ready_ids, remaining_ids = ray.wait(objectids, timeout=1750, num_returns=4) + self.assertLess(time.time() - start_time, 2) self.assertEqual(len(ready_ids), 3) self.assertEqual(len(remaining_ids), 1) ray.wait(objectids) objectids = [f.remote(1.0), f.remote(0.5), f.remote(0.5), f.remote(0.5)] start_time = time.time() - ready_ids, remaining_ids = ray.wait(objectids, timeout=5) + ready_ids, remaining_ids = ray.wait(objectids, timeout=5000) self.assertTrue(time.time() - start_time < 5) self.assertEqual(len(ready_ids), 1) self.assertEqual(len(remaining_ids), 3) @@ -504,150 +421,6 @@ def f(worker): ray.worker.cleanup() - def testComputationGraph(self): - ray.init(start_ray_local=True, num_workers=1) - - @ray.remote - def f(x): - return x - @ray.remote - def g(x, y): - return x, y - a = f.remote(1) - b = f.remote(1) - c = g.remote(a, b) - c = g.remote(a, 1) - # Make sure that we can produce a computation_graph visualization. - ray.visualize_computation_graph(view=False) - - ray.worker.cleanup() - -class ReferenceCountingTest(unittest.TestCase): - - def testDeallocation(self): - reload(test_functions) - for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]: - reload(module) - ray.init(start_ray_local=True, num_workers=1) - - def check_not_deallocated(object_ids): - reference_counts = ray.scheduler_info()["reference_counts"] - for object_id in object_ids: - self.assertGreater(reference_counts[object_id.id], 0) - - def check_everything_deallocated(): - reference_counts = ray.scheduler_info()["reference_counts"] - self.assertEqual(reference_counts, len(reference_counts) * [-1]) - - z = da.zeros.remote([da.BLOCK_SIZE, 2 * da.BLOCK_SIZE]) - time.sleep(0.1) - objectid_val = z.id - time.sleep(0.1) - check_not_deallocated([z]) - del z - time.sleep(0.1) - check_everything_deallocated() - - x = ra.zeros.remote([10, 10]) - y = ra.zeros.remote([10, 10]) - z = ra.dot.remote(x, y) - objectid_val = x.id - time.sleep(0.1) - check_not_deallocated([x, y, z]) - del x - time.sleep(0.1) - check_not_deallocated([y, z]) - del y - time.sleep(0.1) - check_not_deallocated([z]) - del z - time.sleep(0.1) - check_everything_deallocated() - - z = da.zeros.remote([4 * da.BLOCK_SIZE]) - time.sleep(0.1) - check_not_deallocated(ray.get(z).objectids.tolist()) - del z - time.sleep(0.1) - check_everything_deallocated() - - ray.worker.cleanup() - - def testGet(self): - ray.init(start_ray_local=True, num_workers=3) - - for cls in [Foo, Bar, Baz, Qux, SubQux, Exception, CustomError, Point, NamedTupleExample]: - ray.register_class(cls) - - # Remote objects should be deallocated when the corresponding ObjectID goes - # out of scope, and all results of ray.get called on the ID go out of scope. - for val in RAY_TEST_OBJECTS: - x = ray.put(val) - objectid = x.id - xval = ray.get(x) - del x, xval - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1) - - # Remote objects that do not contain numpy arrays should be deallocated when - # the corresponding ObjectID goes out of scope, even if ray.get has been - # called on the ObjectID. - for val in [True, False, None, 1, 1.0, 1L, "hi", u"hi", [1, 2, 3], (1, 2, 3), [(), {(): ()}]]: - x = ray.put(val) - objectid = x.id - xval = ray.get(x) - del x - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1) - - # Remote objects that contain numpy arrays should not be deallocated when - # the corresponding ObjectID goes out of scope, if ray.get has been called - # on the ObjectID and the result of that call is still in scope. - for val in [np.zeros(10), [np.zeros(10)], (((np.zeros(10)),),), {(): np.zeros(10)}, [1, 2, 3, np.zeros(1)]]: - x = ray.put(val) - objectid = x.id - xval = ray.get(x) - del x - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1) - - # Getting an object multiple times should not be a problem. And the remote - # object should not be deallocated until both of the results are out of scope. - for val in [np.zeros(10), [np.zeros(10)], (((np.zeros(10)),),), {(): np.zeros(10)}, [1, 2, 3, np.zeros(1)]]: - x = ray.put(val) - objectid = x.id - xval1 = ray.get(x) - xval2 = ray.get(x) - del xval1 - # Make sure we can still access xval2. - xval2 - del xval2 - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1) - xval3 = ray.get(x) - xval4 = ray.get(x) - xval5 = ray.get(x) - del x - del xval4, xval5 - # Make sure we can still access xval3. - xval3 - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1) - del xval3 - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1) - - # Getting an object multiple times and assigning it to the same name should - # work. This was a problem in https://github.com/ray-project/ray/issues/159. - for val in [np.zeros(10), [np.zeros(10)], (((np.zeros(10)),),), {(): np.zeros(10)}, [1, 2, 3, np.zeros(1)]]: - x = ray.put(val) - objectid = x.id - xval = ray.get(x) - xval = ray.get(x) - xval = ray.get(x) - xval = ray.get(x) - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1) - del x - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1) - del xval - self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1) - - ray.worker.cleanup() - class PythonModeTest(unittest.TestCase): def testPythonMode(self): @@ -712,18 +485,18 @@ def use_l(): class PythonCExtensionTest(unittest.TestCase): - def testReferenceCountNone(self): - ray.init(start_ray_local=True, num_workers=1) - - # Make sure that we aren't accidentally messing up Python's reference counts. - @ray.remote - def f(): - return sys.getrefcount(None) - first_count = ray.get(f.remote()) - second_count = ray.get(f.remote()) - self.assertEqual(first_count, second_count) - - ray.worker.cleanup() + # def testReferenceCountNone(self): + # ray.init(start_ray_local=True, num_workers=1) + # + # # Make sure that we aren't accidentally messing up Python's reference counts. + # @ray.remote + # def f(): + # return sys.getrefcount(None) + # first_count = ray.get(f.remote()) + # second_count = ray.get(f.remote()) + # self.assertEqual(first_count, second_count) + # + # ray.worker.cleanup() def testReferenceCountTrue(self): ray.init(start_ray_local=True, num_workers=1) @@ -867,43 +640,5 @@ def use_foo(): ray.worker.cleanup() -class ClusterAttachingTest(unittest.TestCase): - - def testAttachingToCluster(self): - node_ip_address = "127.0.0.1" - scheduler_port = np.random.randint(40000, 50000) - scheduler_address = "{}:{}".format(node_ip_address, scheduler_port) - ray.services.start_scheduler(scheduler_address, cleanup=True) - time.sleep(0.1) - ray.services.start_node(scheduler_address, node_ip_address, num_workers=1, cleanup=True) - - ray.init(node_ip_address=node_ip_address, scheduler_address=scheduler_address) - - @ray.remote - def f(x): - return x + 1 - self.assertEqual(ray.get(f.remote(0)), 1) - - ray.worker.cleanup() - - def testAttachingToClusterWithMultipleObjectStores(self): - node_ip_address = "127.0.0.1" - scheduler_port = np.random.randint(40000, 50000) - scheduler_address = "{}:{}".format(node_ip_address, scheduler_port) - ray.services.start_scheduler(scheduler_address, cleanup=True) - time.sleep(0.1) - ray.services.start_node(scheduler_address, node_ip_address, num_workers=5, cleanup=True) - ray.services.start_node(scheduler_address, node_ip_address, num_workers=5, cleanup=True) - ray.services.start_node(scheduler_address, node_ip_address, num_workers=5, cleanup=True) - - ray.init(node_ip_address=node_ip_address, scheduler_address=scheduler_address) - - @ray.remote - def f(x): - return x + 1 - self.assertEqual(ray.get(f.remote(0)), 1) - - ray.worker.cleanup() - if __name__ == "__main__": unittest.main(verbosity=2)