From df2eee17f9557883c5b287ba1e673f911af22d3b Mon Sep 17 00:00:00 2001 From: Chengsong Zhang Date: Thu, 1 Jun 2023 23:25:22 +0800 Subject: [PATCH] local groundingdino --- README.md | 27 +- local_groundingdino/datasets/__init__.py | 0 .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 205 bytes .../__pycache__/transforms.cpython-38.pyc | Bin 0 -> 10355 bytes local_groundingdino/datasets/transforms.py | 311 ++++++++ local_groundingdino/models/__init__.py | 18 + .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 534 bytes .../__pycache__/registry.cpython-38.pyc | Bin 0 -> 2138 bytes local_groundingdino/models/registry.py | 66 ++ .../requirements_groundingdino.txt | 5 + local_groundingdino/util/__init__.py | 1 + .../util/__pycache__/__init__.cpython-38.pyc | Bin 0 -> 201 bytes .../util/__pycache__/box_ops.cpython-38.pyc | Bin 0 -> 3878 bytes .../__pycache__/get_tokenlizer.cpython-38.pyc | Bin 0 -> 1150 bytes .../util/__pycache__/misc.cpython-38.pyc | Bin 0 -> 20384 bytes .../util/__pycache__/slconfig.cpython-38.pyc | Bin 0 -> 13160 bytes .../util/__pycache__/utils.cpython-38.pyc | Bin 0 -> 19603 bytes local_groundingdino/util/box_ops.py | 140 ++++ local_groundingdino/util/get_tokenlizer.py | 29 + local_groundingdino/util/inference.py | 244 ++++++ local_groundingdino/util/misc.py | 717 ++++++++++++++++++ local_groundingdino/util/slconfig.py | 427 +++++++++++ local_groundingdino/util/slio.py | 177 +++++ local_groundingdino/util/utils.py | 608 +++++++++++++++ scripts/dino.py | 87 ++- scripts/sam.py | 25 +- 26 files changed, 2840 insertions(+), 42 deletions(-) create mode 100644 local_groundingdino/datasets/__init__.py create mode 100644 local_groundingdino/datasets/__pycache__/__init__.cpython-38.pyc create mode 100644 local_groundingdino/datasets/__pycache__/transforms.cpython-38.pyc create mode 100644 local_groundingdino/datasets/transforms.py create mode 100644 local_groundingdino/models/__init__.py create mode 100644 local_groundingdino/models/__pycache__/__init__.cpython-38.pyc create mode 100644 local_groundingdino/models/__pycache__/registry.cpython-38.pyc create mode 100644 local_groundingdino/models/registry.py create mode 100644 local_groundingdino/requirements_groundingdino.txt create mode 100644 local_groundingdino/util/__init__.py create mode 100644 local_groundingdino/util/__pycache__/__init__.cpython-38.pyc create mode 100644 local_groundingdino/util/__pycache__/box_ops.cpython-38.pyc create mode 100644 local_groundingdino/util/__pycache__/get_tokenlizer.cpython-38.pyc create mode 100644 local_groundingdino/util/__pycache__/misc.cpython-38.pyc create mode 100644 local_groundingdino/util/__pycache__/slconfig.cpython-38.pyc create mode 100644 local_groundingdino/util/__pycache__/utils.cpython-38.pyc create mode 100644 local_groundingdino/util/box_ops.py create mode 100644 local_groundingdino/util/get_tokenlizer.py create mode 100644 local_groundingdino/util/inference.py create mode 100644 local_groundingdino/util/misc.py create mode 100644 local_groundingdino/util/slconfig.py create mode 100644 local_groundingdino/util/slio.py create mode 100644 local_groundingdino/util/utils.py diff --git a/README.md b/README.md index 2304d05..424e86f 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,21 @@ This extension aim for connecting [AUTOMATIC1111 Stable Diffusion WebUI](https:/ ## News -- `2023/04/10`: [Release] SAM extension released! You can click on the image to generate segmentation masks. -- `2023/04/12`: [Feature] Mask expansion released by [@jordan-barrett-jm](https://github.com/jordan-barrett-jm)! You can expand masks to overcome edge problems of SAM. -- `2023/04/15`: [Feature] [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO) support released! You can enter text prompts to generate bounding boxes and segmentation masks. -- `2023/04/15`: [Feature] API support released by [@jordan-barrett-jm](https://github.com/jordan-barrett-jm)! -- `2023/04/18`: [Feature] [ControlNet V1.1](https://github.com/lllyasviel/ControlNet-v1-1-nightly) inpainting support released! You can copy SAM generated masks to ControlNet to do inpainting. Note that you **must** update [ControlNet extension](https://github.com/Mikubill/sd-webui-controlnet) to use it. ControlNet inpainting has far better performance compared to general-purposed models, and you do not need to download inpainting-specific models anymore. -- `2023/04/24`: [Feature] Automatic segmentation support released! Functionalities with * require you to have [ControlNet extension](https://github.com/Mikubill/sd-webui-controlnet) installed. Last commit: `724b4db`. This update includes support for +- `2023/04/10`: [v1.0.0](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.0.0) SAM extension released! You can click on the image to generate segmentation masks. +- `2023/04/12`: [v1.0.1](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.0.1) Mask expansion released by [@jordan-barrett-jm](https://github.com/jordan-barrett-jm)! You can expand masks to overcome edge problems of SAM. +- `2023/04/15`: [v1.1.0](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.1.0) [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO) support released! You can enter text prompts to generate bounding boxes and segmentation masks. +- `2023/04/15`: [v1.2.0](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.2.0) API support released by [@jordan-barrett-jm](https://github.com/jordan-barrett-jm)! +- `2023/04/18`: [v1.3.0](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.3.0) [ControlNet V1.1](https://github.com/lllyasviel/ControlNet-v1-1-nightly) inpainting support released! You can copy SAM generated masks to ControlNet to do inpainting. Note that you **must** update [ControlNet extension](https://github.com/Mikubill/sd-webui-controlnet) to use it. ControlNet inpainting has far better performance compared to general-purposed models, and you do not need to download inpainting-specific models anymore. +- `2023/04/24`: [v1.4.0](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.4.0) Automatic segmentation support released! Functionalities with * require you to have [ControlNet extension](https://github.com/Mikubill/sd-webui-controlnet) installed. Last commit: `724b4db`. This update includes support for - *[ControlNet V1.1](https://github.com/lllyasviel/ControlNet-v1-1-nightly) semantic segmentation - [EditAnything](https://github.com/sail-sg/EditAnything) un-semantic segmentation - Image layout generation (single image + batch process) - *Image masking with categories (single image + batch process) - *Inpaint not masked for ControlNet inpainting on txt2img panel -- `2023/04/29`: [Feature] API has been completely refactored. You can access all features for **single image process** through API. API documentation has been moved to [wiki](https://github.com/continue-revolution/sd-webui-segment-anything/wiki/API). -- `2023/05/22`: [Feature] [EditAnything](https://github.com/sail-sg/EditAnything) is ready to use! You can generate random segmentation and copy the output to EditAnything ControlNet. -- `2023/05/29`: [Feature] You may now do SAM inference on CPU. This is for some MAC users who are not able to do SAM inference on GPU. I discourage other users from using this feature because it is significantly slower than CUDA. Last commit: `89a2213`. +- `2023/04/29`: [v1.4.1](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.4.1) API has been completely refactored. You can access all features for **single image process** through API. API documentation has been moved to [wiki](https://github.com/continue-revolution/sd-webui-segment-anything/wiki/API). +- `2023/05/22`: [v1.4.2](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.4.2) [EditAnything](https://github.com/sail-sg/EditAnything) is ready to use! You can generate random segmentation and copy the output to EditAnything ControlNet. +- `2023/05/29`: [v1.4.3](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.4.3) You may now do SAM inference on CPU by checking "Use CPU for SAM". This is for some MAC users who are not able to do SAM inference on GPU. I discourage other users from using this feature because it is significantly slower than CUDA. Last commit: `89a2213`. +- `2023/06/01`: [v1.5.0](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.5.0) You may now choose to use local GroundingDINO to bypass C++ problem. See [FAQ](#faq)-1 for more detail. ## TODO @@ -36,7 +37,11 @@ There are already at least two great tutorials on how to use this extension. Che You should know the following before submitting an issue. -1. I observe some common problems for Windows users: +1. Due to the overwhemling complaints about GroundingDINO installment and the lack of substitution of similar high-performance text-to-bounding-box library, I decide to modify the source code of GroundingDINO and push to this repository. Starting from [v1.5.0](https://github.com/continue-revolution/sd-webui-segment-anything/releases/tag/v1.5.0), you can choose to use local GroundingDINO by checking `Use local groundingdino to bypass C++ problem` on `Settings/Segment Anything`. This change should solve all problems about ninja, pycocotools, _C and any other problems related to C++/CUDA compilation. + + If you did not modify the setting described above, This script will firstly try to install GroundingDINO and check if your environment has successfully built the C++ dynamic library (the annoying `_C`). If so, this script will use the official implementation of GroundingDINO. This is to show respect to the authors of GroundingDINO. If the script failed to install GroundingDINO, it will use local GroundingDINO instead. + + If you'd still like to resolve the install problem of GroundingDINO, I observe some common problems for Windows users: - `pycocotool`: [here](https://github.com/cocodataset/cocoapi/issues/415#issuecomment-627313816). - `_C`: [here](https://github.com/continue-revolution/sd-webui-segment-anything/issues/32#issuecomment-1513873296). DO NOT skip steps. @@ -80,7 +85,7 @@ GroundingDINO has been supported in this extension. It has the following functio However, there are some existing problems with GroundingDINO: 1. GroundingDINO will be install when you firstly use GroundingDINO features, instead of when you initiate the WebUI. Make sure that your terminal can have access to GitHub, otherwise you have to install GroundingDINO manually. GroundingDINO models will be automatically downloaded from [huggingFace](https://huggingface.co/ShilongLiu/GroundingDINO/tree/main). If your terminal cannot visit HuggingFace, please manually download the model and put it under `${sd-webui-sam}/models/grounding-dino`. -2. GroundingDINO requires your device to compile C++, which might take a long time and throw tons of exceptions. If you encounter `_C` problem, it's most probably because you did not install CUDA Toolkit. Follow steps decribed [here](https://github.com/continue-revolution/sd-webui-segment-anything/issues/32#issuecomment-1513873296). DO NOT skip steps. Otherwise, please go to [Grounded-SAM Issue Page](https://github.com/IDEA-Research/Grounded-Segment-Anything/issues) and submit an issue there. Despite of this, you can still use this extension for point prompts->segmentation masks even if you cannot install GroundingDINO, don't worry. +2. **If you want to use local groundingdino to bypass ALL the painful C++/CUDA/ninja/pycocotools problems, please read [FAQ](#faq)-1.** GroundingDINO requires your device to compile C++, which might take a long time and throw tons of exceptions. If you encounter `_C` problem, it's most probably because you did not install CUDA Toolkit. Follow steps decribed [here](https://github.com/continue-revolution/sd-webui-segment-anything/issues/32#issuecomment-1513873296). DO NOT skip steps. Otherwise, please go to [Grounded-SAM Issue Page](https://github.com/IDEA-Research/Grounded-Segment-Anything/issues) and submit an issue there. Despite of this, you can still use this extension for point prompts->segmentation masks even if you cannot install GroundingDINO, don't worry. 3. If you want to use point prompts, SAM can at most accept one bounding box. This extension will check if there are multiple bounding boxes. If multiple bounding boxes, this extension will disgard all point prompts; otherwise all point prompts will be effective. You may always select one bounding box you want. For more detail, check [How to Use](#how-to-use) and [Demo](#demo). diff --git a/local_groundingdino/datasets/__init__.py b/local_groundingdino/datasets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/local_groundingdino/datasets/__pycache__/__init__.cpython-38.pyc b/local_groundingdino/datasets/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de10bd45f714ad1bb3e84b1396f5e4c6b2703894 GIT binary patch literal 205 zcmYk0y$!-J5QQCy0wHA(3Udn}!~`^m+H&Gcu;kn!|3EShC4;a8EhErTVT%Hu^nIuI zsyo#6Lgd`9qbc`U^q(r1Z6?gJjM$51wYv#l=AYgkjyxc=Odih+A?n7VG`8zfuuSrT zO|pPvL@GT%FyBoE><6SMHBIqosRzfccB=QB$mE`3FgoghF@RE*Y*cD>njZ7@z$Q7| JlwV#M@c~qJIzIpa literal 0 HcmV?d00001 diff --git a/local_groundingdino/datasets/__pycache__/transforms.cpython-38.pyc b/local_groundingdino/datasets/__pycache__/transforms.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d2d745b3e3854d4139807347c8c004967791868 GIT binary patch literal 10355 zcmb_iTZ|;vS+09m*L3&J%+Bu4%&wizVn=Sr-nBQ05@2yA-t{^r(b#0y0XjD2p01jn z?&;f}s`c(px+Czcb3-;8qyR!zGy@XSg2Y2Wyzs&cZy@o&1C)3`{HIQx%YXm>>Q{?JPs8=s`*(u>d0f+eMBw0M0a!zc)^tq^w5k^9 zfzj5hy4;PbA$PNC%H673a<@B9HK(Uz-Hu~uy4J~8J?SyqbDcu9AZ@E%?3Ah{wC%vT zr&Uh`xxmGJK6pIH2i`rsx)96-1@tVws0GEKbWaOP;mKXIx&-({Fb{ZM;8TDXf>jveZ z7T3zPTa8ZGjcakU*Dd2&d7~F^m7ASfBP@Tiys^={bHRH^vaDD1WD%f$=kA@me%$lx zcj|Xh<6I38B6;HcXU}B6mU0jclYK3XI}?@o#2k|>*^Ao zoLIpMTmyTgqCTHR)TiDZwJIA&q zr!a=a?ozt9y<$+<*Y`@ShsA+=ReQZWEDcJd6Z~+X4@$sE5e2Qw+I{`z591T9`Mrf; zKG61!U}4+1Z>UnRC~vSOk`Mz97Ww9`aaG$TT^Dgm(0`M>d1^heqUKJRIPKa-*p3pb zrovh^*Np0_cRQ#S`n{+bH{T5XPOYEVV8SqZpeJ?^-|dGF-^cY3%%v&l2@U*ccADLX z<95*OJftQ`>`pD(j;f0j(=|N0kS`tr-`FZx$sc>dKF6YHh*YcE&lqgXY&jmRIb zR_7e!i+s&_uA@ZDs2bWpdwu>jW2g^}fuXE{{ypQSbzOU)sYN`ln2B-WnZ)p$Rl{#d zmsCHFFK+cZ;l+BdtHL*X7o)hg(GH&unwy)qBFNy=x5JHF&5PlkIP4ORE=EDx_jDAB zs6Jim-i^1g(~Iq1z1H>{s&}g!07mIuq)W6`!KGvIw-b(bXM8BOlo6WEtM2dhH#JK`4ek948c$G!h}NV6rL%PcmX9^L#65ShN~CMMWcYbt1s!N@mJ74tSSTh8F`npK&j?r z&hG-st!R4=l@;3%OQLMRpU`3S7QsOh{yNV)GrwO)616O}a_uh9*7ua4j z6i3y$`uG|fA8t1~kDbG4!-2S`e+P#9TQEX2D);ihvIUIu)aI@^G-8;f0U#I$8~D(+ zUG+TiO^n25>MX=CEO-NNrRr=~A36gEG|LTL(99N^rTBd>#qTf=qvXH<4FuC!xH9k2gN~YWFYgPC1w#5vl{0y)*Blc z?i#f9_VF&AnRm<)-xS#&l(xZPfrpuKQla?5`jq|719h3s`wIXAZN!)c5R+FoVlCF8 z#jQf33z}z zV7Z&z9OvEEX!GbJ!2Vd6#pEnQm-cJuO0S7tOL_&>eDi_6Ua=wO)KIB@GNBd{6K+dl zi}I}G6HhdW=!%qypT&76oLOCfQSp1K*=Tm5b48^rP`ONXL66-=*_vVDWEHwg-tliY z<1N1y^}~AXD>_|OvM0qv6P})jMwy51DC^{F3!H61_v}0gg$gY;(bC6HI7Pv+)jG*l zo`z*NmE7n5u3)Oe`7* zn1WmVc33ghWz0Ik*aKaCo*guSqDT{Wywxtc#O`Q%NiP{Y%g3GRNV^;H?enN$byvZK zP?S8&u8tbqcUJo5#j_`RlsJ(PYnC?o%Z;!b-s!97b}%p^ zzgwE4?X1jhFB_L?DQfWa+*E@}t{cK>iDLL4R;L$)qAULuPjLP7=PO`A{LiH?h{A*^ zBT3cs7@(eDHLW()S@wytm4lp^=+6nd6%}K(gsI>OWaiKj&We&bxr66|cq=KSz*g97 zY{hbfYGL%WYdMqPBkQk5*qS;%>X8xrIACA5ASCAx%4GrgsfQ_{sW8s~&@sh*uo zJ;QWL?;;H4Gw(Z=;`>v4O02smTq61=s?;1pVS^uO^iV@(ZCX&vkbP0hPrw{MI-&0;p6^49Z?)Mj`2NjXwRZYM{R$>hUt={na&~TY0}uXyOYsz-mMJ}T08wlZA)L$UzBH4+4L%o5aiAM|2uc=xmN?BzYy$}ZyPSO6Qg!?30GaDtE{}e^A z2`^mVncnJ%a_Y6|cPHd{0}uXqJb{pdJ%`9)qjL=<`ZTIcZ1SOrKPxcaOGU@7+9C8u zD;K(f{hk3wJ9p2j<^vbv)h8CPaYgb3I;2vB+V-LG+Sus>@%Rwu*OJCZV>UXpLT%2!t zdviFgBylj(jp6U&-0rRkGoOc#thI{jKGyWoHQ^o_(S4?N0&`h|i?}?<54=r-slMyl z%@G8WSZVz7Q1mwAmr5xy+Xy*rRqHm8ZNLq3g=iIbID<-XDFLi4HdFALI3tr;!wkaE zYJ)8@u_Df4)u%bR>WStPB3BD^68rG)H`n8IuUdIIL*I4$#E{s}9yps8l_Z?&2KRz%4YEx67b>WqXG=6>eul4*XeB|${#**A%$dwF zA>xk!Pk_|zbEO?4g_D<7JKT(q;;X;IvJoW(5jL|I_?yAf2kKh7=zqg$co@;O>^Z~< zCOX$pgo`sy$T7={th;1-!4p>C07-7LlxnLlBF@_BAy3`D#x%n(a$=z+8obzMxq>3c zNcHD|a3In9&~PHDU&o+X98gpKBVU$T^_@qKTK8yP;el9W?J^p0UNywrFp&sYm%$^@ zXS)nxac4l9dNHPIVXTtPe1@%GWi_L~zlOelLdgOr@A$1{0xsw|-X=26`3KIK(#bOw zd+I1t7mu`t5|vS9baFB}<(QKP#qixlC{;)?mb!S;i3Co?#RNm=(V$m14nw2G*nA5v zIcyx01DX#9evBhWtzoqK0;}gyC8aM!QK;xqzn~OSkT$JrB#IiLo>2E294r=b2A3r9 zk5ERqT%<8Pb#T*1lb0K(e;}JreIKUkS_YDJrsN!$^MJ|QMfNu(tu3TYocdO;39T6)@_Y)1x0;eWjVzBR_5gGI%Xl8-C2hr?I?B{#09r%&FzPv< z;_Q`L@KHrDmm0%hEQr5>6=x4lD*Xp{!Yx5nFJ`M9O&*daBTt&**kjm80?{!=UNwWP zj9g4vrCK=sYG9?pxQ)q_?255h#h%r`Ho`UkP9nFvvpc7({wGQ%t1!b;yZ=B9`s;Z6 z-#J4nnFt=(bBI_bI@eGnACgHXb*C#K9%VB9s%Zx~$Ywqn$>#O2jl|Q=TV5Ggir4nt zGGaLNM&*q#z8!|$^5o0Gv+|vA0$+Mc-eeN1jFkRHZKK(4#&^qo`EocJm%jJJv*oAD z{mRUToQ#k_Kx4igzB5Dj??)+!NU>3#5+Ef zq9w)D(HuwgbGNe7PoOcS?^g$!1V-RKXnYOcFtk3~scIN8nrvsXyW>>9p=Y~8f+f=d z+3u!QK632SeqECD{h(L({SOFQ@8J;tjUtFZ+R=uZU|$_&>LP*GP@*MN6V4lZao{y8 zac}f)$oHW-5jDQpX1RhArGY*3Fv!pxFo{N*Qpw}%a{68ASeuw9Vz?VexTapF^immQYDTfdkztSn_NSgb{xkcgp;`=)a$td<-$)|-ZA;J2uXB4#)PMQ0x|Y1d|QG)i6keUy^W-0 z8j}PjKCn1y2X7$li6|T&Uid*~V8^+Z%TFpS4GT~{u--Awre6Xv8p+O)2v+QM^;_7< z!=K`VdYa>AN*2N58QhY5cZ^RpLBtB5*z&Dc5v%D}zq4#Tf(i;rT(a!T-+`1eqpMj& z(t@rMk<|YhLJ%+m{uW1fCP1fAo3a*spyM5&cQMncF>x@PrPyD2w}V|W!=)WyB0;Tabo+3Nc;wXyc&;HW-$f*^amd5_b_Az*G&AXu&;doS)PzN zAm&#*FdV<|A3SF|Z2gd}Jz!#gFuj0i3(32F9}k`wqmYWuwH5p5#Fy2#_}b&FBrxo= zwTh~GT7Gu%CceS-x)&zAql9T{7?FJZ=n)qtpC8mOak?{{u6)GrMlRfno9zooS^2$w zB%)k>y{xHYne9NpPc$M&mb}eyw8!h;glJ{0`Z~%o(F&7|#rUW43*ou!K23IVd z2&UlU0Ch6tCGO=kA^#l3A$kUtWdeutgNy<`los#<4fj*-N$)wgIPcEq=U3(n^WOX% L%EG*jo|FFz-DhP# literal 0 HcmV?d00001 diff --git a/local_groundingdino/datasets/transforms.py b/local_groundingdino/datasets/transforms.py new file mode 100644 index 0000000..8f0cbf3 --- /dev/null +++ b/local_groundingdino/datasets/transforms.py @@ -0,0 +1,311 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Transforms and data augmentation for both image + bbox. +""" +import os +import random + +import PIL +import torch +import torchvision.transforms as T +import torchvision.transforms.functional as F + +from local_groundingdino.util.box_ops import box_xyxy_to_cxcywh +from local_groundingdino.util.misc import interpolate + + +def crop(image, target, region): + cropped_image = F.crop(image, *region) + + target = target.copy() + i, j, h, w = region + + # should we do something wrt the original size? + target["size"] = torch.tensor([h, w]) + + fields = ["labels", "area", "iscrowd", "positive_map"] + + if "boxes" in target: + boxes = target["boxes"] + max_size = torch.as_tensor([w, h], dtype=torch.float32) + cropped_boxes = boxes - torch.as_tensor([j, i, j, i]) + cropped_boxes = torch.min(cropped_boxes.reshape(-1, 2, 2), max_size) + cropped_boxes = cropped_boxes.clamp(min=0) + area = (cropped_boxes[:, 1, :] - cropped_boxes[:, 0, :]).prod(dim=1) + target["boxes"] = cropped_boxes.reshape(-1, 4) + target["area"] = area + fields.append("boxes") + + if "masks" in target: + # FIXME should we update the area here if there are no boxes? + target["masks"] = target["masks"][:, i : i + h, j : j + w] + fields.append("masks") + + # remove elements for which the boxes or masks that have zero area + if "boxes" in target or "masks" in target: + # favor boxes selection when defining which elements to keep + # this is compatible with previous implementation + if "boxes" in target: + cropped_boxes = target["boxes"].reshape(-1, 2, 2) + keep = torch.all(cropped_boxes[:, 1, :] > cropped_boxes[:, 0, :], dim=1) + else: + keep = target["masks"].flatten(1).any(1) + + for field in fields: + if field in target: + target[field] = target[field][keep] + + if os.environ.get("IPDB_SHILONG_DEBUG", None) == "INFO": + # for debug and visualization only. + if "strings_positive" in target: + target["strings_positive"] = [ + _i for _i, _j in zip(target["strings_positive"], keep) if _j + ] + + return cropped_image, target + + +def hflip(image, target): + flipped_image = F.hflip(image) + + w, h = image.size + + target = target.copy() + if "boxes" in target: + boxes = target["boxes"] + boxes = boxes[:, [2, 1, 0, 3]] * torch.as_tensor([-1, 1, -1, 1]) + torch.as_tensor( + [w, 0, w, 0] + ) + target["boxes"] = boxes + + if "masks" in target: + target["masks"] = target["masks"].flip(-1) + + return flipped_image, target + + +def resize(image, target, size, max_size=None): + # size can be min_size (scalar) or (w, h) tuple + + def get_size_with_aspect_ratio(image_size, size, max_size=None): + w, h = image_size + if max_size is not None: + min_original_size = float(min((w, h))) + max_original_size = float(max((w, h))) + if max_original_size / min_original_size * size > max_size: + size = int(round(max_size * min_original_size / max_original_size)) + + if (w <= h and w == size) or (h <= w and h == size): + return (h, w) + + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + + return (oh, ow) + + def get_size(image_size, size, max_size=None): + if isinstance(size, (list, tuple)): + return size[::-1] + else: + return get_size_with_aspect_ratio(image_size, size, max_size) + + size = get_size(image.size, size, max_size) + rescaled_image = F.resize(image, size) + + if target is None: + return rescaled_image, None + + ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(rescaled_image.size, image.size)) + ratio_width, ratio_height = ratios + + target = target.copy() + if "boxes" in target: + boxes = target["boxes"] + scaled_boxes = boxes * torch.as_tensor( + [ratio_width, ratio_height, ratio_width, ratio_height] + ) + target["boxes"] = scaled_boxes + + if "area" in target: + area = target["area"] + scaled_area = area * (ratio_width * ratio_height) + target["area"] = scaled_area + + h, w = size + target["size"] = torch.tensor([h, w]) + + if "masks" in target: + target["masks"] = ( + interpolate(target["masks"][:, None].float(), size, mode="nearest")[:, 0] > 0.5 + ) + + return rescaled_image, target + + +def pad(image, target, padding): + # assumes that we only pad on the bottom right corners + padded_image = F.pad(image, (0, 0, padding[0], padding[1])) + if target is None: + return padded_image, None + target = target.copy() + # should we do something wrt the original size? + target["size"] = torch.tensor(padded_image.size[::-1]) + if "masks" in target: + target["masks"] = torch.nn.functional.pad(target["masks"], (0, padding[0], 0, padding[1])) + return padded_image, target + + +class ResizeDebug(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + return resize(img, target, self.size) + + +class RandomCrop(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + region = T.RandomCrop.get_params(img, self.size) + return crop(img, target, region) + + +class RandomSizeCrop(object): + def __init__(self, min_size: int, max_size: int, respect_boxes: bool = False): + # respect_boxes: True to keep all boxes + # False to tolerence box filter + self.min_size = min_size + self.max_size = max_size + self.respect_boxes = respect_boxes + + def __call__(self, img: PIL.Image.Image, target: dict): + init_boxes = len(target["boxes"]) + max_patience = 10 + for i in range(max_patience): + w = random.randint(self.min_size, min(img.width, self.max_size)) + h = random.randint(self.min_size, min(img.height, self.max_size)) + region = T.RandomCrop.get_params(img, [h, w]) + result_img, result_target = crop(img, target, region) + if ( + not self.respect_boxes + or len(result_target["boxes"]) == init_boxes + or i == max_patience - 1 + ): + return result_img, result_target + return result_img, result_target + + +class CenterCrop(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + image_width, image_height = img.size + crop_height, crop_width = self.size + crop_top = int(round((image_height - crop_height) / 2.0)) + crop_left = int(round((image_width - crop_width) / 2.0)) + return crop(img, target, (crop_top, crop_left, crop_height, crop_width)) + + +class RandomHorizontalFlip(object): + def __init__(self, p=0.5): + self.p = p + + def __call__(self, img, target): + if random.random() < self.p: + return hflip(img, target) + return img, target + + +class RandomResize(object): + def __init__(self, sizes, max_size=None): + assert isinstance(sizes, (list, tuple)) + self.sizes = sizes + self.max_size = max_size + + def __call__(self, img, target=None): + size = random.choice(self.sizes) + return resize(img, target, size, self.max_size) + + +class RandomPad(object): + def __init__(self, max_pad): + self.max_pad = max_pad + + def __call__(self, img, target): + pad_x = random.randint(0, self.max_pad) + pad_y = random.randint(0, self.max_pad) + return pad(img, target, (pad_x, pad_y)) + + +class RandomSelect(object): + """ + Randomly selects between transforms1 and transforms2, + with probability p for transforms1 and (1 - p) for transforms2 + """ + + def __init__(self, transforms1, transforms2, p=0.5): + self.transforms1 = transforms1 + self.transforms2 = transforms2 + self.p = p + + def __call__(self, img, target): + if random.random() < self.p: + return self.transforms1(img, target) + return self.transforms2(img, target) + + +class ToTensor(object): + def __call__(self, img, target): + return F.to_tensor(img), target + + +class RandomErasing(object): + def __init__(self, *args, **kwargs): + self.eraser = T.RandomErasing(*args, **kwargs) + + def __call__(self, img, target): + return self.eraser(img), target + + +class Normalize(object): + def __init__(self, mean, std): + self.mean = mean + self.std = std + + def __call__(self, image, target=None): + image = F.normalize(image, mean=self.mean, std=self.std) + if target is None: + return image, None + target = target.copy() + h, w = image.shape[-2:] + if "boxes" in target: + boxes = target["boxes"] + boxes = box_xyxy_to_cxcywh(boxes) + boxes = boxes / torch.tensor([w, h, w, h], dtype=torch.float32) + target["boxes"] = boxes + return image, target + + +class Compose(object): + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, image, target): + for t in self.transforms: + image, target = t(image, target) + return image, target + + def __repr__(self): + format_string = self.__class__.__name__ + "(" + for t in self.transforms: + format_string += "\n" + format_string += " {0}".format(t) + format_string += "\n)" + return format_string diff --git a/local_groundingdino/models/__init__.py b/local_groundingdino/models/__init__.py new file mode 100644 index 0000000..e341396 --- /dev/null +++ b/local_groundingdino/models/__init__.py @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from .GroundingDINO import build_groundingdino + + +def build_model(args): + # we use register to maintain models from catdet6 on. + from .registry import MODULE_BUILD_FUNCS + + assert args.modelname in MODULE_BUILD_FUNCS._module_dict + build_func = MODULE_BUILD_FUNCS.get(args.modelname) + model = build_func(args) + return model diff --git a/local_groundingdino/models/__pycache__/__init__.cpython-38.pyc b/local_groundingdino/models/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a27e94d721b8cd774d1c5b6d08678e538894cfb0 GIT binary patch literal 534 zcmY*Wy-ve05cW@s3T+XBg@G5yP_i&0#7_YsK^ah&ELIx3X&|wY?Esb11$Yn`c@SPA zD-$at6X%3VaMGRb$M@a&yHT&Vf5?_v#cixzZs^mnceD literal 0 HcmV?d00001 diff --git a/local_groundingdino/models/__pycache__/registry.cpython-38.pyc b/local_groundingdino/models/__pycache__/registry.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2cb3cf7562019c26a733a9a8be482f8e000f1372 GIT binary patch literal 2138 zcmb7F&2rl|5C%w!qG&nsZ_`QB3=ch+ZEWSxlPB$@P2x_|M4f5knT{?9O~5j3ic|rp zv1IvD``UM~j((871g|~y6?$=ZK~Ys~_fX6%xC<=s?H`uAi;F&i@yFztGmnrzQJ5bN z2v333W0*MMG$kGUS}E;W9gC9h2)DU&PPilNGrQvmhu6+%r^e|U61soEDbfyUQSV1m zCs7*X!u+s7cnYL`hDneP<)mYA%B^$KvCFGFa3yy^t_hddxpz(gpbmV4`-b=U0&l{8 zgRk!d7B9hR>l$p)+iKqZPbTAZO^B>xWt#`iXeVqvayG2a!`+6-z=;l^9Rh#9Bu z0E13lKtXUau`1`RGP8D9z{4mLYeSZ=JNhvk-Ai3*}riii*At`3q66v9SPPcqdP&}aQbRo51cx5K`8 zA!RP*U5IGmXpkkAVOeqB4`Ntu<0A?kljzR+tv1U4%D;R7XSaY97AUFt)S?b`udFMl z=0P{TLszJ8&9$5=FwhZq3^k2Oh0m&A1cSc?`(+TNWiOOXxL0C}mLI`Xczv0JA9`rs zzQhJ65nE!jG3Lybs3}&gQ;BZ^e^+V^CXafK=X$n9eM6q@+eTxA?u+zVL8#D`58+tR zFu~39RF%)fg(`rso1?|b>mQ%*zIwsF-TnU6bGEbFe)i@zOA7Bv8RD-o6$*QuN1OK2 G<9`6SR_c@h literal 0 HcmV?d00001 diff --git a/local_groundingdino/models/registry.py b/local_groundingdino/models/registry.py new file mode 100644 index 0000000..2d22a59 --- /dev/null +++ b/local_groundingdino/models/registry.py @@ -0,0 +1,66 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# -*- coding: utf-8 -*- +# @Author: Yihao Chen +# @Date: 2021-08-16 16:03:17 +# @Last Modified by: Shilong Liu +# @Last Modified time: 2022-01-23 15:26 +# modified from mmcv + +import inspect +from functools import partial + + +class Registry(object): + def __init__(self, name): + self._name = name + self._module_dict = dict() + + def __repr__(self): + format_str = self.__class__.__name__ + "(name={}, items={})".format( + self._name, list(self._module_dict.keys()) + ) + return format_str + + def __len__(self): + return len(self._module_dict) + + @property + def name(self): + return self._name + + @property + def module_dict(self): + return self._module_dict + + def get(self, key): + return self._module_dict.get(key, None) + + def registe_with_name(self, module_name=None, force=False): + return partial(self.register, module_name=module_name, force=force) + + def register(self, module_build_function, module_name=None, force=False): + """Register a module build function. + Args: + module (:obj:`nn.Module`): Module to be registered. + """ + if not inspect.isfunction(module_build_function): + raise TypeError( + "module_build_function must be a function, but got {}".format( + type(module_build_function) + ) + ) + if module_name is None: + module_name = module_build_function.__name__ + if not force and module_name in self._module_dict: + raise KeyError("{} is already registered in {}".format(module_name, self.name)) + self._module_dict[module_name] = module_build_function + + return module_build_function + + +MODULE_BUILD_FUNCS = Registry("model build functions") diff --git a/local_groundingdino/requirements_groundingdino.txt b/local_groundingdino/requirements_groundingdino.txt new file mode 100644 index 0000000..713223b --- /dev/null +++ b/local_groundingdino/requirements_groundingdino.txt @@ -0,0 +1,5 @@ +torchvision +addict +yapf +opencv-python +supervision==0.6.0 diff --git a/local_groundingdino/util/__init__.py b/local_groundingdino/util/__init__.py new file mode 100644 index 0000000..168f997 --- /dev/null +++ b/local_groundingdino/util/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved diff --git a/local_groundingdino/util/__pycache__/__init__.cpython-38.pyc b/local_groundingdino/util/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f0d5d56a2ea536661608ef64ad4a946e1f67327 GIT binary patch literal 201 zcmYk0F%H5o3`J9k0U`Au48a#bh>eW}F-L9VwvkdBH3_uGVdNlOf|Vn%G2w~@GlW=_9<_CKoq}W1 z7c7$l93s->0fPN*I$+ZwMP1MsTSpCet_p9Oo)cMdfgwfb!5Bv_8nZf$kNLV|n>=o& IKVE|R0F+TU=Kufz literal 0 HcmV?d00001 diff --git a/local_groundingdino/util/__pycache__/box_ops.cpython-38.pyc b/local_groundingdino/util/__pycache__/box_ops.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76e737df73e066a1e372174445ff715bc6f54131 GIT binary patch literal 3878 zcmcgvOK&4t6|P$^mz~6M(mmbDbOXhR#~7SA$uyf53^W6p8HowfKnuw;EmQWrwyR@T znOo(K9hEms_p({AVQVk5V*!5v34XyyNVP&j0%gG(iN(NoZaEb@9cd5?%Bpj2ojP?N z=X~ek#~T|Z4c8y%WBzDU)BZ{i%a@6V@8OPq4Z^jC#u?Z9tiiyIzCJJ-2Gh97t>;?9 z{zU&+Z)`RyYUQoQme`tY&o!*7|3s_aIeV-0IQIK~?2E|lgwkz=qk#KC7lW}o z@B)80>U*&t2Cf%y_ucoy$NQxV97a2?vebc6)04t$GZHN>8=UTUa7RA^NwgUbK4%}B zQ#RG7#?(w;hQzq7?P!UPk*P)t+1*@rSL>QnD`rpi#3F;eq%E7r7yri9#k{!w^4)Wm z8r&b88>u}CT7JM&D-LD5ms(NmwNI*c%EqZNZnjfn0>VyH)=OFIT$2@i|F1#47Y;qE-#wAfLopAQw(C9<`bX-oPirLo9PKc zg=;T#qMW_ZJH{^F<}#A#yV_izn$R|uw4rlNJCkJER$?XQj`q~#Iv)bPVm<{<)i=B5Hc4)QyYL>XA{(jqGuj~4klI!|EC;b>+2Lh0+C_}`? z%v_p3`yxX@HfuhILV+=}mb9}SE^Fs|hV~^2Y;JIKfr869>3oJlo4we5_QtwG0sL(s zstVjDpYLe!cH!&~B^TGbA`sH+`)7i?$dhhIh6A@3$HVAhz21cf_K{rzBw4;M_^4h| zAN{l^DAgkSM!j&<=Wa{5e&BvIKB&19+~Y&=!zYxR2VR_g^Pz}GGKgHy{ph&ne(=N{ zdcHjMBY}h+OMhH*PkTZN_t^c8+ZVx`8JMbSJ<8%AEX$8F`4VE~E}Sp#5xEaiEu@Cm z@2BNQQ6walf*(p5N`)CoI8oS@U!>8OK+YZcWg5RhdlA46*>rlPb zy)t2Q0bA0J2+i8s0Xs0t`XzAX0c0znT}dv#!`shEE{jC+U!;~ijpVwyWC>EQCKi>b z@-$;o>d2DGY64kF9@fpS_yW!&T_P&9uj0G1FjVsr1>c9#RiNuN1bKzI`nv8gd5Cu_ zHd_hwJO773Pc)Qsl=Krl8mhoY0$&Pzb2XH+#F8hI)vz~bb{_V5i08VwWXVH?SAyP9 z>HIR>ttAgF0(}wW_Sy6QJ;>h=hNJkQ`koTSUKa6)Y!#VD58dP11NHXDqxhO8sYZBNe|3gzgVqEqgZ<~0c2zaj2&IDLo@GB{BHvI;BbkdklV-Ab>r z20`BYeTE0>RpzJ^XjJc0Cx>oNdun9897ZNJzb_CL$otdMs#m$nLWuFU;Aj#GN^^_pXLIZ_U=QkM+5>sqx}mUubOP z8U@T@CGsy3*{)!LAi0+svUtGPdWl4pBAQd0$^K}$$C>|}G*+&4+Biq*1B zL_;+6Uf)I6yl86oZ>lQ~p19QXhB87u@#Eg&092|_ps4S=EJ}oF)$3Gs#V)9w#NPv( zi)VgYTzrOpo&GLZJe~lWsu{!gfNvPP2nI@N7wW%+;}Dc7qA&DROMfoL{U6V~MFypn3gRgrF({*brQVkF{b wEWNf^I^$KralSnW`KT}6rWQ7$x^Ef|TQ?lrDL7lsJ*VRAg0gk|%FNOK0}u9)fdBvi literal 0 HcmV?d00001 diff --git a/local_groundingdino/util/__pycache__/get_tokenlizer.cpython-38.pyc b/local_groundingdino/util/__pycache__/get_tokenlizer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7687e8504370f642a35ec05d38dac9275e5bdb69 GIT binary patch literal 1150 zcmZuxOLNmO5SC;+FOrl(2MUy#TsVXw}9(~@2q+RWP_I|BaAuztLj`&VM$PZL54+hE$nC2+}C!EG4 zpINKx5;!oI3nt`kBq*lFG6CtCS z(60PaX!}-i5xW|S7oO@n)dxeEJ2y_XbN_B=Y-d36)YE~9!Uf1Hm}U-O$(&faVCJJ` zb4Yx_^wyjlcqc@@o|C1|DQ7FDpXBn2g8h*V4hqZSoUEwsWO6B&*=07o{HVsgMB%X>QFXNZKz@6 zpEb`KpHF7@hLH^8|2vy&t`eC@8XL-iaGtT+7000ot<|~+_jcY$SjUuc{xq~>7et0f z+7-l6WUO(O4`G~&H(D#O4V6wp>wHYd1ydbKtFe%7ZKze!n`&Wo7)imKEI+$!&wBTb z=v%Pzmu-(#BH9BbwfLmkkXAnyEglVrsfmTRg)zt;+?X+@0|K&S!c4=G1G)sx01F;wvUe)`%K)W4T*)R(Z`;C|#TJ1d z8C#GWoB#*>&TJhy-0Fu$v{E^M7oSyiPVCG&+d3!2-(&$EG|RdQY3Uk59bpZ@L)bt- zzU%7<>i|uf^(bpg4!6PT?qjJgI@yauIZDHk=q32w;S!Vm0%(I`E<8(`Zrnt literal 0 HcmV?d00001 diff --git a/local_groundingdino/util/__pycache__/misc.cpython-38.pyc b/local_groundingdino/util/__pycache__/misc.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..738d86ac979d5b1a4eeffbac8833942fe71c854a GIT binary patch literal 20384 zcmb7sdvILWdEb5Q-6tLdKoX=VYAMk&A(Nm)$+FDQ5-F0hR8x>5(w4oJa==3%`H5eM3FAU@v{V2;crkk^sS=$N^ zg!@+yZatvYWxa@0}vp<5pMe_TDPK8#kLP{-69?x)ma>La*6sgA1?xId-l)k)kxsvcKQ;Qq8a zrJlt7W8w7HGnmH(^-=ZoEgLO;JT3W{dPZh;T3uA9)sJB&KZX&WQJ=sFKXKEkoKY{R z=hRu0e?py8&!eqpm2=se`(*bg3oqBBT6t-`Rg3HGR&=~vZ`B&>s@_^It9lgc`r>*V zs`5(M=!80&FBD#GM{#4LTx)kWPIRhK9F~`KyIGFgy0&ty9-)!44wph5wrU~j-zDTN z%(+SN6|GRJF4SvrQdHqmb-fWQK4esQbv>MO67LdvOmeSuIFxE*&QAO*VJm9u#Gbzk zShenEk)U`_&@$DQ>!i0a+2#%_21Ts)k`6?Kq=7J%I%paaPF zD^F#xG#)lPt8&O?R9+RZNPd_#fP|&aA>Rj}Jn{lUC@&x{V1#@T`DryHTTxPb)jqT_ zrd%xafn@A*v)ztYLiL$yV?FFfS9G**K`HrRc;r8z8$(439JXveEba<_OGK#Bi)kd(a(K~@2SNr68TyLCg0x#w}8%Zt* z>aBVl1dpQUsD#9_bGB!fY#)E!eIxri->p8$;$mtyOQ?!Ba*# z%DwGw+OfZt-L`BC_gp$bdn>>eQyq>iU~=~4%Bf$ z%07vrh^uHh`|R$X(Y?U?_aE$We8a(RIgnPYZAUp+r3`wmN zFTeQGrB@a{8Jv6h)x>_{dne8x`lCO1`rvcRxBvLpw|@A$zxM20CUM$Pk_}te>cEJ^ zT@GWR0|K&r;B&)Hw0kmzA$h*UIhlQ!vV8~pFgrT0Vbzi8kvfnn2t}Zz2|j@d-UQ(2 z19$`@cR}!Fk?+`o2fIe0pGCDRxFWV=IlC7WUw?3q=nvp}fXn49+KgVQr z1N8$a`z)@0Z_}gG6q0#Qf0Q(2hGk^f-zv9iK<|*x;DRW8J0qi*%Ls7mr%_#hf(a)n zeFOf;yb^z@t((>Oee2zC@JTiZfNDYT23m;5kyzk5UjOg^boY)NH=D028td9fNnn1zT6*ly#gwgxtO9KviXu7b-$ z=mkL+?xL%$wMd$}0;x)e5v0{}Ju26#H3)G=ph~<}t=339HU=>VTtV6cC{R{NionVs zElcv0SY;4NCcVA>%4?S{+;7p6eE~mB@sF*WFPG4&;ojMjU2^m*cpd@V2^0{cxn|M7 z0HTsvlPt=L&Gf8gu9b~4Cw8`6Fl0COwgAXIciVv!;*#cXdJqB~2!YQlx8*4Br!CaU z_PlL-EBCeqj**GI)qF3rRoF&*Z{zK1QDwmP{jXR{w(?c>RxU2}KoBl~&2MJY*8XNZ zwl&_%_I&o#nvEw^?j|Ikcx+WlQ7eDjg&tSH_f{`!zKth)S+tRh_w@X&sckEszUf3s zJo806o;D?T>n~fId4BQc;mtxXzqJ=5+Ski1*|_h28@k|amonDXKk50s+?xB2t)DkF zdj2d%oYpE~21Vp&dzoHw$)T_hmXa}}$2S!jKjtmtpOkUqsXuOvp%fp${PH&)jO5ZA z&gPh`)Iro4>*ac5v(~a{)m38$EWE=GU=<$#r#-aIb>&_>h@G%E$9tu%y}fac`W^e~ zja~_mJ@M{8F_66uu!hvJT!px=3CmTzSdVp8ZGJSs1)FF~143VeOVDB}*KWDx^1t@UQu z(41J3!6q+t+fCqyKS%lRVRayPM=7g2)0TMT0X^tUtqUvz@AQOWrSjzIt5Vd0?KE4 zUatU(4VxcE<4NB+@GI~c^iquXlI{ytJ_V{=0)O+i_JQi|$2Cj4V6XHg2Nc=q6)_HP z>i}n-(%w?92;wrXU&WeM3g=(_bnx263!grJQGW~tLe7MmY3}&QMv#J0(H5*fgp6SQ z%#ec7@Oj%wsh4Z07YU;@1#&hZVVaWi{jmRcP(Hx^?%^S%-H%q?Gq}GzNz$L=0I2)l zZxp|~a}+{@G-XvYp*N9Rm^-LHj(g&RoIqm{pw)qt2*V1Yu>+GMLwbs&G?zhQV7E$& zWFmpVU9H$`$b{5KQyS9COde&j!bD_l5)j>B(qz(N(q=-cp|2uIvT=Kn^e)Mectmq! znwY6$ChJlmG9xfuU1RxmCO4RDFd;3_T_$fa5vnU}MG_*v4Cz;qiKtdudEm4oSnn2q z^ONojF`Ss5VtG!myd)StYtK0H{gs;NNNfN<@84>VcID&v@>8T#TErzPO zZT(C~fZqiIcrh$ATORog(CrXJ8W$p8h5!ykpzsG9j3C?Nu;KVUPlReB5{ZB(^=z$y zWX^+VO{}F>45HBUh*T87UbjAK{kZiSw5*;Fbn&ZCgYw)*$f&nypQX{ae2nQFB#uv&$Iz)hQqk~YftrH0J%vBh?~fzPh5gd+aHG^*Nzx3Gam>&;YPVFMz{r%l&e z#%zRsX0C@NJ7qWQ8>HekAi$sAL=UXNF# zW(0++yi^Yx3h#_EFu#` z&9_94-!$sUOli(sLC)i6@;boDQ6$H_196O#%eo%T-~#{q8E`2 zJOq1)4e$xL2JYb{pn_w;N03u_+17ya>a|Pl zWynYG+W8TL=5F&k&yHu1#MIAd9OZ_lW_Dv%@ia#$N;1uDfSm{;NZ6=1VRO^q82nuS zbtb#$5QWbFjZ2t?g~jOZ-8rU#(Ru{f2x#OHpW=tK6+u|Fwk?p5X8~{yv=zJO#?GpH z)6qvkQb1tRvP|qRIndLdjDs&iu5&bsb?wU6%{|Hx9Kw<#|C}~aE-RY6pr$%q? zLjT7+=PtYQJTlyiRcp=H(4klHMAMU$*(f`PGU(VWi$RFDpvO_lf`Q21c5d2Vgu+4v zWubfI!*6YVxNNkl@=BFv?Q%6veQagEobzRxwAK*WV4)d5*RNot<8v5I;?cmZIl6S) z6OZlentxIZXh-0iAaFsE({(5S0{GXR9a)DRmG1$BTz@exAOoHv%R$`*;Bd@aqH7Lm&Dc#K8tx3h5+rFN&GA-i{-43l)ub^UBF4p+B647M6+n(e{5t( zA8g_0!1_Pt2Lq+qj}AIQ=^1M{G+Tz2`ib;5YU%roD#=5)31lJnb8K7#3r?<)kqxRW z)QBYg-!w8sTs2(L_wh9(3HlPpV;DPHkSkVUI#Q4pdTvx!Bp?nYmCbB=&&93+mA!3m z<_Ecgg0U%O(-yc=O4iQQrIe*QT1}OA{+I_KZCu7acOtC-*fhfLGm3t@_IfX9-~wf zv(}meqtQ2e`PGssqp@km+MK|AVDgG5dK0KOc^kDiC;KzO7cD3}rBx_N=~t76m>|vH zgsG}mK+1XV*|vIP4_Io?Z5YU4PgL}X_uT{$uj8&2A~KH$xQjh9_$3c;aodyB7+1xfwQ5fV8bQ zPR~ESv`Oh7zsJuRD>qEAQ3zWu1o}+kqIE+|JvJx;KKUj;1_}AQ_|9PKuP_%O=^lA& zvwVf#=yEy|`m>E+J1UHzN&l zhA(fiF1eC;2$CG;7P6}(&lJ2ILl*1LIkq=LBZL`n04WuYYDiae(}wxw$n@EAQVWfS$Xqo-} zMdqGoLOQSi7L(s#GUNi16Fu=8c=1DA;@`nwyj&K4`J9(?HQ$k<4t^xLWXCS9={MXtF0s#TDL&pr1oF;tz`Zv*s*#WM5;stuW)g%O_ zd~Pqqr2r1?q0Yk^U3cN+IRvOQ;3kHvwH|l#2s=UWla9i~d-3AA3k(ZbT9W3w1!hkm zCMUen-A5r0xQFh6XsTRWp^-G|mQI|gE-tbWgi3WY=<#&9TR2_@ddBOK$uGj7T3b0? zhGZnx6R!hZ1s=EU+BZ`4AGqJl5#cUa=05B<-X z^G(A^7HU=m8;e;FSs7WBY`O-kTo=)$V2UWY1}`1RDO0$JE#ttceYXDBX!M?4C2*d{ z6&*!Fm8NHXjg|rk$N(dV%2aU_1QgmP#B+QC6+xEgaT6smIot{nT&B|aE|51Ef_B(q z5v#G@X@uC_%wjc0V8{@$WEzE9yU_r_43=6y#hK6+XE`vt*!uU78yPWCikFO-NNdJO zDHqzGMdKjlT#1N+K#%-E$hPJo z+;ENjxEp72MKqc9HQUe)5r?DfZ-Goe!}XxiG7gR!tr{z~6n;=dIl)yuv3anPBfeQf@K1{bXYG0D4tfG)Ama4|O_e9elr| z&cQgZe8`QI>(LtTtHm8ETNW`z)%Cc2mje}>%v}oWvZvh-8DNAcTW^+|>+parhUF(2 zhWAt%F_X|~jzO)NE5X(%Zj(s=M}GSVlB584PCLTxuOT9mrAL_j1Yfx@U?m={TuI?G zj0ha6eflIS^dqm}l0Lm9L(7-j6D648L&Hfe)v2RWLW-dl zLEerTON_~i%f2wMuBQgUmRT$MENYO_9=HL_AG1-p=#5W1#l78tw zVIr%(%$#5_QMM1m-)AWiwURU29Mmg_502J?`tFS>?%0^$;Eqhh=>#Rl7Layl z-)~<=HUytTKurOf;xa;?0A%r)3!_Fd^-yN)u3mXVs`bMR(b!<3C8zC=!wE=`oCY-kSTT8 zfz|LkJ9^&DE=-MkI(GZBo=RtJ0_PC6VSfvAXIQ4w9ZPxQyhM+73j_d@2Y(Y_8j}9| z3{b%8_XcBgx>F;oB8@K0P3%_BNY#XE7)9-m_$cI7fcC%f@smuBF!}FH{*;LfWw<$# zql~FZn}3F*U=h^MN$6}vXl|2?X3OEvv84`lM!F0~mWrHbKp{LdVq{LlHBzSef$`$$ z!+rTU_7rSm2#kUxN-G#uH%iXSU^}pdK~a0dqeQxyfqhLR>{TD3S03W%cmp9(aG1kE4$56z9V5pV`pvw*BaeeQ zhN2_Ht4M3FLcmk{&As&;$jM)C7jcQQ*JWI?+~Q6j z7h%L^@ac{a8dzQDQBC&kjtQaBf6Rj69?JH87G>Y%_OW5{d!1^1pw2N=3x_d4#^X(v zq`!>hh3=Tds;_rC;tZSH1M4&-M8uH{56HkU58DtzwraGS91Ys7)(xC!`5<*i1bhAW z*-e>=faoL0B^i+=6R(X1^dItFn+buIhblT7f&Lx~M35LazUv!&SromA>Mr89?Zw1~ ziD)IUukYyKC_fYe#r2cO7qY#?Zth;o@9;g>QZz;fPHlky6z(OV(~~f{@3T+YN1>@` z*2NwXB@$mtpytS4lUx>Yk&;A@qS3z4N$@iPQvl90h)aZOfXfqW2bGwdK@W%>h^||O zSvSqm1WbWNVp?FbpMm>YkO(b{i=5xR`HL`51I=z(Jfwgcnbjgt%`q-+v{zbX1S3q? zFrdO^E_VR8+%d8xL^cBDT=ate5ZUOgPO$a?<|fhF9+gGJN^Z~+&drUswB$kM^0z%i z#CT{WFSD59EHLIJ&QziU3zCCy1rzvmuh1*fax+Apk+tmOjhI*n|9!lHG6h4+0c$7T z*oM^o0M!sEC(`tg*b!Ua9l@4o{xX1jKb#pEA$NIz*)#%~@VKE|A5E28KZHC_(GI2dd6+$ggZRNZ=jUp z{D;v#{taIZvCL_;l0yFk9gZ@&kh}DMW}U~F2&xKV3JR8Xpdk332{Q}oNs-bT>>a?b zs8UKF$QL6+yclW%TkoKfraUwFN~HfGy2`e}nrEWW^#8)Q|C4*i0a+qFdG_J=JjeSB z&xKRer*$U>DfgINaAxhJu=qf_a^$;vHj5i(P!kzOF*PK_cvZ}fUx9!loGM42)U$DB zjG&q6UJ%Lg$_hfL`%A?GhO}Tuh`+$Z=eA^ zj+MIORK{?ItbtMCbPh%bQ+Y%k3qvySCM<-`fI|c|iWWRmm9x7?BapOCfDYipr3q3_ zP!91xn!_Q9T-tslE(O5!lx%2#a3|AmATLaBd-{8s?7eUEGaLlvTFXO_ku#prd-27{ zO7SIX8IhF60TWCg;jAAV*U(6|VT5H~cFbLP;R?p|DC+5dfFv2m{CN;AJu9^X>4r3d z>wlCZF}f3v$??)GfB&&iP~3T{j3W_2(HPHM`P_^oK-K&9%ooSmv)Kq9k&qCT|KDD< z*dcdxndE2>Wd!qm`=yuk)UJM6_Wu1EdHg0|bPbqc`m;5^xnBp#gn*qup&Vp*hMnO8 zv}qiV9_@(bF0NN_iIl`;)l=r6-Vgzd&g07W#$mOH55SUfKHJiQ1G1-sjZMZG4-9gj zV!#;=f>=lUl0Lv;;2QjxCoG=F6@3#=O5(N=D)9x0-v&#BlWxyVB>I)a_$6)m0i;S~zezg!6*(Q9KuwveZwHCh7FRL8O z(ElH#E#`LhnCJ739?ZB3gK^*-@Mw>*2m`Qxz!icXj$%wD@rRCKgcERn@J!NLvmzN&3w(RIdvKe9$dg_2b}-ib#fhNox&Fk^Cw(( z9~lOSpN79zI>6n;fu~Pb^oy)o?vX|HDL78X*AchKDhya*pY%Q~HP)jQcrN}5zAzzd z9;+lpIW-L5_DWlIAAUXrKS2B-m4Nvjjn1bhNsrBSAG(N>-ycw6Zl5@U^l5fPMy`)w zLZ(Z1xwD?6vFz;KaE^1RgqWEK$E3d?S;iFY^ic0yL!8MU&K2G@P(AZHDe*;+9%Mp* z8=uy7BbkyJram|mXRklZagp^v=M`$Qi;(P6 z+5b_~7%1X@2M@-uX$*CwoRmj_HVD?1(^Ih8())>8knJIZg9yPOC!s6K+x8&0QU#X- z&~nEC=&d|NBE-VO)!=XQ8x#Ew&En0ccydYky*!;vs7D7y5pn5Y%BWXBWc(O{QN|J2 z;vwV$WzYl~y&TwMuKVx}?MuYL318E{M*k`^K&PhvmBLcf+2QcV=}#T>ElFd0R+ zt;VEc{e8Ch6(lfqo1$PRH>M<(e0fn!+|w*8HDBDH?lMO84O7F?5;u%o^KbfU$eln)YSArf1crb zI?Hk5#2r+r)pm#zvO*L66#^d*SK9GATqqHja)+@{`YapBw!$i0De*{T7b^`=hsqo0xBj!33# zFLRUjs|?g_#G%s}5*JpBtEJuTD25OC-Cvg#yLffIj`(137@D(Nt@*x2QEkjaR1bBh zP1o;n@xw_pc4X`^f*5kLs4O5SEDispq>P`aCK7&*z?4>uQ?WQOw-_RzJT>K%Th+LJ zEoA9N`PGdpB;^R=CeemgBW=_gRs6e-S{3I@5!rdYx?!4LUXtG*K4(zx%UHpMyMMt` z%VWvmAlP9Uf)xlVAZ@zUI9O^_mv`V|7W<=f+$6A;!6Y@UB}03cCOOl8h6KlvIuQ^I z|Hfj@r}_#~VGy_!;a^_BFlWjROZ|VPLgUabK++d5hF>7? z(1vM&zLwxu9EbJ@jgjwaEZ+GeZk@#?`dQ!qOdai8ko}Yg-!sUg^5I8@-w^wpO@QYO z=ugo`#3RfgdZP$SV5A%^2>V53jN=&A9EXNeHm#t1GDrHDZ0K-DFXXoe}NE*e+dJj3SyW?b17S-0ahsSApp7+sWv>)ybrA( zG@-EWMwAp*8n9)y1HHVM_@)#Np&1N8zlM41&oTKn9us$E9kFGAvy~XON_-h1+65sI z@GI>GG1Bi}<5bv=Iq}Z3ar^Lx|BCUR?{=r|o!SUZ=j!qj5aBL>@&HU2q;1q6TSkV} zWEXG@svElo!N?jQN+Sq zq*?O4AKm%W@4USD>T@I0U=?064RQibL*U!RS+{gj5;RMQw`OIM@(f6alM#fNas$~@ zYFeiNB>$qJ-V&w`unN#UOdyDGsx~_&LcvAA8(58fL!UwY#JK@W+l}3pHYlfbDn=pP zrpwKCUHvv^B~}Q?VG?T-qUjN+ADPfuv@Iu?5Ya(S0%Sd&Mw2`67~A9$;tx5+Xb=;L z1O6-j8V5cEM4Dq_h_vDUcC+Tc8q8x$xmWtuf~rCZr%%Rt2%kY#=J|b34kVKj z?6srnKjX|uY6*;#P-)lJD_l7H3YPE$bKb&&K8Flx)=#kDIsF1h@;dVRSCLEd!l@A4 z5$T`gn-B5L$C>*nCcnkxSCGKVkMJePKERF0OvmLrt{x<)jSjRNonZ-ivz%$um)X`M ztgz=EEkYmT%ST!F&zO*ailD|b0^CCM1D{`q#ph8qnb^UU=D|pWuT*AsmClp92nT|A zx5&fd=6FiJq1S;sh{}`yQD9E|t+4U0Gx-J+#*IlRmT_&+HY%Qh#9gh&_E##)_%}!e-L9-pMH6j!^FVt%Z z-i32*)vPAZx5KF72Fukr3MyU@MvW+mg3Mc~H=IjzwRpANj23I{mWpn+7vrS5+=!kI>nkg3F+lb7t!R0z zzKFd>Eq=#~aj46lj-%CP)JmSNw$_uY_15ZQqg|^uf>p)I@r|dwxR%r#i*W>TT|lpr!;$B*!t*jEPxS#16U+kQDP$5O9fDL%qX08fKG0(5%NX_8J5aqac>EG`j~!bC<`OY+FYT?MZ&zTbv;2?oW&ahrMBK;;6UG zp7pZPy5JazeZ8PQZyNkdoDeOp<*vEk2Wh1yQuE#1hS{@UHCj)iZ60k`Orr;)?0Tuw zvtKqo1@i1U-*8?r(8hhmNUe)TYHJiK*-Q7$x$eVretY>^R7?D;RidU}O%hdKUP~f> zuHLK>c-K;j&qRfhJXJ+)CsXoRUqNO5y)gwrb;fae#4AV0An=|GyvuGZ)oSje}uBzlJ zAFZ8FC=##*(Z7o(1#TE@<%`)jR5jRy83GfVO~o5&itYjzM6^-RweI6Q!& zg1~I{&qzinTCpQ1v_kuiv0+wn$U9+9@-Fgjn3sGWd2l?*d*M@p(BOS{z&E2pI33PN z%_Q>Vf*ffIdEtY|PiqcXnbDlDvIpGqFbJapT4Jg*VFdJPC) z@bT01?USnGvXr!2jddT0LoVxs@TJZNdi9&O+PBIDX{n->lYX;VyICf)h|9Wxt~-0X z3SJeRl!lz>%$YNOZDrLz?dv7Xle^AWPoG%umrqN6`NZkho>}luTrQPLJG+&u!;%xj zTz|;&^04HMx#bgcCw)CkZ&*BRSY96W){Sv1W;h@JgkG>Ap*;?$VHc2&JB*+$bAN2UJ>tN0GBx-(gSSo9o8-Dg14Zn|T z*!r(^s>#)p{?9x6<&*ws0mt8%tR;}xSjlUNX*)|P1+ z;Ht;LDIIXi^W$9g`^LK#MJu?Z^$K_TMsKu&rXh-Wp&s zkIRR|aoU}zl{qS^hM8TBlPuS%syNCfFG@$@jwM^faAvM#c_0CVYBI)kM5ycwz!d91 zlPKt{2YOQ3B!!h>70g*M!WsN^j}GXtyi1pDz#>l@0;>q3rCx!@qCNpp10h4;nE>IY zvt?}NLX@g3aX0f@#@j#WnTdPNOHGR0x2&Fpo)f8ct$?0(GI^~K+Se`hdTMP>ZBa-~ zZ_cEgVNYtN_Lj9}q0HDUB5#JyLl8#E-fOd)`$7j|j&=Jt4{#mG9ZVevvz*i)3LTb( zxzN31q`7N{Hy_$EUcf4QF6xh@&S{7;J>%%0TMNS-?VQKv>YdC2H_$uZ=nHmtX2dlRXbPHcW`%lI_PN8=`b>Rxx>v6P3MJj&Jf3aPt! zDDjb-Opd3M$;UUWcTDxm>14}ECs$ee@eQLlCFyiJ`GRrzncf8U`j|kbkWK(H=Id5( zrne`Z+I&2n;cnAu0)CR}wqX-VkLt@sJh6|Y4!pU17<+-z0^$SlKx4(}rq=hY8xBS{Gb;$F zTtmH7NqnoGTrJs|Rena)0CgN=>?DK2DRT2#yV*JOI_L3_xTXu8$uhcgBg`lDJZ~uKRHIvuW@aHXRd@b{wN9g61DX2QqjkToRr)GMyoH9vo2#g*V^?~=EYdPHe2HLICEra zZoP$7C8~sFsS`|i)|uUDcc{z}%+RKCWax30*XmVVapFe3txmFK{)=A`KBt~!=cicY zeg3(NXM-2dzMAEMSzKdQs6%H4Dzs{_qZ%-RpsWqgF>|XOFwzi~6sF5;$imD4h-0m8 zx@#Ts#%xcZO{{8AZzb(uxiqT{oIQkHpam=ztSZIv0?x9%gbV7I^wmXKqKyy0Y-Lq8 ztv4s11esdI#wvMTtFmbVUp63vO%Hkn+*(CG*-JjNiC|D&(OFhi^bmuZ%&hGo2lX;$ zJdY-v{O$*W&%yrWy!@m5VUUWcTYKgsLBZU-4>Ld_HY`;Mp zD3|1ry6*~8PbM}CsjX|bFjH!sGHx8>hzUkhKj}^-lPM&9&w(cJSYHFkhk6bVph# zEh=ek+3)|v9unn(-EP$z4ZnReQnwUH&cCvJWqWIqH1ui}CqBln-fY)HOeN#`z~A}` z`4fzGtxI!?=hSLEQl?o__2*wBr@LH&!W%|$P1VItyFAne*=qSiTMe6Ch>|tc+A;cT z1OfEH-A6um-#p!YR#d60^+q^QsA|;~-6QoDRIhsCQ-c;(>MuvND%hsqEtB6?HAJC^ zC08TAHa-GOSGH)cXeQo0#yLo({VB@+tL-(vS%rlX;e@*WY3iL<2*vtps|{t3LymRm z?pxMVlB5?fS{U8-yN~+u)%F?|z6?RpLTTdPf{Knz2MgdLN>l3VAQkm0lFZ?Gsju>T ziQk16&YnN}lJG`#i3Jlc;UI*6!o7s7=EIj+Wn*yYm}WIImoxKvX5P%qaf4NCRkr+f zJn=jdW2Rs_ApN3w%sOgY=KKeaOQN@gMOfzS2L<~W*hBa5h~T%g)Ln(XQ2n@TN{f9> zu~4JTw`|dz;c&WwMF>+yUL*>cSYHEH0V}127~GPG{_71arawjt?HID4?Cu?|a!|ii z8yMYaXyoxw#kKLoe})9=s&U;@#l(OTWwwsOw*r+H{A3}4?+6Mnygmb+KQ+Opj`W<= zfnDILXOdj%fTLL2BDnEP;(`a!GvkGJGQpmrxQg8JVsjYrH0Uru; zP%CGke7T#G;EK7CO4)(~0IKe2jC`8o7-D&NOWj8Y(!g)7fxwAkg77eOfmo{`knU09 zq>3Q2Z$jsQqi;Y#6mx1BK!CVmo^nZ@;y1JnaaIB`iIW;TSDCxY>gfw-FP^<{>Ff&> z*cV0ODjv3+ADus-m%j6dj**d=FXgpcXLPJ|H)b{jx8l)=$W-E189V9#!n+QYg6Acg zqyc&40ZC`&U9=7&=RjULK-&Fu`i#(E(03b_QnTYDSjrW^TLzib_v@(+O5kP^2h?`7 z6FmJfs4dX)ox8aeOU$E*H_dS0UGuIbWloq6y*m?c+H|7dG2Vo8{H`Ijw$!_HtfL;r-!+(l zl-2hj(dcLrheOfClv z0BeSsVS)A_RYMDA;t|mi;NuKo4)(P7mQFFqT`}%Jo7z%`DPf1^A|rhbD5tCLWdXOu9xx#8_Ig}{ek}lC4JXu0rcSr-7wdE{39*wT#d&y_>o$M z$4C4Xm5B=5VK|+&s~HFZ>lt5ApMV(Z0@{emrcTl^4VG!XYqwl+V?)8xOf-es_O1=4 zIIGLi${v*6gJpjYbM$g4-S*Csq96Ufb<$TMGB_WK&&-Y!^`5I5$g3s`CZPrdO&ALB zy~B$fpgzl8mzO8}2?gl*f#u$?D7Q49)rThXrB7h0g--eagdYkTPn=qfTCj!H zng8avWFHQx>hwp0HfeHb`s;kxDlII@@&^pVx*5QMF*U#=wi0Ahm4(V)1mDOjeAuRyrm z02Yf%cn;9xhO#Nx|0GEmD9~~3%p?3rypBZb5$qH=r$vj7OzDhQOsMC&qj)X~%mEf6gnrjUgI z$Og#y@{b`$b9meyG(LoNQ5xEZjq5qMZz}WX3Gj4xC`wxFNFTrVGcx_-+&B* zSFH6bdpRVEePYC0t=9GlzD@)drmra5BOfOuR9VUFJ5yaD+iwDP58WQC8s2MQ%?n zsT(|Z^9?P!-^PcQel%Aq|axFm*? zdW8du{G5h!iPo36ZL+C0Vwf>JFVW4YAquei6*m1Rw6lkKO zvu%Y-4J(zWL<(8eg$6+k^;NQ2>RL^7floWoNxc;CI2xo}O?SJq4Bx&A*# z8y$H#9B<=9e1U__n+{@3;-K8pjasG*}vCw z4?hAJS{CeB{QZxco3*<8cVlGRSQZXMsP%b{NC_r!bc#-D!Q!!^&xbsJsXOz$gy;Nu z9scUtaE%Pi)@AgiD^x`tM!3arrGy}@JP5vGx|Kf0RbLZr`=QcHrJEqu@ zT~lmPP>Dx!7F?JfLE-{@l2nBTmbivSPD^7!XR&+bFnDX@J$rPWVYfJjVk5*EjhGrB zbxUviT89Sp|La=&rh6ZrzlW9VJj~xm+1GyhI{Pq1qv4PFpN_nlgKkPA(Js_NT4IAB zxWvgh)((=KhxT$ff53j5zfQ8GRS!i0Vzg)2l3#3bT>0^ zfv(AE#F3HZ#LtQ%a)`_kFA81#J7GmCeU-~)R|EXrr8+e-<#`2c+k&~+ZKiykqS4$m z#6i9UWx$5QKr6`J#L{6HL#Wf#|2ACQP#KO;Rlr8_iM;{$GaOC1G>=GX?nW8yY~&!U zSq3-sR05~EIIG;yVT^D1orl|;g4#+w9hD8K_eu*kEO-DdYG3=KDL^wXpye z)62E0dA*1Qc%mCCC0rNlvt7A<6$JBh!+6oyO_9OOxMqcL-SDvd)2Fp2wJW4PbcX`s z(*Q664gxjthI6OGaUP5t4312o2LUNa0f#%(XIyQG=}H%vi2HsNw#dx#()%CaCmf+P zp%uGtfIbi$juNOORiKmHu^Mwc7I%V0*s7wx8$$@S=KBk(?9rJa*l=fa4Y#}Kh!od> ziMSpu3F12A+{>&Hz2tEWr%9bky_!T)C6&_wYV38Oz0t)|l)xLDlnBDXn{E_781ta- zK!a^efV_>IOSskyGW3s8^l6{(z(;u#H87N9Q#k}l!5p{CUEXE`sH7?gzBR@+4~$9u zQHuw(jctq2qMsqVk;Y+bcl|O}q9P`{Fm9>%_Sobj_Qx>C|7lVfgi+LyXvuCf(_4IR zZ03 zT(?3XCRrB;v36yPz@V0=e#nGskfJ)Gk&=Q`{Q(n-3Be-ur_2c?xppjRn9zs;$^_s1B|5CU?Rjmr^_p5T1nshEE1Os5GgT00*n1~QW z5Kdou?)=x!GBVMW&b(Ct0ir{er_m+!b+BLJm%>9}d*HT1**{NFG+;71%qBx14$>jp zLr>l%Y&NUloLCX)B=mLo6!E!o&pjFNC2Uio~}1GvoC-+#NEoeWjnLyZiMJ;V=TTUzS6Chw2C2;3annHkFe= zM8VZtZGf5o6_)Z693{AL4C#jyybHdcI*PNxCEk|R@xj~*uC65Nmr#zIEH*YihET+% z>;R26cu8>RV+3DHy^!jS zhs#)6sqv_x--C9Z<~HnMj7(S03e^ zc>c3|U;oE#7g*;5Vdl!wsceFQJBdfY*5={I*>qpL(Mi}^Reyw$GVA(HwZ=WVzKy5aF zi`%`xVzO{j1|89I=v3^o=%y63gKfTGHuG|*?d?>+Z*?z!il$L;-ee}CG*^_vgY)bwS;_$FUEFB>oC zaeF^&8iq2ISut|qubDID-O5?`wJNcyowH5m#VYY?BA1Y7yOOM?a;a)huBV#Lr6tX* z^yd1cY=5pFzwydIbw_T8NTJJg`s zS&A)KZvWB{Z?)^5rFN^KH{-d-K4GZG)SfpDwdbyt+oSfXeJHzEol^(YuQHPNJIMNTRr;+|N(w|Tl)idfS(w|hv)G+Egs50tf_Pa`oElYQcsdM>aWw&q3H0*mr6YG^xo40zsiu%Ng;GcHHmzpx zHiNfgZrbf}huzeIwcprpsM&j_I;l>*Y3DL1|1sq4P#?b+Q>WD#q&=(5t48MX24`z7 zV+OsiXys~GJy$OJcugtST`AUAZf9aa>_Y8!U^64IUn+ad?!U0=*RR!=-CDWfYT{pg z=~BJ6P%Z`LTG3*q(G|yU689zC-fsZ-#=7B~OO_v7vKOqomOkpo*Uc}0H{P+bPm%8kA3pK{y zv59}$u7KEq&ANiPF4RhH5UZAJL9F7|f}~do%h-w& z6EFL2)yu?n54&O43RO2S%YnHZm^a@w^dY2vxO`%ox7(te-%)<}2=TRbE(F z^~&|y@mucvYWak_=A(mn_fB{!OgrwmrK(%=j~8mU{l#*vbfQu(7ApCYuCLY<-f`DY ztor4O057|8TOUA+J8*kL0EW5C95M&+Z;yG(*)G#=tt+J2tmL-fJYQb+uY? zvl&a1H}w+$89PXW1iqUW0uI#NTlx08eu{Vw0cjV4!*~h?ixsy}>p*=Hsa_v|VJ0xL zL9-7d)JO4NO8@>pSUHpa#<|k(-2BqtJ8>X*KJuJn~J%XKbe1+j9q6eP6kuj*Pc#;T)>t0{q7NYn@7fjGtkkYm~MhNEW? z#T)ib6${_@VAcEti5TJOdw8~#{lJt`*MS*truZylJwS;Ct5wcc36%tX>`u&&-%qTY z56nBJJXv@m$*m!*y1m97d)>Zo6IUf3#ICY$7M{X(ejDwinkf%yhhUVxvWcyd z5A3VhH!$5ja~mYq2AQaF2sF&9a&Ge z8y<}?-4K&Vho4c0v(E4{US@;H8|0pj5@U}lkx2$Iv>yyy@H|)hSW%a?#=^JRSP=7k z9VAy=U0wBEJ%D;OmkU<#G5iE}6--7Xl}n+6c_QYLeA2GBnI*v~wYsj7<-B14BR&i! z+-0WB^!HQt0N62&_W`S!G8m=kN#GV)pz|UL&*Sz!2H+apr?E$y%EEpUQ#SUFn2M_e z_At&@3cqpLOL~GGm+Hm(rAoo`UM{Sl$8+zRn*Hfo<)q`Ma0?6n0UlU2D`usvQ1x)j z4!7%YvsSyfJ#%B`hOG8;Gv+qT4Vc?5H(Ygu8!b0kZm`^1xuL2P+)TNRa@*u)sb;w; zs*iIUgpzO$8~O!o(wDI9URKYmPpB8vi|Ui=N7PH|Wi_W>QLm~`-FtQ2$Q$a4y80%V zdMT1ecd@AQmb$i0YE0Pi({uV+j8BgcWC%#n4X)u0E@J&OpHc*r+L}AaMlS)S$wof` z8DoQtu`x)nlVBIYZUV~IjmHS~5bPz`N3frOB6#C*0?NA$%BhWm1e9$X4#6P;inR^t z%Mm_3LqJ~IASZ5c0yo(E4e32cw80T=94E*UoFEt_7$X=bm>?j3ZA=kN6U-3I5}YJB zMeuQg(*$P-&Jvs>I8Sha;35IpW#c)5%LLC8e1hNwf)@!sN$?{CFA=;W19K)YX9M$OV2zF99>YC>dlL5)?rGdJxMy(#XB;@=z!@LM z4ZLySjRS8SIOD*XK;8uMCXhFQyb0t@ppFUTPcR>KOrnlS;7tN=5_prqn*`1za3+B> z37o0XoHaFudmQ%!Zs1H!;ddJM4DMOnz?%l%H1MW@H;uZcC-DosY1B1?ycwj=p!^Kd zXV8}!l%GNQ8I+$v`C0U37UgHrms#Mgh|lFjGKwX&bjXHSbh;+@U53qQQaWOwQDfmV;Dw|C93?_sa?KLeWn=LEk> zz?IV00`#xp_3z+r^)C&DA~#rsK2fUc+xco?1uB9TO(S@$uFIuzZJeVj6n$nwItGtL z@8e;1lngrAo?BVi)MAioj+QII`$qtVWtsn9|NG_Cl|5!--@~S@%TBN`9x|4Uj#7_; z);Gya`@w1NnsZRz{w31&7XjY2^n3ULN2#^Vgc|6k_TL4b$J!0Ead2~2x>0|a4(Z=O zjp)*s@FQK)|B6rDY!VD~Pw_pVJhU&}sF?(3oO*mdAEfj7P=m#DUp{|hwNMFj^!se| zHwk{LyLssdSzS~9Uc@b|nlcAIlZr{&meF!ea|*mg05OrE(i&p}~~^5V$5B6(C5sPHAIyGwpj#qXy=GwuPV^(UZMKxs?NbuCrS z6g$HG&Eu|CqHr02#tFglmT}km+agqi&{!%xem$?;o8_VlmiQMy*Az&|NQ9~3gpdwF z`ZVn+Qu*vZ#Jgt$Bn-=FJlU$FLy)v^yLw0RP$Eh1xT}gF843l-I{Rp`kC6zGIrulg z3Ps3Q_|%>kiV!a2mI!%RS6TwGF@C8~sW?{`%awYqN34{_34fT2ql|YnXurJCr(urfLuhsNHjQ|HUe7PFi`c3iZ^)0(8oPuQ>^@ zNo-n}`l~D>W027ZgZ(bj?&J2Tm>6RIq@jJL%{>rMeIIG&Un`Q_&5QaCCs`Kz&*H&1 zmY_}1l6%{N84=Tmkz%de&Db?!a(ag2e~RFX0<7J)l3X z!}Pe%bQm6YlFU~L#| z(+;i+y5yabC%&yY!wZ#q!Oxs>UaYOaL?~AaCD$9ixOQpng|*i*EyTiF*>i@m;{o&9 zqU)4v%3X6nQ|FdjE-ix04t4w%zD!|$bE$A*rEJ)nD`FISr`ol&D`w#l2R#WzB`p1r+Vh`Z(qxlM+UO;jvnUi=n>YR|{*J+#aMV z@MK{7a`o5nDvUrAb;SiO`o!&^|0D6z?SW{j_we2o{Bab9Lt<4^RiqxEg7|#FhdU%$ zTrAXTZUr6;5-8VCqU2ZVUYQnC$ZD-M6YMOxe!f&#^}I5$i26@Zp~wlNcfhjEL2C~g zPYj$vFrofKT)3Xb6E1O2$PU*f9sEDFrLsa#c?RRM20p6)UUCg}RT7E&`YsGjFj) zNVWCmGze|MoMKvc(>X|;t`w^Cs&Mv?P~tndjYhUb=xye#Q!fY$2K`u-;6?XFzEUrZ z2R*gbs=RtZ9}@Cj0oy4iyu`|Jo&O;G#Tqf;=JnkbFG~5=xu6f$K)H@>9_-RyS16MzJY*z3 zfHK?Os&$fkxYLQpTC~nSIG?F=njJc4NbgYB46KD3RZU^b$ScQj>3<7Aaw@$ zfjb$hsg!Pfgewz^HegtinpHH#1>zVHF|mRS&1QnQ)b$r^h@*#rGtk}4947i-BSTYI z>fa%FpMbNVDT+#-2?3=h~2jfzo)3jiw| ziW*$=&bmc!vO=s49$K3oX%*jxC8}&je{jdKjVA6T;O|W=KqCE&Ngpx{3;GpJ$e}ot zKFAKLV{@Uh5@WA3Xz!ZnEg0aR;zj=tg8xat<*fgV;CldFeI}7wt0|2aNGJ4$hD|8B zO(6BW(zop=N+#@?T?D zt3T~+l%s22d~PxGFX3g~T!NTgG7w4`^d)>^!yfTt_Yo>&*o2nKvh~2aAAcYg?hG`8 zJi?d2N_Jsk8EnFWO#-Y`goS+6wG|8YN(WZh;xDm}%*BVw1G36&Nhe3Zl9r zPRWNEB9X`8wv8f7NY>RHq&$V6&JqleyaRX&da;?68S8^s*T2ua|3DyXU=Uc4SML$e zW-EbJ@;6OLkXT(&P=O9`PPpWZ_@H@I{xt^Mvl1c|DZiUnQhakzvHj8<^F88xmQ8HTOZMi{4|tc^_=I9(!D5UnU;O)@M-(gx@~=Akth|gZ9K{ z907I(Fb*yx}IRr>oktxC-9CC;wCQ>w~tuk7iEovF;xfd z5(oO$Ld#{MV4pkG*(3YL&6e7$=gn;?ZE zD+Q1+=$V5fP^|C3zKN93cSB+ZJ^A87Nm|hV4)6Nk6Nt}-=0DopJTJ02G}h2Jxn27f^ET6`uX2G`bWLloo7qzq+$G*b68q?(K}&HJ&vRx29oFrJ|(fg zYMxq2i#sEfW`BSjVO_nS32}N49o8E8To?K^AUQ53H+b#EV#m1?dG&#<061aIN`^ zbwzaalZ)M5C_t~0YPzTrA>Jy(u^I`7vhO*lsN^q3!r<~6tvR>7(kT!^Vn1K64>yO9 zX(ME_Sq>kqI?WjvW>9A<3elo=kCkf+^<&aSrm~TRdfhqWu#xtr&2-E&jubeUonfG4 zoHJ*fW2cG6Dl%ugD~XUfb%LD*`)c2#L>ulX(Fl))e2jq{ zqc|sZ(0q5h4@VdJ2)+#9ktAq5f&}5D3n%{IIiK1@tcDP)v58n6(>-5kMALn-&={U- z&h?gUl=Au#Wz5F_xJfFUozr`K(}yI+IXVVxAaAf~+F!8Vu_PQ5Dw7kS+bOIk5f+CX zc!H}1eVf$eic_vv)a7|Tip&Y)Qb97Gmk~FfdM;eOTy#y=Z>~lM7)C z;GqFsftwf4mMtO11qBJ3Bo6;sau6`)$9y=$NF=1$?<8M?o?#n)d^xRs6D~8;XdLkq z(DYO5F+`1fRQzoiy?*+BFOA8B@tW~hjK5El7l!SR$9@!VHS0(51|gpO@JS&r$rDi& z0+eqdAwM)ve`pRj;$U~*`OrKI_d==}RGx#Mol2B`mVkDB5LRV+*Rk+b~bDe96)v1+C6PE!*s)Od)oBfKcpA-G$@lt2s@oX*B&%?l~6MBznwVVU8F^*H04ayYQ# zEmWe*RIJ=?&fsPq2(^nV?gEaQ<4kl1ji4}re$JxruXv|UoG6w3N?|^`T=pT!+$^H@ zIJaB{%eeaZO&qGO*G|wIhO^Tr=D{;3ss;Gebt{c$kzsj}4{;dz>6gZ5MjhvAr&hmF zIOSZP8Xc2PbG;2>0U(Yks6nFY^C^f+*)VYaMI!WePzw(H7zVz1wWnb^tZFxpte@v1 zq_^JK1AiOc+lWl+=a4FPO)LeUBBMsE*_iiV6D*YbqWRsPV9rBN1VwF^~1HX5$87UwQ>9+8Y{;gq6NGii~dl& zDgE2Tn_*fgdb|_!(sy#_*u=@vN9f&HM9>N+TI-BjuLqx;i`1#H@vd~P$TQApSDG}0 z)R4*Z^2%jB64o-pW!$YKi+MOKi0zEf-W`7Z_3_!!?8%ekQzOp!%xw1L>?r6rF*=)_ z9-ExS)AV$9Y-al88zZewP%!X%eSB;>dva`S0$3BH+3Axrz#ku<%#Kb@PGeT4rn6JC zGqc;3oE)9XPE1drrLmLK*@^LS%-iJ2iR{G5Db$ALot-&3IsV2QVRz-AZRFG%!2%22 z35*)R&B=8b*s{(OkQ(^9A`}8i3Blv+W)dG+l&d@d0UsD2Lcdp9V)`NuVT;G`fIuFn zL0>|RIoq$yYr_1Rz}q={Uh^3cA9x5cH_ME^&MEn)sKukd&Dfnn5bTI4|EwW|!tf8F zF+B&2A`v<-YPY;RI@(*#gx=N}te$O-YvY79-3p(BR-0qUyDvz+Tvu-8a<%Y5?$5s% zeEu*0^xRz6i5<2=vBlf_B!JJuc{u9C01tF9oWhaw&`=PTY&v`p1hU}7hnNB6Ih>?2 zmV5N`%wNJH$GILe44S_S7e5aZfv9nB6|SV^dr_7ASu+vAe0etwCSx$LwRkEfBOIdosyK) zmvDOv01PfZF#Z1D zfIE|s@f$~9Gp2Al{)z!}0a}*+6uJwR6<2(+R>v1N1r@{^oIpnRKjV$8d7sMYPT*INU2V42UdI#cJR z);9e@B8gEdH4h>GTezEX*BzVc-ZCt%A7LJPX2Ae-#q;ze?3_nY15IGKXouzbNmD!K zudgF>)8uK6_|KRwBR<&bmFN0fhbzZhv?98@wJw|ibN%liy*b9d&e{+rcolhbI50*- zbaQ~2^wT5~4L2j`j|_V_A1knB4zYxQ*N7q%<(mIG?#Or<=uh?FoGMd`!0x(8i}Sd> z&j7R}-cBCoxrdLLR01Dz;;f&1nqu>763Du^n~+?5SAq{%lF=JJ2*q2c*F z#ME{LgBRhLLJSjT#Z}nU5c51f5yUI-sk>2Flg!$@_-x^pSsunibUX-BvKiJCiOBEQ ztmTAv#J8gpu(3G}mjWX%W&A)0R&mM`ZjuzXA;b?{FG$U!FY=U>CwC=C;9M3oUh!~8 zK{&mP;<`p~2tJ1qfMdTtbJTUd=cCD>cmSj z*wNm0oVJaVh_v9WNrT+=$XBF=GVg_yc-LxAi)QG0yW!&KYUs7x%`SwVLff>zzs;~C zL@Yiz9Yt=<^lmHDge63#jiO_r8%7*5a*o<#R}L|w4G!~l5gFgbEe2uA989GWspLRx zASO9Qpmbd=;YKB^CENyiZV$qZ@gNGfozd6rcJy^S#T)bSje8{C_%@(+P`oLu&rFv< zdjU%hvZFQP=OcN>#+w(PTT{U>8w^t{mb+Xorf79gTy(Z0HP-QLx1;WH8E7QKAlZT> zJTf>ES%{MO)EBXLX%*tN>0N7!Gdg@X)AfCH>2XePxaRs~#r1W`v~ZYhnYMX&d>2l; zh6i$*H$+}TY#amS%EJIBkrHdBK&kD{r?MMiZ_bl%BCW<`%L-~nmqK?h1Rmm1@SHI? z#xG7iKMZm?xZot$GUindTzKvlpVk1H)Dc+}cUS7g#ayE7k)WGN#Da%OA_?$9)ukT- zUtwEN;yOfAM5b*-6Uw&F>FXwu216n|fRHcKhi=}7HcwrqnNAg$XYJ&$Cv)LyklZ+} zg)O-_ihyjfknSyyaV4!_yq=BH8wad(l~w<*qcf8 zYe<_Si6|uaomt9gjI?EAM4CuQbf{3RZ!)^%5Xe`?aE_8(H= zVf2{kZVpbeHZOWDh9vEUAvIYO^7B0;z%#aGBiAf5$~uJ&u};%q!-Mj#Qy_QKx@(ZA zpuI)z!i#*fAyETzo}9vmszFm9Q;zePJZFb^zRjZ;uo&TV^`slyZ5FauonrrRb8qa} zt6CD|A$t`|hy>YgbCk_tyUmfi=9!KLJ2vB1gAx$H-rknc8S{CVg4=1^A|PaDWM_JG zl)y*h2iQ2}CibIbdLWg?Zp3sDFUcCP_SM6f>mY%7~f-&lo6gv0MemD}O- zFO7K7S0OjuU=Y+{oU5IZ2vLGQ95yS!aFH+Pa=jkEXM&?SUq&tA$9_1=#?R=HkgUz; z#if8Td&^AeZKJ4MTB^+B)D8aHMiG8~bf#TD{YG6pN>-rVE5_>)JjfB^C+;+<0h-}^ zIFoTl%Es_WamMPZgY=~!k~(NZh;Vb7pO@haS&K~wPI>$6NW_j$kGA?jLR!gYLIi>A2#vFiEF zU10k9Z-9NI7h0<43Fs@Vp0XN;O^wFEM^_$>z|wz>^|0$v1hLF zc-S?ky6X81+Jr}jL#MSkF8S$Js^I12ufoV_%`T<^F+?~D)6jn?j5l}(J`;t%1H{kc zB4)$qk`m4Wt4FNxWuU{|@%gQj_XcS!lLzyw@hnfWtt@IREie26C+zfgY|RWvINHg@Mm!9HT)T&I?i9!TWk&QtwsePJ3Lxk$)+-epqk@?53mHvMps4-@hN5=)b!nk=Td?t-|u zlg0lWI{Eq-fbtlMK)6FEiF0V1Kem%)fGg`&5Xr>fT+-Zq8ez776^v-5&#boFi?iMzyWnUS@1MsS4tVFk1W{H x_V@dq82GW&K>9#>&%mMd(7^t5GPNtUW8gyisq`~L2M3-WICfyK4Mw#-`afDOufzZV literal 0 HcmV?d00001 diff --git a/local_groundingdino/util/box_ops.py b/local_groundingdino/util/box_ops.py new file mode 100644 index 0000000..781068d --- /dev/null +++ b/local_groundingdino/util/box_ops.py @@ -0,0 +1,140 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Utilities for bounding box manipulation and GIoU. +""" +import torch +from torchvision.ops.boxes import box_area + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + # import ipdb; ipdb.set_trace() + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / (union + 1e-6) + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + The boxes should be in [x0, y0, x1, y1] format + + Returns a [N, M] pairwise matrix, where N = len(boxes1) + and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + # except: + # import ipdb; ipdb.set_trace() + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / (area + 1e-6) + + +# modified from torchvision to also return the union +def box_iou_pairwise(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, :2], boxes2[:, :2]) # [N,2] + rb = torch.min(boxes1[:, 2:], boxes2[:, 2:]) # [N,2] + + wh = (rb - lt).clamp(min=0) # [N,2] + inter = wh[:, 0] * wh[:, 1] # [N] + + union = area1 + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou_pairwise(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + Input: + - boxes1, boxes2: N,4 + Output: + - giou: N, 4 + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + assert boxes1.shape == boxes2.shape + iou, union = box_iou_pairwise(boxes1, boxes2) # N, 4 + + lt = torch.min(boxes1[:, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,2] + area = wh[:, 0] * wh[:, 1] + + return iou - (area - union) / area + + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = masks * x.unsqueeze(0) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = masks * y.unsqueeze(0) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) + + +if __name__ == "__main__": + x = torch.rand(5, 4) + y = torch.rand(3, 4) + iou, union = box_iou(x, y) + import ipdb + + ipdb.set_trace() diff --git a/local_groundingdino/util/get_tokenlizer.py b/local_groundingdino/util/get_tokenlizer.py new file mode 100644 index 0000000..dd2d972 --- /dev/null +++ b/local_groundingdino/util/get_tokenlizer.py @@ -0,0 +1,29 @@ +from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast +import os + +def get_tokenlizer(text_encoder_type): + if not isinstance(text_encoder_type, str): + # print("text_encoder_type is not a str") + if hasattr(text_encoder_type, "text_encoder_type"): + text_encoder_type = text_encoder_type.text_encoder_type + elif text_encoder_type.get("text_encoder_type", False): + text_encoder_type = text_encoder_type.get("text_encoder_type") + elif os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type): + pass + else: + raise ValueError( + "Unknown type of text_encoder_type: {}".format(type(text_encoder_type)) + ) + print("final text_encoder_type: {}".format(text_encoder_type)) + + tokenizer = AutoTokenizer.from_pretrained(text_encoder_type) + return tokenizer + + +def get_pretrained_language_model(text_encoder_type): + if text_encoder_type == "bert-base-uncased" or (os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type)): + return BertModel.from_pretrained(text_encoder_type) + if text_encoder_type == "roberta-base": + return RobertaModel.from_pretrained(text_encoder_type) + + raise ValueError("Unknown text_encoder_type {}".format(text_encoder_type)) diff --git a/local_groundingdino/util/inference.py b/local_groundingdino/util/inference.py new file mode 100644 index 0000000..5a36833 --- /dev/null +++ b/local_groundingdino/util/inference.py @@ -0,0 +1,244 @@ +from typing import Tuple, List + +import cv2 +import numpy as np +import supervision as sv +import torch +from PIL import Image +from torchvision.ops import box_convert + +import local_groundingdino.datasets.transforms as T +from local_groundingdino.models import build_model +from local_groundingdino.util.misc import clean_state_dict +from local_groundingdino.util.slconfig import SLConfig +from local_groundingdino.util.utils import get_phrases_from_posmap + +# ---------------------------------------------------------------------------------------------------------------------- +# OLD API +# ---------------------------------------------------------------------------------------------------------------------- + + +def preprocess_caption(caption: str) -> str: + result = caption.lower().strip() + if result.endswith("."): + return result + return result + "." + + +def load_model(model_config_path: str, model_checkpoint_path: str, device: str = "cuda"): + args = SLConfig.fromfile(model_config_path) + args.device = device + model = build_model(args) + checkpoint = torch.load(model_checkpoint_path, map_location="cpu") + model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False) + model.eval() + return model + + +def load_image(image_path: str) -> Tuple[np.array, torch.Tensor]: + transform = T.Compose( + [ + T.RandomResize([800], max_size=1333), + T.ToTensor(), + T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + image_source = Image.open(image_path).convert("RGB") + image = np.asarray(image_source) + image_transformed, _ = transform(image_source, None) + return image, image_transformed + + +def predict( + model, + image: torch.Tensor, + caption: str, + box_threshold: float, + text_threshold: float, + device: str = "cuda" +) -> Tuple[torch.Tensor, torch.Tensor, List[str]]: + caption = preprocess_caption(caption=caption) + + model = model.to(device) + image = image.to(device) + + with torch.no_grad(): + outputs = model(image[None], captions=[caption]) + + prediction_logits = outputs["pred_logits"].cpu().sigmoid()[0] # prediction_logits.shape = (nq, 256) + prediction_boxes = outputs["pred_boxes"].cpu()[0] # prediction_boxes.shape = (nq, 4) + + mask = prediction_logits.max(dim=1)[0] > box_threshold + logits = prediction_logits[mask] # logits.shape = (n, 256) + boxes = prediction_boxes[mask] # boxes.shape = (n, 4) + + tokenizer = model.tokenizer + tokenized = tokenizer(caption) + + phrases = [ + get_phrases_from_posmap(logit > text_threshold, tokenized, tokenizer).replace('.', '') + for logit + in logits + ] + + return boxes, logits.max(dim=1)[0], phrases + + +def annotate(image_source: np.ndarray, boxes: torch.Tensor, logits: torch.Tensor, phrases: List[str]) -> np.ndarray: + h, w, _ = image_source.shape + boxes = boxes * torch.Tensor([w, h, w, h]) + xyxy = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy() + detections = sv.Detections(xyxy=xyxy) + + labels = [ + f"{phrase} {logit:.2f}" + for phrase, logit + in zip(phrases, logits) + ] + + box_annotator = sv.BoxAnnotator() + annotated_frame = cv2.cvtColor(image_source, cv2.COLOR_RGB2BGR) + annotated_frame = box_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels) + return annotated_frame + + +# ---------------------------------------------------------------------------------------------------------------------- +# NEW API +# ---------------------------------------------------------------------------------------------------------------------- + + +class Model: + + def __init__( + self, + model_config_path: str, + model_checkpoint_path: str, + device: str = "cuda" + ): + self.model = load_model( + model_config_path=model_config_path, + model_checkpoint_path=model_checkpoint_path, + device=device + ).to(device) + self.device = device + + def predict_with_caption( + self, + image: np.ndarray, + caption: str, + box_threshold: float = 0.35, + text_threshold: float = 0.25 + ) -> Tuple[sv.Detections, List[str]]: + """ + import cv2 + + image = cv2.imread(IMAGE_PATH) + + model = Model(model_config_path=CONFIG_PATH, model_checkpoint_path=WEIGHTS_PATH) + detections, labels = model.predict_with_caption( + image=image, + caption=caption, + box_threshold=BOX_THRESHOLD, + text_threshold=TEXT_THRESHOLD + ) + + import supervision as sv + + box_annotator = sv.BoxAnnotator() + annotated_image = box_annotator.annotate(scene=image, detections=detections, labels=labels) + """ + processed_image = Model.preprocess_image(image_bgr=image).to(self.device) + boxes, logits, phrases = predict( + model=self.model, + image=processed_image, + caption=caption, + box_threshold=box_threshold, + text_threshold=text_threshold, + device=self.device) + source_h, source_w, _ = image.shape + detections = Model.post_process_result( + source_h=source_h, + source_w=source_w, + boxes=boxes, + logits=logits) + return detections, phrases + + def predict_with_classes( + self, + image: np.ndarray, + classes: List[str], + box_threshold: float, + text_threshold: float + ) -> sv.Detections: + """ + import cv2 + + image = cv2.imread(IMAGE_PATH) + + model = Model(model_config_path=CONFIG_PATH, model_checkpoint_path=WEIGHTS_PATH) + detections = model.predict_with_classes( + image=image, + classes=CLASSES, + box_threshold=BOX_THRESHOLD, + text_threshold=TEXT_THRESHOLD + ) + + + import supervision as sv + + box_annotator = sv.BoxAnnotator() + annotated_image = box_annotator.annotate(scene=image, detections=detections) + """ + caption = ". ".join(classes) + processed_image = Model.preprocess_image(image_bgr=image).to(self.device) + boxes, logits, phrases = predict( + model=self.model, + image=processed_image, + caption=caption, + box_threshold=box_threshold, + text_threshold=text_threshold, + device=self.device) + source_h, source_w, _ = image.shape + detections = Model.post_process_result( + source_h=source_h, + source_w=source_w, + boxes=boxes, + logits=logits) + class_id = Model.phrases2classes(phrases=phrases, classes=classes) + detections.class_id = class_id + return detections + + @staticmethod + def preprocess_image(image_bgr: np.ndarray) -> torch.Tensor: + transform = T.Compose( + [ + T.RandomResize([800], max_size=1333), + T.ToTensor(), + T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + image_pillow = Image.fromarray(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)) + image_transformed, _ = transform(image_pillow, None) + return image_transformed + + @staticmethod + def post_process_result( + source_h: int, + source_w: int, + boxes: torch.Tensor, + logits: torch.Tensor + ) -> sv.Detections: + boxes = boxes * torch.Tensor([source_w, source_h, source_w, source_h]) + xyxy = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy() + confidence = logits.numpy() + return sv.Detections(xyxy=xyxy, confidence=confidence) + + @staticmethod + def phrases2classes(phrases: List[str], classes: List[str]) -> np.ndarray: + class_ids = [] + for phrase in phrases: + try: + class_ids.append(classes.index(phrase)) + except ValueError: + class_ids.append(None) + return np.array(class_ids) diff --git a/local_groundingdino/util/misc.py b/local_groundingdino/util/misc.py new file mode 100644 index 0000000..d64b84e --- /dev/null +++ b/local_groundingdino/util/misc.py @@ -0,0 +1,717 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import colorsys +import datetime +import functools +import io +import json +import os +import pickle +import subprocess +import time +from collections import OrderedDict, defaultdict, deque +from typing import List, Optional + +import numpy as np +import torch +import torch.distributed as dist + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision +from torch import Tensor + +__torchvision_need_compat_flag = float(torchvision.__version__.split(".")[1]) < 7 +if __torchvision_need_compat_flag: + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + if d.shape[0] == 0: + return 0 + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + if os.environ.get("SHILONG_AMP", None) == "1": + eps = 1e-4 + else: + eps = 1e-6 + return self.total / (self.count + eps) + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value, + ) + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + """ + + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + + return dist.group.WORLD + + +def all_gather_cpu(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + + world_size = get_world_size() + if world_size == 1: + return [data] + + cpu_group = _get_global_gloo_group() + + buffer = io.BytesIO() + torch.save(data, buffer) + data_view = buffer.getbuffer() + device = "cuda" if cpu_group is None else "cpu" + tensor = torch.ByteTensor(data_view).to(device) + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device=device, dtype=torch.long) + size_list = [torch.tensor([0], device=device, dtype=torch.long) for _ in range(world_size)] + if cpu_group is None: + dist.all_gather(size_list, local_size) + else: + print("gathering on cpu") + dist.all_gather(size_list, local_size, group=cpu_group) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + assert isinstance(local_size.item(), int) + local_size = int(local_size.item()) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device=device)) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device=device) + tensor = torch.cat((tensor, padding), dim=0) + if cpu_group is None: + dist.all_gather(tensor_list, tensor) + else: + dist.all_gather(tensor_list, tensor, group=cpu_group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + tensor = torch.split(tensor, [size, max_size - size], dim=0)[0] + buffer = io.BytesIO(tensor.cpu().numpy()) + obj = torch.load(buffer) + data_list.append(obj) + + return data_list + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + + if os.getenv("CPU_REDUCE") == "1": + return all_gather_cpu(data) + + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + # print(name, str(meter)) + # import ipdb;ipdb.set_trace() + if meter.count > 0: + loss_str.append("{}: {}".format(name, str(meter))) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None, logger=None): + if logger is None: + print_func = print + else: + print_func = logger.info + + i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ] + ) + else: + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + ] + ) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + # import ipdb; ipdb.set_trace() + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print_func( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + ) + ) + else: + print_func( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + ) + ) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print_func( + "{} Total time: {} ({:.4f} s / it)".format( + header, total_time_str, total_time / len(iterable) + ) + ) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode("ascii").strip() + + sha = "N/A" + diff = "clean" + branch = "N/A" + try: + sha = _run(["git", "rev-parse", "HEAD"]) + subprocess.check_output(["git", "diff"], cwd=cwd) + diff = _run(["git", "diff-index", "HEAD"]) + diff = "has uncommited changes" if diff else "clean" + branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + # import ipdb; ipdb.set_trace() + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + if mask == "auto": + self.mask = torch.zeros_like(tensors).to(tensors.device) + if self.mask.dim() == 3: + self.mask = self.mask.sum(0).to(bool) + elif self.mask.dim() == 4: + self.mask = self.mask.sum(1).to(bool) + else: + raise ValueError( + "tensors dim must be 3 or 4 but {}({})".format( + self.tensors.dim(), self.tensors.shape + ) + ) + + def imgsize(self): + res = [] + for i in range(self.tensors.shape[0]): + mask = self.mask[i] + maxH = (~mask).sum(0).max() + maxW = (~mask).sum(1).max() + res.append(torch.Tensor([maxH, maxW])) + return res + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def to_img_list_single(self, tensor, mask): + assert tensor.dim() == 3, "dim of tensor should be 3 but {}".format(tensor.dim()) + maxH = (~mask).sum(0).max() + maxW = (~mask).sum(1).max() + img = tensor[:, :maxH, :maxW] + return img + + def to_img_list(self): + """remove the padding and convert to img list + + Returns: + [type]: [description] + """ + if self.tensors.dim() == 3: + return self.to_img_list_single(self.tensors, self.mask) + else: + res = [] + for i in range(self.tensors.shape[0]): + tensor_i = self.tensors[i] + mask_i = self.mask[i] + res.append(self.to_img_list_single(tensor_i, mask_i)) + return res + + @property + def device(self): + return self.tensors.device + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + @property + def shape(self): + return {"tensors.shape": self.tensors.shape, "mask.shape": self.mask.shape} + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max( + torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32) + ).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "WORLD_SIZE" in os.environ and os.environ["WORLD_SIZE"] != "": # 'RANK' in os.environ and + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = args.local_rank = int(os.environ["LOCAL_RANK"]) + + # launch by torch.distributed.launch + # Single node + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 1 --rank 0 ... + # Multi nodes + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 2 --rank 0 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' ... + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 2 --rank 1 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' ... + # args.rank = int(os.environ.get('OMPI_COMM_WORLD_RANK')) + # local_world_size = int(os.environ['GPU_PER_NODE_COUNT']) + # args.world_size = args.world_size * local_world_size + # args.gpu = args.local_rank = int(os.environ['LOCAL_RANK']) + # args.rank = args.rank * local_world_size + args.local_rank + print( + "world size: {}, rank: {}, local rank: {}".format( + args.world_size, args.rank, args.local_rank + ) + ) + print(json.dumps(dict(os.environ), indent=2)) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.local_rank = int(os.environ["SLURM_LOCALID"]) + args.world_size = int(os.environ["SLURM_NPROCS"]) + + print( + "world size: {}, world rank: {}, local rank: {}, device_count: {}".format( + args.world_size, args.rank, args.local_rank, torch.cuda.device_count() + ) + ) + else: + print("Not using distributed mode") + args.distributed = False + args.world_size = 1 + args.rank = 0 + args.local_rank = 0 + return + + print("world_size:{} rank:{} local_rank:{}".format(args.world_size, args.rank, args.local_rank)) + args.distributed = True + torch.cuda.set_device(args.local_rank) + args.dist_backend = "nccl" + print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) + + torch.distributed.init_process_group( + backend=args.dist_backend, + world_size=args.world_size, + rank=args.rank, + init_method=args.dist_url, + ) + + print("Before torch.distributed.barrier()") + torch.distributed.barrier() + print("End torch.distributed.barrier()") + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +@torch.no_grad() +def accuracy_onehot(pred, gt): + """_summary_ + + Args: + pred (_type_): n, c + gt (_type_): n, c + """ + tp = ((pred - gt).abs().sum(-1) < 1e-4).float().sum() + acc = tp / gt.shape[0] * 100 + return acc + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if __torchvision_need_compat_flag < 0.7: + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) + + +class color_sys: + def __init__(self, num_colors) -> None: + self.num_colors = num_colors + colors = [] + for i in np.arange(0.0, 360.0, 360.0 / num_colors): + hue = i / 360.0 + lightness = (50 + np.random.rand() * 10) / 100.0 + saturation = (90 + np.random.rand() * 10) / 100.0 + colors.append( + tuple([int(j * 255) for j in colorsys.hls_to_rgb(hue, lightness, saturation)]) + ) + self.colors = colors + + def __call__(self, idx): + return self.colors[idx] + + +def inverse_sigmoid(x, eps=1e-3): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def clean_state_dict(state_dict): + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k[:7] == "module.": + k = k[7:] # remove `module.` + new_state_dict[k] = v + return new_state_dict diff --git a/local_groundingdino/util/slconfig.py b/local_groundingdino/util/slconfig.py new file mode 100644 index 0000000..672e72e --- /dev/null +++ b/local_groundingdino/util/slconfig.py @@ -0,0 +1,427 @@ +# ========================================================== +# Modified from mmcv +# ========================================================== +import ast +import os +import os.path as osp +import shutil +import sys +import tempfile +from argparse import Action +from importlib import import_module + +from addict import Dict +from yapf.yapflib.yapf_api import FormatCode + +BASE_KEY = "_base_" +DELETE_KEY = "_delete_" +RESERVED_KEYS = ["filename", "text", "pretty_text", "get", "dump", "merge_from_dict"] + + +def check_file_exist(filename, msg_tmpl='file "{}" does not exist'): + if not osp.isfile(filename): + raise FileNotFoundError(msg_tmpl.format(filename)) + + +class ConfigDict(Dict): + def __missing__(self, name): + raise KeyError(name) + + def __getattr__(self, name): + try: + value = super(ConfigDict, self).__getattr__(name) + except KeyError: + ex = AttributeError(f"'{self.__class__.__name__}' object has no " f"attribute '{name}'") + except Exception as e: + ex = e + else: + return value + raise ex + + +class SLConfig(object): + """ + config files. + only support .py file as config now. + + ref: mmcv.utils.config + + Example: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/kchen/projects/mmcv/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: " + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + """ + + @staticmethod + def _validate_py_syntax(filename): + with open(filename) as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError: + raise SyntaxError("There are syntax errors in config " f"file {filename}") + + @staticmethod + def _file2dict(filename): + filename = osp.abspath(osp.expanduser(filename)) + check_file_exist(filename) + if filename.lower().endswith(".py"): + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile(dir=temp_config_dir, suffix=".py") + temp_config_name = osp.basename(temp_config_file.name) + if os.name == 'nt': + temp_config_file.close() + shutil.copyfile(filename, osp.join(temp_config_dir, temp_config_name)) + temp_module_name = osp.splitext(temp_config_name)[0] + sys.path.insert(0, temp_config_dir) + SLConfig._validate_py_syntax(filename) + mod = import_module(temp_module_name) + sys.path.pop(0) + cfg_dict = { + name: value for name, value in mod.__dict__.items() if not name.startswith("__") + } + # delete imported module + del sys.modules[temp_module_name] + # close temp file + temp_config_file.close() + elif filename.lower().endswith((".yml", ".yaml", ".json")): + from .slio import slload + + cfg_dict = slload(filename) + else: + raise IOError("Only py/yml/yaml/json type are supported now!") + + cfg_text = filename + "\n" + with open(filename, "r") as f: + cfg_text += f.read() + + # parse the base file + if BASE_KEY in cfg_dict: + cfg_dir = osp.dirname(filename) + base_filename = cfg_dict.pop(BASE_KEY) + base_filename = base_filename if isinstance(base_filename, list) else [base_filename] + + cfg_dict_list = list() + cfg_text_list = list() + for f in base_filename: + _cfg_dict, _cfg_text = SLConfig._file2dict(osp.join(cfg_dir, f)) + cfg_dict_list.append(_cfg_dict) + cfg_text_list.append(_cfg_text) + + base_cfg_dict = dict() + for c in cfg_dict_list: + if len(base_cfg_dict.keys() & c.keys()) > 0: + raise KeyError("Duplicate key is not allowed among bases") + # TODO Allow the duplicate key while warnning user + base_cfg_dict.update(c) + + base_cfg_dict = SLConfig._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = base_cfg_dict + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = "\n".join(cfg_text_list) + + return cfg_dict, cfg_text + + @staticmethod + def _merge_a_into_b(a, b): + """merge dict `a` into dict `b` (non-inplace). + values in `a` will overwrite `b`. + copy first to avoid inplace modification + + Args: + a ([type]): [description] + b ([type]): [description] + + Returns: + [dict]: [description] + """ + # import ipdb; ipdb.set_trace() + if not isinstance(a, dict): + return a + + b = b.copy() + for k, v in a.items(): + if isinstance(v, dict) and k in b and not v.pop(DELETE_KEY, False): + + if not isinstance(b[k], dict) and not isinstance(b[k], list): + # if : + # import ipdb; ipdb.set_trace() + raise TypeError( + f"{k}={v} in child config cannot inherit from base " + f"because {k} is a dict in the child config but is of " + f"type {type(b[k])} in base config. You may set " + f"`{DELETE_KEY}=True` to ignore the base config" + ) + b[k] = SLConfig._merge_a_into_b(v, b[k]) + elif isinstance(b, list): + try: + _ = int(k) + except: + raise TypeError( + f"b is a list, " f"index {k} should be an int when input but {type(k)}" + ) + b[int(k)] = SLConfig._merge_a_into_b(v, b[int(k)]) + else: + b[k] = v + + return b + + @staticmethod + def fromfile(filename): + cfg_dict, cfg_text = SLConfig._file2dict(filename) + return SLConfig(cfg_dict, cfg_text=cfg_text, filename=filename) + + def __init__(self, cfg_dict=None, cfg_text=None, filename=None): + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError("cfg_dict must be a dict, but " f"got {type(cfg_dict)}") + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f"{key} is reserved for config file") + + super(SLConfig, self).__setattr__("_cfg_dict", ConfigDict(cfg_dict)) + super(SLConfig, self).__setattr__("_filename", filename) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, "r") as f: + text = f.read() + else: + text = "" + super(SLConfig, self).__setattr__("_text", text) + + @property + def filename(self): + return self._filename + + @property + def text(self): + return self._text + + @property + def pretty_text(self): + + indent = 4 + + def _indent(s_, num_spaces): + s = s_.split("\n") + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(num_spaces * " ") + line for line in s] + s = "\n".join(s) + s = first + "\n" + s + return s + + def _format_basic_types(k, v, use_mapping=False): + if isinstance(v, str): + v_str = f"'{v}'" + else: + v_str = str(v) + + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: {v_str}" + else: + attr_str = f"{str(k)}={v_str}" + attr_str = _indent(attr_str, indent) + + return attr_str + + def _format_list(k, v, use_mapping=False): + # check if all items in the list are dict + if all(isinstance(_, dict) for _ in v): + v_str = "[\n" + v_str += "\n".join( + f"dict({_indent(_format_dict(v_), indent)})," for v_ in v + ).rstrip(",") + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: {v_str}" + else: + attr_str = f"{str(k)}={v_str}" + attr_str = _indent(attr_str, indent) + "]" + else: + attr_str = _format_basic_types(k, v, use_mapping) + return attr_str + + def _contain_invalid_identifier(dict_str): + contain_invalid_identifier = False + for key_name in dict_str: + contain_invalid_identifier |= not str(key_name).isidentifier() + return contain_invalid_identifier + + def _format_dict(input_dict, outest_level=False): + r = "" + s = [] + + use_mapping = _contain_invalid_identifier(input_dict) + if use_mapping: + r += "{" + for idx, (k, v) in enumerate(input_dict.items()): + is_last = idx >= len(input_dict) - 1 + end = "" if outest_level or is_last else "," + if isinstance(v, dict): + v_str = "\n" + _format_dict(v) + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: dict({v_str}" + else: + attr_str = f"{str(k)}=dict({v_str}" + attr_str = _indent(attr_str, indent) + ")" + end + elif isinstance(v, list): + attr_str = _format_list(k, v, use_mapping) + end + else: + attr_str = _format_basic_types(k, v, use_mapping) + end + + s.append(attr_str) + r += "\n".join(s) + if use_mapping: + r += "}" + return r + + cfg_dict = self._cfg_dict.to_dict() + text = _format_dict(cfg_dict, outest_level=True) + # copied from setup.cfg + yapf_style = dict( + based_on_style="pep8", + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True, + ) + text, _ = FormatCode(text, style_config=yapf_style, verify=True) + + return text + + def __repr__(self): + return f"Config (path: {self.filename}): {self._cfg_dict.__repr__()}" + + def __len__(self): + return len(self._cfg_dict) + + def __getattr__(self, name): + # # debug + # print('+'*15) + # print('name=%s' % name) + # print("addr:", id(self)) + # # print('type(self):', type(self)) + # print(self.__dict__) + # print('+'*15) + # if self.__dict__ == {}: + # raise ValueError + + return getattr(self._cfg_dict, name) + + def __getitem__(self, name): + return self._cfg_dict.__getitem__(name) + + def __setattr__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setattr__(name, value) + + def __setitem__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setitem__(name, value) + + def __iter__(self): + return iter(self._cfg_dict) + + def dump(self, file=None): + # import ipdb; ipdb.set_trace() + if file is None: + return self.pretty_text + else: + with open(file, "w") as f: + f.write(self.pretty_text) + + def merge_from_dict(self, options): + """Merge list into cfg_dict + + Merge the dict parsed by MultipleKVAction into this cfg. + + Examples: + >>> options = {'model.backbone.depth': 50, + ... 'model.backbone.with_cp':True} + >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet')))) + >>> cfg.merge_from_dict(options) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict( + ... model=dict(backbone=dict(depth=50, with_cp=True))) + + Args: + options (dict): dict of configs to merge from. + """ + option_cfg_dict = {} + for full_key, v in options.items(): + d = option_cfg_dict + key_list = full_key.split(".") + for subkey in key_list[:-1]: + d.setdefault(subkey, ConfigDict()) + d = d[subkey] + subkey = key_list[-1] + d[subkey] = v + + cfg_dict = super(SLConfig, self).__getattribute__("_cfg_dict") + super(SLConfig, self).__setattr__( + "_cfg_dict", SLConfig._merge_a_into_b(option_cfg_dict, cfg_dict) + ) + + # for multiprocess + def __setstate__(self, state): + self.__init__(state) + + def copy(self): + return SLConfig(self._cfg_dict.copy()) + + def deepcopy(self): + return SLConfig(self._cfg_dict.deepcopy()) + + +class DictAction(Action): + """ + argparse action to split an argument into KEY=VALUE form + on the first = and append to a dictionary. List options should + be passed as comma separated values, i.e KEY=V1,V2,V3 + """ + + @staticmethod + def _parse_int_float_bool(val): + try: + return int(val) + except ValueError: + pass + try: + return float(val) + except ValueError: + pass + if val.lower() in ["true", "false"]: + return True if val.lower() == "true" else False + if val.lower() in ["none", "null"]: + return None + return val + + def __call__(self, parser, namespace, values, option_string=None): + options = {} + for kv in values: + key, val = kv.split("=", maxsplit=1) + val = [self._parse_int_float_bool(v) for v in val.split(",")] + if len(val) == 1: + val = val[0] + options[key] = val + setattr(namespace, self.dest, options) diff --git a/local_groundingdino/util/slio.py b/local_groundingdino/util/slio.py new file mode 100644 index 0000000..72c1f0f --- /dev/null +++ b/local_groundingdino/util/slio.py @@ -0,0 +1,177 @@ +# ========================================================== +# Modified from mmcv +# ========================================================== + +import json +import pickle +from abc import ABCMeta, abstractmethod +from pathlib import Path + +import yaml + +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + + +# =========================== +# Rigister handler +# =========================== + + +class BaseFileHandler(metaclass=ABCMeta): + @abstractmethod + def load_from_fileobj(self, file, **kwargs): + pass + + @abstractmethod + def dump_to_fileobj(self, obj, file, **kwargs): + pass + + @abstractmethod + def dump_to_str(self, obj, **kwargs): + pass + + def load_from_path(self, filepath, mode="r", **kwargs): + with open(filepath, mode) as f: + return self.load_from_fileobj(f, **kwargs) + + def dump_to_path(self, obj, filepath, mode="w", **kwargs): + with open(filepath, mode) as f: + self.dump_to_fileobj(obj, f, **kwargs) + + +class JsonHandler(BaseFileHandler): + def load_from_fileobj(self, file): + return json.load(file) + + def dump_to_fileobj(self, obj, file, **kwargs): + json.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + return json.dumps(obj, **kwargs) + + +class PickleHandler(BaseFileHandler): + def load_from_fileobj(self, file, **kwargs): + return pickle.load(file, **kwargs) + + def load_from_path(self, filepath, **kwargs): + return super(PickleHandler, self).load_from_path(filepath, mode="rb", **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault("protocol", 2) + return pickle.dumps(obj, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault("protocol", 2) + pickle.dump(obj, file, **kwargs) + + def dump_to_path(self, obj, filepath, **kwargs): + super(PickleHandler, self).dump_to_path(obj, filepath, mode="wb", **kwargs) + + +class YamlHandler(BaseFileHandler): + def load_from_fileobj(self, file, **kwargs): + kwargs.setdefault("Loader", Loader) + return yaml.load(file, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault("Dumper", Dumper) + yaml.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault("Dumper", Dumper) + return yaml.dump(obj, **kwargs) + + +file_handlers = { + "json": JsonHandler(), + "yaml": YamlHandler(), + "yml": YamlHandler(), + "pickle": PickleHandler(), + "pkl": PickleHandler(), +} + +# =========================== +# load and dump +# =========================== + + +def is_str(x): + """Whether the input is an string instance. + + Note: This method is deprecated since python 2 is no longer supported. + """ + return isinstance(x, str) + + +def slload(file, file_format=None, **kwargs): + """Load data from json/yaml/pickle files. + + This method provides a unified api for loading data from serialized files. + + Args: + file (str or :obj:`Path` or file-like object): Filename or a file-like + object. + file_format (str, optional): If not specified, the file format will be + inferred from the file extension, otherwise use the specified one. + Currently supported formats include "json", "yaml/yml" and + "pickle/pkl". + + Returns: + The content from the file. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None and is_str(file): + file_format = file.split(".")[-1] + if file_format not in file_handlers: + raise TypeError(f"Unsupported format: {file_format}") + + handler = file_handlers[file_format] + if is_str(file): + obj = handler.load_from_path(file, **kwargs) + elif hasattr(file, "read"): + obj = handler.load_from_fileobj(file, **kwargs) + else: + raise TypeError('"file" must be a filepath str or a file-object') + return obj + + +def sldump(obj, file=None, file_format=None, **kwargs): + """Dump data to json/yaml/pickle strings or files. + + This method provides a unified api for dumping data as strings or to files, + and also supports custom arguments for each file format. + + Args: + obj (any): The python object to be dumped. + file (str or :obj:`Path` or file-like object, optional): If not + specified, then the object is dump to a str, otherwise to a file + specified by the filename or file-like object. + file_format (str, optional): Same as :func:`load`. + + Returns: + bool: True for success, False otherwise. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None: + if is_str(file): + file_format = file.split(".")[-1] + elif file is None: + raise ValueError("file_format must be specified since file is None") + if file_format not in file_handlers: + raise TypeError(f"Unsupported format: {file_format}") + + handler = file_handlers[file_format] + if file is None: + return handler.dump_to_str(obj, **kwargs) + elif is_str(file): + handler.dump_to_path(obj, file, **kwargs) + elif hasattr(file, "write"): + handler.dump_to_fileobj(obj, file, **kwargs) + else: + raise TypeError('"file" must be a filename str or a file-object') diff --git a/local_groundingdino/util/utils.py b/local_groundingdino/util/utils.py new file mode 100644 index 0000000..9b37abe --- /dev/null +++ b/local_groundingdino/util/utils.py @@ -0,0 +1,608 @@ +import argparse +import json +import warnings +from collections import OrderedDict +from copy import deepcopy +from typing import Any, Dict, List + +import numpy as np +import torch +from transformers import AutoTokenizer + +from local_groundingdino.util.slconfig import SLConfig + + +def slprint(x, name="x"): + if isinstance(x, (torch.Tensor, np.ndarray)): + print(f"{name}.shape:", x.shape) + elif isinstance(x, (tuple, list)): + print("type x:", type(x)) + for i in range(min(10, len(x))): + slprint(x[i], f"{name}[{i}]") + elif isinstance(x, dict): + for k, v in x.items(): + slprint(v, f"{name}[{k}]") + else: + print(f"{name}.type:", type(x)) + + +def clean_state_dict(state_dict): + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k[:7] == "module.": + k = k[7:] # remove `module.` + new_state_dict[k] = v + return new_state_dict + + +def renorm( + img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] +) -> torch.FloatTensor: + # img: tensor(3,H,W) or tensor(B,3,H,W) + # return: same as img + assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim() + if img.dim() == 3: + assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % ( + img.size(0), + str(img.size()), + ) + img_perm = img.permute(1, 2, 0) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(2, 0, 1) + else: # img.dim() == 4 + assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % ( + img.size(1), + str(img.size()), + ) + img_perm = img.permute(0, 2, 3, 1) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(0, 3, 1, 2) + + +class CocoClassMapper: + def __init__(self) -> None: + self.category_map_str = { + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + "7": 7, + "8": 8, + "9": 9, + "10": 10, + "11": 11, + "13": 12, + "14": 13, + "15": 14, + "16": 15, + "17": 16, + "18": 17, + "19": 18, + "20": 19, + "21": 20, + "22": 21, + "23": 22, + "24": 23, + "25": 24, + "27": 25, + "28": 26, + "31": 27, + "32": 28, + "33": 29, + "34": 30, + "35": 31, + "36": 32, + "37": 33, + "38": 34, + "39": 35, + "40": 36, + "41": 37, + "42": 38, + "43": 39, + "44": 40, + "46": 41, + "47": 42, + "48": 43, + "49": 44, + "50": 45, + "51": 46, + "52": 47, + "53": 48, + "54": 49, + "55": 50, + "56": 51, + "57": 52, + "58": 53, + "59": 54, + "60": 55, + "61": 56, + "62": 57, + "63": 58, + "64": 59, + "65": 60, + "67": 61, + "70": 62, + "72": 63, + "73": 64, + "74": 65, + "75": 66, + "76": 67, + "77": 68, + "78": 69, + "79": 70, + "80": 71, + "81": 72, + "82": 73, + "84": 74, + "85": 75, + "86": 76, + "87": 77, + "88": 78, + "89": 79, + "90": 80, + } + self.origin2compact_mapper = {int(k): v - 1 for k, v in self.category_map_str.items()} + self.compact2origin_mapper = {int(v - 1): int(k) for k, v in self.category_map_str.items()} + + def origin2compact(self, idx): + return self.origin2compact_mapper[int(idx)] + + def compact2origin(self, idx): + return self.compact2origin_mapper[int(idx)] + + +def to_device(item, device): + if isinstance(item, torch.Tensor): + return item.to(device) + elif isinstance(item, list): + return [to_device(i, device) for i in item] + elif isinstance(item, dict): + return {k: to_device(v, device) for k, v in item.items()} + else: + raise NotImplementedError( + "Call Shilong if you use other containers! type: {}".format(type(item)) + ) + + +# +def get_gaussian_mean(x, axis, other_axis, softmax=True): + """ + + Args: + x (float): Input images(BxCxHxW) + axis (int): The index for weighted mean + other_axis (int): The other index + + Returns: weighted index for axis, BxC + + """ + mat2line = torch.sum(x, axis=other_axis) + # mat2line = mat2line / mat2line.mean() * 10 + if softmax: + u = torch.softmax(mat2line, axis=2) + else: + u = mat2line / (mat2line.sum(2, keepdim=True) + 1e-6) + size = x.shape[axis] + ind = torch.linspace(0, 1, size).to(x.device) + batch = x.shape[0] + channel = x.shape[1] + index = ind.repeat([batch, channel, 1]) + mean_position = torch.sum(index * u, dim=2) + return mean_position + + +def get_expected_points_from_map(hm, softmax=True): + """get_gaussian_map_from_points + B,C,H,W -> B,N,2 float(0, 1) float(0, 1) + softargmax function + + Args: + hm (float): Input images(BxCxHxW) + + Returns: + weighted index for axis, BxCx2. float between 0 and 1. + + """ + # hm = 10*hm + B, C, H, W = hm.shape + y_mean = get_gaussian_mean(hm, 2, 3, softmax=softmax) # B,C + x_mean = get_gaussian_mean(hm, 3, 2, softmax=softmax) # B,C + # return torch.cat((x_mean.unsqueeze(-1), y_mean.unsqueeze(-1)), 2) + return torch.stack([x_mean, y_mean], dim=2) + + +# Positional encoding (section 5.1) +# borrow from nerf +class Embedder: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.create_embedding_fn() + + def create_embedding_fn(self): + embed_fns = [] + d = self.kwargs["input_dims"] + out_dim = 0 + if self.kwargs["include_input"]: + embed_fns.append(lambda x: x) + out_dim += d + + max_freq = self.kwargs["max_freq_log2"] + N_freqs = self.kwargs["num_freqs"] + + if self.kwargs["log_sampling"]: + freq_bands = 2.0 ** torch.linspace(0.0, max_freq, steps=N_freqs) + else: + freq_bands = torch.linspace(2.0**0.0, 2.0**max_freq, steps=N_freqs) + + for freq in freq_bands: + for p_fn in self.kwargs["periodic_fns"]: + embed_fns.append(lambda x, p_fn=p_fn, freq=freq: p_fn(x * freq)) + out_dim += d + + self.embed_fns = embed_fns + self.out_dim = out_dim + + def embed(self, inputs): + return torch.cat([fn(inputs) for fn in self.embed_fns], -1) + + +def get_embedder(multires, i=0): + import torch.nn as nn + + if i == -1: + return nn.Identity(), 3 + + embed_kwargs = { + "include_input": True, + "input_dims": 3, + "max_freq_log2": multires - 1, + "num_freqs": multires, + "log_sampling": True, + "periodic_fns": [torch.sin, torch.cos], + } + + embedder_obj = Embedder(**embed_kwargs) + embed = lambda x, eo=embedder_obj: eo.embed(x) + return embed, embedder_obj.out_dim + + +class APOPMeter: + def __init__(self) -> None: + self.tp = 0 + self.fp = 0 + self.tn = 0 + self.fn = 0 + + def update(self, pred, gt): + """ + Input: + pred, gt: Tensor() + """ + assert pred.shape == gt.shape + self.tp += torch.logical_and(pred == 1, gt == 1).sum().item() + self.fp += torch.logical_and(pred == 1, gt == 0).sum().item() + self.tn += torch.logical_and(pred == 0, gt == 0).sum().item() + self.tn += torch.logical_and(pred == 1, gt == 0).sum().item() + + def update_cm(self, tp, fp, tn, fn): + self.tp += tp + self.fp += fp + self.tn += tn + self.tn += fn + + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def get_raw_dict(args): + """ + return the dicf contained in args. + + e.g: + >>> with open(path, 'w') as f: + json.dump(get_raw_dict(args), f, indent=2) + """ + if isinstance(args, argparse.Namespace): + return vars(args) + elif isinstance(args, dict): + return args + elif isinstance(args, SLConfig): + return args._cfg_dict + else: + raise NotImplementedError("Unknown type {}".format(type(args))) + + +def stat_tensors(tensor): + assert tensor.dim() == 1 + tensor_sm = tensor.softmax(0) + entropy = (tensor_sm * torch.log(tensor_sm + 1e-9)).sum() + + return { + "max": tensor.max(), + "min": tensor.min(), + "mean": tensor.mean(), + "var": tensor.var(), + "std": tensor.var() ** 0.5, + "entropy": entropy, + } + + +class NiceRepr: + """Inherit from this class and define ``__nice__`` to "nicely" print your + objects. + + Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function + Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``. + If the inheriting class has a ``__len__``, method then the default + ``__nice__`` method will return its length. + + Example: + >>> class Foo(NiceRepr): + ... def __nice__(self): + ... return 'info' + >>> foo = Foo() + >>> assert str(foo) == '' + >>> assert repr(foo).startswith('>> class Bar(NiceRepr): + ... pass + >>> bar = Bar() + >>> import pytest + >>> with pytest.warns(None) as record: + >>> assert 'object at' in str(bar) + >>> assert 'object at' in repr(bar) + + Example: + >>> class Baz(NiceRepr): + ... def __len__(self): + ... return 5 + >>> baz = Baz() + >>> assert str(baz) == '' + """ + + def __nice__(self): + """str: a "nice" summary string describing this module""" + if hasattr(self, "__len__"): + # It is a common pattern for objects to use __len__ in __nice__ + # As a convenience we define a default __nice__ for these objects + return str(len(self)) + else: + # In all other cases force the subclass to overload __nice__ + raise NotImplementedError(f"Define the __nice__ method for {self.__class__!r}") + + def __repr__(self): + """str: the string of the module""" + try: + nice = self.__nice__() + classname = self.__class__.__name__ + return f"<{classname}({nice}) at {hex(id(self))}>" + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) + + def __str__(self): + """str: the string of the module""" + try: + classname = self.__class__.__name__ + nice = self.__nice__() + return f"<{classname}({nice})>" + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) + + +def ensure_rng(rng=None): + """Coerces input into a random number generator. + + If the input is None, then a global random state is returned. + + If the input is a numeric value, then that is used as a seed to construct a + random state. Otherwise the input is returned as-is. + + Adapted from [1]_. + + Args: + rng (int | numpy.random.RandomState | None): + if None, then defaults to the global rng. Otherwise this can be an + integer or a RandomState class + Returns: + (numpy.random.RandomState) : rng - + a numpy random number generator + + References: + .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270 # noqa: E501 + """ + + if rng is None: + rng = np.random.mtrand._rand + elif isinstance(rng, int): + rng = np.random.RandomState(rng) + else: + rng = rng + return rng + + +def random_boxes(num=1, scale=1, rng=None): + """Simple version of ``kwimage.Boxes.random`` + + Returns: + Tensor: shape (n, 4) in x1, y1, x2, y2 format. + + References: + https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390 + + Example: + >>> num = 3 + >>> scale = 512 + >>> rng = 0 + >>> boxes = random_boxes(num, scale, rng) + >>> print(boxes) + tensor([[280.9925, 278.9802, 308.6148, 366.1769], + [216.9113, 330.6978, 224.0446, 456.5878], + [405.3632, 196.3221, 493.3953, 270.7942]]) + """ + rng = ensure_rng(rng) + + tlbr = rng.rand(num, 4).astype(np.float32) + + tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2]) + tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3]) + br_x = np.maximum(tlbr[:, 0], tlbr[:, 2]) + br_y = np.maximum(tlbr[:, 1], tlbr[:, 3]) + + tlbr[:, 0] = tl_x * scale + tlbr[:, 1] = tl_y * scale + tlbr[:, 2] = br_x * scale + tlbr[:, 3] = br_y * scale + + boxes = torch.from_numpy(tlbr) + return boxes + + +class ModelEma(torch.nn.Module): + def __init__(self, model, decay=0.9997, device=None): + super(ModelEma, self).__init__() + # make a copy of the model for accumulating moving average of weights + self.module = deepcopy(model) + self.module.eval() + + # import ipdb; ipdb.set_trace() + + self.decay = decay + self.device = device # perform ema on different device from model if set + if self.device is not None: + self.module.to(device=device) + + def _update(self, model, update_fn): + with torch.no_grad(): + for ema_v, model_v in zip( + self.module.state_dict().values(), model.state_dict().values() + ): + if self.device is not None: + model_v = model_v.to(device=self.device) + ema_v.copy_(update_fn(ema_v, model_v)) + + def update(self, model): + self._update(model, update_fn=lambda e, m: self.decay * e + (1.0 - self.decay) * m) + + def set(self, model): + self._update(model, update_fn=lambda e, m: m) + + +class BestMetricSingle: + def __init__(self, init_res=0.0, better="large") -> None: + self.init_res = init_res + self.best_res = init_res + self.best_ep = -1 + + self.better = better + assert better in ["large", "small"] + + def isbetter(self, new_res, old_res): + if self.better == "large": + return new_res > old_res + if self.better == "small": + return new_res < old_res + + def update(self, new_res, ep): + if self.isbetter(new_res, self.best_res): + self.best_res = new_res + self.best_ep = ep + return True + return False + + def __str__(self) -> str: + return "best_res: {}\t best_ep: {}".format(self.best_res, self.best_ep) + + def __repr__(self) -> str: + return self.__str__() + + def summary(self) -> dict: + return { + "best_res": self.best_res, + "best_ep": self.best_ep, + } + + +class BestMetricHolder: + def __init__(self, init_res=0.0, better="large", use_ema=False) -> None: + self.best_all = BestMetricSingle(init_res, better) + self.use_ema = use_ema + if use_ema: + self.best_ema = BestMetricSingle(init_res, better) + self.best_regular = BestMetricSingle(init_res, better) + + def update(self, new_res, epoch, is_ema=False): + """ + return if the results is the best. + """ + if not self.use_ema: + return self.best_all.update(new_res, epoch) + else: + if is_ema: + self.best_ema.update(new_res, epoch) + return self.best_all.update(new_res, epoch) + else: + self.best_regular.update(new_res, epoch) + return self.best_all.update(new_res, epoch) + + def summary(self): + if not self.use_ema: + return self.best_all.summary() + + res = {} + res.update({f"all_{k}": v for k, v in self.best_all.summary().items()}) + res.update({f"regular_{k}": v for k, v in self.best_regular.summary().items()}) + res.update({f"ema_{k}": v for k, v in self.best_ema.summary().items()}) + return res + + def __repr__(self) -> str: + return json.dumps(self.summary(), indent=2) + + def __str__(self) -> str: + return self.__repr__() + + +def targets_to(targets: List[Dict[str, Any]], device): + """Moves the target dicts to the given device.""" + excluded_keys = [ + "questionId", + "tokens_positive", + "strings_positive", + "tokens", + "dataset_name", + "sentence_id", + "original_img_id", + "nb_eval", + "task_id", + "original_id", + "token_span", + "caption", + "dataset_type", + ] + return [ + {k: v.to(device) if k not in excluded_keys else v for k, v in t.items()} for t in targets + ] + + +def get_phrases_from_posmap( + posmap: torch.BoolTensor, tokenized: Dict, tokenizer: AutoTokenizer +): + assert isinstance(posmap, torch.Tensor), "posmap must be torch.Tensor" + if posmap.dim() == 1: + non_zero_idx = posmap.nonzero(as_tuple=True)[0].tolist() + token_ids = [tokenized["input_ids"][i] for i in non_zero_idx] + return tokenizer.decode(token_ids) + else: + raise NotImplementedError("posmap must be 1-dim") diff --git a/scripts/dino.py b/scripts/dino.py index 55ec3cb..a2d1f91 100644 --- a/scripts/dino.py +++ b/scripts/dino.py @@ -7,6 +7,7 @@ from modules import scripts, shared from modules.devices import device, torch_gc, cpu +import local_groundingdino dino_model_cache = OrderedDict() @@ -25,23 +26,68 @@ }, } -dino_install_issue_text = "submit an issue to https://github.com/IDEA-Research/Grounded-Segment-Anything/issues." +dino_install_issue_text = "permanently switch to local groundingdino on Settings/Segment Anything or submit an issue to https://github.com/IDEA-Research/Grounded-Segment-Anything/issues." + + +def install_groundingdino_dependencies(): + import launch + local_groundingdino_dir = os.path.join(scripts.basedir(), "local_groundingdino") + local_groundingdino_req_file = os.path.join(local_groundingdino_dir, "requirements_groundingdino.txt") + with open(local_groundingdino_req_file) as file: + for lib in file: + lib = lib.strip() + if not launch.is_installed(lib): + launch.run_pip( + f"install {lib}", + f"groundingdino requirement: {lib}") def install_goundingdino(): + if shared.opts.data.get("sam_use_local_groundingdino", False): + print("Using local groundingdino.") + return False + + def verify_dll(install_local=True): + try: + from groundingdino import _C + print("GroundingDINO dynamic library have been successfully built.") + return True + except Exception: + import traceback + traceback.print_exc() + def run_pip_uninstall(command, desc=None): + from launch import python, run + default_command_live = (os.environ.get('WEBUI_LAUNCH_LIVE_OUTPUT') == "1") + return run(f'"{python}" -m pip uninstall -y {command}', desc=f"Uninstalling {desc}", errdesc=f"Couldn't uninstall {desc}", live=default_command_live) + if install_local: + print(f"Failed to build dymanic library. Will uninstall GroundingDINO from pip and fall back to local groundingdino this time. Please {dino_install_issue_text}") + run_pip_uninstall( + f"groundingdino", + f"sd-webui-segment-anything requirement: groundingdino") + install_groundingdino_dependencies() + else: + print(f"Failed to build dymanic library. Will uninstall GroundingDINO from pip and re-try installing from GitHub source code. Please {dino_install_issue_text}") + run_pip_uninstall( + f"uninstall groundingdino", + f"sd-webui-segment-anything requirement: groundingdino") + return False + import launch if launch.is_installed("groundingdino"): - return True + print("Found GroundingDINO in pip. Verifying if dynamic library build success.") + if verify_dll(install_local=False): + return True try: launch.run_pip( f"install git+https://github.com/IDEA-Research/GroundingDINO", f"sd-webui-segment-anything requirement: groundingdino") - print("GroundingDINO install success.") - return True + print("GroundingDINO install success. Verifying if dynamic library build success.") + return verify_dll() except Exception: import traceback - print(traceback.print_exc()) - print(f"GroundingDINO install failed. Please {dino_install_issue_text}") + traceback.print_exc() + print(f"GroundingDINO install failed. Will fall back to local groundingdino this time. Please {dino_install_issue_text}") + install_groundingdino_dependencies() return False @@ -68,7 +114,7 @@ def clear_dino_cache(): torch_gc() -def load_dino_model(dino_checkpoint): +def load_dino_model(dino_checkpoint, dino_install_success): print(f"Initializing GroundingDINO {dino_checkpoint}") if dino_checkpoint in dino_model_cache: dino = dino_model_cache[dino_checkpoint] @@ -76,9 +122,14 @@ def load_dino_model(dino_checkpoint): dino.to(device=device) else: clear_dino_cache() - from groundingdino.models import build_model - from groundingdino.util.slconfig import SLConfig - from groundingdino.util.utils import clean_state_dict + if dino_install_success: + from groundingdino.models import build_model + from groundingdino.util.slconfig import SLConfig + from groundingdino.util.utils import clean_state_dict + else: + from local_groundingdino.models import build_model + from local_groundingdino.util.slconfig import SLConfig + from local_groundingdino.util.utils import clean_state_dict args = SLConfig.fromfile(dino_model_info[dino_checkpoint]["config"]) dino = build_model(args) checkpoint = torch.hub.load_state_dict_from_url( @@ -91,8 +142,11 @@ def load_dino_model(dino_checkpoint): return dino -def load_dino_image(image_pil): - import groundingdino.datasets.transforms as T +def load_dino_image(image_pil, dino_install_success): + if dino_install_success: + import groundingdino.datasets.transforms as T + else: + from local_groundingdino.datasets import transforms as T transform = T.Compose( [ T.RandomResize([800], max_size=1333), @@ -129,11 +183,10 @@ def get_grounding_output(model, image, caption, box_threshold): def dino_predict_internal(input_image, dino_model_name, text_prompt, box_threshold): install_success = install_goundingdino() - if not install_success: - return None, False print("Running GroundingDINO Inference") - dino_image = load_dino_image(input_image.convert("RGB")) - dino_model = load_dino_model(dino_model_name) + dino_image = load_dino_image(input_image.convert("RGB"), install_success) + dino_model = load_dino_model(dino_model_name, install_success) + install_success = install_success or shared.opts.data.get("sam_use_local_groundingdino", False) boxes_filt = get_grounding_output( dino_model, dino_image, text_prompt, box_threshold @@ -146,4 +199,4 @@ def dino_predict_internal(input_image, dino_model_name, text_prompt, box_thresho boxes_filt[i][2:] += boxes_filt[i][:2] gc.collect() torch_gc() - return boxes_filt, True + return boxes_filt, install_success diff --git a/scripts/sam.py b/scripts/sam.py index 6fd39c1..ad5b02c 100644 --- a/scripts/sam.py +++ b/scripts/sam.py @@ -195,14 +195,9 @@ def sam_predict(sam_model_name, input_image, positive_points, negative_points, sam_predict_result = " done." if dino_enabled: boxes_filt, install_success = dino_predict_internal(input_image, dino_model_name, text_prompt, box_threshold) - if install_success and dino_preview_checkbox is not None and dino_preview_checkbox and dino_preview_boxes_selection is not None: + if dino_preview_checkbox is not None and dino_preview_checkbox and dino_preview_boxes_selection is not None: valid_indices = [int(i) for i in dino_preview_boxes_selection if int(i) < boxes_filt.shape[0]] boxes_filt = boxes_filt[valid_indices] - if not install_success: - if len(positive_points) == 0 and len(negative_points) == 0: - return [], f"GroundingDINO installment has failed. Check your terminal for more detail and {dino_install_issue_text}. " - else: - sam_predict_result += f" However, GroundingDINO installment has failed. Your process automatically fall back to point prompt only. Check your terminal for more detail and {dino_install_issue_text}. " sam = init_sam_model(sam_model_name) print(f"Running SAM Inference {image_np_rgb.shape}") predictor = SamPredictor(sam) @@ -237,7 +232,7 @@ def sam_predict(sam_model_name, input_image, positive_points, negative_points, multimask_output=True) masks = masks[:, None, ...] garbage_collect(sam) - return create_mask_output(image_np, masks, boxes_filt), sam_predict_status + sam_predict_result + return create_mask_output(image_np, masks, boxes_filt), sam_predict_status + (sam_predict_result + "" if install_success else f" However, GroundingDINO installment has failed. Your process automatically fall back to local groundingdino. Check your terminal for more detail and {dino_install_issue_text}.") def dino_predict(input_image, dino_model_name, text_prompt, box_threshold): @@ -247,11 +242,9 @@ def dino_predict(input_image, dino_model_name, text_prompt, box_threshold): return None, gr.update(), gr.update(visible=True, value=f"GroundingDINO requires text prompt.") image_np = np.array(input_image) boxes_filt, install_success = dino_predict_internal(input_image, dino_model_name, text_prompt, box_threshold) - if not install_success: - return None, gr.update(), gr.update(visible=True, value=f"GroundingDINO installment failed. Preview failed. See your terminal for more detail and {dino_install_issue_text}") boxes_filt = boxes_filt.numpy() boxes_choice = [str(i) for i in range(boxes_filt.shape[0])] - return Image.fromarray(show_boxes(image_np, boxes_filt.astype(int), show_index=True)), gr.update(choices=boxes_choice, value=boxes_choice), gr.update(visible=False) + return Image.fromarray(show_boxes(image_np, boxes_filt.astype(int), show_index=True)), gr.update(choices=boxes_choice, value=boxes_choice), gr.update(visible=False) if install_success else gr.update(visible=True, value=f"GroundingDINO installment failed. Your process automatically fall back to local groundingdino. See your terminal for more detail and {dino_install_issue_text}") def dino_batch_process( @@ -277,9 +270,6 @@ def dino_batch_process( image_np_rgb = image_np[..., :3] boxes_filt, install_success = dino_predict_internal(input_image, batch_dino_model_name, batch_text_prompt, batch_box_threshold) - if not install_success: - return f"GroundingDINO installment failed. Batch processing failed. See your terminal for more detail and {dino_install_issue_text}" - if boxes_filt is None or boxes_filt.shape[0] == 0: msg = f"GroundingDINO generated 0 box for image {input_image_file}, please lower the box threshold if you want any segmentation for this image. " print(msg) @@ -303,7 +293,7 @@ def dino_batch_process( dino_batch_save_image, dino_batch_save_mask, dino_batch_save_background, dino_batch_save_image_with_mask) garbage_collect(sam) - return process_info + "Done" + return process_info + "Done" + ("" if install_success else f". However, GroundingDINO installment has failed. Your process automatically fall back to local groundingdino. See your terminal for more detail and {dino_install_issue_text}") def cnet_seg( @@ -792,4 +782,11 @@ def on_after_component(component, **_kwargs): return + +def on_ui_settings(): + section = ('segment_anything', "Segment Anything") + shared.opts.add_option("sam_use_local_groundingdino", shared.OptionInfo(False, "Use local groundingdino to bypass C++ problem", section=section)) + + +script_callbacks.on_ui_settings(on_ui_settings) script_callbacks.on_after_component(on_after_component)