From 40a77442a756ab160ae3465b26322f6e480405d9 Mon Sep 17 00:00:00 2001
From: Luke <lukeandshuo@sina.com>
Date: Tue, 16 Jul 2019 01:50:00 -0700
Subject: [PATCH] fixed the python memory leak (#844)

Summary:
related to the issue https://github.com/facebookresearch/fastText/issues/841
Pull Request resolved: https://github.com/facebookresearch/fastText/pull/844

Test Plan:
Detect leaks with valgrind and python compiled in debug mode; with docker and ubuntu 16.04;

`docker run -it -v ~/development/otherstuff/dockerlinux/:/home/  ubuntu:16.04 bash`
```
apt update
apt upgrade
apt-get install g++ valgrind -y
apt-get install libssl-dev zlib1g-dev   # needed by ssl and zlib enabled python (in order to run get-pip.py and get pip)
```

Download python's source code.
Do these modifications :
 - Uncomment Py_USING_MEMORY_DEBUGGER in Objects/obmalloc.c
 - Uncomment the lines in Misc/valgrind-python.supp that suppress the warnings for PyObject_Free and PyObject_Realloc
 - Uncomment ssl part in Modules/Setup.dist:
```
SSL=/usr
_ssl _ssl.c \
        -DUSE_SSL -I$(SSL)/include -I$(SSL)/include/openssl \
        -L$(SSL)/lib -lssl -lcrypto

```

now compile Python with
```
./configure --with-pydebug --without-pymalloc --with-valgrind --prefix /opt/debugpython/ OPT=-g
make install
```

get `get-pip.py` from https://bootstrap.pypa.io/get-pip.py
run `/opt/debugpython/bin/python get-pip.py`

Now you have custom Python with pip.
Install fasttext.

Run valgrind on this custom built Python :
```
valgrind --tool=memcheck --leak-check=full --suppressions=/root/Python-2.7.8/Misc/valgrind-python.supp /opt/debugpython/bin/python memleak.py
```

With memleak.py :
```
#!/usr/bin/env python

import fasttext

ft = fasttext.load_model("dbpedia.bin")

def leaker():
    for i in range(1000000):
        _ = ft.predict("hello, world!")

if __name__ == '__main__':
    leaker()
```

 ------------

References:
 - https://stackoverflow.com/questions/5937337/building-python-with-ssl-support-in-non-standard-location
 - https://www.1stbyte.com/2005/06/26/configure-and-compile-python-with-zlib/
 - https://askubuntu.com/questions/597906/how-to-install-pip-and-a-python-package-for-self-installed-python
 - https://stackoverflow.com/questions/20112989/how-to-use-valgrind-with-python

Reviewed By: EdouardGrave

Differential Revision: D16161994

Pulled By: Celebio

fbshipit-source-id: 31995d810a15a64139670abeff48f0e8e1ad5968
---
 .../fasttext_module/fasttext/pybind/fasttext_pybind.cc   | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/python/fasttext_module/fasttext/pybind/fasttext_pybind.cc b/python/fasttext_module/fasttext/pybind/fasttext_pybind.cc
index 76c25b88d..4b2992275 100644
--- a/python/fasttext_module/fasttext/pybind/fasttext_pybind.cc
+++ b/python/fasttext_module/fasttext/pybind/fasttext_pybind.cc
@@ -31,10 +31,15 @@ py::str castToPythonString(const std::string& s, const char* onUnicodeError) {
   // for python 2 and not encoded for python 3 :
   // https://github.com/pybind/pybind11/blob/ccbe68b084806dece5863437a7dc93de20bd9b15/include/pybind11/pytypes.h#L930
 #if PY_MAJOR_VERSION < 3
-  handle = PyUnicode_AsEncodedString(handle, "utf-8", onUnicodeError);
+  PyObject* handle_encoded =
+      PyUnicode_AsEncodedString(handle, "utf-8", onUnicodeError);
+  Py_DECREF(handle);
+  handle = handle_encoded;
 #endif
 
-  return py::str(handle);
+  py::str handle_str = py::str(handle);
+  Py_DECREF(handle);
+  return handle_str;
 }
 
 std::pair<std::vector<py::str>, std::vector<py::str>> getLineText(