Fixed CallingConvention's leaks/issues.

src/core/modules/memory/memory_function.cpp

@@ -155,20 +155,17 @@ CFunction::CFunction(unsigned long ulAddr, object oCallingConvention, object oAr
 		// A custom calling convention will be used...
 		m_eCallingConvention = CONV_CUSTOM;
 		m_oCallingConvention = oCallingConvention(m_tArgs, m_eReturnType);
+		m_pCallingConvention = extract<ICallingConvention*>(m_oCallingConvention);
 
-		// FIXME:
-		// This is required to fix a crash, but it will also cause a memory leak,
-		// because no calling convention object that is created via this method will ever be deleted.
-		// TODO: Pretty sure this was required due to the missing held type definition. It was added, but wasn't tested yet.
+		// Reserve a Python reference for DynamicHooks.
 		Py_INCREF(m_oCallingConvention.ptr());
-		m_pCallingConvention = extract<ICallingConvention*>(m_oCallingConvention);
+
+		// Initialize our wrapper so that Python overrides are properly resolved.
+		detail::initialize_wrapper(m_oCallingConvention.ptr(), get_pointer((ICallingConventionWrapper *)m_pCallingConvention));
 	}
 
 	// Step 4: Get the DynCall calling convention
 	m_iCallingConvention = GetDynCallConvention(m_eCallingConvention);
-
-	// We allocated the calling convention, we are responsible to cleanup.
-	m_bAllocatedCallingConvention = true;
 }
 
 CFunction::CFunction(unsigned long ulAddr, Convention_t eCallingConvention,
@@ -180,33 +177,22 @@ CFunction::CFunction(unsigned long ulAddr, Convention_t eCallingConvention,
 	m_pCallingConvention = NULL;
 	m_oCallingConvention = object();
 
-	// We didn't allocate the calling convention, someone else is responsible for it.
-	m_bAllocatedCallingConvention = false;
-
 	m_tArgs = tArgs;
 	m_eReturnType = eReturnType;
 	m_oConverter = oConverter;
 }
 
 CFunction::~CFunction()
 {
-	// If we didn't allocate the calling convention, then it is not our responsibility.
-	if (!m_bAllocatedCallingConvention)
-		return;
-
-	// If we created calling convention, clean it up.
-	// This does not apply to hooked calling convention.
-	if (m_oCallingConvention.is_none())
-	{
-		delete m_pCallingConvention;
-	}
-	else
+	// If the convention isn't flagged as hooked, then we need to take care of it.
+	if (m_pCallingConvention && !m_pCallingConvention->m_bHooked)
 	{
-		ICallingConventionWrapper* _pCallingConventionWrapper = extract<ICallingConventionWrapper*>(m_oCallingConvention);
-
-		Py_DECREF(m_oCallingConvention.ptr());
-
-		delete _pCallingConventionWrapper;
+		// If we don't have a Python instance, then we can safely delete it.
+		if (m_oCallingConvention.is_none())
+			delete m_pCallingConvention;
+		// Otherwise, just release our reference and let Python take care of it.
+		else if (Py_REFCNT(m_oCallingConvention.ptr()) > 1)
+			Py_DECREF(m_oCallingConvention.ptr());
 	}
 
 	m_pCallingConvention = NULL;
@@ -398,9 +384,6 @@ void CFunction::AddHook(HookType_t eType, PyObject* pCallable)
 
 	if (!pHook) {
 		pHook = HookFunctionHelper((void *) m_ulAddr, m_pCallingConvention);
-
-		// DynamicHooks will handle our convention from there, regardless if we allocated it or not.
-		m_bAllocatedCallingConvention = false;
 	}
 
 	// Add the hook handler. If it's already added, it won't be added twice
@@ -420,9 +403,6 @@ bool CFunction::AddHook(HookType_t eType, HookHandlerFn* pFunc)
 
 		if (!pHook)
 			return false;
-
-		// DynamicHooks will handle our convention from there, regardless if we allocated it or not.
-		m_bAllocatedCallingConvention = false;
 	}
 
 	pHook->AddCallback(eType, pFunc);
@@ -446,6 +426,19 @@ void CFunction::DeleteHook()
 		return;
 
 	g_mapCallbacks.erase(pHook);
+
+	// Flag the convention as no longer hooked and being taken care of by DynamicHooks.
+	pHook->m_pCallingConvention->m_bHooked = false;
+
+	// Release the Python reference we reserved for DynamicHooks.
+	ICallingConventionWrapper *pConv = dynamic_cast<ICallingConventionWrapper *>(pHook->m_pCallingConvention);
+	if (pConv)
+	{
+		PyObject *pOwner = detail::wrapper_base_::owner(pConv);
+		if (pOwner && Py_REFCNT(pOwner))
+			Py_DECREF(pOwner);
+	}
+
 	// Set the calling convention to NULL, because DynamicHooks will delete it otherwise.
 	pHook->m_pCallingConvention = NULL;
 	GetHookManager()->UnhookFunction((void *) m_ulAddr);

src/core/modules/memory/memory_wrap.cpp

@@ -827,20 +827,24 @@ void export_registers(scope _memory)
 // ============================================================================
 void export_calling_convention(scope _memory)
 {
-	class_<ICallingConventionWrapper, ICallingConventionWrapper *, boost::noncopyable>(
+	class_<ICallingConventionWrapper, boost::shared_ptr<ICallingConventionWrapper>, boost::noncopyable>(
 		"CallingConvention",
 		"An an abstract class that is used to create custom calling "
 		"conventions (only available for hooking function and not for"
 		" calling functions).\n",
-		init< object, DataType_t, optional<int, Convention_t> >(
-			(arg("arg_types"), arg("return_type"), arg("alignment")=4, arg("default_convention")=CONV_CUSTOM),
+		no_init)
+
+		.def("__init__",
+			make_constructor(&ICallingConventionWrapper::__init__,
+				default_call_policies(),
+				(arg("arg_types"), arg("return_type"), arg("alignment")=4, arg("default_convention")=CONV_CUSTOM)
+			),
 			"Initialize the calling convention.\n"
 			"\n"
 			":param iterable arg_types: A list of :class:`DataType` values that define the argument types of a function.\n"
 			":param DataType return_type: The return type of a function.\n"
 			":param int alignment: The stack alignment.\n"
 			":param Convention_t default_convention: The default convention for un override function."
-			)
 		)
 
 		.def("get_registers",

src/core/modules/memory/memory_wrap.h

@@ -62,12 +62,39 @@ class ICallingConventionWrapper: public ICallingConvention, public wrapper<ICall
 		}
 	}
 
-	~ICallingConventionWrapper()
+	virtual ~ICallingConventionWrapper()
 	{
+		// If we are still flagged as hooked, then that means DynamicHooks is done with us.
+		if (m_bHooked)
+		{
+			// Release the Python reference we reserved for DynamicHooks.
+			PyObject *pOwner = detail::wrapper_base_::owner(this);
+			if (pOwner && Py_REFCNT(pOwner))
+				Py_DECREF(pOwner);
+		}
+
 		delete m_pDefaultCallingConvention;
 		m_pDefaultCallingConvention = nullptr;
 	}
 
+	static void Deleter(ICallingConventionWrapper *pThis)
+	{
+		// If we are still hooked, DynamicHooks will take care of us.
+		if (pThis->m_bHooked)
+			return;
+
+		// We are not hooked, nor referenced anymore so we can be deleted.
+		delete pThis;
+	}
+
+	static boost::shared_ptr<ICallingConventionWrapper> __init__(
+		object oArgTypes, DataType_t returnType, int iAlignment=4, Convention_t eDefaultConv=CONV_CUSTOM)
+	{
+		return boost::shared_ptr<ICallingConventionWrapper>(
+			new ICallingConventionWrapper(oArgTypes, returnType, iAlignment, eDefaultConv), &Deleter
+		);
+	}
+
 	virtual std::list<Register_t> GetRegisters()
 	{
 		override get_registers = get_override("get_registers");

src/thirdparty/DynamicHooks/include/convention.h

@@ -143,8 +143,14 @@ class ICallingConvention
 		m_vecArgTypes = vecArgTypes;
 		m_returnType = returnType;
 		m_iAlignment = iAlignment;
+		m_bHooked = false;
 	}
 
+	/*
+	Destructs the calling convention.
+	*/
+	virtual ~ICallingConvention() {};
+
 	/*
 	This should return a list of Register_t values. These registers will be
 	saved for later access.
@@ -187,6 +193,7 @@ class ICallingConvention
 	std::vector<DataType_t> m_vecArgTypes;
 	DataType_t m_returnType;
 	int m_iAlignment;
+	bool m_bHooked;
 };
 
 #endif // _CONVENTION_H