Add default convention to CallingConvention.

src/core/modules/memory/memory_function.cpp

@@ -35,6 +35,7 @@
 #include "memory_function.h"
 #include "memory_utilities.h"
 #include "memory_hooks.h"
+#include "memory_wrap.h"
 
 // DynamicHooks
 #include "conventions/x86MsCdecl.h"
@@ -87,7 +88,7 @@ int GetDynCallConvention(Convention_t eConv)
 // ============================================================================
 // >> MakeDynamicHooksConvention
 // ============================================================================
-ICallingConvention* MakeDynamicHooksConvention(Convention_t eConv, std::vector<DataType_t> vecArgTypes, DataType_t returnType, int iAlignment=4)
+ICallingConvention* MakeDynamicHooksConvention(Convention_t eConv, std::vector<DataType_t> vecArgTypes, DataType_t returnType, int iAlignment)
 {
 #ifdef _WIN32
 	switch (eConv)
@@ -149,17 +150,17 @@ CFunction::CFunction(unsigned long ulAddr, object oCallingConvention, object oAr
 	catch( ... )
 	{
 		PyErr_Clear();
-	
+
 		// A custom calling convention will be used...
 		m_eCallingConvention = CONV_CUSTOM;
-		object _oCallingConvention = oCallingConvention(m_tArgs, m_eReturnType);
+		m_oCallingConvention = oCallingConvention(m_tArgs, m_eReturnType);
 
 		// FIXME:
 		// This is required to fix a crash, but it will also cause a memory leak,
 		// because no calling convention object that is created via this method will ever be deleted.
 		// TODO: Pretty sure this was required due to the missing held type definition. It was added, but wasn't tested yet.
-		Py_INCREF(_oCallingConvention.ptr());
-		m_pCallingConvention = extract<ICallingConvention*>(_oCallingConvention);
+		Py_INCREF(m_oCallingConvention.ptr());
+		m_pCallingConvention = extract<ICallingConvention*>(m_oCallingConvention);
 
 		// We didn't allocate the calling convention, someone else is responsible for it.
 		m_bAllocatedCallingConvention = false;
@@ -188,6 +189,20 @@ CFunction::CFunction(unsigned long ulAddr, Convention_t eCallingConvention,
 
 CFunction::~CFunction()
 {
+	// If we created custom calling convention, clean it up.
+	// This does not apply to hooked calling convention.
+	if (!m_oCallingConvention.is_none())
+	{
+		CHook* pHook = GetHookManager()->FindHook((void *) m_ulAddr);
+		if (!pHook || pHook->m_pCallingConvention != m_pCallingConvention)
+		{
+			Py_DECREF(m_oCallingConvention.ptr());
+			m_pCallingConvention = NULL;
+		}
+
+		return;
+	}
+
 	// If we didn't allocate the calling convention, then it is not our responsibility.
 	if (!m_bAllocatedCallingConvention)
 		return;

src/core/modules/memory/memory_function.h

@@ -102,7 +102,15 @@ class CFunction: public CPointer, private boost::noncopyable
 	// DynamicHooks calling convention (built-in and custom)
 	ICallingConvention*		m_pCallingConvention;
 	bool					m_bAllocatedCallingConvention;
+
+	// Custom calling convention
+	object					m_oCallingConvention = object();
 };
 
 
+//---------------------------------------------------------------------------------
+// Functions
+//---------------------------------------------------------------------------------
+ICallingConvention* MakeDynamicHooksConvention(Convention_t eConv, std::vector<DataType_t> vecArgTypes, DataType_t returnType, int iAlignment=4);
+
 #endif // _MEMORY_FUNCTION_H

src/core/modules/memory/memory_wrap.cpp

@@ -820,18 +820,19 @@ void export_registers(scope _memory)
 // ============================================================================
 void export_calling_convention(scope _memory)
 {
-	class_<ICallingConventionWrapper, ICallingConventionWrapper *, boost::noncopyable>(
+	class_<ICallingConventionWrapper, boost::noncopyable>(
 		"CallingConvention",
 		"An an abstract class that is used to create custom calling "
 		"conventions (only available for hooking function and not for"
 		" calling functions).\n",
-		init< object, DataType_t, optional<int> >(
-			(arg("arg_types"), arg("return_type"), arg("alignment")),
+		init< object, DataType_t, optional<int, Convention_t> >(
+			(arg("arg_types"), arg("return_type"), arg("alignment")=4, arg("default_convention")=CONV_CUSTOM),
 			"Initialize the calling convention.\n"
 			"\n"
 			":param iterable arg_types: A list of :class:`DataType` values that define the argument types of a function.\n"
 			":param DataType return_type: The return type of a function.\n"
-			":param int alignment: The stack alignment."
+			":param int alignment: The stack alignment.\n"
+			":param Convention_t default_convention: The default convention for un override function."
 			)
 		)
 
@@ -844,14 +845,15 @@ void export_calling_convention(scope _memory)
 			&ICallingConventionWrapper::GetPopSize,
 			"Return the number of bytes that should be added to the stack to clean up."
 		)
-		
+
 		.def("get_argument_ptr",
-			&ICallingConventionWrapper::GetArgumentPtrWrapper,
+			&ICallingConventionWrapper::GetArgumentPtr,
 			(arg("index"), arg("registers")),
 			"Return a pointer to the argument at the given index.\n"
 			"\n"
 			":param int index: The index of the argument.\n"
-			":param Registers registers: A snapshot of all saved registers."
+			":param Registers registers: A snapshot of all saved registers.",
+			return_by_value_policy()
 		)
 
 		.def("argument_ptr_changed",
@@ -865,11 +867,12 @@ void export_calling_convention(scope _memory)
 		)
 
 		.def("get_return_ptr",
-			&ICallingConventionWrapper::GetReturnPtrWrapper,
+			&ICallingConventionWrapper::GetReturnPtr,
 			(arg("registers")),
 			"Return a pointer to the return value.\n"
 			"\n"
-			":param Registers registers: A snapshot of all saved registers."
+			":param Registers registers: A snapshot of all saved registers.",
+			return_by_value_policy()
 		)
 
 		.def("return_ptr_changed",

src/core/modules/memory/memory_wrap.h

@@ -32,6 +32,14 @@
 // ============================================================================
 // DynamicHooks
 #include "convention.h"
+#include "conventions/x86MsCdecl.h"
+#include "conventions/x86MsThiscall.h"
+#include "conventions/x86MsStdcall.h"
+#include "conventions/x86GccCdecl.h"
+#include "conventions/x86GccThiscall.h"
+
+// Memory
+#include "memory_function.h"
 
 // Utilities
 #include "memory_utilities.h"
@@ -48,14 +56,26 @@ using namespace boost::python;
 class ICallingConventionWrapper: public ICallingConvention, public wrapper<ICallingConvention>
 {
 public:
-	ICallingConventionWrapper(object oArgTypes, DataType_t returnType, int iAlignment=4)
+	ICallingConventionWrapper(object oArgTypes, DataType_t returnType, int iAlignment=4, Convention_t eDefaultConv=CONV_CUSTOM)
 		:ICallingConvention(ObjectToDataTypeVector(oArgTypes), returnType, iAlignment)
 	{
+		if (eDefaultConv != CONV_CUSTOM) {
+			m_pDefaultCallingConvention = MakeDynamicHooksConvention(eDefaultConv, m_vecArgTypes, m_returnType, m_iAlignment);
+		}
+	}
+
+	~ICallingConventionWrapper()
+	{
+		delete m_pDefaultCallingConvention;
+		m_pDefaultCallingConvention = nullptr;
 	}
 
 	virtual std::list<Register_t> GetRegisters()
 	{
 		override get_registers = get_override("get_registers");
+		if (!get_registers && m_pDefaultCallingConvention) {
+			return m_pDefaultCallingConvention->GetRegisters();
+		}
 		CHECK_OVERRIDE(get_registers);
 
 		object registers = get_registers();
@@ -71,47 +91,60 @@ class ICallingConventionWrapper: public ICallingConvention, public wrapper<ICall
 	virtual int GetPopSize()
 	{
 		override get_pop_size = get_override("get_pop_size");
+		if (!get_pop_size && m_pDefaultCallingConvention) {
+			return m_pDefaultCallingConvention->GetPopSize();
+		}
 		CHECK_OVERRIDE(get_pop_size);
+
 		return get_pop_size();
 	}
-
-	virtual void* GetArgumentPtr(int iIndex, CRegisters* pRegisters)
-	{
-		CPointer* ptr = extract<CPointer*>(GetArgumentPtrWrapper(iIndex, pRegisters));
-		return (void *) ptr->m_ulAddr;
-	}
 
-	object GetArgumentPtrWrapper(int iIndex, CRegisters* pRegisters)
+	virtual void* GetArgumentPtr(int iIndex, CRegisters* pRegisters)
 	{
 		override get_argument_ptr = get_override("get_argument_ptr");
+		if (!get_argument_ptr && m_pDefaultCallingConvention) {
+			return m_pDefaultCallingConvention->GetArgumentPtr(iIndex, pRegisters);
+		}
 		CHECK_OVERRIDE(get_argument_ptr);
-		return get_argument_ptr(iIndex, ptr(pRegisters));
+
+		object argument_ptr = get_argument_ptr(iIndex, ptr(pRegisters));
+		CPointer* _ptr = extract<CPointer*>(argument_ptr);
+		return (void *) _ptr->m_ulAddr;
 	}
 
 	virtual void ArgumentPtrChanged(int iIndex, CRegisters* pRegisters, void* pArgumentPtr)
 	{
 		override argument_ptr_changed = get_override("argument_ptr_changed");
+		if (!argument_ptr_changed && m_pDefaultCallingConvention) {
+			m_pDefaultCallingConvention->ArgumentPtrChanged(iIndex, pRegisters, pArgumentPtr);
+			return;
+		}
 		CHECK_OVERRIDE(argument_ptr_changed);
 		argument_ptr_changed(iIndex, ptr(pRegisters), CPointer((unsigned long) pArgumentPtr));
 	}
 
 	virtual void* GetReturnPtr(CRegisters* pRegisters)
-	{
-		CPointer* ptr = extract<CPointer*>(GetReturnPtrWrapper(pRegisters));
-		return (void *) ptr->m_ulAddr;
-	}
-
-	object GetReturnPtrWrapper(CRegisters* pRegisters)
 	{
 		override get_return_ptr = get_override("get_return_ptr");
-		CHECK_OVERRIDE(get_return_ptr);
-		return get_return_ptr(ptr(pRegisters));
+		if (!get_return_ptr && m_pDefaultCallingConvention) {
+			return m_pDefaultCallingConvention->GetReturnPtr(pRegisters);
+		}
+		CHECK_OVERRIDE(get_return_ptr)
+
+		object return_ptr = get_return_ptr(ptr(pRegisters));
+		CPointer* _ptr = extract<CPointer*>(return_ptr);
+		return (void *) _ptr->m_ulAddr;
 	}
 
 	virtual void ReturnPtrChanged(CRegisters* pRegisters, void* pReturnPtr)
 	{
 		override return_ptr_changed = get_override("return_ptr_changed");
+		if (!return_ptr_changed && m_pDefaultCallingConvention) {
+			m_pDefaultCallingConvention->ReturnPtrChanged(pRegisters, pReturnPtr);
+			return;
+		}
 		CHECK_OVERRIDE(return_ptr_changed);
+
 		return_ptr_changed(ptr(pRegisters), CPointer((unsigned long) pReturnPtr));
 	}
 
@@ -125,6 +158,9 @@ class ICallingConventionWrapper: public ICallingConvention, public wrapper<ICall
 
 		return tuple(argumentTypes);
 	}
+
+public:
+	ICallingConvention* m_pDefaultCallingConvention = nullptr;
 };