Userdata-classes are now the assumed type for any unmatching T which are not derived from sol::reference or not one of the basic types

tests added to confirm userdata can be passed into C++ function types
demangle is now named lua_demangle, and the core demangle without any replacements (to fit lua) is just named demangle
Formattings fixes everywhere

sol/demangle.hpp

@@ -33,9 +33,26 @@ namespace sol {
 namespace detail {
 #ifdef _MSC_VER
 std::string demangle(const std::type_info& id) {
+    return id.name();
+}
+
+#elif defined(__GNUC__) || defined(__clang__)
+std::string demangle(const std::type_info& id) {
+    int status;
+    char* unmangled = abi::__cxa_demangle(id.name(), 0, 0, &status);
+    std::string realname = unmangled;
+    free(unmangled);
+    return realname;
+}
+
+#else
+#error Compiler not supported for demangling
+#endif // compilers
+
+std::string lua_demangle(const std::type_info& id) {
+    std::string realname = demangle(id);
     const static std::array<std::string, 2> removals = { "struct ", "class " };
     const static std::array<std::string, 2> replacements = { "::", "_" };
-    std::string realname = id.name();
     for(std::size_t r = 0; r < removals.size(); ++r) {
         auto found = realname.find(removals[r]);
         while (found != std::string::npos) {
@@ -52,19 +69,6 @@ std::string demangle(const std::type_info& id) {
     }
     return realname;
 }
-
-#elif defined(__GNUC__) || defined(__clang__)
-std::string demangle(const std::type_info& id) {
-    int status;
-    char* unmangled = abi::__cxa_demangle(id.name(), 0, 0, &status);
-    std::string realname = unmangled;
-    free(unmangled);
-    return realname;
-}
-
-#else
-#error Compiler not supported for demangling
-#endif // compilers
 } // detail
 } // sol
 

sol/function_types.hpp

@@ -270,20 +270,6 @@ struct userdata_function : public base_function {
     template<typename... FxArgs>
     userdata_function(FxArgs&&... fxargs): fx(std::forward<FxArgs>(fxargs)...) {}
 
-    template<typename... Args>
-    int operator()(types<void>, types<Args...> t, lua_State* L) {
-        stack::get_call(L, fx, t);
-        std::ptrdiff_t nargs = sizeof...(Args);
-        lua_pop(L, nargs);
-        return 0;
-    }
-
-    template<typename... Args>
-    int operator()(types<>, types<Args...> t, lua_State* L) {
-        return (*this)(types<void>(), t, L);
-    }
-
-
     template<typename Return, typename Raw = Unqualified<Return>>
     typename std::enable_if<std::is_same<T, Raw>::value, void>::type special_push(lua_State*, Return&&) {
         // push nothing
@@ -294,6 +280,14 @@ struct userdata_function : public base_function {
         stack::push(L, std::forward<Return>(r));
     }
 
+    template<typename... Args>
+    int operator()(types<void>, types<Args...> t, lua_State* L) {
+        stack::get_call(L, 2, fx, t);
+        std::ptrdiff_t nargs = sizeof...(Args);
+        lua_pop(L, nargs);
+        return 0;
+    }
+
     template<typename... Ret, typename... Args>
     int operator()(types<Ret...>, types<Args...> t, lua_State* L) {
         typedef typename return_type<Ret...>::type return_type;
@@ -305,6 +299,11 @@ struct userdata_function : public base_function {
         return sizeof...(Ret);
     }
 
+    template<typename... Args>
+    int operator()(types<>, types<Args...> t, lua_State* L) {
+        return (*this)(types<void>(), t, L);
+    }
+
     virtual int operator()(lua_State* L) override {
         fx.prepare(L);
         return (*this)(tuple_types<typename traits_type::return_type>(), typename traits_type::args_type(), L);

sol/stack.hpp

@@ -61,16 +61,21 @@ inline type get(types<type>, lua_State* L, int index = -1) {
     return static_cast<type>(lua_type(L, index));
 }
 
+template <typename T, typename U = typename std::remove_reference<T>::type>
+inline U get_sol_type(std::true_type, types<T>, lua_State* L, int index = -1) {
+    return U(L, index);
+}
+
 template <typename T>
-inline T& get(types<userdata<T>>, lua_State* L, int index = -1) {
+inline T& get_sol_type(std::false_type, types<T>, lua_State* L, int index = -1) {
     userdata_t udata = get(types<userdata_t>{}, L, index);
     T* obj = static_cast<T*>(udata.value);
     return *obj;
 }
 
-template <typename T, typename U = typename std::remove_reference<T>::type>
-inline U get(types<T>, lua_State* L, int index = -1) {
-    return U(L, index);
+template <typename T, typename U = Unqualified<T>>
+inline auto get(types<T> t, lua_State* L, int index = -1) -> decltype(get_sol_type(std::is_base_of<sol::reference, U>(), t, L, index)) {
+    return get_sol_type(std::is_base_of<sol::reference, U>(), t, L, index);
 }
 
 template<typename T>

sol/userdata.hpp

@@ -42,7 +42,7 @@ struct userdata_traits {
 };
 
 template<typename T>
-const std::string userdata_traits<T>::name = detail::demangle(typeid(T));
+const std::string userdata_traits<T>::name = detail::lua_demangle(typeid(T));
 
 template<typename T>
 const std::string userdata_traits<T>::metatable = std::string("sol.stateful.").append(name);

tests.cpp

@@ -2,13 +2,13 @@
 #include <catch.hpp>
 #include <sol.hpp>
 
-void test_free_func( std::function<void( )> f ) {
-    f( );
+void test_free_func(std::function<void()> f) {
+    f();
 }
 
-void test_free_func2( std::function<int( int )> f, int arg1 ) {
-    int val = f( arg1 );
-    assert( arg1 == val );
+void test_free_func2(std::function<int(int)> f, int arg1) {
+    int val = f(arg1);
+    assert(arg1 == val);
 }
 
 std::string free_function() {
@@ -16,6 +16,25 @@ std::string free_function() {
     return "test";
 }
 
+struct self_test {
+    int bark;
+
+    self_test() : bark(100) {
+
+    }
+
+    void g(const std::string& str) {
+        std::cout << str << '\n';
+        bark += 1;
+    }
+
+    void f(const self_test& t) {
+        std::cout << "got test" << '\n';
+        assert(t.bark == bark);
+        assert(&t == this);
+    }
+};
+
 struct object {
     std::string operator() () {
         std::cout << "member_test()" << std::endl;
@@ -284,7 +303,7 @@ TEST_CASE("tables/functions_variables", "Check if tables and function calls work
             std::cout << "stateless lambda()" << std::endl;
             return "test";
         }
- );
+);
     REQUIRE_NOTHROW(run_script(lua));
 
     lua.get<sol::table>("os").set_function("fun", &free_function);
@@ -302,7 +321,7 @@ TEST_CASE("tables/functions_variables", "Check if tables and function calls work
         std::cout << "stateless lambda()" << std::endl;
         return "test";
     }
- );
+);
     REQUIRE_NOTHROW(run_script(lua));
 
     // r-value, cannot optimise
@@ -333,12 +352,12 @@ TEST_CASE("functions/return_order_and_multi_get", "Check if return order is in t
     REQUIRE(tluaget == triple);
 }
 
-TEST_CASE( "functions/sol::function to std::function", "check if conversion to std::function works properly and calls with correct arguments" ) {
+TEST_CASE("functions/sol::function to std::function", "check if conversion to std::function works properly and calls with correct arguments") {
     sol::state lua;
-    lua.open_libraries( sol::lib::base );
+    lua.open_libraries(sol::lib::base);
 
-    lua.set_function( "testFunc", test_free_func );
-    lua.set_function( "testFunc2", test_free_func2 );
+    lua.set_function("testFunc", test_free_func);
+    lua.set_function("testFunc2", test_free_func2);
     lua.script(
         "testFunc(function() print(\"hello std::function\") end)"
        );
@@ -521,3 +540,17 @@ TEST_CASE("tables/userdata utility derived", "userdata classes must play nice wh
     REQUIRE((lua.get<int>("dgn10") == 70));
     REQUIRE((lua.get<int>("dgn") == 7));
 }
+
+
+TEST_CASE("tables/self-referential userdata", "userdata classes must play nice when C++ object types are requested for C++ code") {
+    sol::state lua;
+    lua.open_libraries(sol::lib::base);
+
+    lua.new_userdata<self_test>("test", "g", &self_test::g, "f", &self_test::f);
+
+    lua.script(
+        "local a = test.new()\n"
+        "a:g(\"woof\")\n"
+        "a:f(a)\n"
+       );
+}