Merge pull request #50 from georgemoralis/skmp/auto-aerolib-stubs

Add auto stubs, update aerolib, faster init

CMakeLists.txt

@@ -61,6 +61,9 @@ add_executable(shadps4
     src/Core/virtual_memory.h
     src/Core/PS4/Linker.cpp
     src/Core/PS4/Linker.h
+    src/Core/PS4/Stubs.cpp
+    src/Core/PS4/Stubs.h
+    src/Core/PS4/Util/aerolib.cpp
     src/Lib/Threads.cpp
     src/Lib/Threads.h
     src/Core/PS4/HLE/Kernel/Objects/physical_memory.h

scripts/ps4_names2stubs.py

@@ -0,0 +1,45 @@
+# Helper script that generates stub implementation of all known nids + lookup tables
+# for shadps4
+
+import sys, os
+import struct
+#from hashlib import sha1
+import hashlib
+from base64 import b64encode as base64enc
+from binascii import unhexlify as uhx
+
+#ref https://github.com/SocraticBliss/ps4_name2nid_plugin
+#ref derived from https://github.com/zecoxao/sce_symbols.git
+
+# needs ps4_names.txt (see: https://github.com/zecoxao/sce_symbols.git for full list)
+# generates "stubs.inl" to include in Core\PS4\Util
+
+NEW_NIDS = {}
+NAMES   = 'ps4_names.txt'
+
+def name2nid(name):
+    symbol = hashlib.sha1(name.encode() + uhx('518D64A635DED8C1E6B039B1C3E55230')).digest()
+    id     = struct.unpack('<Q', symbol[:8])[0]
+    nid    = base64enc(uhx('%016x' % id), b'+-').rstrip(b'=')
+    NEW_NIDS[nid]=name
+
+def save_stubs(NIDS):
+    nidsSorted=sorted(NIDS.items(), key=lambda x: x[0])
+
+    nidsFile=open("aerolib.inl", "w")
+    nidsFile.writelines('// generated using ps4_names2stubs.py\n\n')
+    for nid, name in nidsSorted:
+        nidsFile.writelines('STUB("%s", %s)\n' % (str(nid,'utf-8'), name))
+
+    nidsFile.close()
+
+
+
+f = open(NAMES,"r")
+for line in f.readlines():
+    line = line.strip()
+    name2nid(line)
+
+f.close()
+
+save_stubs(NEW_NIDS)

src/Core/PS4/Linker.cpp

@@ -6,7 +6,7 @@
 #include "Util/aerolib.h"
 #include "Loader/SymbolsResolver.h"
 #include "HLE/Kernel/ThreadManagement.h"
-
+#include "Stubs.h"
 
 constexpr bool debug_loader = true;
 
@@ -467,14 +467,18 @@ void Linker::LoadSymbols(Module* m)
 				//if st_value!=0 then it's export symbol
 				bool is_sym_export = sym->st_value != 0;
 				std::string nidName = "";
-				if (aerolib::symbolsMap.find(ids.at(0)) != aerolib::symbolsMap.end())
+
+				auto aeronid = aerolib::find_by_nid(ids.at(0).c_str());
+
+				if (aeronid != nullptr)
 				{
-					nidName = aerolib::symbolsMap.at(ids.at(0));
+                    nidName = aeronid->name;
 				}
 				else
 				{
 					nidName = "UNK";
 				}
+
 				SymbolRes sym_r{};
 				sym_r.name = ids.at(0);
 				sym_r.nidName = nidName;
@@ -589,7 +593,6 @@ void Linker::Relocate(Module* m)
 	}
 }
 
-
 void Linker::Resolve(const std::string& name, int Symtype, Module* m, SymbolRecord* return_info) { 
 	auto ids = StringUtil::split_string(name, '#');
 
@@ -616,8 +619,15 @@ void Linker::Resolve(const std::string& name, int Symtype, Module* m, SymbolReco
             if (rec != nullptr) {
                 *return_info = *rec;
             } else {
-                return_info->virtual_address = 0;
-                return_info->name = "Unresolved!!!";
+                auto aeronid = aerolib::find_by_nid(sr.name.c_str());
+                if (aeronid) {
+                    return_info->name = aeronid->name;
+                    return_info->virtual_address = GetStub(aeronid->nid);
+                    LOG_ERROR_IF(debug_loader, "Linker: Stub resolved {} as {} (lib: {}, mod: {}) \n", sr.name, return_info->name, library->name, module->name);
+                } else {
+                    return_info->virtual_address = (u64)&UnresolvedStub;
+                    return_info->name = "Unresolved!!!";
+				}
             }
 		}
 		else

src/Core/PS4/Stubs.cpp

@@ -0,0 +1,71 @@
+#include "Stubs.h"
+
+#include "Util/aerolib.h"
+
+#include "Util/log.h"
+
+// Helper to provide stub implementations for missing functions
+// 
+// This works by pre-compiling generic stub functions ("slots"), and then
+// on lookup, setting up the nid_entry they are matched with
+// 
+// If it runs out of stubs with name information, it will return
+// a default implemetnation without function name details
+
+// Up to 512, larger values lead to more resolve stub slots
+// and to longer compile / CI times
+// 
+// Must match STUBS_LIST define
+#define MAX_STUBS 128
+
+u64 UnresolvedStub() {
+    LOG_ERROR("Unknown Stub: called, returning zero\n");
+    return 0;
+}
+
+static u64 UnknownStub() {
+    LOG_ERROR("Unknown Stub: called, returning zero\n");
+    return 0;
+}
+
+
+static aerolib::nid_entry* stub_nids[MAX_STUBS];
+
+template <int stub_index>
+static u64 CommonStub() {
+    auto entry = stub_nids[stub_index];
+
+    LOG_ERROR("Stub: {} ({}) called, returning zero\n", entry->name, entry->nid);
+    return 0;
+}
+
+static u32 UsedStubEntries;
+
+#define XREP_1(x)        \
+    &CommonStub<x>,
+
+#define XREP_2(x) XREP_1(x) XREP_1(x + 1)
+#define XREP_4(x) XREP_2(x) XREP_2(x + 2)
+#define XREP_8(x) XREP_4(x) XREP_4(x + 4)
+#define XREP_16(x) XREP_8(x) XREP_8(x + 8)
+#define XREP_32(x) XREP_16(x) XREP_16(x + 16)
+#define XREP_64(x) XREP_32(x) XREP_32(x + 32)
+#define XREP_128(x) XREP_64(x) XREP_64(x + 64)
+#define XREP_256(x) XREP_128(x) XREP_128(x + 128)
+#define XREP_512(x) XREP_256(x) XREP_256(x + 256)
+
+#define STUBS_LIST XREP_128(0)
+
+static u64 (*stub_handlers[MAX_STUBS])() = {
+    STUBS_LIST
+};
+
+u64 GetStub(const char* nid) {
+	auto entry = aerolib::find_by_nid(nid);
+    if (!entry || UsedStubEntries >= MAX_STUBS) {
+        return (u64)&UnknownStub;
+    } else {
+        stub_nids[UsedStubEntries] = entry;
+        return (u64)stub_handlers[UsedStubEntries++];
+    }
+}

src/Core/PS4/Stubs.h

@@ -0,0 +1,4 @@
+#include "types.h"
+
+u64 UnresolvedStub();
+u64 GetStub(const char *nid);

src/Core/PS4/Util/aerolib.cpp

@@ -0,0 +1,38 @@
+#include "aerolib.h"
+
+#include "types.h"
+
+#include <string.h>
+
+#include "Util/log.h"
+
+namespace aerolib {
+
+// Use a direct table here + binary search as contents are static
+nid_entry nids[] = {
+#define STUB(nid, name) \
+    { nid, #name },
+#include "aerolib.inl"
+#undef STUB
+};
+
+nid_entry* find_by_nid(const char* nid) {
+    s64 l = 0;
+    s64 r = sizeof(nids) / sizeof(nids[0]) - 1;
+
+    while (l <= r) {
+        size_t m = l + (r - l) / 2;
+
+        int cmp = strcmp(nids[m].nid, nid);
+
+        if (cmp == 0)
+            return &nids[m];
+        else if (cmp < 0)
+            l = m + 1;
+        else
+            r = m - 1;
+    }
+
+    return nullptr;
+}
+}  // namespace aerolib