Merge pull request #50 from georgemoralis/skmp/auto-aerolib-stubs

Add auto stubs, update aerolib, faster init
This commit is contained in:
georgemoralis 2023-10-11 12:18:22 +03:00 committed by GitHub
commit a25c5f738e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 22633 additions and 11232 deletions

View File

@ -61,6 +61,9 @@ add_executable(shadps4
src/Core/virtual_memory.h src/Core/virtual_memory.h
src/Core/PS4/Linker.cpp src/Core/PS4/Linker.cpp
src/Core/PS4/Linker.h src/Core/PS4/Linker.h
src/Core/PS4/Stubs.cpp
src/Core/PS4/Stubs.h
src/Core/PS4/Util/aerolib.cpp
src/Lib/Threads.cpp src/Lib/Threads.cpp
src/Lib/Threads.h src/Lib/Threads.h
src/Core/PS4/HLE/Kernel/Objects/physical_memory.h src/Core/PS4/HLE/Kernel/Objects/physical_memory.h

11223
scripts/ps4_names.txt Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,45 @@
# Helper script that generates stub implementation of all known nids + lookup tables
# for shadps4
import sys, os
import struct
#from hashlib import sha1
import hashlib
from base64 import b64encode as base64enc
from binascii import unhexlify as uhx
#ref https://github.com/SocraticBliss/ps4_name2nid_plugin
#ref derived from https://github.com/zecoxao/sce_symbols.git
# needs ps4_names.txt (see: https://github.com/zecoxao/sce_symbols.git for full list)
# generates "stubs.inl" to include in Core\PS4\Util
NEW_NIDS = {}
NAMES = 'ps4_names.txt'
def name2nid(name):
symbol = hashlib.sha1(name.encode() + uhx('518D64A635DED8C1E6B039B1C3E55230')).digest()
id = struct.unpack('<Q', symbol[:8])[0]
nid = base64enc(uhx('%016x' % id), b'+-').rstrip(b'=')
NEW_NIDS[nid]=name
def save_stubs(NIDS):
nidsSorted=sorted(NIDS.items(), key=lambda x: x[0])
nidsFile=open("aerolib.inl", "w")
nidsFile.writelines('// generated using ps4_names2stubs.py\n\n')
for nid, name in nidsSorted:
nidsFile.writelines('STUB("%s", %s)\n' % (str(nid,'utf-8'), name))
nidsFile.close()
f = open(NAMES,"r")
for line in f.readlines():
line = line.strip()
name2nid(line)
f.close()
save_stubs(NEW_NIDS)

View File

@ -6,7 +6,7 @@
#include "Util/aerolib.h" #include "Util/aerolib.h"
#include "Loader/SymbolsResolver.h" #include "Loader/SymbolsResolver.h"
#include "HLE/Kernel/ThreadManagement.h" #include "HLE/Kernel/ThreadManagement.h"
#include "Stubs.h"
constexpr bool debug_loader = true; constexpr bool debug_loader = true;
@ -467,14 +467,18 @@ void Linker::LoadSymbols(Module* m)
//if st_value!=0 then it's export symbol //if st_value!=0 then it's export symbol
bool is_sym_export = sym->st_value != 0; bool is_sym_export = sym->st_value != 0;
std::string nidName = ""; std::string nidName = "";
if (aerolib::symbolsMap.find(ids.at(0)) != aerolib::symbolsMap.end())
auto aeronid = aerolib::find_by_nid(ids.at(0).c_str());
if (aeronid != nullptr)
{ {
nidName = aerolib::symbolsMap.at(ids.at(0)); nidName = aeronid->name;
} }
else else
{ {
nidName = "UNK"; nidName = "UNK";
} }
SymbolRes sym_r{}; SymbolRes sym_r{};
sym_r.name = ids.at(0); sym_r.name = ids.at(0);
sym_r.nidName = nidName; sym_r.nidName = nidName;
@ -589,7 +593,6 @@ void Linker::Relocate(Module* m)
} }
} }
void Linker::Resolve(const std::string& name, int Symtype, Module* m, SymbolRecord* return_info) { void Linker::Resolve(const std::string& name, int Symtype, Module* m, SymbolRecord* return_info) {
auto ids = StringUtil::split_string(name, '#'); auto ids = StringUtil::split_string(name, '#');
@ -616,8 +619,15 @@ void Linker::Resolve(const std::string& name, int Symtype, Module* m, SymbolReco
if (rec != nullptr) { if (rec != nullptr) {
*return_info = *rec; *return_info = *rec;
} else { } else {
return_info->virtual_address = 0; auto aeronid = aerolib::find_by_nid(sr.name.c_str());
return_info->name = "Unresolved!!!"; if (aeronid) {
return_info->name = aeronid->name;
return_info->virtual_address = GetStub(aeronid->nid);
LOG_ERROR_IF(debug_loader, "Linker: Stub resolved {} as {} (lib: {}, mod: {}) \n", sr.name, return_info->name, library->name, module->name);
} else {
return_info->virtual_address = (u64)&UnresolvedStub;
return_info->name = "Unresolved!!!";
}
} }
} }
else else

71
src/Core/PS4/Stubs.cpp Normal file
View File

@ -0,0 +1,71 @@
#include "Stubs.h"
#include "Util/aerolib.h"
#include "Util/log.h"
// Helper to provide stub implementations for missing functions
//
// This works by pre-compiling generic stub functions ("slots"), and then
// on lookup, setting up the nid_entry they are matched with
//
// If it runs out of stubs with name information, it will return
// a default implemetnation without function name details
// Up to 512, larger values lead to more resolve stub slots
// and to longer compile / CI times
//
// Must match STUBS_LIST define
#define MAX_STUBS 128
u64 UnresolvedStub() {
LOG_ERROR("Unknown Stub: called, returning zero\n");
return 0;
}
static u64 UnknownStub() {
LOG_ERROR("Unknown Stub: called, returning zero\n");
return 0;
}
static aerolib::nid_entry* stub_nids[MAX_STUBS];
template <int stub_index>
static u64 CommonStub() {
auto entry = stub_nids[stub_index];
LOG_ERROR("Stub: {} ({}) called, returning zero\n", entry->name, entry->nid);
return 0;
}
static u32 UsedStubEntries;
#define XREP_1(x) \
&CommonStub<x>,
#define XREP_2(x) XREP_1(x) XREP_1(x + 1)
#define XREP_4(x) XREP_2(x) XREP_2(x + 2)
#define XREP_8(x) XREP_4(x) XREP_4(x + 4)
#define XREP_16(x) XREP_8(x) XREP_8(x + 8)
#define XREP_32(x) XREP_16(x) XREP_16(x + 16)
#define XREP_64(x) XREP_32(x) XREP_32(x + 32)
#define XREP_128(x) XREP_64(x) XREP_64(x + 64)
#define XREP_256(x) XREP_128(x) XREP_128(x + 128)
#define XREP_512(x) XREP_256(x) XREP_256(x + 256)
#define STUBS_LIST XREP_128(0)
static u64 (*stub_handlers[MAX_STUBS])() = {
STUBS_LIST
};
u64 GetStub(const char* nid) {
auto entry = aerolib::find_by_nid(nid);
if (!entry || UsedStubEntries >= MAX_STUBS) {
return (u64)&UnknownStub;
} else {
stub_nids[UsedStubEntries] = entry;
return (u64)stub_handlers[UsedStubEntries++];
}
}

4
src/Core/PS4/Stubs.h Normal file
View File

@ -0,0 +1,4 @@
#include "types.h"
u64 UnresolvedStub();
u64 GetStub(const char *nid);

View File

@ -0,0 +1,38 @@
#include "aerolib.h"
#include "types.h"
#include <string.h>
#include "Util/log.h"
namespace aerolib {
// Use a direct table here + binary search as contents are static
nid_entry nids[] = {
#define STUB(nid, name) \
{ nid, #name },
#include "aerolib.inl"
#undef STUB
};
nid_entry* find_by_nid(const char* nid) {
s64 l = 0;
s64 r = sizeof(nids) / sizeof(nids[0]) - 1;
while (l <= r) {
size_t m = l + (r - l) / 2;
int cmp = strcmp(nids[m].nid, nid);
if (cmp == 0)
return &nids[m];
else if (cmp < 0)
l = m + 1;
else
r = m - 1;
}
return nullptr;
}
} // namespace aerolib

File diff suppressed because it is too large Load Diff

11225
src/Core/PS4/Util/aerolib.inl Normal file

File diff suppressed because it is too large Load Diff