LLE libc + other fixes part1 (#97)

* app0 folder is absolute

* some improvements on symbols types

* clang format

* missing libs.h

* improved symbols_resolver

* moved config to config folder

* functions to dump import functions

* improved logging output

* option for debugdump and improvements

* Apply suggestions from code review

Co-authored-by: GPUCode <47210458+GPUCode@users.noreply.github.com>

* clang format

---------

Co-authored-by: GPUCode <47210458+GPUCode@users.noreply.github.com>

CMakeLists.txt

@@ -186,6 +186,8 @@ set(COMMON src/common/logging/backend.cpp
            src/common/assert.h
            src/common/bounded_threadsafe_queue.h
            src/common/concepts.h
+           src/common/config.cpp
+           src/common/config.h
            src/common/debug.h
            src/common/disassembler.cpp
            src/common/disassembler.h
@@ -229,10 +231,6 @@ set(FILE_FORMAT src/core/file_format/pfs.h
                 src/core/file_format/psf.h
 )
 
-set(UTILITIES   src/Util/config.cpp
-                src/Util/config.h
-)
-
 if(ENABLE_QT_GUI)
 qt_add_executable(shadps4
     ${QT_GUI}
@@ -240,7 +238,6 @@ qt_add_executable(shadps4
     ${CORE}
     ${CRYPTO}
     ${FILE_FORMAT}
-    ${UTILITIES}
 )
 else()
 add_executable(shadps4
@@ -308,7 +305,6 @@ add_executable(shadps4
     ${CORE}
     ${CRYPTO}
     ${FILE_FORMAT}
-    ${UTILITIES}
 )
 endif()
 

src/common/config.cpp

@@ -13,6 +13,7 @@ bool isNeo = false;
 u32 screenWidth = 1280;
 u32 screenHeight = 720;
 std::string logFilter;
+bool isDebugDump = false;
 
 bool isNeoMode() {
     return isNeo;
@@ -30,6 +31,10 @@ std::string getLogFilter() {
     return logFilter;
 }
 
+bool debugDump() {
+    return isDebugDump;
+}
+
 void load(const std::filesystem::path& path) {
     // If the configuration file does not exist, create it and return
     std::error_code error;
@@ -65,6 +70,14 @@ void load(const std::filesystem::path& path) {
             screenHeight = toml::find_or<toml::integer>(general, "screenHeight", false);
         }
     }
+    if (data.contains("Debug")) {
+        auto debugResult = toml::expect<toml::value>(data.at("Debug"));
+        if (debugResult.is_ok()) {
+            auto debug = debugResult.unwrap();
+
+            isDebugDump = toml::find_or<toml::boolean>(debug, "DebugDump", false);
+        }
+    }
 }
 void save(const std::filesystem::path& path) {
     toml::basic_value<toml::preserve_comments> data;
@@ -89,6 +102,7 @@ void save(const std::filesystem::path& path) {
     data["General"]["logFilter"] = logFilter;
     data["GPU"]["screenWidth"] = screenWidth;
     data["GPU"]["screenHeight"] = screenHeight;
+    data["Debug"]["DebugDump"] = isDebugDump;
 
     std::ofstream file(path, std::ios::out);
     file << data;

src/common/config.h

@@ -4,7 +4,7 @@
 #pragma once
 
 #include <filesystem>
-#include "common/types.h"
+#include "types.h"
 
 namespace Config {
 void load(const std::filesystem::path& path);
@@ -16,4 +16,6 @@ std::string getLogFilter();
 u32 getScreenWidth();
 u32 getScreenHeight();
 
+bool debugDump();
+
 }; // namespace Config

src/common/logging/backend.cpp

@@ -6,13 +6,13 @@
 #include <thread>
 
 #include <fmt/format.h>
-#include "Util/config.h"
 
 #ifdef _WIN32
 #include <windows.h> // For OutputDebugStringW
 #endif
 
 #include "common/bounded_threadsafe_queue.h"
+#include "common/config.h"
 #include "common/io_file.h"
 #include "common/logging/backend.h"
 #include "common/logging/log.h"

src/common/path_util.cpp

@@ -35,7 +35,6 @@ static auto UserPaths = [] {
     create_path(PathType::LogDir, user_dir / LOG_DIR);
     create_path(PathType::ScreenshotsDir, user_dir / SCREENSHOTS_DIR);
     create_path(PathType::ShaderDir, user_dir / SHADER_DIR);
-    create_path(PathType::App0, user_dir / APP0_DIR);
 
     return paths;
 }();

src/core/PS4/HLE/Graphics/video_out.cpp

@@ -4,7 +4,7 @@
 #include <cstdio>
 #include <string>
 #include "Objects/video_out_ctx.h"
-#include "Util/config.h"
+#include "common/config.h"
 #include "common/debug.h"
 #include "common/logging/log.h"
 #include "common/singleton.h"

src/core/aerolib/stubs.cpp

@@ -19,7 +19,7 @@ namespace Core::AeroLib {
 // and to longer compile / CI times
 //
 // Must match STUBS_LIST define
-constexpr u32 MAX_STUBS = 128;
+constexpr u32 MAX_STUBS = 512;
 
 u64 UnresolvedStub() {
     LOG_ERROR(Core, "Returning zero to {}", __builtin_return_address(0));
@@ -61,7 +61,7 @@ static u32 UsedStubEntries;
 #define XREP_256(x) XREP_128(x) XREP_128(x + 128)
 #define XREP_512(x) XREP_256(x) XREP_256(x + 256)
 
-#define STUBS_LIST XREP_128(0)
+#define STUBS_LIST XREP_512(0)
 
 static u64 (*stub_handlers[MAX_STUBS])() = {STUBS_LIST};
 

src/core/hle/kernel/cpu_management.cpp

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "Util/config.h"
+#include "common/config.h"
 #include "common/logging/log.h"
 #include "core/hle/kernel/cpu_management.h"
 

src/core/hle/libraries/libs.h

@@ -8,28 +8,28 @@
 
 #define LIB_FUNCTION(nid, lib, libversion, mod, moduleVersionMajor, moduleVersionMinor, function)  \
     {                                                                                              \
-        Core::Loader::SymbolRes sr{};                                                              \
+        Core::Loader::SymbolResolver sr{};                                                         \
         sr.name = nid;                                                                             \
         sr.library = lib;                                                                          \
         sr.library_version = libversion;                                                           \
         sr.module = mod;                                                                           \
         sr.module_version_major = moduleVersionMajor;                                              \
         sr.module_version_minor = moduleVersionMinor;                                              \
-        sr.type = STT_FUN;                                                                         \
+        sr.type = Core::Loader::SymbolType::Function;                                              \
         auto func = reinterpret_cast<u64>(function);                                               \
         sym->AddSymbol(sr, func);                                                                  \
     }
 
 #define LIB_OBJ(nid, lib, libversion, mod, moduleVersionMajor, moduleVersionMinor, function)       \
     {                                                                                              \
-        Core::Loader::SymbolRes sr{};                                                              \
+        Core::Loader::SymbolResolver sr{};                                                         \
         sr.name = nid;                                                                             \
         sr.library = lib;                                                                          \
         sr.library_version = libversion;                                                           \
         sr.module = mod;                                                                           \
         sr.module_version_major = moduleVersionMajor;                                              \
         sr.module_version_minor = moduleVersionMinor;                                              \
-        sr.type = STT_OBJECT;                                                                      \
+        sr.type = Core::Loader::SymbolType::Object;                                                \
         auto func = reinterpret_cast<u64>(function);                                               \
         sym->AddSymbol(sr, func);                                                                  \
     }

src/core/linker.cpp

@@ -2,7 +2,9 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include <Zydis/Zydis.h>
+#include "common/config.h"
 #include "common/logging/log.h"
+#include "common/path_util.h"
 #include "common/string_util.h"
 #include "core/aerolib/aerolib.h"
 #include "core/aerolib/stubs.h"
@@ -66,6 +68,7 @@ Module* Linker::LoadModule(const std::filesystem::path& elf_name) {
     auto& m = m_modules.emplace_back();
     m = std::make_unique<Module>();
     m->elf.Open(elf_name);
+    m->file_name = std::filesystem::path(elf_name).filename().string();
 
     if (m->elf.IsElfFile()) {
         LoadModuleToMemory(m.get());
@@ -434,7 +437,7 @@ void Linker::LoadSymbols(Module* m) {
                     nidName = "UNK";
                 }
 
-                Loader::SymbolRes sym_r{};
+                Loader::SymbolResolver sym_r{};
                 sym_r.name = ids.at(0);
                 sym_r.nidName = nidName;
                 sym_r.library = library->name;
@@ -442,7 +445,20 @@ void Linker::LoadSymbols(Module* m) {
                 sym_r.module = module->name;
                 sym_r.module_version_major = module->version_major;
                 sym_r.module_version_minor = module->version_minor;
-                sym_r.type = type;
+                switch (type) {
+                case STT_NOTYPE:
+                    sym_r.type = Loader::SymbolType::NoType;
+                    break;
+                case STT_FUN:
+                    sym_r.type = Loader::SymbolType::Function;
+                    break;
+                case STT_OBJECT:
+                    sym_r.type = Loader::SymbolType::Object;
+                    break;
+                default:
+                    sym_r.type = Loader::SymbolType::Unknown;
+                    break;
+                }
 
                 if (is_sym_export) {
                     m->export_sym.AddSymbol(sym_r, sym->st_value + m->base_virtual_addr);
@@ -471,7 +487,7 @@ void Linker::Relocate(Module* m) {
         u64 rel_base_virtual_addr = m->base_virtual_addr;
         u64 rel_virtual_addr = m->base_virtual_addr + rel->rel_offset;
         bool rel_isResolved = false;
-        u8 rel_sym_type = 0;
+        Loader::SymbolType rel_sym_type = Loader::SymbolType::Unknown;
         std::string rel_name;
 
         switch (type) {
@@ -495,10 +511,10 @@ void Linker::Relocate(Module* m) {
             Loader::SymbolRecord symrec{};
             switch (sym_type) {
             case STT_FUN:
-                rel_sym_type = 2;
+                rel_sym_type = Loader::SymbolType::Function;
                 break;
             case STT_OBJECT:
-                rel_sym_type = 1;
+                rel_sym_type = Loader::SymbolType::Object;
                 break;
             default:
                 LOG_INFO(Core_Linker, "unknown symbol type {}", sym_type);
@@ -558,7 +574,7 @@ void Linker::Relocate(Module* m) {
     }
 }
 
-void Linker::Resolve(const std::string& name, int Symtype, Module* m,
+void Linker::Resolve(const std::string& name, Loader::SymbolType Symtype, Module* m,
                      Loader::SymbolRecord* return_info) {
     const auto ids = Common::SplitString(name, '#');
     if (ids.size() == 3) // symbols are 3 parts name , library , module
@@ -567,7 +583,7 @@ void Linker::Resolve(const std::string& name, int Symtype, Module* m,
         const auto* module = FindModule(*m, ids.at(2));
 
         if (library != nullptr && module != nullptr) {
-            Loader::SymbolRes sr{};
+            Loader::SymbolResolver sr{};
             sr.name = ids.at(0);
             sr.library = library->name;
             sr.library_version = library->version;
@@ -632,6 +648,10 @@ static void RunMainEntry(u64 addr, EntryParams* params, exit_func_t exit_func) {
 }
 
 void Linker::Execute() {
+    if (Config::debugDump()) {
+        DebugDump();
+    }
+
     Core::Libraries::LibKernel::pthreadInitSelfMainThread();
     EntryParams p{};
     p.argc = 1;
@@ -641,4 +661,18 @@ void Linker::Execute() {
     RunMainEntry(module->elf.GetElfEntry() + module->base_virtual_addr, &p, ProgramExitFunc);
 }
 
+void Linker::DebugDump() {
+    std::scoped_lock lock{m_mutex};
+    const auto& log_dir = Common::FS::GetUserPath(Common::FS::PathType::LogDir);
+    const std::filesystem::path debug(log_dir / "debugdump");
+    std::filesystem::create_directory(debug);
+    for (const auto& m : m_modules) {
+        // TODO make a folder with game id for being more unique?
+        const std::filesystem::path filepath(debug / m.get()->file_name);
+        std::filesystem::create_directory(filepath);
+        m.get()->import_sym.DebugDump(filepath / "imports.txt");
+        m.get()->export_sym.DebugDump(filepath / "exports.txt");
+    }
+}
+
 } // namespace Core

src/core/linker.h

@@ -100,6 +100,8 @@ struct Module {
     u64 aligned_base_size = 0;
     u64 base_virtual_addr = 0;
 
+    std::string file_name;
+
     std::vector<u8> m_dynamic;
     std::vector<u8> m_dynamic_data;
     DynamicModuleInfo dynamic_info{};
@@ -124,9 +126,10 @@ public:
         return m_hle_symbols;
     }
     void Relocate(Module* m);
-    void Resolve(const std::string& name, int Symtype, Module* m,
+    void Resolve(const std::string& name, Loader::SymbolType Symtype, Module* m,
                  Loader::SymbolRecord* return_info);
     void Execute();
+    void DebugDump();
 
 private:
     const ModuleInfo* FindModule(const Module& m, const std::string& id);

src/core/loader/symbols_resolver.cpp

@@ -1,25 +1,28 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include "common/io_file.h"
 #include "common/logging/log.h"
+#include "common/string_util.h"
 #include "common/types.h"
+#include "core/aerolib/aerolib.h"
 #include "core/loader/symbols_resolver.h"
 
 namespace Core::Loader {
 
-void SymbolsResolver::AddSymbol(const SymbolRes& s, u64 virtual_addr) {
+void SymbolsResolver::AddSymbol(const SymbolResolver& s, u64 virtual_addr) {
     SymbolRecord r{};
     r.name = GenerateName(s);
     r.virtual_address = virtual_addr;
     m_symbols.push_back(r);
 }
 
-std::string SymbolsResolver::GenerateName(const SymbolRes& s) {
+std::string SymbolsResolver::GenerateName(const SymbolResolver& s) {
-    return fmt::format("{} lib[{}_v{}]mod[{}_v{}.{}]", s.name, s.library, s.library_version,
+    return fmt::format("{}#{}#{}#{}#{}#{}#{}", s.name, s.library, s.library_version, s.module,
-                       s.module, s.module_version_major, s.module_version_minor);
+                       s.module_version_major, s.module_version_minor, SymbolTypeToS(s.type));
 }
 
-const SymbolRecord* SymbolsResolver::FindSymbol(const SymbolRes& s) const {
+const SymbolRecord* SymbolsResolver::FindSymbol(const SymbolResolver& s) const {
     const std::string name = GenerateName(s);
     for (u32 i = 0; i < m_symbols.size(); i++) {
         if (m_symbols[i].name.compare(name) == 0) {
@@ -31,4 +34,22 @@ const SymbolRecord* SymbolsResolver::FindSymbol(const SymbolRes& s) const {
     return nullptr;
 }
 
+void SymbolsResolver::DebugDump(const std::filesystem::path& file_name) {
+    Common::FS::IOFile f{file_name, Common::FS::FileAccessMode::Write,
+                         Common::FS::FileType::TextFile};
+    for (const auto& symbol : m_symbols) {
+        const auto ids = Common::SplitString(symbol.name, '#');
+        std::string nidName = "";
+        auto aeronid = AeroLib::FindByNid(ids.at(0).c_str());
+        if (aeronid != nullptr) {
+            nidName = aeronid->name;
+        } else {
+            nidName = "UNK";
+        }
+        f.WriteString(fmt::format("{:<20} {:<16} {:<60} {:<30} {:<2} {:<30} {:<2} {:<2} {:<10}\n",
+                                  symbol.virtual_address, ids.at(0), nidName, ids.at(1), ids.at(2),
+                                  ids.at(3), ids.at(4), ids.at(5), ids.at(6)));
+    }
+}
+
 } // namespace Core::Loader

src/core/loader/symbols_resolver.h

@@ -10,12 +10,20 @@
 
 namespace Core::Loader {
 
+enum class SymbolType {
+    Unknown,
+    Function,
+    Object,
+    Tls,
+    NoType,
+};
+
 struct SymbolRecord {
     std::string name;
     u64 virtual_address;
 };
 
-struct SymbolRes {
+struct SymbolResolver {
     std::string name;
     std::string nidName;
     std::string library;
@@ -23,7 +31,7 @@ struct SymbolRes {
     std::string module;
     u8 module_version_major;
     u8 module_version_minor;
-    u32 type;
+    SymbolType type;
 };
 
 class SymbolsResolver {
@@ -31,10 +39,27 @@ public:
     SymbolsResolver() = default;
     virtual ~SymbolsResolver() = default;
 
-    void AddSymbol(const SymbolRes& s, u64 virtual_addr);
+    void AddSymbol(const SymbolResolver& s, u64 virtual_addr);
-    const SymbolRecord* FindSymbol(const SymbolRes& s) const;
+    const SymbolRecord* FindSymbol(const SymbolResolver& s) const;
 
-    static std::string GenerateName(const SymbolRes& s);
+    static std::string GenerateName(const SymbolResolver& s);
+
+    static std::string_view SymbolTypeToS(SymbolType sym_type) {
+        switch (sym_type) {
+        case SymbolType::Unknown:
+            return "Unknown";
+        case SymbolType::Function:
+            return "Function";
+        case SymbolType::Object:
+            return "Object";
+        case SymbolType::Tls:
+            return "Tls";
+        case SymbolType::NoType:
+            return "NoType";
+        }
+    }
+
+    void DebugDump(const std::filesystem::path& file_name);
 
 private:
     std::vector<SymbolRecord> m_symbols;

src/main.cpp

@@ -10,7 +10,7 @@
 #include <thread>
 
 #include <core/hle/libraries/libkernel/thread_management.h>
-#include "Util/config.h"
+#include "common/config.h"
 #include "common/discord.h"
 #include "common/logging/backend.h"
 #include "common/path_util.h"