core/kernel: Miscellaneous memory fixes (#557)

* core/libraries/kernel: Fix inaccurate direct memory size

* core/memory: Fix available dmem query on non-free dmem areas

* core/kernel: return ENOMEM if memory area size is zero

* core/kernel: Fix returns on `sceKernelAvailableDirectMemorySize`

* core/memory: Remove unneeded size alignment

src/core/libraries/kernel/memory_management.cpp

@@ -75,19 +75,28 @@ s32 PS4_SYSV_ABI sceKernelAvailableDirectMemorySize(u64 searchStart, u64 searchE
     LOG_WARNING(Kernel_Vmm, "called searchStart = {:#x}, searchEnd = {:#x}, alignment = {:#x}",
                 searchStart, searchEnd, alignment);
 
-    if (searchEnd <= searchStart) {
+    if (physAddrOut == nullptr || sizeOut == nullptr) {
         return ORBIS_KERNEL_ERROR_EINVAL;
     }
     if (searchEnd > SCE_KERNEL_MAIN_DMEM_SIZE) {
         return ORBIS_KERNEL_ERROR_EINVAL;
     }
+    if (searchEnd <= searchStart) {
+        return ORBIS_KERNEL_ERROR_ENOMEM;
+    }
 
     auto* memory = Core::Memory::Instance();
 
-    PAddr physAddr;
-    s32 result =
-        memory->DirectQueryAvailable(searchStart, searchEnd, alignment, &physAddr, sizeOut);
+    PAddr physAddr{};
+    size_t size{};
+    s32 result = memory->DirectQueryAvailable(searchStart, searchEnd, alignment, &physAddr, &size);
+
+    if (size == 0) {
+        return ORBIS_KERNEL_ERROR_ENOMEM;
+    }
+
     *physAddrOut = static_cast<u64>(physAddr);
+    *sizeOut = size;
 
     return result;
 }

src/core/libraries/kernel/memory_management.h

@@ -6,7 +6,7 @@
 #include "common/bit_field.h"
 #include "common/types.h"
 
-constexpr u64 SCE_KERNEL_MAIN_DMEM_SIZE = 6_GB; // ~ 6GB
+constexpr u64 SCE_KERNEL_MAIN_DMEM_SIZE = 4608_MB; // ~ 4.5GB
 
 namespace Libraries::Kernel {
 

src/core/memory.cpp

@@ -328,6 +328,11 @@ int MemoryManager::DirectQueryAvailable(PAddr search_start, PAddr search_end, si
     PAddr paddr{};
     size_t max_size{};
     while (dmem_area != dmem_map.end() && dmem_area->second.GetEnd() <= search_end) {
+        if (!dmem_area->second.is_free) {
+            dmem_area++;
+            continue;
+        }
+
         if (dmem_area->second.size > max_size) {
             paddr = dmem_area->second.base;
             max_size = dmem_area->second.size;