Page out RW-trapped memory in NCE Memory Trapping

To cut down memory usage we now page out memory that is RW trapped via the NCE memory trapping API, the callbacks are supposed to page in the memory. This behavior is backed up by Texture/Buffer syncing which would read the host copies of data and write it to the guest, by paging the corresponding data on the guest we're avoiding redundant memory usage.

app/src/main/cpp/skyline/kernel/memory.cpp

@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: MPL-2.0
 // Copyright © 2020 Skyline Team and Contributors (https://github.com/skyline-emu/)
 
-#include <android/sharedmem.h>
+#include <asm-generic/unistd.h>
+#include <fcntl.h>
 #include "memory.h"
 #include "types/KProcess.h"
 
@@ -62,11 +63,14 @@ namespace skyline::kernel {
         if (!base.address)
             throw exception("Cannot find a suitable carveout for the guest address space");
 
-        memoryFd = ASharedMemory_create("HOS-AS", base.size);
+        memoryFd = static_cast<int>(syscall(__NR_memfd_create, "HOS-AS", MFD_CLOEXEC)); // We need to use memfd directly as ASharedMemory doesn't always use it while we depend on it for FreeMemory (using FALLOC_FL_PUNCH_HOLE) to work
-        if (memoryFd < 0)
+        if (memoryFd == -1)
-            throw exception("Failed to create shared memory for guest address space: {}", strerror(errno));
+            throw exception("Failed to create memfd for guest address space: {}", strerror(errno));
 
-        auto result{mmap(reinterpret_cast<void *>(base.address), base.size, PROT_NONE, MAP_FIXED | MAP_SHARED, memoryFd, 0)};
+        if (ftruncate(memoryFd, static_cast<off_t>(base.size)) == -1)
+            throw exception("Failed to resize memfd for guest address space: {}", strerror(errno));
+
+        auto result{mmap(reinterpret_cast<void *>(base.address), base.size, PROT_WRITE, MAP_FIXED | MAP_SHARED, memoryFd, 0)};
         if (result == MAP_FAILED)
             throw exception("Failed to mmap guest address space: {}", strerror(errno));
 
@@ -188,6 +192,20 @@ namespace skyline::kernel {
         return span<u8>{reinterpret_cast<u8 *>(mirrorBase), totalSize};
     }
 
+    void MemoryManager::FreeMemory(u8 *pointer, size_t size) {
+        auto address{reinterpret_cast<u64>(pointer)};
+        if (address < base.address || address + size > base.address + base.size)
+            throw exception("Mapping is outside of VMM base: 0x{:X} - 0x{:X}", address, address + size);
+
+        size_t offset{address - base.address};
+        if (!util::IsPageAligned(offset) || !util::IsPageAligned(size))
+            throw exception("Mapping is not aligned to a page: 0x{:X}-0x{:X} (0x{:X})", address, address + size, offset);
+
+        // We need to use fallocate(FALLOC_FL_PUNCH_HOLE) to free the backing memory rather than madvise(MADV_REMOVE) as the latter fails when the memory doesn't have write permissions, we generally need to free memory after reprotecting it to disallow accesses between the two calls which would cause UB
+        if (fallocate(*memoryFd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, static_cast<off_t>(offset), static_cast<off_t>(size)) != 0)
+            throw exception("Failed to free memory at 0x{:X}-0x{:X} (0x{:X}): {}", address, address + size, offset, strerror(errno));
+    }
+
     void MemoryManager::InsertChunk(const ChunkDescriptor &chunk) {
         std::unique_lock lock(mutex);
 

app/src/main/cpp/skyline/kernel/memory.h

@@ -258,6 +258,12 @@ namespace skyline {
              */
             span<u8> CreateMirrors(const std::vector<span<u8>>& regions);
 
+            /**
+             * @brief Frees the underlying physical memory for a page-aligned mapping in the guest address space
+             * @note All subsequent accesses to freed memory will return 0s
+             */
+            void FreeMemory(u8* pointer, size_t size);
+
             void InsertChunk(const ChunkDescriptor &chunk);
 
             std::optional<ChunkDescriptor> Get(void *ptr);

app/src/main/cpp/skyline/nce.cpp

@@ -383,7 +383,7 @@ namespace skyline::nce {
                     if (entryProtection > lowestProtection) {
                         lowestProtection = entryProtection;
                         if (entryProtection == TrapProtection::ReadWrite)
-                            return PROT_EXEC;
+                            return PROT_NONE;
                     }
                 }
 
@@ -393,7 +393,7 @@ namespace skyline::nce {
                     case TrapProtection::WriteOnly:
                         return PROT_READ | PROT_EXEC;
                     case TrapProtection::ReadWrite:
-                        return PROT_EXEC;
+                        return PROT_NONE;
                 }
             });
         } else if (protection == TrapProtection::WriteOnly) {
@@ -401,14 +401,24 @@ namespace skyline::nce {
                 auto entries{trapMap.GetRange(region)};
                 for (const auto &entry : entries)
                     if (entry.get().protection == TrapProtection::ReadWrite)
-                        return PROT_EXEC;
+                        return PROT_NONE;
 
                 return PROT_READ | PROT_EXEC;
             });
-        } else {
+        } else if (protection == TrapProtection::ReadWrite) {
             reprotectIntervalsWithFunction([&](auto region) {
-                return PROT_EXEC; // No checks are needed as this is already the highest level of protection
+                return PROT_NONE; // No checks are needed as this is already the highest level of protection
             });
+
+            // Page out regions that are no longer accessible, these should be paged back in by a callback
+            for (auto region : intervals) {
+                auto freeStart{util::AlignUp(region.start, PAGE_SIZE)}, freeEnd{util::AlignDown(region.end, PAGE_SIZE)}; // We want to avoid the first and last page as they may contain data that won't be paged back in by the callback
+                ssize_t freeSize{freeEnd - freeStart};
+
+                constexpr ssize_t MinimumPageoutSize{PAGE_SIZE}; //!< The minimum size to page out, we don't want to page out small intervals for performance reasons
+                if (freeSize > MinimumPageoutSize)
+                    state.process->memory.FreeMemory(freeStart, static_cast<size_t>(freeSize));
+            }
         }
     }
 

app/src/main/cpp/skyline/nce.h

@@ -93,12 +93,14 @@ namespace skyline::nce {
          * @param writeOnly If the trap is optimally for write-only accesses initially, this is not guarenteed
          * @note The handle **must** be deleted using DeleteTrap before the NCE instance is destroyed
          * @note It is UB to supply a region of host memory rather than guest memory
+         * @note Any regions trapped without writeOnly may have their data (except border pages) paged out and it needs to be paged back in inside the callbacks
          */
         TrapHandle TrapRegions(span<span<u8>> regions, bool writeOnly, const TrapCallback& readCallback, const TrapCallback& writeCallback);
 
         /**
          * @brief Re-traps a region of memory after protections were removed
          * @param writeOnly If the trap is optimally for write-only accesses, this is not guarenteed
+         * @note Any regions trapped without writeOnly may have their data (except border pages) paged out and it needs to be paged back in inside the callbacks
          */
         void RetrapRegions(TrapHandle handle, bool writeOnly);