Implement filesystem backends for RomFS and Partition FS

RomFS is a hierarchial filesystem where each level is made up of a
linked list of files and child directories. It is used in NCAs to store
the applications icon as well as by applications themselves for
accessing assets.

Partition FS encapsulates both the HFS0 found in XCIs and the PFS0 used
for ExeFS images and NSPs, it is purely file based and has no support at
all for directories aside from the root.

app/CMakeLists.txt

@@ -100,6 +100,8 @@ add_library(skyline SHARED
         ${source_DIR}/skyline/services/visrv/IManagerDisplayService.cpp
         ${source_DIR}/skyline/services/visrv/IManagerRootService.cpp
         ${source_DIR}/skyline/services/visrv/ISystemDisplayService.cpp
+        ${source_DIR}/skyline/vfs/partition_filesystem.cpp
+        ${source_DIR}/skyline/vfs/rom_filesystem.cpp
         ${source_DIR}/skyline/vfs/os_backing.cpp
         ${source_DIR}/skyline/vfs/nacp.cpp
         )

app/src/main/cpp/skyline/vfs/partition_filesystem.cpp

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: MPL-2.0
+// Copyright © 2020 Skyline Team and Contributors (https://github.com/skyline-emu/)
+
+#include "region_backing.h"
+#include "partition_filesystem.h"
+
+namespace skyline::vfs {
+    PartitionFileSystem::PartitionFileSystem(std::shared_ptr<Backing> backing) : FileSystem(), backing(backing) {
+        backing->Read(&header);
+
+        if (header.magic == util::MakeMagic<u32>("PFS0"))
+            hashed = false;
+        else if (header.magic == util::MakeMagic<u32>("HFS0"))
+            hashed = true;
+        else
+            throw exception("Invalid filesystem magic: {}", header.magic);
+
+        size_t entrySize = hashed ? sizeof(HashedFileEntry) : sizeof(PartitionFileEntry);
+        size_t stringTableOffset = sizeof(FsHeader) + (header.numFiles * entrySize);
+        fileDataOffset = stringTableOffset + header.stringTableSize;
+
+        std::vector<char> stringTable(header.stringTableSize);
+        backing->Read(stringTable.data(), stringTableOffset, header.stringTableSize);
+
+        for (u32 i = 0; i < header.numFiles; i++) {
+            PartitionFileEntry entry{};
+            backing->Read(&entry, sizeof(FsHeader) + i * entrySize);
+
+            std::string name(&stringTable[entry.stringTableOffset]);
+            fileMap.emplace(name, std::move(entry));
+        }
+    }
+
+    std::shared_ptr<Backing> PartitionFileSystem::OpenFile(std::string path, Backing::Mode mode) {
+        try {
+            auto &entry = fileMap.at(path);
+            return std::make_shared<RegionBacking>(backing, fileDataOffset + entry.offset, entry.size, mode);
+        } catch (std::out_of_range &e) {
+            return nullptr;
+        }
+    }
+
+    bool PartitionFileSystem::FileExists(std::string path) {
+        return fileMap.count(path);
+    }
+
+    std::shared_ptr<Directory> PartitionFileSystem::OpenDirectory(std::string path, Directory::ListMode listMode) {
+        // PFS doesn't have directories
+        if (path != "")
+            return nullptr;
+
+        std::vector<Directory::Entry> fileList;
+        for (const auto &file : fileMap)
+            fileList.emplace_back(Directory::Entry{file.first, Directory::EntryType::File});
+
+        return std::make_shared<PartitionFileSystemDirectory>(fileList, listMode);
+    }
+
+    PartitionFileSystemDirectory::PartitionFileSystemDirectory(const std::vector<Entry> &fileList, ListMode listMode) : Directory(listMode), fileList(fileList) {}
+
+    std::vector<Directory::Entry> PartitionFileSystemDirectory::Read() {
+        if (listMode.file)
+            return fileList;
+        else
+            return std::vector<Entry>();
+    }
+}

app/src/main/cpp/skyline/vfs/partition_filesystem.h

@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: MPL-2.0
+// Copyright © 2020 Skyline Team and Contributors (https://github.com/skyline-emu/)
+
+#pragma once
+
+#include <array>
+#include "filesystem.h"
+
+namespace skyline::vfs {
+    /**
+     * @brief The PartitionFileSystem class abstracts a partition filesystem using the vfs::FileSystem api
+     */
+    class PartitionFileSystem : public FileSystem {
+      private:
+        /**
+         * @brief This holds the header of the filesystem
+         */
+        struct FsHeader {
+            u32 magic; //!< The filesystem magic: 'PFS0' or 'HFS0'
+            u32 numFiles; //!< The number of files in the filesystem
+            u32 stringTableSize; //!< The size of the filesystem's string table
+            u32 _pad_;
+        } header{};
+        static_assert(sizeof(FsHeader) == 0x10);
+
+        /**
+         * @brief This holds a file entry in a partition filesystem
+         */
+        struct PartitionFileEntry {
+            u64 offset; //!< The offset of the file in the backing
+            u64 size; //!< The size of the file
+            u32 stringTableOffset; //!< The offset of the file in the string table
+            u32 _pad_;
+        };
+        static_assert(sizeof(PartitionFileEntry) == 0x18);
+
+        /**
+         * @brief This holds a file entry in a hashed filesystem
+         */
+        struct HashedFileEntry {
+            PartitionFileEntry entry; //!< The base file entry
+            u32 _pad_;
+            std::array<u8, 0x20> hash; //!< The hash of the file
+        };
+        static_assert(sizeof(HashedFileEntry) == 0x40);
+
+        bool hashed; //!< Whether the filesystem contains hash data
+        size_t fileDataOffset; //!< The offset from the backing to the base of the file data
+        std::shared_ptr<Backing> backing; //!< The backing file of the filesystem
+        std::unordered_map<std::string, PartitionFileEntry> fileMap; //!< A map that maps file names to their corresponding entry
+
+      public:
+        PartitionFileSystem(std::shared_ptr<Backing> backing);
+
+        std::shared_ptr<Backing> OpenFile(std::string path, Backing::Mode mode = {true, false, false});
+
+        bool FileExists(std::string path);
+
+        std::shared_ptr<Directory> OpenDirectory(std::string path, Directory::ListMode listMode);
+    };
+
+    /**
+     * @brief The PartitionFileSystemDirectory provides access to the root directory of a partition filesystem
+     */
+    class PartitionFileSystemDirectory : public Directory {
+      private:
+        std::vector<Entry> fileList; //!< A list of every file in the PFS root directory
+
+      public:
+        PartitionFileSystemDirectory(const std::vector<Entry> &fileList, ListMode listMode);
+
+        std::vector<Entry> Read();
+    };
+}

app/src/main/cpp/skyline/vfs/rom_filesystem.cpp

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: MPL-2.0
+// Copyright © 2020 Skyline Team and Contributors (https://github.com/skyline-emu/)
+
+#include "region_backing.h"
+#include "rom_filesystem.h"
+
+namespace skyline::vfs {
+    RomFileSystem::RomFileSystem(std::shared_ptr <Backing> backing) : FileSystem(), backing(backing) {
+        backing->Read(&header);
+
+        TraverseDirectory(0, "");
+    }
+
+    void RomFileSystem::TraverseFiles(u32 offset, std::string path) {
+        RomFsFileEntry entry{};
+
+        do {
+            backing->Read(&entry, header.fileMetaTableOffset + offset);
+
+            if (entry.nameSize) {
+                std::vector<char> name(entry.nameSize);
+                backing->Read(name.data(), header.fileMetaTableOffset + offset + sizeof(RomFsFileEntry), entry.nameSize);
+
+                std::string fullPath = path + (path.empty() ? "" : "/") + std::string(name.data(), entry.nameSize);
+                fileMap.emplace(fullPath, entry);
+            }
+
+            offset = entry.siblingOffset;
+        } while (offset != constant::RomFsEmptyEntry);
+    }
+
+    void RomFileSystem::TraverseDirectory(u32 offset, std::string path) {
+        RomFsDirectoryEntry entry{};
+        backing->Read(&entry, header.dirMetaTableOffset + offset);
+
+        std::string childPath = path;
+        if (entry.nameSize) {
+            std::vector<char> name(entry.nameSize);
+            backing->Read(name.data(), header.dirMetaTableOffset + offset + sizeof(RomFsDirectoryEntry), entry.nameSize);
+            childPath = path + (path.empty() ? "" : "/") + std::string(name.data(), entry.nameSize);
+        }
+
+        directoryMap.emplace(childPath, entry);
+
+        if (entry.fileOffset != constant::RomFsEmptyEntry)
+            TraverseFiles(entry.fileOffset, childPath);
+
+        if (entry.childOffset != constant::RomFsEmptyEntry)
+            TraverseDirectory(entry.childOffset, childPath);
+
+        if (entry.siblingOffset != constant::RomFsEmptyEntry)
+            TraverseDirectory(entry.siblingOffset, path);
+    }
+
+    std::shared_ptr <Backing> RomFileSystem::OpenFile(std::string path, Backing::Mode mode) {
+        try {
+            const auto &entry = fileMap.at(path);
+            return std::make_shared<RegionBacking>(backing, header.dataOffset + entry.offset, entry.size, mode);
+        } catch (std::out_of_range &e) {
+            return nullptr;
+        }
+    }
+
+    bool RomFileSystem::FileExists(std::string path) {
+        return fileMap.count(path);
+    }
+
+    std::shared_ptr <Directory> RomFileSystem::OpenDirectory(std::string path, Directory::ListMode listMode) {
+        try {
+            auto &entry = directoryMap.at(path);
+            return std::make_shared<RomFileSystemDirectory>(backing, header, entry, listMode);
+        } catch (std::out_of_range &e) {
+            return nullptr;
+        }
+    }
+
+    RomFileSystemDirectory::RomFileSystemDirectory(const std::shared_ptr <Backing> &backing, const RomFileSystem::RomFsHeader &header, const RomFileSystem::RomFsDirectoryEntry &ownEntry, ListMode listMode) : Directory(listMode), backing(backing), header(header), ownEntry(ownEntry) {}
+
+    std::vector <RomFileSystemDirectory::Entry> RomFileSystemDirectory::Read() {
+        std::vector <Entry> contents;
+
+        if (listMode.file) {
+            RomFileSystem::RomFsFileEntry romFsFileEntry;
+            u32 offset = ownEntry.fileOffset;
+
+            do {
+                backing->Read(&romFsFileEntry, header.fileMetaTableOffset + offset);
+
+                if (romFsFileEntry.nameSize) {
+                    std::vector<char> name(romFsFileEntry.nameSize);
+                    backing->Read(name.data(), header.fileMetaTableOffset + offset + sizeof(RomFileSystem::RomFsFileEntry), romFsFileEntry.nameSize);
+
+                    contents.emplace_back(Entry{std::string(name.data(), romFsFileEntry.nameSize), EntryType::File});
+                }
+
+                offset = romFsFileEntry.siblingOffset;
+            } while (offset != constant::RomFsEmptyEntry);
+        }
+
+        if (listMode.directory) {
+            RomFileSystem::RomFsDirectoryEntry romFsDirectoryEntry;
+            u32 offset = ownEntry.childOffset;
+
+            do {
+                backing->Read(&romFsDirectoryEntry, header.dirMetaTableOffset + offset);
+
+                if (romFsDirectoryEntry.nameSize) {
+                    std::vector<char> name(romFsDirectoryEntry.nameSize);
+                    backing->Read(name.data(), header.dirMetaTableOffset + offset + sizeof(RomFileSystem::RomFsDirectoryEntry), romFsDirectoryEntry.nameSize);
+
+                    contents.emplace_back(Entry{std::string(name.data(), romFsDirectoryEntry.nameSize), EntryType::Directory});
+                }
+
+                offset = romFsDirectoryEntry.siblingOffset;
+            } while (offset != constant::RomFsEmptyEntry);
+        }
+
+        return contents;
+    }
+}

app/src/main/cpp/skyline/vfs/rom_filesystem.h

@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: MPL-2.0
+// Copyright © 2020 Skyline Team and Contributors (https://github.com/skyline-emu/)
+
+#pragma once
+
+#include "filesystem.h"
+
+namespace skyline {
+    namespace constant {
+        constexpr u32 RomFsEmptyEntry = 0xffffffff; //!< The value a RomFS entry has it's offset set to if it is empty
+    }
+
+    namespace vfs {
+        /**
+         * @brief The RomFileSystem class abstracts access to a RomFS image using the vfs::FileSystem api
+         */
+        class RomFileSystem : public FileSystem {
+          private:
+            std::shared_ptr<Backing> backing; //!< The backing file of the filesystem
+
+            /**
+             * @brief Traverses the sibling files of the given file and adds them to the file map
+             * @param offset The offset of the file entry to traverses the siblings of
+             * @param path The path to the parent directory of the supplied file entry
+             */
+            void TraverseFiles(u32 offset, std::string path);
+
+            /**
+             * @brief Traverses the directories within the given directory, adds them to the directory map and calls TraverseFiles on them
+             * @param offset The offset of the directory entry to traverses the directories in
+             * @param path The path to the supplied directory entry
+             */
+            void TraverseDirectory(u32 offset, std::string path);
+
+          public:
+            /**
+             * @brief This holds the header of a RomFS image
+             */
+            struct RomFsHeader {
+                u64 headerSize; //!< The size of the header
+                u64 dirHashTableOffset; //!< The offset of the directory hash table
+                u64 dirHashTableSize; //!< The size of the directory hash table
+                u64 dirMetaTableOffset; //!< The offset of the directory metadata table
+                u64 dirMetaTableSize; //!< The size of the directory metadata table
+                u64 fileHashTableOffset; //!< The offset of the file hash table
+                u64 fileHashTableSize; //!< The size of the file hash table
+                u64 fileMetaTableOffset; //!< The offset of the file metadata table
+                u64 fileMetaTableSize; //!< The size of the file metadata table
+                u64 dataOffset; //!< The offset of the file data
+            } header{};
+            static_assert(sizeof(RomFsHeader) == 0x50);
+
+            /**
+             * @brief This holds a directory entry in a RomFS image
+             */
+            struct RomFsDirectoryEntry {
+                u32 parentOffset; //!< The offset from the directory metadata base of the parent directory
+                u32 siblingOffset; //!< The offset from the directory metadata base of a sibling directory
+                u32 childOffset; //!< The offset from the directory metadata base of a child directory
+                u32 fileOffset; //!< The offset from the file metadata base of a child file
+                u32 hash; //!< The hash of the directory
+                u32 nameSize; //!< The size of the directory's name in bytes
+            };
+
+            /**
+             * @brief This holds a file entry in a RomFS image
+             */
+            struct RomFsFileEntry {
+                u32 parentOffset; //!< The offset from the directory metadata base of the parent directory
+                u32 siblingOffset; //!< The offset from the file metadata base of a sibling file
+                u64 offset; //!< The offset from the file data base of the file contents
+                u64 size; //!< The size of the file in bytes
+                u32 hash; //!< The hash of the file
+                u32 nameSize; //!< The size of the file's name in bytes
+            };
+
+            std::unordered_map<std::string, RomFsFileEntry> fileMap; //!< A map that maps file names to their corresponding entry
+            std::unordered_map<std::string, RomFsDirectoryEntry> directoryMap; //!< A map that maps directory names to their corresponding entry
+
+            RomFileSystem(std::shared_ptr<Backing> backing);
+
+            std::shared_ptr<Backing> OpenFile(std::string path, Backing::Mode mode = {true, false, false});
+
+            bool FileExists(std::string path);
+
+            std::shared_ptr<Directory> OpenDirectory(std::string path, Directory::ListMode listMode);
+        };
+
+        /**
+         * @brief The RomFileSystemDirectory provides access to directories within a RomFS
+         */
+        class RomFileSystemDirectory : public Directory {
+          private:
+            RomFileSystem::RomFsDirectoryEntry ownEntry; //!< This directory's entry in the RomFS header
+            RomFileSystem::RomFsHeader header; //!< A header of this files parent RomFS image
+            std::shared_ptr<Backing> backing; //!< The backing of the RomFS image
+
+          public:
+            RomFileSystemDirectory(const std::shared_ptr<Backing> &backing, const RomFileSystem::RomFsHeader &header, const RomFileSystem::RomFsDirectoryEntry &ownEntry, ListMode listMode);
+
+            std::vector<Entry> Read();
+        };
+    }
+}