FileSys: split the constructor into an Open method, in order to notify the opener something went wrong.

Kernel: Return an invalid handle to OpenFile when it failed to open.

src/core/file_sys/archive_sdmc.cpp

@@ -46,6 +46,8 @@ bool Archive_SDMC::Initialize() {
 std::unique_ptr<File> Archive_SDMC::OpenFile(const std::string& path, const Mode mode) const {
     DEBUG_LOG(FILESYS, "called path=%s mode=%d", path.c_str(), mode);
     File_SDMC* file = new File_SDMC(this, path, mode);
+    if (!file->Open())
+        return nullptr;
     return std::unique_ptr<File>(file);
 }
 

src/core/file_sys/file.h

@@ -18,6 +18,12 @@ public:
     File() { }
     virtual ~File() { }
 
+    /**
+     * Open the file
+     * @return true if the file opened correctly
+     */
+    virtual bool Open() = 0;
+
     /**
      * Read data from the file
      * @param offset Offset in bytes to start reading data from

src/core/file_sys/file_romfs.cpp

@@ -17,6 +17,14 @@ File_RomFS::File_RomFS() {
 File_RomFS::~File_RomFS() {
 }
 
+/**
+ * Open the file
+ * @return true if the file opened correctly
+ */
+bool File_RomFS::Open() {
+    return false;
+}
+
 /**
  * Read data from the file
  * @param offset Offset in bytes to start reading data from

src/core/file_sys/file_romfs.h

@@ -19,6 +19,12 @@ public:
     File_RomFS();
     ~File_RomFS() override;
 
+    /**
+     * Open the file
+     * @return true if the file opened correctly
+     */
+    bool Open() override;
+
     /**
      * Read data from the file
      * @param offset Offset in bytes to start reading data from

src/core/file_sys/file_sdmc.cpp

@@ -19,26 +19,36 @@ File_SDMC::File_SDMC(const Archive_SDMC* archive, const std::string& path, const
     // TODO(Link Mauve): normalize path into an absolute path without "..", it can currently bypass
     // the root directory we set while opening the archive.
     // For example, opening /../../etc/passwd can give the emulated program your users list.
-    std::string real_path = archive->GetMountPoint() + path;
-
-    if (!mode.create_flag && !FileUtil::Exists(real_path)) {
-        file = nullptr;
-        return;
-    }
-
-    std::string mode_string;
-    if (mode.read_flag)
-        mode_string += "r";
-    if (mode.write_flag)
-        mode_string += "w";
-
-    file = new FileUtil::IOFile(real_path, mode_string.c_str());
+    this->path = archive->GetMountPoint() + path;
+    this->mode.hex = mode.hex;
 }
 
 File_SDMC::~File_SDMC() {
     Close();
 }
 
+/**
+ * Open the file
+ * @return true if the file opened correctly
+ */
+bool File_SDMC::Open() {
+    if (!mode.create_flag && !FileUtil::Exists(path)) {
+        ERROR_LOG(FILESYS, "Non-existing file %s can’t be open without mode create.", path.c_str());
+        return false;
+    }
+
+    std::string mode_string;
+    if (mode.read_flag && mode.write_flag)
+        mode_string = "w+";
+    else if (mode.read_flag)
+        mode_string = "r";
+    else if (mode.write_flag)
+        mode_string = "w";
+
+    file = new FileUtil::IOFile(path, mode_string.c_str());
+    return true;
+}
+
 /**
  * Read data from the file
  * @param offset Offset in bytes to start reading data from

src/core/file_sys/file_sdmc.h

@@ -22,6 +22,12 @@ public:
     File_SDMC(const Archive_SDMC* archive, const std::string& path, const Mode mode);
     ~File_SDMC() override;
 
+    /**
+     * Open the file
+     * @return true if the file opened correctly
+     */
+    bool Open() override;
+
     /**
      * Read data from the file
      * @param offset Offset in bytes to start reading data from
@@ -61,6 +67,8 @@ public:
     bool Close() const override;
 
 private:
+    std::string path;
+    Mode mode;
     FileUtil::IOFile* file;
 };
 

src/core/hle/kernel/archive.cpp

@@ -374,6 +374,9 @@ Handle OpenFileFromArchive(Handle archive_handle, const std::string& path, const
     file->path = path;
     file->backend = archive->backend->OpenFile(path, mode);
 
+    if (!file->backend)
+        return 0;
+
     return handle;
 }