Merge pull request #8538 from JosJuice/wia

Add support for the WIA and RVZ disc image formats

Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java

@@ -22,7 +22,7 @@ import java.util.List;
 public final class FileBrowserHelper
 {
   public static final HashSet<String> GAME_EXTENSIONS = new HashSet<>(Arrays.asList(
-          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wad", "dol", "elf", "dff"));
+          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "wad", "dol", "elf", "dff"));
 
   public static final HashSet<String> RAW_EXTENSION = new HashSet<>(Collections.singletonList(
           "raw"));

Source/Core/Core/Boot/Boot.cpp

@@ -159,7 +159,7 @@ BootParameters::GenerateFromFile(std::vector<std::string> paths,
     paths.clear();
 
   static const std::unordered_set<std::string> disc_image_extensions = {
-      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".dol", ".elf"}};
+      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".dol", ".elf"}};
   if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive)
   {
     std::unique_ptr<DiscIO::VolumeDisc> disc = DiscIO::CreateDisc(path);

Source/Core/DiscIO/Blob.cpp

@@ -20,6 +20,7 @@
 #include "DiscIO/DriveBlob.h"
 #include "DiscIO/FileBlob.h"
 #include "DiscIO/TGCBlob.h"
+#include "DiscIO/WIABlob.h"
 #include "DiscIO/WbfsBlob.h"
 
 namespace DiscIO
@@ -205,6 +206,10 @@ std::unique_ptr<BlobReader> CreateBlobReader(const std::string& filename)
     return TGCFileReader::Create(std::move(file));
   case WBFS_MAGIC:
     return WbfsFileReader::Create(std::move(file), filename);
+  case WIA_MAGIC:
+    return WIAFileReader::Create(std::move(file), filename);
+  case RVZ_MAGIC:
+    return RVZFileReader::Create(std::move(file), filename);
   default:
     if (auto directory_blob = DirectoryBlobReader::Create(filename))
       return std::move(directory_blob);

Source/Core/DiscIO/Blob.h

@@ -25,6 +25,8 @@
 
 namespace DiscIO
 {
+enum class WIARVZCompressionType : u32;
+
 // Increment CACHE_REVISION (GameFileCache.cpp) if the enum below is modified
 enum class BlobType
 {
@@ -34,7 +36,9 @@ enum class BlobType
   GCZ,
   CISO,
   WBFS,
-  TGC
+  TGC,
+  WIA,
+  RVZ,
 };
 
 class BlobReader
@@ -172,5 +176,9 @@ bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path,
 bool ConvertToPlain(BlobReader* infile, const std::string& infile_path,
                     const std::string& outfile_path, CompressCB callback = nullptr,
                     void* arg = nullptr);
+bool ConvertToWIAOrRVZ(BlobReader* infile, const std::string& infile_path,
+                       const std::string& outfile_path, bool rvz,
+                       WIARVZCompressionType compression_type, int compression_level,
+                       int chunk_size, CompressCB callback = nullptr, void* arg = nullptr);
 
 }  // namespace DiscIO

Source/Core/DiscIO/CMakeLists.txt

@@ -21,6 +21,8 @@ add_library(discio
   FileSystemGCWii.h
   Filesystem.cpp
   Filesystem.h
+  LaggedFibonacciGenerator.cpp
+  LaggedFibonacciGenerator.h
   MultithreadedCompressor.h
   NANDImporter.cpp
   NANDImporter.h
@@ -42,6 +44,10 @@ add_library(discio
   VolumeWii.h
   WbfsBlob.cpp
   WbfsBlob.h
+  WIABlob.cpp
+  WIABlob.h
+  WIACompression.cpp
+  WIACompression.h
   WiiEncryptionCache.cpp
   WiiEncryptionCache.h
   WiiSaveBanner.cpp
@@ -49,6 +55,11 @@ add_library(discio
 )
 
 target_link_libraries(discio
+PUBLIC
+  BZip2::BZip2
+  LibLZMA::LibLZMA
+  zstd
+
 PRIVATE
   minizip
   pugixml

Source/Core/DiscIO/DiscIO.vcxproj

@@ -55,6 +55,7 @@
     <ClCompile Include="FileBlob.cpp" />
     <ClCompile Include="Filesystem.cpp" />
     <ClCompile Include="FileSystemGCWii.cpp" />
+    <ClCompile Include="LaggedFibonacciGenerator.cpp" />
     <ClCompile Include="NANDImporter.cpp" />
     <ClCompile Include="ScrubbedBlob.cpp" />
     <ClCompile Include="TGCBlob.cpp" />
@@ -65,6 +66,8 @@
     <ClCompile Include="VolumeWad.cpp" />
     <ClCompile Include="VolumeWii.cpp" />
     <ClCompile Include="WbfsBlob.cpp" />
+    <ClCompile Include="WIABlob.cpp" />
+    <ClCompile Include="WIACompression.cpp" />
     <ClCompile Include="WiiEncryptionCache.cpp" />
     <ClCompile Include="WiiSaveBanner.cpp" />
   </ItemGroup>
@@ -80,6 +83,7 @@
     <ClInclude Include="FileBlob.h" />
     <ClInclude Include="Filesystem.h" />
     <ClInclude Include="FileSystemGCWii.h" />
+    <ClInclude Include="LaggedFibonacciGenerator.h" />
     <ClInclude Include="MultithreadedCompressor.h" />
     <ClInclude Include="NANDImporter.h" />
     <ClInclude Include="ScrubbedBlob.h" />
@@ -91,6 +95,8 @@
     <ClInclude Include="VolumeWad.h" />
     <ClInclude Include="VolumeWii.h" />
     <ClInclude Include="WbfsBlob.h" />
+    <ClInclude Include="WIABlob.h" />
+    <ClInclude Include="WIACompression.h" />
     <ClInclude Include="WiiEncryptionCache.h" />
     <ClInclude Include="WiiSaveBanner.h" />
   </ItemGroup>
@@ -110,6 +116,15 @@
     <ProjectReference Include="$(ExternalsDir)pugixml\pugixml.vcxproj">
       <Project>{38fee76f-f347-484b-949c-b4649381cffb}</Project>
     </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)bzip2\bzip2.vcxproj">
+      <Project>{055a775f-b4f5-4970-9240-f6cf7661f37b}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)liblzma\liblzma.vcxproj">
+      <Project>{1d8c51d2-ffa4-418e-b183-9f42b6a6717e}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(ExternalsDir)zstd\zstd.vcxproj">
+      <Project>{1bea10f3-80ce-4bc4-9331-5769372cdf99}</Project>
+    </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

Source/Core/DiscIO/DiscIO.vcxproj.filters

@@ -90,6 +90,15 @@
     <ClCompile Include="ScrubbedBlob.cpp">
       <Filter>Volume\Blob</Filter>
     </ClCompile>
+    <ClCompile Include="WIABlob.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
+    <ClCompile Include="LaggedFibonacciGenerator.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
+    <ClCompile Include="WIACompression.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="DiscScrubber.h">
@@ -164,6 +173,15 @@
     <ClInclude Include="MultithreadedCompressor.h">
       <Filter>Volume\Blob</Filter>
     </ClInclude>
+    <ClInclude Include="WIABlob.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
+    <ClInclude Include="LaggedFibonacciGenerator.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
+    <ClInclude Include="WIACompression.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Text Include="CMakeLists.txt" />

Source/Core/DiscIO/LaggedFibonacciGenerator.cpp

@@ -0,0 +1,212 @@
+// This file is under the public domain.
+
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+
+#include "Common/Align.h"
+#include "Common/Assert.h"
+#include "Common/CommonTypes.h"
+#include "Common/Swap.h"
+
+namespace DiscIO
+{
+void LaggedFibonacciGenerator::SetSeed(const u32 seed[SEED_SIZE])
+{
+  SetSeed(reinterpret_cast<const u8*>(seed));
+}
+
+void LaggedFibonacciGenerator::SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)])
+{
+  m_position_bytes = 0;
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    m_buffer[i] = Common::swap32(seed + i * sizeof(u32));
+
+  Initialize(false);
+}
+
+size_t LaggedFibonacciGenerator::GetSeed(const u8* data, size_t size, size_t data_offset,
+                                         u32 seed_out[SEED_SIZE])
+{
+  if ((reinterpret_cast<uintptr_t>(data) - data_offset) % alignof(u32) != 0)
+  {
+    ASSERT(false);
+    return 0;
+  }
+
+  // For code simplicity, only include whole u32 words when regenerating the seed. It would be
+  // possible to get rid of this restriction and use a few additional bytes, but it's probably more
+  // effort than it's worth considering that junk data often starts or ends on 4-byte offsets.
+  const size_t bytes_to_skip = Common::AlignUp(data_offset, sizeof(u32)) - data_offset;
+  const u32* u32_data = reinterpret_cast<const u32*>(data + bytes_to_skip);
+  const size_t u32_size = (size - bytes_to_skip) / sizeof(u32);
+  const size_t u32_data_offset = (data_offset + bytes_to_skip) / sizeof(u32);
+
+  LaggedFibonacciGenerator lfg;
+  if (!GetSeed(u32_data, u32_size, u32_data_offset, &lfg, seed_out))
+    return false;
+
+  lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32));
+
+  const u8* end = data + size;
+  size_t reconstructed_bytes = 0;
+  while (data < end && lfg.GetByte() == *data)
+  {
+    ++reconstructed_bytes;
+    ++data;
+  }
+  return reconstructed_bytes;
+}
+
+bool LaggedFibonacciGenerator::GetSeed(const u32* data, size_t size, size_t data_offset,
+                                       LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE])
+{
+  if (size < LFG_K)
+    return false;
+
+  // If the data doesn't look like something we can regenerate, return early to save time
+  if (!std::all_of(data, data + LFG_K, [](u32 x) {
+        return (Common::swap32(x) & 0x00C00000) == (Common::swap32(x) >> 2 & 0x00C00000);
+      }))
+  {
+    return false;
+  }
+
+  const size_t data_offset_mod_k = data_offset % LFG_K;
+  const size_t data_offset_div_k = data_offset / LFG_K;
+
+  std::copy(data, data + LFG_K - data_offset_mod_k, lfg->m_buffer.data() + data_offset_mod_k);
+  std::copy(data + LFG_K - data_offset_mod_k, data + LFG_K, lfg->m_buffer.data());
+
+  lfg->Backward(0, data_offset_mod_k);
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Backward();
+
+  if (!lfg->Reinitialize(seed_out))
+    return false;
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Forward();
+
+  return true;
+}
+
+void LaggedFibonacciGenerator::GetBytes(size_t count, u8* out)
+{
+  while (count > 0)
+  {
+    const size_t length = std::min(count, LFG_K * sizeof(u32) - m_position_bytes);
+
+    std::memcpy(out, reinterpret_cast<u8*>(m_buffer.data()) + m_position_bytes, length);
+
+    m_position_bytes += length;
+    count -= length;
+    out += length;
+
+    if (m_position_bytes == LFG_K * sizeof(u32))
+    {
+      Forward();
+      m_position_bytes = 0;
+    }
+  }
+}
+
+u8 LaggedFibonacciGenerator::GetByte()
+{
+  const u8 result = reinterpret_cast<u8*>(m_buffer.data())[m_position_bytes];
+
+  ++m_position_bytes;
+
+  if (m_position_bytes == LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes = 0;
+  }
+
+  return result;
+}
+
+void LaggedFibonacciGenerator::Forward(size_t count)
+{
+  m_position_bytes += count;
+  while (m_position_bytes >= LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes -= LFG_K * sizeof(u32);
+  }
+}
+
+void LaggedFibonacciGenerator::Forward()
+{
+  for (size_t i = 0; i < LFG_J; ++i)
+    m_buffer[i] ^= m_buffer[i + LFG_K - LFG_J];
+
+  for (size_t i = LFG_J; i < LFG_K; ++i)
+    m_buffer[i] ^= m_buffer[i - LFG_J];
+}
+
+void LaggedFibonacciGenerator::Backward(size_t start_word, size_t end_word)
+{
+  const size_t loop_end = std::max(LFG_J, start_word);
+  for (size_t i = std::min(end_word, LFG_K); i > loop_end; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 - LFG_J];
+
+  for (size_t i = std::min(end_word, LFG_J); i > start_word; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 + LFG_K - LFG_J];
+}
+
+bool LaggedFibonacciGenerator::Reinitialize(u32 seed_out[SEED_SIZE])
+{
+  for (size_t i = 0; i < 4; ++i)
+    Backward();
+
+  for (u32& x : m_buffer)
+    x = Common::swap32(x);
+
+  // Reconstruct the bits which are missing due to the output code shifting by 18 instead of 16.
+  // Unfortunately we can't reconstruct bits 16 and 17 (counting LSB as 0) for the first word,
+  // but the observable result (when shifting by 18 instead of 16) is not affected by this.
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+  {
+    m_buffer[i] = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000) |
+                  ((m_buffer[i + 16] ^ m_buffer[i + 15]) << 9 & 0x00030000);
+  }
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    seed_out[i] = Common::swap32(m_buffer[i]);
+
+  return Initialize(true);
+}
+
+bool LaggedFibonacciGenerator::Initialize(bool check_existing_data)
+{
+  for (size_t i = SEED_SIZE; i < LFG_K; ++i)
+  {
+    const u32 calculated = (m_buffer[i - 17] << 23) ^ (m_buffer[i - 16] >> 9) ^ m_buffer[i - 1];
+
+    if (check_existing_data)
+    {
+      const u32 actual = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000);
+      if ((calculated & 0xFFFCFFFF) != actual)
+        return false;
+    }
+
+    m_buffer[i] = calculated;
+  }
+
+  // Instead of doing the "shift by 18 instead of 16" oddity when actually outputting the data,
+  // we can do the shifting (and byteswapping) at this point to make the output code simpler.
+  for (u32& x : m_buffer)
+    x = Common::swap32((x & 0xFF00FFFF) | ((x >> 2) & 0x00FF0000));
+
+  for (size_t i = 0; i < 4; ++i)
+    Forward();
+
+  return true;
+}
+
+}  // namespace DiscIO

Source/Core/DiscIO/LaggedFibonacciGenerator.h

@@ -0,0 +1,51 @@
+// This file is under the public domain.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+
+#include "Common/CommonTypes.h"
+
+namespace DiscIO
+{
+class LaggedFibonacciGenerator
+{
+public:
+  static constexpr size_t SEED_SIZE = 17;
+
+  // Reconstructs a seed and writes it to seed_out, then returns the number of bytes which can
+  // be reconstructed using that seed. Can return any number between 0 and size, inclusive.
+  // data - data_offset must be 4-byte aligned.
+  static size_t GetSeed(const u8* data, size_t size, size_t data_offset, u32 seed_out[SEED_SIZE]);
+
+  // SetSeed must be called before using the functions below
+  void SetSeed(const u32 seed[SEED_SIZE]);
+  void SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)]);
+
+  // Outputs a number of bytes and advances the internal state by the same amount.
+  void GetBytes(size_t count, u8* out);
+  u8 GetByte();
+
+  // Advances the internal state like GetBytes, but without outputting data. O(N), like GetBytes.
+  void Forward(size_t count);
+
+private:
+  static bool GetSeed(const u32* data, size_t size, size_t data_offset,
+                      LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE]);
+
+  void Forward();
+  void Backward(size_t start_word = 0, size_t end_word = LFG_K);
+
+  bool Reinitialize(u32 seed_out[SEED_SIZE]);
+  bool Initialize(bool check_existing_data);
+
+  static constexpr size_t LFG_K = 521;
+  static constexpr size_t LFG_J = 32;
+
+  std::array<u32, LFG_K> m_buffer;
+
+  size_t m_position_bytes = 0;
+};
+
+}  // namespace DiscIO

Source/Core/DiscIO/VolumeWii.cpp

@@ -201,18 +201,9 @@ bool VolumeWii::Read(u64 offset, u64 length, u8* buffer, const Partition& partit
       if (!m_reader->Read(block_offset_on_disc, BLOCK_TOTAL_SIZE, read_buffer.data()))
         return false;
 
-      // Decrypt the block's data.
-      // 0x3D0 - 0x3DF in read_buffer will be overwritten,
-      // but that won't affect anything, because we won't
-      // use the content of read_buffer anymore after this
-      mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_DATA_SIZE, &read_buffer[0x3D0],
-                            &read_buffer[BLOCK_HEADER_SIZE], m_last_decrypted_block_data);
+      // Decrypt the block's data
+      DecryptBlockData(read_buffer.data(), m_last_decrypted_block_data, aes_context);
       m_last_decrypted_block = block_offset_on_disc;
-
-      // The only thing we currently use from the 0x000 - 0x3FF part
-      // of the block is the IV (at 0x3D0), but it also contains SHA-1
-      // hashes that IOS uses to check that discs aren't tampered with.
-      // http://wiibrew.org/wiki/Wii_Disc#Encrypted
     }
 
     // Copy the decrypted data
@@ -482,14 +473,10 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encr
     return false;
 
   HashBlock hashes;
-  u8 iv[16] = {0};
-  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(HashBlock), iv,
-                        encrypted_data.data(), reinterpret_cast<u8*>(&hashes));
+  DecryptBlockHashes(encrypted_data.data(), &hashes, aes_context);
 
   u8 cluster_data[BLOCK_DATA_SIZE];
-  std::memcpy(iv, encrypted_data.data() + 0x3D0, 16);
-  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(cluster_data), iv,
-                        encrypted_data.data() + sizeof(HashBlock), cluster_data);
+  DecryptBlockData(encrypted_data.data(), cluster_data, aes_context);
 
   for (u32 hash_index = 0; hash_index < 31; ++hash_index)
   {
@@ -532,54 +519,33 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition)
   return CheckBlockIntegrity(block_index, cluster, partition);
 }
 
-bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
-                             u64 partition_data_decrypted_size,
-                             const std::array<u8, AES_KEY_SIZE>& key, BlobReader* blob,
-                             std::array<u8, GROUP_TOTAL_SIZE>* out)
+bool VolumeWii::HashGroup(const std::array<u8, BLOCK_DATA_SIZE> in[BLOCKS_PER_GROUP],
+                          HashBlock out[BLOCKS_PER_GROUP],
+                          const std::function<bool(size_t block)>& read_function)
 {
-  std::vector<std::array<u8, BLOCK_DATA_SIZE>> unencrypted_data(BLOCKS_PER_GROUP);
-  std::vector<HashBlock> unencrypted_hashes(BLOCKS_PER_GROUP);
-
   std::array<std::future<void>, BLOCKS_PER_GROUP> hash_futures;
-  bool error_occurred = false;
+  bool success = true;
 
   for (size_t i = 0; i < BLOCKS_PER_GROUP; ++i)
   {
-    if (!error_occurred)
-    {
-      if (offset + (i + 1) * BLOCK_DATA_SIZE <= partition_data_decrypted_size)
-      {
-        if (!blob->ReadWiiDecrypted(offset + i * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE,
-                                    unencrypted_data[i].data(), partition_data_offset))
-        {
-          error_occurred = true;
-        }
-      }
-      else
-      {
-        unencrypted_data[i].fill(0);
-      }
-    }
+    if (read_function && success)
+      success = read_function(i);
 
-    hash_futures[i] = std::async(std::launch::async, [&unencrypted_data, &unencrypted_hashes,
-                                                      &hash_futures, error_occurred, i]() {
+    hash_futures[i] = std::async(std::launch::async, [&in, &out, &hash_futures, success, i]() {
       const size_t h1_base = Common::AlignDown(i, 8);
 
-      if (!error_occurred)
+      if (success)
       {
         // H0 hashes
         for (size_t j = 0; j < 31; ++j)
-        {
-          mbedtls_sha1_ret(unencrypted_data[i].data() + j * 0x400, 0x400,
-                           unencrypted_hashes[i].h0[j]);
-        }
+          mbedtls_sha1_ret(in[i].data() + j * 0x400, 0x400, out[i].h0[j]);
 
         // H0 padding
-        std::memset(unencrypted_hashes[i].padding_0, 0, sizeof(HashBlock::padding_0));
+        std::memset(out[i].padding_0, 0, sizeof(HashBlock::padding_0));
 
         // H1 hash
-        mbedtls_sha1_ret(reinterpret_cast<u8*>(unencrypted_hashes[i].h0), sizeof(HashBlock::h0),
-                         unencrypted_hashes[h1_base].h1[i - h1_base]);
+        mbedtls_sha1_ret(reinterpret_cast<u8*>(out[i].h0), sizeof(HashBlock::h0),
+                         out[h1_base].h1[i - h1_base]);
       }
 
       if (i % 8 == 7)
@@ -587,21 +553,18 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
         for (size_t j = 0; j < 7; ++j)
           hash_futures[h1_base + j].get();
 
-        if (!error_occurred)
+        if (success)
         {
           // H1 padding
-          std::memset(unencrypted_hashes[h1_base].padding_1, 0, sizeof(HashBlock::padding_1));
+          std::memset(out[h1_base].padding_1, 0, sizeof(HashBlock::padding_1));
 
           // H1 copies
           for (size_t j = 1; j < 8; ++j)
-          {
-            std::memcpy(unencrypted_hashes[h1_base + j].h1, unencrypted_hashes[h1_base].h1,
-                        sizeof(HashBlock::h1));
-          }
+            std::memcpy(out[h1_base + j].h1, out[h1_base].h1, sizeof(HashBlock::h1));
 
           // H2 hash
-          mbedtls_sha1_ret(reinterpret_cast<u8*>(unencrypted_hashes[i].h1), sizeof(HashBlock::h1),
-                           unencrypted_hashes[0].h2[h1_base / 8]);
+          mbedtls_sha1_ret(reinterpret_cast<u8*>(out[i].h1), sizeof(HashBlock::h1),
+                           out[0].h2[h1_base / 8]);
         }
 
         if (i == BLOCKS_PER_GROUP - 1)
@@ -609,17 +572,14 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
           for (size_t j = 0; j < 7; ++j)
             hash_futures[j * 8 + 7].get();
 
-          if (!error_occurred)
+          if (success)
           {
             // H2 padding
-            std::memset(unencrypted_hashes[0].padding_2, 0, sizeof(HashBlock::padding_2));
+            std::memset(out[0].padding_2, 0, sizeof(HashBlock::padding_2));
 
             // H2 copies
             for (size_t j = 1; j < BLOCKS_PER_GROUP; ++j)
-            {
-              std::memcpy(unencrypted_hashes[j].h2, unencrypted_hashes[0].h2,
-                          sizeof(HashBlock::h2));
-            }
+              std::memcpy(out[j].h2, out[0].h2, sizeof(HashBlock::h2));
           }
         }
       }
@@ -629,9 +589,41 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
   // Wait for all the async tasks to finish
   hash_futures.back().get();
 
-  if (error_occurred)
+  return success;
+}
+
+bool VolumeWii::EncryptGroup(
+    u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size,
+    const std::array<u8, AES_KEY_SIZE>& key, BlobReader* blob,
+    std::array<u8, GROUP_TOTAL_SIZE>* out,
+    const std::function<void(HashBlock hash_blocks[BLOCKS_PER_GROUP])>& hash_exception_callback)
+{
+  std::vector<std::array<u8, BLOCK_DATA_SIZE>> unencrypted_data(BLOCKS_PER_GROUP);
+  std::vector<HashBlock> unencrypted_hashes(BLOCKS_PER_GROUP);
+
+  const bool success =
+      HashGroup(unencrypted_data.data(), unencrypted_hashes.data(), [&](size_t block) {
+        if (offset + (block + 1) * BLOCK_DATA_SIZE <= partition_data_decrypted_size)
+        {
+          if (!blob->ReadWiiDecrypted(offset + block * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE,
+                                      unencrypted_data[block].data(), partition_data_offset))
+          {
+            return false;
+          }
+        }
+        else
+        {
+          unencrypted_data[block].fill(0);
+        }
+        return true;
+      });
+
+  if (!success)
     return false;
 
+  if (hash_exception_callback)
+    hash_exception_callback(unencrypted_hashes.data());
+
   const unsigned int threads =
       std::min(BLOCKS_PER_GROUP, std::max<unsigned int>(1, std::thread::hardware_concurrency()));
 
@@ -667,4 +659,20 @@ bool VolumeWii::EncryptGroup(u64 offset, u64 partition_data_offset,
   return true;
 }
 
+void VolumeWii::DecryptBlockHashes(const u8* in, HashBlock* out, mbedtls_aes_context* aes_context)
+{
+  std::array<u8, 16> iv;
+  iv.fill(0);
+  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(HashBlock), iv.data(), in,
+                        reinterpret_cast<u8*>(out));
+}
+
+void VolumeWii::DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context)
+{
+  std::array<u8, 16> iv;
+  std::copy(&in[0x3d0], &in[0x3e0], iv.data());
+  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_DATA_SIZE, iv.data(),
+                        &in[BLOCK_HEADER_SIZE], out);
+}
+
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeWii.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <array>
+#include <functional>
 #include <map>
 #include <memory>
 #include <optional>
@@ -97,9 +98,22 @@ public:
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const;
 
+  // The in parameter can either contain all the data to begin with,
+  // or read_function can write data into the in parameter when called.
+  // The latter lets reading run in parallel with hashing.
+  // This function returns false iff read_function returns false.
+  static bool HashGroup(const std::array<u8, BLOCK_DATA_SIZE> in[BLOCKS_PER_GROUP],
+                        HashBlock out[BLOCKS_PER_GROUP],
+                        const std::function<bool(size_t block)>& read_function = {});
+
   static bool EncryptGroup(u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size,
                            const std::array<u8, AES_KEY_SIZE>& key, BlobReader* blob,
-                           std::array<u8, GROUP_TOTAL_SIZE>* out);
+                           std::array<u8, GROUP_TOTAL_SIZE>* out,
+                           const std::function<void(HashBlock hash_blocks[BLOCKS_PER_GROUP])>&
+                               hash_exception_callback = {});
+
+  static void DecryptBlockHashes(const u8* in, HashBlock* out, mbedtls_aes_context* aes_context);
+  static void DecryptBlockData(const u8* in, u8* out, mbedtls_aes_context* aes_context);
 
 protected:
   u32 GetOffsetShift() const override { return 2; }

Source/Core/DiscIO/WIABlob.h

@@ -0,0 +1,390 @@
+// Copyright 2018 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <array>
+#include <limits>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <type_traits>
+#include <utility>
+
+#include "Common/CommonTypes.h"
+#include "Common/File.h"
+#include "Common/Swap.h"
+#include "DiscIO/Blob.h"
+#include "DiscIO/MultithreadedCompressor.h"
+#include "DiscIO/WIACompression.h"
+#include "DiscIO/WiiEncryptionCache.h"
+
+namespace DiscIO
+{
+class FileSystem;
+class VolumeDisc;
+
+enum class WIARVZCompressionType : u32
+{
+  None = 0,
+  Purge = 1,
+  Bzip2 = 2,
+  LZMA = 3,
+  LZMA2 = 4,
+  Zstd = 5,
+};
+
+std::pair<int, int> GetAllowedCompressionLevels(WIARVZCompressionType compression_type);
+
+constexpr u32 WIA_MAGIC = 0x01414957;  // "WIA\x1" (byteswapped to little endian)
+constexpr u32 RVZ_MAGIC = 0x015A5652;  // "RVZ\x1" (byteswapped to little endian)
+
+template <bool RVZ>
+class WIARVZFileReader : public BlobReader
+{
+public:
+  ~WIARVZFileReader();
+
+  static std::unique_ptr<WIARVZFileReader> Create(File::IOFile file, const std::string& path);
+
+  BlobType GetBlobType() const override;
+
+  u64 GetRawSize() const override { return Common::swap64(m_header_1.wia_file_size); }
+  u64 GetDataSize() const override { return Common::swap64(m_header_1.iso_file_size); }
+  bool IsDataSizeAccurate() const override { return true; }
+
+  u64 GetBlockSize() const override { return Common::swap32(m_header_2.chunk_size); }
+  bool HasFastRandomAccessInBlock() const override { return false; }
+
+  bool Read(u64 offset, u64 size, u8* out_ptr) override;
+  bool SupportsReadWiiDecrypted() const override;
+  bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override;
+
+  static ConversionResultCode Convert(BlobReader* infile, const VolumeDisc* infile_volume,
+                                      File::IOFile* outfile, WIARVZCompressionType compression_type,
+                                      int compression_level, int chunk_size, CompressCB callback,
+                                      void* arg);
+
+private:
+  using SHA1 = std::array<u8, 20>;
+  using WiiKey = std::array<u8, 16>;
+
+  // See docs/WIA.md for details about the format
+
+#pragma pack(push, 1)
+  struct WIAHeader1
+  {
+    u32 magic;
+    u32 version;
+    u32 version_compatible;
+    u32 header_2_size;
+    SHA1 header_2_hash;
+    u64 iso_file_size;
+    u64 wia_file_size;
+    SHA1 header_1_hash;
+  };
+  static_assert(sizeof(WIAHeader1) == 0x48, "Wrong size for WIA header 1");
+
+  struct WIAHeader2
+  {
+    u32 disc_type;
+    u32 compression_type;
+    u32 compression_level;  // Informative only
+    u32 chunk_size;
+
+    std::array<u8, 0x80> disc_header;
+
+    u32 number_of_partition_entries;
+    u32 partition_entry_size;
+    u64 partition_entries_offset;
+    SHA1 partition_entries_hash;
+
+    u32 number_of_raw_data_entries;
+    u64 raw_data_entries_offset;
+    u32 raw_data_entries_size;
+
+    u32 number_of_group_entries;
+    u64 group_entries_offset;
+    u32 group_entries_size;
+
+    u8 compressor_data_size;
+    u8 compressor_data[7];
+  };
+  static_assert(sizeof(WIAHeader2) == 0xdc, "Wrong size for WIA header 2");
+
+  struct PartitionDataEntry
+  {
+    u32 first_sector;
+    u32 number_of_sectors;
+    u32 group_index;
+    u32 number_of_groups;
+  };
+  static_assert(sizeof(PartitionDataEntry) == 0x10, "Wrong size for WIA partition data entry");
+
+  struct PartitionEntry
+  {
+    WiiKey partition_key;
+    std::array<PartitionDataEntry, 2> data_entries;
+  };
+  static_assert(sizeof(PartitionEntry) == 0x30, "Wrong size for WIA partition entry");
+
+  struct RawDataEntry
+  {
+    u64 data_offset;
+    u64 data_size;
+    u32 group_index;
+    u32 number_of_groups;
+  };
+  static_assert(sizeof(RawDataEntry) == 0x18, "Wrong size for WIA raw data entry");
+
+  struct WIAGroupEntry
+  {
+    u32 data_offset;  // >> 2
+    u32 data_size;
+  };
+  static_assert(sizeof(WIAGroupEntry) == 0x08, "Wrong size for WIA group entry");
+
+  struct RVZGroupEntry
+  {
+    u32 data_offset;  // >> 2
+    u32 data_size;
+    u32 rvz_packed_size;
+  };
+  static_assert(sizeof(RVZGroupEntry) == 0x0c, "Wrong size for RVZ group entry");
+
+  using GroupEntry = std::conditional_t<RVZ, RVZGroupEntry, WIAGroupEntry>;
+
+  struct HashExceptionEntry
+  {
+    u16 offset;
+    SHA1 hash;
+  };
+  static_assert(sizeof(HashExceptionEntry) == 0x16, "Wrong size for WIA hash exception entry");
+#pragma pack(pop)
+
+  struct DataEntry
+  {
+    u32 index;
+    bool is_partition;
+    u8 partition_data_index;
+
+    DataEntry(size_t index_) : index(static_cast<u32>(index_)), is_partition(false) {}
+    DataEntry(size_t index_, size_t partition_data_index_)
+        : index(static_cast<u32>(index_)), is_partition(true),
+          partition_data_index(static_cast<u8>(partition_data_index_))
+    {
+    }
+  };
+
+  class Chunk
+  {
+  public:
+    Chunk();
+    Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
+          u32 exception_lists, bool compressed_exception_lists, u32 rvz_packed_size,
+          u64 data_offset, std::unique_ptr<Decompressor> decompressor);
+
+    bool Read(u64 offset, u64 size, u8* out_ptr);
+
+    // This can only be called once at least one byte of data has been read
+    void GetHashExceptions(std::vector<HashExceptionEntry>* exception_list,
+                           u64 exception_list_index, u16 additional_offset) const;
+
+    template <typename T>
+    bool ReadAll(std::vector<T>* vector)
+    {
+      return Read(0, vector->size() * sizeof(T), reinterpret_cast<u8*>(vector->data()));
+    }
+
+  private:
+    bool Decompress();
+    bool HandleExceptions(const u8* data, size_t bytes_allocated, size_t bytes_written,
+                          size_t* bytes_used, bool align);
+
+    DecompressionBuffer m_in;
+    DecompressionBuffer m_out;
+    size_t m_in_bytes_read = 0;
+
+    std::unique_ptr<Decompressor> m_decompressor = nullptr;
+    File::IOFile* m_file = nullptr;
+    u64 m_offset_in_file = 0;
+
+    size_t m_out_bytes_allocated_for_exceptions = 0;
+    size_t m_out_bytes_used_for_exceptions = 0;
+    size_t m_in_bytes_used_for_exceptions = 0;
+    u32 m_exception_lists = 0;
+    bool m_compressed_exception_lists = false;
+    u32 m_rvz_packed_size = 0;
+    u64 m_data_offset = 0;
+  };
+
+  explicit WIARVZFileReader(File::IOFile file, const std::string& path);
+  bool Initialize(const std::string& path);
+  bool HasDataOverlap() const;
+
+  bool ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chunk_size, u32 sector_size,
+                      u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups,
+                      u32 exception_lists);
+  Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
+                            WIARVZCompressionType compression_type, u32 exception_lists = 0,
+                            u32 rvz_packed_size = 0, u64 data_offset = 0);
+
+  static bool ApplyHashExceptions(const std::vector<HashExceptionEntry>& exception_list,
+                                  VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]);
+
+  static std::string VersionToString(u32 version);
+
+  struct ReuseID
+  {
+    bool operator==(const ReuseID& other) const
+    {
+      return std::tie(partition_key, data_size, encrypted, value) ==
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
+    }
+    bool operator<(const ReuseID& other) const
+    {
+      return std::tie(partition_key, data_size, encrypted, value) <
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
+    }
+    bool operator>(const ReuseID& other) const
+    {
+      return std::tie(partition_key, data_size, encrypted, value) >
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
+    }
+    bool operator!=(const ReuseID& other) const { return !operator==(other); }
+    bool operator>=(const ReuseID& other) const { return !operator<(other); }
+    bool operator<=(const ReuseID& other) const { return !operator>(other); }
+
+    const WiiKey* partition_key;
+    u64 data_size;
+    bool encrypted;
+    u8 value;
+  };
+
+  struct CompressThreadState
+  {
+    using WiiBlockData = std::array<u8, VolumeWii::BLOCK_DATA_SIZE>;
+
+    std::unique_ptr<Compressor> compressor;
+
+    std::vector<WiiBlockData> decryption_buffer =
+        std::vector<WiiBlockData>(VolumeWii::BLOCKS_PER_GROUP);
+
+    std::vector<VolumeWii::HashBlock> hash_buffer =
+        std::vector<VolumeWii::HashBlock>(VolumeWii::BLOCKS_PER_GROUP);
+  };
+
+  struct CompressParameters
+  {
+    std::vector<u8> data;
+    const DataEntry* data_entry;
+    u64 data_offset;
+    u64 bytes_read;
+    size_t group_index;
+  };
+
+  struct WIAOutputParametersEntry
+  {
+    std::vector<u8> exception_lists;
+    std::vector<u8> main_data;
+    std::optional<ReuseID> reuse_id;
+    std::optional<GroupEntry> reused_group;
+  };
+
+  struct RVZOutputParametersEntry
+  {
+    std::vector<u8> exception_lists;
+    std::vector<u8> main_data;
+    std::optional<ReuseID> reuse_id;
+    std::optional<GroupEntry> reused_group;
+    size_t rvz_packed_size = 0;
+    bool compressed = false;
+  };
+
+  using OutputParametersEntry =
+      std::conditional_t<RVZ, RVZOutputParametersEntry, WIAOutputParametersEntry>;
+
+  struct OutputParameters
+  {
+    std::vector<OutputParametersEntry> entries;
+    u64 bytes_read;
+    size_t group_index;
+  };
+
+  static bool PadTo4(File::IOFile* file, u64* bytes_written);
+  static void AddRawDataEntry(u64 offset, u64 size, int chunk_size, u32* total_groups,
+                              std::vector<RawDataEntry>* raw_data_entries,
+                              std::vector<DataEntry>* data_entries);
+  static PartitionDataEntry
+  CreatePartitionDataEntry(u64 offset, u64 size, u32 index, int chunk_size, u32* total_groups,
+                           const std::vector<PartitionEntry>& partition_entries,
+                           std::vector<DataEntry>* data_entries);
+  static ConversionResultCode SetUpDataEntriesForWriting(
+      const VolumeDisc* volume, int chunk_size, u64 iso_size, u32* total_groups,
+      std::vector<PartitionEntry>* partition_entries, std::vector<RawDataEntry>* raw_data_entries,
+      std::vector<DataEntry>* data_entries, std::vector<const FileSystem*>* partition_file_systems);
+  static std::optional<std::vector<u8>> Compress(Compressor* compressor, const u8* data,
+                                                 size_t size);
+  static bool WriteHeader(File::IOFile* file, const u8* data, size_t size, u64 upper_bound,
+                          u64* bytes_written, u64* offset_out);
+
+  static void SetUpCompressor(std::unique_ptr<Compressor>* compressor,
+                              WIARVZCompressionType compression_type, int compression_level,
+                              WIAHeader2* header_2);
+  static bool TryReuse(std::map<ReuseID, GroupEntry>* reusable_groups,
+                       std::mutex* reusable_groups_mutex, OutputParametersEntry* entry);
+  static ConversionResult<OutputParameters>
+  ProcessAndCompress(CompressThreadState* state, CompressParameters parameters,
+                     const std::vector<PartitionEntry>& partition_entries,
+                     const std::vector<DataEntry>& data_entries, const FileSystem* file_system,
+                     std::map<ReuseID, GroupEntry>* reusable_groups,
+                     std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group,
+                     u64 exception_lists_per_chunk, bool compressed_exception_lists,
+                     bool compression);
+  static ConversionResultCode Output(std::vector<OutputParametersEntry>* entries,
+                                     File::IOFile* outfile,
+                                     std::map<ReuseID, GroupEntry>* reusable_groups,
+                                     std::mutex* reusable_groups_mutex, GroupEntry* group_entry,
+                                     u64* bytes_written);
+  static ConversionResultCode RunCallback(size_t groups_written, u64 bytes_read, u64 bytes_written,
+                                          u32 total_groups, u64 iso_size, CompressCB callback,
+                                          void* arg);
+
+  bool m_valid;
+  WIARVZCompressionType m_compression_type;
+
+  File::IOFile m_file;
+  Chunk m_cached_chunk;
+  u64 m_cached_chunk_offset = std::numeric_limits<u64>::max();
+  WiiEncryptionCache m_encryption_cache;
+
+  std::vector<HashExceptionEntry> m_exception_list;
+  bool m_write_to_exception_list = false;
+  u64 m_exception_list_last_group_index;
+
+  WIAHeader1 m_header_1;
+  WIAHeader2 m_header_2;
+  std::vector<PartitionEntry> m_partition_entries;
+  std::vector<RawDataEntry> m_raw_data_entries;
+  std::vector<GroupEntry> m_group_entries;
+
+  std::map<u64, DataEntry> m_data_entries;
+
+  // Perhaps we could set WIA_VERSION_WRITE_COMPATIBLE to 0.9, but WIA version 0.9 was never in
+  // any official release of wit, and interim versions (either source or binaries) are hard to find.
+  // Since we've been unable to check if we're write compatible with 0.9, we set it 1.0 to be safe.
+
+  static constexpr u32 WIA_VERSION = 0x01000000;
+  static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000;
+  static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000;
+
+  static constexpr u32 RVZ_VERSION = 0x01000000;
+  static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00030000;
+  static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00030000;
+};
+
+using WIAFileReader = WIARVZFileReader<false>;
+using RVZFileReader = WIARVZFileReader<true>;
+
+}  // namespace DiscIO

Source/Core/DiscIO/WIACompression.cpp

@@ -0,0 +1,810 @@
+// Copyright 2020 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#include "DiscIO/WIACompression.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include <bzlib.h>
+#include <lzma.h>
+#include <mbedtls/sha1.h>
+#include <zstd.h>
+
+#include "Common/Assert.h"
+#include "Common/CommonTypes.h"
+#include "Common/Swap.h"
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+namespace DiscIO
+{
+static u32 LZMA2DictionarySize(u8 p)
+{
+  return (static_cast<u32>(2) | (p & 1)) << (p / 2 + 11);
+}
+
+Decompressor::~Decompressor() = default;
+
+bool NoneDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                                  size_t* in_bytes_read)
+{
+  const size_t length =
+      std::min(in.bytes_written - *in_bytes_read, out->data.size() - out->bytes_written);
+
+  std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read, length);
+
+  *in_bytes_read += length;
+  out->bytes_written += length;
+
+  m_done = in.data.size() == *in_bytes_read;
+  return true;
+}
+
+PurgeDecompressor::PurgeDecompressor(u64 decompressed_size) : m_decompressed_size(decompressed_size)
+{
+  mbedtls_sha1_init(&m_sha1_context);
+}
+
+bool PurgeDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                                   size_t* in_bytes_read)
+{
+  if (!m_started)
+  {
+    mbedtls_sha1_starts_ret(&m_sha1_context);
+
+    // Include the exception lists in the SHA-1 calculation (but not in the compression...)
+    mbedtls_sha1_update_ret(&m_sha1_context, in.data.data(), *in_bytes_read);
+
+    m_started = true;
+  }
+
+  while (!m_done && in.bytes_written != *in_bytes_read &&
+         (m_segment_bytes_written < sizeof(m_segment) || out->data.size() != out->bytes_written))
+  {
+    if (m_segment_bytes_written == 0 && *in_bytes_read == in.data.size() - sizeof(SHA1))
+    {
+      const size_t zeroes_to_write = std::min<size_t>(m_decompressed_size - m_out_bytes_written,
+                                                      out->data.size() - out->bytes_written);
+
+      std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write);
+
+      out->bytes_written += zeroes_to_write;
+      m_out_bytes_written += zeroes_to_write;
+
+      if (m_out_bytes_written == m_decompressed_size && in.bytes_written == in.data.size())
+      {
+        SHA1 actual_hash;
+        mbedtls_sha1_finish_ret(&m_sha1_context, actual_hash.data());
+
+        SHA1 expected_hash;
+        std::memcpy(expected_hash.data(), in.data.data() + *in_bytes_read, expected_hash.size());
+
+        *in_bytes_read += expected_hash.size();
+        m_done = true;
+
+        if (actual_hash != expected_hash)
+          return false;
+      }
+
+      return true;
+    }
+
+    if (m_segment_bytes_written < sizeof(m_segment))
+    {
+      const size_t bytes_to_copy =
+          std::min(in.bytes_written - *in_bytes_read, sizeof(m_segment) - m_segment_bytes_written);
+
+      std::memcpy(reinterpret_cast<u8*>(&m_segment) + m_segment_bytes_written,
+                  in.data.data() + *in_bytes_read, bytes_to_copy);
+      mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy);
+
+      *in_bytes_read += bytes_to_copy;
+      m_bytes_read += bytes_to_copy;
+      m_segment_bytes_written += bytes_to_copy;
+    }
+
+    if (m_segment_bytes_written < sizeof(m_segment))
+      return true;
+
+    const size_t offset = Common::swap32(m_segment.offset);
+    const size_t size = Common::swap32(m_segment.size);
+
+    if (m_out_bytes_written < offset)
+    {
+      const size_t zeroes_to_write =
+          std::min(offset - m_out_bytes_written, out->data.size() - out->bytes_written);
+
+      std::memset(out->data.data() + out->bytes_written, 0, zeroes_to_write);
+
+      out->bytes_written += zeroes_to_write;
+      m_out_bytes_written += zeroes_to_write;
+    }
+
+    if (m_out_bytes_written >= offset && m_out_bytes_written < offset + size)
+    {
+      const size_t bytes_to_copy = std::min(
+          std::min(offset + size - m_out_bytes_written, out->data.size() - out->bytes_written),
+          in.bytes_written - *in_bytes_read);
+
+      std::memcpy(out->data.data() + out->bytes_written, in.data.data() + *in_bytes_read,
+                  bytes_to_copy);
+      mbedtls_sha1_update_ret(&m_sha1_context, in.data.data() + *in_bytes_read, bytes_to_copy);
+
+      *in_bytes_read += bytes_to_copy;
+      m_bytes_read += bytes_to_copy;
+      out->bytes_written += bytes_to_copy;
+      m_out_bytes_written += bytes_to_copy;
+    }
+
+    if (m_out_bytes_written >= offset + size)
+      m_segment_bytes_written = 0;
+  }
+
+  return true;
+}
+
+Bzip2Decompressor::~Bzip2Decompressor()
+{
+  if (m_started)
+    BZ2_bzDecompressEnd(&m_stream);
+}
+
+bool Bzip2Decompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                                   size_t* in_bytes_read)
+{
+  if (!m_started)
+  {
+    if (BZ2_bzDecompressInit(&m_stream, 0, 0) != BZ_OK)
+      return false;
+
+    m_started = true;
+  }
+
+  constexpr auto clamped_cast = [](size_t x) {
+    return static_cast<unsigned int>(
+        std::min<size_t>(std::numeric_limits<unsigned int>().max(), x));
+  };
+
+  char* const in_ptr = reinterpret_cast<char*>(const_cast<u8*>(in.data.data() + *in_bytes_read));
+  m_stream.next_in = in_ptr;
+  m_stream.avail_in = clamped_cast(in.bytes_written - *in_bytes_read);
+
+  char* const out_ptr = reinterpret_cast<char*>(out->data.data() + out->bytes_written);
+  m_stream.next_out = out_ptr;
+  m_stream.avail_out = clamped_cast(out->data.size() - out->bytes_written);
+
+  const int result = BZ2_bzDecompress(&m_stream);
+
+  *in_bytes_read += m_stream.next_in - in_ptr;
+  out->bytes_written += m_stream.next_out - out_ptr;
+
+  m_done = result == BZ_STREAM_END;
+  return result == BZ_OK || result == BZ_STREAM_END;
+}
+
+LZMADecompressor::LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size)
+{
+  m_options.preset_dict = nullptr;
+
+  if (!lzma2 && filter_options_size == 5)
+  {
+    // The dictionary size is stored as a 32-bit little endian unsigned integer
+    static_assert(sizeof(m_options.dict_size) == sizeof(u32));
+    std::memcpy(&m_options.dict_size, filter_options + 1, sizeof(u32));
+
+    const u8 d = filter_options[0];
+    if (d >= (9 * 5 * 5))
+    {
+      m_error_occurred = true;
+    }
+    else
+    {
+      m_options.lc = d % 9;
+      const u8 e = d / 9;
+      m_options.pb = e / 5;
+      m_options.lp = e % 5;
+    }
+  }
+  else if (lzma2 && filter_options_size == 1)
+  {
+    const u8 d = filter_options[0];
+    if (d > 40)
+      m_error_occurred = true;
+    else
+      m_options.dict_size = d == 40 ? 0xFFFFFFFF : LZMA2DictionarySize(d);
+  }
+  else
+  {
+    m_error_occurred = true;
+  }
+
+  m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1;
+  m_filters[0].options = &m_options;
+  m_filters[1].id = LZMA_VLI_UNKNOWN;
+  m_filters[1].options = nullptr;
+}
+
+LZMADecompressor::~LZMADecompressor()
+{
+  if (m_started)
+    lzma_end(&m_stream);
+}
+
+bool LZMADecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                                  size_t* in_bytes_read)
+{
+  if (!m_started)
+  {
+    if (m_error_occurred || lzma_raw_decoder(&m_stream, m_filters) != LZMA_OK)
+      return false;
+
+    m_started = true;
+  }
+
+  const u8* const in_ptr = in.data.data() + *in_bytes_read;
+  m_stream.next_in = in_ptr;
+  m_stream.avail_in = in.bytes_written - *in_bytes_read;
+
+  u8* const out_ptr = out->data.data() + out->bytes_written;
+  m_stream.next_out = out_ptr;
+  m_stream.avail_out = out->data.size() - out->bytes_written;
+
+  const lzma_ret result = lzma_code(&m_stream, LZMA_RUN);
+
+  *in_bytes_read += m_stream.next_in - in_ptr;
+  out->bytes_written += m_stream.next_out - out_ptr;
+
+  m_done = result == LZMA_STREAM_END;
+  return result == LZMA_OK || result == LZMA_STREAM_END;
+}
+
+ZstdDecompressor::ZstdDecompressor()
+{
+  m_stream = ZSTD_createDStream();
+}
+
+ZstdDecompressor::~ZstdDecompressor()
+{
+  ZSTD_freeDStream(m_stream);
+}
+
+bool ZstdDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                                  size_t* in_bytes_read)
+{
+  if (!m_stream)
+    return false;
+
+  ZSTD_inBuffer in_buffer{in.data.data(), in.bytes_written, *in_bytes_read};
+  ZSTD_outBuffer out_buffer{out->data.data(), out->data.size(), out->bytes_written};
+
+  const size_t result = ZSTD_decompressStream(m_stream, &out_buffer, &in_buffer);
+
+  *in_bytes_read = in_buffer.pos;
+  out->bytes_written = out_buffer.pos;
+
+  m_done = result == 0;
+  return !ZSTD_isError(result);
+}
+
+RVZPackDecompressor::RVZPackDecompressor(std::unique_ptr<Decompressor> decompressor,
+                                         DecompressionBuffer decompressed, u64 data_offset,
+                                         u32 rvz_packed_size)
+    : m_decompressor(std::move(decompressor)), m_decompressed(std::move(decompressed)),
+      m_data_offset(data_offset), m_rvz_packed_size(rvz_packed_size)
+{
+  m_bytes_read = m_decompressed.bytes_written;
+}
+
+bool RVZPackDecompressor::IncrementBytesRead(size_t x)
+{
+  m_bytes_read += x;
+  return m_bytes_read <= m_rvz_packed_size;
+}
+
+std::optional<bool> RVZPackDecompressor::ReadToDecompressed(const DecompressionBuffer& in,
+                                                            size_t* in_bytes_read,
+                                                            size_t decompressed_bytes_read,
+                                                            size_t bytes_to_read)
+{
+  if (m_decompressed.data.size() < decompressed_bytes_read + bytes_to_read)
+    m_decompressed.data.resize(decompressed_bytes_read + bytes_to_read);
+
+  if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read)
+  {
+    const size_t prev_bytes_written = m_decompressed.bytes_written;
+
+    if (!m_decompressor->Decompress(in, &m_decompressed, in_bytes_read))
+      return false;
+
+    if (!IncrementBytesRead(m_decompressed.bytes_written - prev_bytes_written))
+      return false;
+
+    if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read)
+      return true;
+  }
+
+  return std::nullopt;
+}
+
+bool RVZPackDecompressor::Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                                     size_t* in_bytes_read)
+{
+  while (out->data.size() != out->bytes_written && !Done())
+  {
+    if (m_size == 0)
+    {
+      if (m_decompressed.bytes_written == m_decompressed_bytes_read)
+      {
+        m_decompressed.data.resize(sizeof(u32));
+        m_decompressed.bytes_written = 0;
+        m_decompressed_bytes_read = 0;
+      }
+
+      std::optional<bool> result =
+          ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read, sizeof(u32));
+      if (result)
+        return *result;
+
+      m_size = Common::swap32(m_decompressed.data.data() + m_decompressed_bytes_read);
+
+      m_junk = m_size & 0x80000000;
+      if (m_junk)
+      {
+        m_size &= 0x7FFFFFFF;
+
+        constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32);
+        constexpr size_t BLOCK_SIZE = 0x8000;
+
+        result = ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read + sizeof(u32),
+                                    SEED_SIZE);
+        if (result)
+          return *result;
+
+        m_lfg.SetSeed(m_decompressed.data.data() + m_decompressed_bytes_read + sizeof(u32));
+        m_lfg.Forward(m_data_offset % BLOCK_SIZE);
+
+        m_decompressed_bytes_read += SEED_SIZE;
+      }
+
+      m_decompressed_bytes_read += sizeof(u32);
+    }
+
+    size_t bytes_to_write = std::min<size_t>(m_size, out->data.size() - out->bytes_written);
+    if (m_junk)
+    {
+      m_lfg.GetBytes(bytes_to_write, out->data.data() + out->bytes_written);
+      out->bytes_written += bytes_to_write;
+    }
+    else
+    {
+      if (m_decompressed.bytes_written != m_decompressed_bytes_read)
+      {
+        bytes_to_write =
+            std::min(bytes_to_write, m_decompressed.bytes_written - m_decompressed_bytes_read);
+
+        std::memcpy(out->data.data() + out->bytes_written,
+                    m_decompressed.data.data() + m_decompressed_bytes_read, bytes_to_write);
+
+        m_decompressed_bytes_read += bytes_to_write;
+        out->bytes_written += bytes_to_write;
+      }
+      else
+      {
+        const size_t prev_out_bytes_written = out->bytes_written;
+        const size_t old_out_size = out->data.size();
+        const size_t new_out_size = out->bytes_written + bytes_to_write;
+
+        if (new_out_size < old_out_size)
+          out->data.resize(new_out_size);
+
+        if (!m_decompressor->Decompress(in, out, in_bytes_read))
+          return false;
+
+        out->data.resize(old_out_size);
+
+        bytes_to_write = out->bytes_written - prev_out_bytes_written;
+
+        if (!IncrementBytesRead(bytes_to_write))
+          return false;
+
+        if (bytes_to_write == 0)
+          return true;
+      }
+    }
+
+    m_data_offset += bytes_to_write;
+    m_size -= static_cast<u32>(bytes_to_write);
+  }
+
+  // If out is full but not all data has been read from in, give the decompressor a chance to read
+  // from in anyway. This is needed for the case where zstd has read everything except the checksum.
+  if (out->data.size() == out->bytes_written && in.bytes_written != *in_bytes_read)
+  {
+    if (!m_decompressor->Decompress(in, out, in_bytes_read))
+      return false;
+  }
+
+  return true;
+}
+
+bool RVZPackDecompressor::Done() const
+{
+  return m_size == 0 && m_rvz_packed_size == m_bytes_read &&
+         m_decompressed.bytes_written == m_decompressed_bytes_read && m_decompressor->Done();
+}
+
+Compressor::~Compressor() = default;
+
+PurgeCompressor::PurgeCompressor()
+{
+  mbedtls_sha1_init(&m_sha1_context);
+}
+
+PurgeCompressor::~PurgeCompressor() = default;
+
+bool PurgeCompressor::Start()
+{
+  m_buffer.clear();
+  m_bytes_written = 0;
+
+  mbedtls_sha1_starts_ret(&m_sha1_context);
+
+  return true;
+}
+
+bool PurgeCompressor::AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size)
+{
+  mbedtls_sha1_update_ret(&m_sha1_context, data, size);
+  return true;
+}
+
+bool PurgeCompressor::Compress(const u8* data, size_t size)
+{
+  // We could add support for calling this twice if we're fine with
+  // making the code more complicated, but there's no need to support it
+  ASSERT_MSG(DISCIO, m_bytes_written == 0,
+             "Calling PurgeCompressor::Compress() twice is not supported");
+
+  m_buffer.resize(size + sizeof(PurgeSegment) + sizeof(SHA1));
+
+  size_t bytes_read = 0;
+
+  while (true)
+  {
+    const auto first_non_zero =
+        std::find_if(data + bytes_read, data + size, [](u8 x) { return x != 0; });
+
+    const u32 non_zero_data_start = static_cast<u32>(first_non_zero - data);
+    if (non_zero_data_start == size)
+      break;
+
+    size_t non_zero_data_end = non_zero_data_start;
+    size_t sequence_length = 0;
+    for (size_t i = non_zero_data_start; i < size; ++i)
+    {
+      if (data[i] == 0)
+      {
+        ++sequence_length;
+      }
+      else
+      {
+        sequence_length = 0;
+        non_zero_data_end = i + 1;
+      }
+
+      // To avoid wasting space, only count runs of zeroes that are of a certain length
+      // (unless there is nothing after the run of zeroes, then we might as well always count it)
+      if (sequence_length > sizeof(PurgeSegment))
+        break;
+    }
+
+    const u32 non_zero_data_length = static_cast<u32>(non_zero_data_end - non_zero_data_start);
+
+    const PurgeSegment segment{Common::swap32(non_zero_data_start),
+                               Common::swap32(non_zero_data_length)};
+    std::memcpy(m_buffer.data() + m_bytes_written, &segment, sizeof(segment));
+    m_bytes_written += sizeof(segment);
+
+    std::memcpy(m_buffer.data() + m_bytes_written, data + non_zero_data_start,
+                non_zero_data_length);
+    m_bytes_written += non_zero_data_length;
+
+    bytes_read = non_zero_data_end;
+  }
+
+  return true;
+}
+
+bool PurgeCompressor::End()
+{
+  mbedtls_sha1_update_ret(&m_sha1_context, m_buffer.data(), m_bytes_written);
+
+  mbedtls_sha1_finish_ret(&m_sha1_context, m_buffer.data() + m_bytes_written);
+  m_bytes_written += sizeof(SHA1);
+
+  ASSERT(m_bytes_written <= m_buffer.size());
+
+  return true;
+}
+
+const u8* PurgeCompressor::GetData() const
+{
+  return m_buffer.data();
+}
+
+size_t PurgeCompressor::GetSize() const
+{
+  return m_bytes_written;
+}
+
+Bzip2Compressor::Bzip2Compressor(int compression_level) : m_compression_level(compression_level)
+{
+}
+
+Bzip2Compressor::~Bzip2Compressor()
+{
+  BZ2_bzCompressEnd(&m_stream);
+}
+
+bool Bzip2Compressor::Start()
+{
+  ASSERT_MSG(DISCIO, m_stream.state == nullptr,
+             "Called Bzip2Compressor::Start() twice without calling Bzip2Compressor::End()");
+
+  m_buffer.clear();
+  m_stream.next_out = reinterpret_cast<char*>(m_buffer.data());
+
+  return BZ2_bzCompressInit(&m_stream, m_compression_level, 0, 0) == BZ_OK;
+}
+
+bool Bzip2Compressor::Compress(const u8* data, size_t size)
+{
+  m_stream.next_in = reinterpret_cast<char*>(const_cast<u8*>(data));
+  m_stream.avail_in = static_cast<unsigned int>(size);
+
+  ExpandBuffer(size);
+
+  while (m_stream.avail_in != 0)
+  {
+    if (m_stream.avail_out == 0)
+      ExpandBuffer(0x100);
+
+    if (BZ2_bzCompress(&m_stream, BZ_RUN) != BZ_RUN_OK)
+      return false;
+  }
+
+  return true;
+}
+
+bool Bzip2Compressor::End()
+{
+  bool success = true;
+
+  while (true)
+  {
+    if (m_stream.avail_out == 0)
+      ExpandBuffer(0x100);
+
+    const int result = BZ2_bzCompress(&m_stream, BZ_FINISH);
+    if (result != BZ_FINISH_OK && result != BZ_STREAM_END)
+      success = false;
+    if (result != BZ_FINISH_OK)
+      break;
+  }
+
+  if (BZ2_bzCompressEnd(&m_stream) != BZ_OK)
+    success = false;
+
+  return success;
+}
+
+void Bzip2Compressor::ExpandBuffer(size_t bytes_to_add)
+{
+  const size_t bytes_written = GetSize();
+  m_buffer.resize(m_buffer.size() + bytes_to_add);
+  m_stream.next_out = reinterpret_cast<char*>(m_buffer.data()) + bytes_written;
+  m_stream.avail_out = static_cast<unsigned int>(m_buffer.size() - bytes_written);
+}
+
+const u8* Bzip2Compressor::GetData() const
+{
+  return m_buffer.data();
+}
+
+size_t Bzip2Compressor::GetSize() const
+{
+  return static_cast<size_t>(reinterpret_cast<u8*>(m_stream.next_out) - m_buffer.data());
+}
+
+LZMACompressor::LZMACompressor(bool lzma2, int compression_level, u8 compressor_data_out[7],
+                               u8* compressor_data_size_out)
+{
+  // lzma_lzma_preset returns false on success for some reason
+  if (lzma_lzma_preset(&m_options, static_cast<uint32_t>(compression_level)))
+  {
+    m_initialization_failed = true;
+    return;
+  }
+
+  if (!lzma2)
+  {
+    if (compressor_data_size_out)
+      *compressor_data_size_out = 5;
+
+    if (compressor_data_out)
+    {
+      ASSERT(m_options.lc < 9);
+      ASSERT(m_options.lp < 5);
+      ASSERT(m_options.pb < 5);
+      compressor_data_out[0] =
+          static_cast<u8>((m_options.pb * 5 + m_options.lp) * 9 + m_options.lc);
+
+      // The dictionary size is stored as a 32-bit little endian unsigned integer
+      static_assert(sizeof(m_options.dict_size) == sizeof(u32));
+      std::memcpy(compressor_data_out + 1, &m_options.dict_size, sizeof(u32));
+    }
+  }
+  else
+  {
+    if (compressor_data_size_out)
+      *compressor_data_size_out = 1;
+
+    if (compressor_data_out)
+    {
+      u8 encoded_dict_size = 0;
+      while (encoded_dict_size < 40 && m_options.dict_size > LZMA2DictionarySize(encoded_dict_size))
+        ++encoded_dict_size;
+
+      compressor_data_out[0] = encoded_dict_size;
+    }
+  }
+
+  m_filters[0].id = lzma2 ? LZMA_FILTER_LZMA2 : LZMA_FILTER_LZMA1;
+  m_filters[0].options = &m_options;
+  m_filters[1].id = LZMA_VLI_UNKNOWN;
+  m_filters[1].options = nullptr;
+}
+
+LZMACompressor::~LZMACompressor()
+{
+  lzma_end(&m_stream);
+}
+
+bool LZMACompressor::Start()
+{
+  if (m_initialization_failed)
+    return false;
+
+  m_buffer.clear();
+  m_stream.next_out = m_buffer.data();
+
+  return lzma_raw_encoder(&m_stream, m_filters) == LZMA_OK;
+}
+
+bool LZMACompressor::Compress(const u8* data, size_t size)
+{
+  m_stream.next_in = data;
+  m_stream.avail_in = size;
+
+  ExpandBuffer(size);
+
+  while (m_stream.avail_in != 0)
+  {
+    if (m_stream.avail_out == 0)
+      ExpandBuffer(0x100);
+
+    if (lzma_code(&m_stream, LZMA_RUN) != LZMA_OK)
+      return false;
+  }
+
+  return true;
+}
+
+bool LZMACompressor::End()
+{
+  while (true)
+  {
+    if (m_stream.avail_out == 0)
+      ExpandBuffer(0x100);
+
+    switch (lzma_code(&m_stream, LZMA_FINISH))
+    {
+    case LZMA_OK:
+      break;
+    case LZMA_STREAM_END:
+      return true;
+    default:
+      return false;
+    }
+  }
+}
+
+void LZMACompressor::ExpandBuffer(size_t bytes_to_add)
+{
+  const size_t bytes_written = GetSize();
+  m_buffer.resize(m_buffer.size() + bytes_to_add);
+  m_stream.next_out = m_buffer.data() + bytes_written;
+  m_stream.avail_out = m_buffer.size() - bytes_written;
+}
+
+const u8* LZMACompressor::GetData() const
+{
+  return m_buffer.data();
+}
+
+size_t LZMACompressor::GetSize() const
+{
+  return static_cast<size_t>(m_stream.next_out - m_buffer.data());
+}
+
+ZstdCompressor::ZstdCompressor(int compression_level)
+{
+  m_stream = ZSTD_createCStream();
+
+  if (ZSTD_isError(ZSTD_CCtx_setParameter(m_stream, ZSTD_c_compressionLevel, compression_level)))
+    m_stream = nullptr;
+}
+
+ZstdCompressor::~ZstdCompressor()
+{
+  ZSTD_freeCStream(m_stream);
+}
+
+bool ZstdCompressor::Start()
+{
+  if (!m_stream)
+    return false;
+
+  m_buffer.clear();
+  m_out_buffer = {};
+
+  return !ZSTD_isError(ZSTD_CCtx_reset(m_stream, ZSTD_reset_session_only));
+}
+
+bool ZstdCompressor::Compress(const u8* data, size_t size)
+{
+  ZSTD_inBuffer in_buffer{data, size, 0};
+
+  ExpandBuffer(size);
+
+  while (in_buffer.size != in_buffer.pos)
+  {
+    if (m_out_buffer.size == m_out_buffer.pos)
+      ExpandBuffer(0x100);
+
+    if (ZSTD_isError(ZSTD_compressStream(m_stream, &m_out_buffer, &in_buffer)))
+      return false;
+  }
+
+  return true;
+}
+
+bool ZstdCompressor::End()
+{
+  while (true)
+  {
+    if (m_out_buffer.size == m_out_buffer.pos)
+      ExpandBuffer(0x100);
+
+    const size_t result = ZSTD_endStream(m_stream, &m_out_buffer);
+    if (ZSTD_isError(result))
+      return false;
+    if (result == 0)
+      return true;
+  }
+}
+
+void ZstdCompressor::ExpandBuffer(size_t bytes_to_add)
+{
+  m_buffer.resize(m_buffer.size() + bytes_to_add);
+
+  m_out_buffer.dst = m_buffer.data();
+  m_out_buffer.size = m_buffer.size();
+}
+
+}  // namespace DiscIO

Source/Core/DiscIO/WIACompression.h

@@ -0,0 +1,252 @@
+// Copyright 2020 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include <bzlib.h>
+#include <lzma.h>
+#include <mbedtls/sha1.h>
+#include <zstd.h>
+
+#include "Common/CommonTypes.h"
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+namespace DiscIO
+{
+struct DecompressionBuffer
+{
+  std::vector<u8> data;
+  size_t bytes_written = 0;
+};
+
+using SHA1 = std::array<u8, 20>;
+
+struct PurgeSegment
+{
+  u32 offset;
+  u32 size;
+};
+static_assert(sizeof(PurgeSegment) == 0x08, "Wrong size for WIA purge segment");
+
+class Decompressor
+{
+public:
+  virtual ~Decompressor();
+
+  virtual bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                          size_t* in_bytes_read) = 0;
+  virtual bool Done() const { return m_done; };
+
+protected:
+  bool m_done = false;
+};
+
+class NoneDecompressor final : public Decompressor
+{
+public:
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+};
+
+// This class assumes that more bytes won't be added to in once in.bytes_written == in.data.size()
+// and that *in_bytes_read initially will be equal to the size of the exception lists
+class PurgeDecompressor final : public Decompressor
+{
+public:
+  PurgeDecompressor(u64 decompressed_size);
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  const u64 m_decompressed_size;
+
+  PurgeSegment m_segment = {};
+  size_t m_bytes_read = 0;
+  size_t m_segment_bytes_written = 0;
+  size_t m_out_bytes_written = 0;
+  bool m_started = false;
+
+  mbedtls_sha1_context m_sha1_context;
+};
+
+class Bzip2Decompressor final : public Decompressor
+{
+public:
+  ~Bzip2Decompressor();
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  bz_stream m_stream = {};
+  bool m_started = false;
+};
+
+class LZMADecompressor final : public Decompressor
+{
+public:
+  LZMADecompressor(bool lzma2, const u8* filter_options, size_t filter_options_size);
+  ~LZMADecompressor();
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  lzma_stream m_stream = LZMA_STREAM_INIT;
+  lzma_options_lzma m_options = {};
+  lzma_filter m_filters[2];
+  bool m_started = false;
+  bool m_error_occurred = false;
+};
+
+class ZstdDecompressor final : public Decompressor
+{
+public:
+  ZstdDecompressor();
+  ~ZstdDecompressor();
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+private:
+  ZSTD_DStream* m_stream;
+};
+
+class RVZPackDecompressor final : public Decompressor
+{
+public:
+  RVZPackDecompressor(std::unique_ptr<Decompressor> decompressor, DecompressionBuffer decompressed,
+                      u64 data_offset, u32 rvz_packed_size);
+
+  bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                  size_t* in_bytes_read) override;
+
+  bool Done() const override;
+
+private:
+  bool IncrementBytesRead(size_t x);
+  std::optional<bool> ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read,
+                                         size_t decompressed_bytes_read, size_t bytes_to_read);
+
+  std::unique_ptr<Decompressor> m_decompressor;
+  DecompressionBuffer m_decompressed;
+  size_t m_decompressed_bytes_read = 0;
+  size_t m_bytes_read;
+  u64 m_data_offset;
+  u32 m_rvz_packed_size;
+
+  u32 m_size = 0;
+  bool m_junk;
+  LaggedFibonacciGenerator m_lfg;
+};
+
+class Compressor
+{
+public:
+  virtual ~Compressor();
+
+  // First call Start, then AddDataOnlyForPurgeHashing/Compress any number of times,
+  // then End, then GetData/GetSize any number of times.
+
+  virtual bool Start() = 0;
+  virtual bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) { return true; }
+  virtual bool Compress(const u8* data, size_t size) = 0;
+  virtual bool End() = 0;
+
+  virtual const u8* GetData() const = 0;
+  virtual size_t GetSize() const = 0;
+};
+
+class PurgeCompressor final : public Compressor
+{
+public:
+  PurgeCompressor();
+  ~PurgeCompressor();
+
+  bool Start() override;
+  bool AddPrecedingDataOnlyForPurgeHashing(const u8* data, size_t size) override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override;
+  size_t GetSize() const override;
+
+private:
+  std::vector<u8> m_buffer;
+  size_t m_bytes_written;
+  mbedtls_sha1_context m_sha1_context;
+};
+
+class Bzip2Compressor final : public Compressor
+{
+public:
+  Bzip2Compressor(int compression_level);
+  ~Bzip2Compressor();
+
+  bool Start() override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override;
+  size_t GetSize() const override;
+
+private:
+  void ExpandBuffer(size_t bytes_to_add);
+
+  bz_stream m_stream = {};
+  std::vector<u8> m_buffer;
+  int m_compression_level;
+};
+
+class LZMACompressor final : public Compressor
+{
+public:
+  LZMACompressor(bool lzma2, int compression_level, u8 compressor_data_out[7],
+                 u8* compressor_data_size_out);
+  ~LZMACompressor();
+
+  bool Start() override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override;
+  size_t GetSize() const override;
+
+private:
+  void ExpandBuffer(size_t bytes_to_add);
+
+  lzma_stream m_stream = LZMA_STREAM_INIT;
+  lzma_options_lzma m_options = {};
+  lzma_filter m_filters[2];
+  std::vector<u8> m_buffer;
+  bool m_initialization_failed = false;
+};
+
+class ZstdCompressor final : public Compressor
+{
+public:
+  ZstdCompressor(int compression_level);
+  ~ZstdCompressor();
+
+  bool Start() override;
+  bool Compress(const u8* data, size_t size) override;
+  bool End() override;
+
+  const u8* GetData() const override { return m_buffer.data(); }
+  size_t GetSize() const override { return m_out_buffer.pos; }
+
+private:
+  void ExpandBuffer(size_t bytes_to_add);
+
+  ZSTD_CStream* m_stream;
+  ZSTD_outBuffer m_out_buffer;
+  std::vector<u8> m_buffer;
+};
+
+}  // namespace DiscIO

Source/Core/DiscIO/WiiEncryptionCache.cpp

@@ -24,7 +24,8 @@ WiiEncryptionCache::~WiiEncryptionCache() = default;
 
 const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>*
 WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset,
-                                 u64 partition_data_decrypted_size, const Key& key)
+                                 u64 partition_data_decrypted_size, const Key& key,
+                                 const HashExceptionCallback& hash_exception_callback)
 {
   // Only allocate memory if this function actually ends up getting called
   if (!m_cache)
@@ -40,8 +41,20 @@ WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset,
 
   if (m_cached_offset != group_offset_on_disc)
   {
+    std::function<void(VolumeWii::HashBlock * hash_blocks)> hash_exception_callback_2;
+
+    if (hash_exception_callback)
+    {
+      hash_exception_callback_2 =
+          [offset, &hash_exception_callback](
+              VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]) {
+            return hash_exception_callback(hash_blocks, offset);
+          };
+    }
+
     if (!VolumeWii::EncryptGroup(group_offset_in_partition, partition_data_offset,
-                                 partition_data_decrypted_size, key, m_blob, m_cache.get()))
+                                 partition_data_decrypted_size, key, m_blob, m_cache.get(),
+                                 hash_exception_callback_2))
     {
       m_cached_offset = std::numeric_limits<u64>::max();  // Invalidate the cache
       return nullptr;
@@ -54,13 +67,14 @@ WiiEncryptionCache::EncryptGroup(u64 offset, u64 partition_data_offset,
 }
 
 bool WiiEncryptionCache::EncryptGroups(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset,
-                                       u64 partition_data_decrypted_size, const Key& key)
+                                       u64 partition_data_decrypted_size, const Key& key,
+                                       const HashExceptionCallback& hash_exception_callback)
 {
   while (size > 0)
   {
     const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>* group =
         EncryptGroup(Common::AlignDown(offset, VolumeWii::GROUP_TOTAL_SIZE), partition_data_offset,
-                     partition_data_decrypted_size, key);
+                     partition_data_decrypted_size, key, hash_exception_callback);
 
     if (!group)
       return false;

Source/Core/DiscIO/WiiEncryptionCache.h

@@ -19,6 +19,8 @@ class WiiEncryptionCache
 {
 public:
   using Key = std::array<u8, VolumeWii::AES_KEY_SIZE>;
+  using HashExceptionCallback = std::function<void(
+      VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP], u64 offset)>;
 
   // The blob pointer is kept around for the lifetime of this object.
   explicit WiiEncryptionCache(BlobReader* blob);
@@ -28,15 +30,15 @@ public:
   // If the returned pointer is nullptr, reading from the blob failed.
   // If the returned pointer is not nullptr, it is guaranteed to be valid until
   // the next call of this function or the destruction of this object.
-  const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>* EncryptGroup(u64 offset,
-                                                                  u64 partition_data_offset,
-                                                                  u64 partition_data_decrypted_size,
-                                                                  const Key& key);
+  const std::array<u8, VolumeWii::GROUP_TOTAL_SIZE>*
+  EncryptGroup(u64 offset, u64 partition_data_offset, u64 partition_data_decrypted_size,
+               const Key& key, const HashExceptionCallback& hash_exception_callback = {});
 
   // Encrypts a variable number of groups, as determined by the offset and size parameters.
   // Supports reading groups partially.
   bool EncryptGroups(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset,
-                     u64 partition_data_decrypted_size, const Key& key);
+                     u64 partition_data_decrypted_size, const Key& key,
+                     const HashExceptionCallback& hash_exception_callback = {});
 
 private:
   BlobReader* m_blob;

Source/Core/DolphinQt/ConvertDialog.cpp

@@ -26,6 +26,7 @@
 #include "Common/Logging/Log.h"
 #include "DiscIO/Blob.h"
 #include "DiscIO/ScrubbedBlob.h"
+#include "DiscIO/WIABlob.h"
 #include "DolphinQt/QtUtils/ModalMessageBox.h"
 #include "DolphinQt/QtUtils/ParallelProgressDialog.h"
 #include "UICommon/GameFile.h"
@@ -57,6 +58,8 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
   m_format = new QComboBox;
   m_format->addItem(QStringLiteral("ISO"), static_cast<int>(DiscIO::BlobType::PLAIN));
   m_format->addItem(QStringLiteral("GCZ"), static_cast<int>(DiscIO::BlobType::GCZ));
+  m_format->addItem(QStringLiteral("WIA"), static_cast<int>(DiscIO::BlobType::WIA));
+  m_format->addItem(QStringLiteral("RVZ"), static_cast<int>(DiscIO::BlobType::RVZ));
   if (std::all_of(m_files.begin(), m_files.end(),
                   [](const auto& file) { return file->GetBlobType() == DiscIO::BlobType::PLAIN; }))
   {
@@ -69,9 +72,17 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
   grid_layout->addWidget(new QLabel(tr("Block Size:")), 1, 0);
   grid_layout->addWidget(m_block_size, 1, 1);
 
+  m_compression = new QComboBox;
+  grid_layout->addWidget(new QLabel(tr("Compression:")), 2, 0);
+  grid_layout->addWidget(m_compression, 2, 1);
+
+  m_compression_level = new QComboBox;
+  grid_layout->addWidget(new QLabel(tr("Compression Level:")), 3, 0);
+  grid_layout->addWidget(m_compression_level, 3, 1);
+
   m_scrub = new QCheckBox;
-  grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 2, 0);
-  grid_layout->addWidget(m_scrub, 2, 1);
+  grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 4, 0);
+  grid_layout->addWidget(m_scrub, 4, 1);
   m_scrub->setEnabled(
       std::none_of(m_files.begin(), m_files.end(), std::mem_fn(&UICommon::GameFile::IsDatelDisc)));
 
@@ -83,12 +94,17 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
   QGroupBox* options_group = new QGroupBox(tr("Options"));
   options_group->setLayout(options_layout);
 
-  QLabel* info_text =
-      new QLabel(tr("ISO: A simple and robust format which is supported by many programs. "
-                    "It takes up more space than any other format.\n\n"
-                    "GCZ: A basic compressed format which is compatible with most versions of "
-                    "Dolphin and some other programs. It can't efficiently compress junk data "
-                    "(unless removed) or encrypted Wii data."));
+  QLabel* info_text = new QLabel(
+      tr("ISO: A simple and robust format which is supported by many programs. It takes up more "
+         "space than any other format.\n\n"
+         "GCZ: A basic compressed format which is compatible with most versions of Dolphin and "
+         "some other programs. It can't efficiently compress junk data (unless removed) or "
+         "encrypted Wii data.\n\n"
+         "WIA: An advanced compressed format which is compatible with recent versions of Dolphin "
+         "and a few other programs. It can efficiently compress encrypted Wii data, but not junk "
+         "data (unless removed).\n\n"
+         "RVZ: An advanced compressed format which is compatible with recent versions of Dolphin. "
+         "It can efficiently compress both junk data and encrypted Wii data."));
   info_text->setWordWrap(true);
 
   QVBoxLayout* info_layout = new QVBoxLayout;
@@ -104,14 +120,34 @@ ConvertDialog::ConvertDialog(QList<std::shared_ptr<const UICommon::GameFile>> fi
 
   connect(m_format, QOverload<int>::of(&QComboBox::currentIndexChanged), this,
           &ConvertDialog::OnFormatChanged);
+  connect(m_compression, QOverload<int>::of(&QComboBox::currentIndexChanged), this,
+          &ConvertDialog::OnCompressionChanged);
   connect(convert_button, &QPushButton::clicked, this, &ConvertDialog::Convert);
 
   OnFormatChanged();
+  OnCompressionChanged();
 }
 
 void ConvertDialog::AddToBlockSizeComboBox(int size)
 {
   m_block_size->addItem(QString::fromStdString(UICommon::FormatSize(size, 0)), size);
+
+  // Select 128 KiB by default, or if it is not available, the size closest to it.
+  // This code assumes that sizes get added to the combo box in increasing order.
+  constexpr int DEFAULT_SIZE = 0x20000;
+  if (size <= DEFAULT_SIZE)
+    m_block_size->setCurrentIndex(m_block_size->count() - 1);
+}
+
+void ConvertDialog::AddToCompressionComboBox(const QString& name,
+                                             DiscIO::WIARVZCompressionType type)
+{
+  m_compression->addItem(name, static_cast<int>(type));
+}
+
+void ConvertDialog::AddToCompressionLevelComboBox(int level)
+{
+  m_compression_level->addItem(QString::number(level), level);
 }
 
 void ConvertDialog::OnFormatChanged()
@@ -127,6 +163,9 @@ void ConvertDialog::OnFormatChanged()
   const DiscIO::BlobType format = static_cast<DiscIO::BlobType>(m_format->currentData().toInt());
 
   m_block_size->clear();
+  m_compression->clear();
+
+  // Populate m_block_size
   switch (format)
   {
   case DiscIO::BlobType::GCZ:
@@ -166,11 +205,90 @@ void ConvertDialog::OnFormatChanged()
 
     break;
   }
+  case DiscIO::BlobType::WIA:
+    m_block_size->setEnabled(true);
+
+    // This is the smallest block size supported by WIA. For performance, larger sizes are avoided.
+    AddToBlockSizeComboBox(0x200000);
+
+    break;
+  case DiscIO::BlobType::RVZ:
+    m_block_size->setEnabled(true);
+
+    for (int block_size = MIN_BLOCK_SIZE; block_size <= MAX_BLOCK_SIZE; block_size *= 2)
+      AddToBlockSizeComboBox(block_size);
+
+    break;
   default:
     break;
   }
 
+  // Populate m_compression
+  switch (format)
+  {
+  case DiscIO::BlobType::GCZ:
+    m_compression->setEnabled(true);
+    AddToCompressionComboBox(QStringLiteral("Deflate"), DiscIO::WIARVZCompressionType::None);
+    break;
+  case DiscIO::BlobType::WIA:
+  case DiscIO::BlobType::RVZ:
+  {
+    m_compression->setEnabled(true);
+
+    // i18n: %1 is the name of a compression method (e.g. LZMA)
+    const QString slow = tr("%1 (slow)");
+
+    AddToCompressionComboBox(tr("No Compression"), DiscIO::WIARVZCompressionType::None);
+
+    if (format == DiscIO::BlobType::WIA)
+      AddToCompressionComboBox(QStringLiteral("Purge"), DiscIO::WIARVZCompressionType::Purge);
+
+    AddToCompressionComboBox(slow.arg(QStringLiteral("bzip2")),
+                             DiscIO::WIARVZCompressionType::Bzip2);
+
+    AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA")), DiscIO::WIARVZCompressionType::LZMA);
+
+    AddToCompressionComboBox(slow.arg(QStringLiteral("LZMA2")),
+                             DiscIO::WIARVZCompressionType::LZMA2);
+
+    if (format == DiscIO::BlobType::RVZ)
+    {
+      AddToCompressionComboBox(QStringLiteral("Zstandard"), DiscIO::WIARVZCompressionType::Zstd);
+      m_compression->setCurrentIndex(m_compression->count() - 1);
+    }
+
+    break;
+  }
+  default:
+    m_compression->setEnabled(false);
+    break;
+  }
+
   m_block_size->setEnabled(m_block_size->count() > 1);
+  m_compression->setEnabled(m_compression->count() > 1);
+
+  m_scrub->setEnabled(format != DiscIO::BlobType::RVZ);
+  if (format == DiscIO::BlobType::RVZ)
+    m_scrub->setChecked(false);
+}
+
+void ConvertDialog::OnCompressionChanged()
+{
+  m_compression_level->clear();
+
+  const auto compression_type =
+      static_cast<DiscIO::WIARVZCompressionType>(m_compression->currentData().toInt());
+
+  const std::pair<int, int> range = DiscIO::GetAllowedCompressionLevels(compression_type);
+
+  for (int i = range.first; i <= range.second; ++i)
+  {
+    AddToCompressionLevelComboBox(i);
+    if (i == 5)
+      m_compression_level->setCurrentIndex(m_compression_level->count() - 1);
+  }
+
+  m_compression_level->setEnabled(m_compression_level->count() > 1);
 }
 
 bool ConvertDialog::ShowAreYouSureDialog(const QString& text)
@@ -189,6 +307,9 @@ void ConvertDialog::Convert()
 {
   const DiscIO::BlobType format = static_cast<DiscIO::BlobType>(m_format->currentData().toInt());
   const int block_size = m_block_size->currentData().toInt();
+  const DiscIO::WIARVZCompressionType compression =
+      static_cast<DiscIO::WIARVZCompressionType>(m_compression->currentData().toInt());
+  const int compression_level = m_compression_level->currentData().toInt();
   const bool scrub = m_scrub->isChecked();
 
   if (scrub && format == DiscIO::BlobType::PLAIN)
@@ -224,7 +345,15 @@ void ConvertDialog::Convert()
     break;
   case DiscIO::BlobType::GCZ:
     extension = QStringLiteral(".gcz");
-    filter = tr("Compressed GC/Wii images (*.gcz)");
+    filter = tr("GCZ GC/Wii images (*.gcz)");
+    break;
+  case DiscIO::BlobType::WIA:
+    extension = QStringLiteral(".wia");
+    filter = tr("WIA GC/Wii images (*.wia)");
+    break;
+  case DiscIO::BlobType::RVZ:
+    extension = QStringLiteral(".rvz");
+    filter = tr("RVZ GC/Wii images (*.rvz)");
     break;
   default:
     ASSERT(false);
@@ -330,8 +459,9 @@ void ConvertDialog::Convert()
     {
       std::future<bool> good;
 
-      if (format == DiscIO::BlobType::PLAIN)
+      switch (format)
       {
+      case DiscIO::BlobType::PLAIN:
         good = std::async(std::launch::async, [&] {
           const bool good =
               DiscIO::ConvertToPlain(blob_reader.get(), original_path, dst_path.toStdString(),
@@ -339,9 +469,9 @@ void ConvertDialog::Convert()
           progress_dialog.Reset();
           return good;
         });
-      }
-      else if (format == DiscIO::BlobType::GCZ)
-      {
+        break;
+
+      case DiscIO::BlobType::GCZ:
         good = std::async(std::launch::async, [&] {
           const bool good =
               DiscIO::ConvertToGCZ(blob_reader.get(), original_path, dst_path.toStdString(),
@@ -350,6 +480,19 @@ void ConvertDialog::Convert()
           progress_dialog.Reset();
           return good;
         });
+        break;
+
+      case DiscIO::BlobType::WIA:
+      case DiscIO::BlobType::RVZ:
+        good = std::async(std::launch::async, [&] {
+          const bool good = DiscIO::ConvertToWIAOrRVZ(
+              blob_reader.get(), original_path, dst_path.toStdString(),
+              format == DiscIO::BlobType::RVZ, compression, compression_level, block_size,
+              &CompressCB, &progress_dialog);
+          progress_dialog.Reset();
+          return good;
+        });
+        break;
       }
 
       progress_dialog.GetRaw()->exec();

Source/Core/DolphinQt/ConvertDialog.h

@@ -14,6 +14,11 @@
 class QCheckBox;
 class QComboBox;
 
+namespace DiscIO
+{
+enum class WIARVZCompressionType : u32;
+}
+
 namespace UICommon
 {
 class GameFile;
@@ -29,15 +34,20 @@ public:
 
 private slots:
   void OnFormatChanged();
+  void OnCompressionChanged();
   void Convert();
 
 private:
   void AddToBlockSizeComboBox(int size);
+  void AddToCompressionComboBox(const QString& name, DiscIO::WIARVZCompressionType type);
+  void AddToCompressionLevelComboBox(int level);
 
   bool ShowAreYouSureDialog(const QString& text);
 
   QComboBox* m_format;
   QComboBox* m_block_size;
+  QComboBox* m_compression;
+  QComboBox* m_compression_level;
   QCheckBox* m_scrub;
   QList<std::shared_ptr<const UICommon::GameFile>> m_files;
 };

Source/Core/DolphinQt/GameList/GameTracker.cpp

@@ -24,6 +24,7 @@ static const QStringList game_filters{
     QStringLiteral("*.[gG][cC][mM]"), QStringLiteral("*.[iI][sS][oO]"),
     QStringLiteral("*.[tT][gG][cC]"), QStringLiteral("*.[cC][iI][sS][oO]"),
     QStringLiteral("*.[gG][cC][zZ]"), QStringLiteral("*.[wW][bB][fF][sS]"),
+    QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[rR][vV][zZ]"),
     QStringLiteral("*.[wW][aA][dD]"), QStringLiteral("*.[eE][lL][fF]"),
     QStringLiteral("*.[dD][oO][lL]")};
 

Source/Core/DolphinQt/Info.plist.in

@@ -14,8 +14,10 @@
                 <string>gcz</string>
                 <string>iso</string>
                 <string>m3u</string>
+                <string>rvz</string>
                 <string>tgc</string>
                 <string>wad</string>
+                <string>wia</string>
                 <string>wbfs</string>
             </array>
             <key>CFBundleTypeIconFile</key>

Source/Core/DolphinQt/MainWindow.cpp

@@ -686,8 +686,8 @@ QStringList MainWindow::PromptFileNames()
   QStringList paths = QFileDialog::getOpenFileNames(
       this, tr("Select a File"),
       settings.value(QStringLiteral("mainwindow/lastdir"), QString{}).toString(),
-      tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wad *.dff *.m3u);;"
-         "All Files (*)"));
+      tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.rvz *.wad "
+         "*.dff *.m3u);;All Files (*)"));
 
   if (!paths.isEmpty())
   {

Source/Core/DolphinQt/Settings/PathPane.cpp

@@ -42,10 +42,10 @@ void PathPane::Browse()
 
 void PathPane::BrowseDefaultGame()
 {
-  QString file = QDir::toNativeSeparators(QFileDialog::getOpenFileName(
-      this, tr("Select a Game"), Settings::Instance().GetDefaultGame(),
-      tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wad *.m3u);;"
-         "All Files (*)")));
+  QString file = QDir::toNativeSeparators(
+      QFileDialog::getOpenFileName(this, tr("Select a Game"), Settings::Instance().GetDefaultGame(),
+                                   tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs "
+                                      "*.ciso *.gcz *.wia *.rvz *.wad *.m3u);;All Files (*)")));
 
   if (!file.isEmpty())
     Settings::Instance().SetDefaultGame(file);

Source/Core/UICommon/GameFileCache.cpp

@@ -33,7 +33,7 @@ std::vector<std::string> FindAllGamePaths(const std::vector<std::string>& direct
                                           bool recursive_scan)
 {
   static const std::vector<std::string> search_extensions = {
-      ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wad", ".dol", ".elf"};
+      ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wia", ".rvz", ".wad", ".dol", ".elf"};
 
   // TODO: We could process paths iteratively as they are found
   return Common::DoFileSearch(directories_to_scan, search_extensions, recursive_scan);

docs/WIA.md

@@ -0,0 +1,247 @@
+# WIA file format description
+
+This document describes the WIA disc image format, version 1.00, as implemented in wit v2.40a. A few notes about Dolphin's implementation of the format are also included, where Dolphin differs from wit. The unique features of WIA compared to older formats like GCZ are:
+
+ - Support for the compression algorithms bzip2, LZMA, and LZMA2
+ - Wii partition data is stored decrypted and without hashes, making it compressible
+ 
+Like essentially all compressed GC/Wii disc image formats, WIA divides the data into blocks (called chunks in wit). Each chunk is compressed separately, making random access of compressed data possible.
+
+The struct names and variable names below are taken directly from wit. Data in WIA files can be stored in any order unless otherwise noted. All integers are big endian unless otherwise noted. The type `sha1_hash_t` refers to an array of 20 bytes.
+
+## `wia_file_head_t`
+
+This struct is stored at offset 0x0 and is 0x48 bytes long. The wit source code says its format will never be changed.
+
+A short note from the wit source code about how version numbers are encoded:
+
+```
+//-----------------------------------------------------
+// Format of version number: AABBCCDD = A.BB | A.BB.CC
+// If D != 0x00 && D != 0xff => append: 'beta' D
+//-----------------------------------------------------
+```
+
+|Type and name|Description|
+|--|--|
+|`char magic[4]`|Always contains `"WIA\x1"`.|
+|`u32 version`|The WIA format version.|
+|`u32 version_compatible`|If the reading program supports the version of WIA indicated here, it can read the file. `version` can be higher than `version_compatible` (wit v2.40a sets the former to `0x01000000` and the latter to `0x00090000`).|
+|`u32 disc_size`|The size of the `wia_disc_t` struct. wit v2.40a always includes the full 7 bytes of `compr_data` when writing this.|
+|`sha1_hash_t disc_hash`|The SHA-1 hash of the `wia_disc_t` struct. The number of bytes to hash is determined by `disc_size`. For instance, you may have to hash all 7 bytes of `compr_data` regardless of what `compr_data_len` says.|
+|`u64 iso_file_size`|The original size of the disc (or in other words, the size of the ISO file that has the same contents as this WIA file).|
+|`u64 wia_file_size`|The size of this file.|
+|`sha1_hash_t file_head_hash`|The SHA-1 hash of this struct, up to but not including `file_head_hash` itself.|
+
+## `wia_disc_t`
+
+This struct is stored at offset 0x48, immediately after `wia_file_head_t`.
+
+|Type and name|Description|
+|--|--|
+|`u32 disc_type`|wit sets this to 0 for "unknown" (does this ever happen in practice?), 1 for GameCube discs, 2 for Wii discs.|
+|`u32 compression`|0 for NONE, 1 for PURGE (see the `wia_exception_t` section), 2 for BZIP2, 3 for LZMA, 4 for LZMA2.
+|`u32 compr_level`|The compression level used by the compressor. The possible values are compressor-specific. For informational purposes only.|
+|`u32 chunk_size`|The size of the chunks that data is divided into. Must be a multiple of 2 MiB.|
+|`u8 dhead[0x80]`|The first 0x80 bytes of the disc image.
+|`u32 n_part`|The number of `wia_part_t` structs.|
+|`u32 part_t_size`|The size of one `wia_part_t` struct. If this is smaller than `sizeof(wia_part_t)`, fill the missing bytes with `0x00`.|
+|`u64 part_off`|The offset in the file where the `wia_part_t` structs are stored (uncompressed).|
+|`sha1_hash_t part_hash`|The SHA-1 hash of the `wia_part_t` structs. The number of bytes to hash is determined by `n_part * part_t_size`.|
+|`u32 n_raw_data`|The number of `wia_raw_data_t` structs.|
+|`u64 raw_data_off`|The offset in the file where the `wia_raw_data_t` structs are stored (compressed).|
+|`u32 raw_data_size`|The total compressed size of the `wia_raw_data_t` structs.|
+|`u32 n_groups`|The number of `wia_group_t` structs.|
+|`u64 group_off`|The offset in the file where the `wia_group_t` structs are stored (compressed).|
+|`u32 group_size`|The total compressed size of the `wia_group_t` structs.|
+|`u8 compr_data_len`|The number of used bytes in the `compr_data` array.|
+|`u8 compr_data[7]`|Compressor specific data (see below).|
+
+If the compression method is NONE, PURGE or BZIP2, `compr_data_len`is 0. If the compression method is LZMA or LZMA2, the compressor specific data is stored in the format used by the 7-Zip SDK. It needs to be converted if you are using e.g. liblzma.
+
+For LZMA, the data is 5 bytes long. The first byte encodes the `lc`, `pb`, and `lp` parameters, and the four other bytes encode the dictionary size in little endian. The first byte can be decoded as follows (code from the 7-Zip SDK):
+
+```
+d = data[0];
+if (d >= (9 * 5 * 5))
+  return SZ_ERROR_UNSUPPORTED;
+
+p->lc = d % 9;
+d /= 9;
+p->pb = d / 5;
+p->lp = d % 5;
+```
+
+For LZMA2, the data consists of a single byte that encodes the dictionary size. It can be decoded as follows (code from the 7-Zip SDK):
+
+```
+#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
+
+if (prop > 40)
+  return SZ_ERROR_UNSUPPORTED;
+dicSize = (prop == 40) ? 0xFFFFFFFF : LZMA2_DIC_SIZE_FROM_PROP(prop);
+```
+
+Preset dictionaries are not used for any compression method.
+
+## `wia_part_data_t`
+
+|Type and name|Description|
+|--|--|
+|`u32 first_sector`|The sector on the disc at which this data starts. One sector is 32 KiB (or 31 KiB excluding hashes).|
+|`u32 n_sectors`|The number of sectors on the disc covered by this struct. One sector is 32 KiB (or 31 KiB excluding hashes).|
+|`u32 group_index`|The index of the first `wia_group_t` struct that points to the data covered by this struct. The other `wia_group_t` indices follow sequentially.|
+|`u32 n_groups`|The number of `wia_group_t` structs used for this data.|
+
+## `wia_part_t`
+
+This struct is used for keeping track of Wii partition data that on the actual disc is encrypted and hashed. This does not include the unencrypted area at the beginning of partitions that contains the ticket, TMD, certificate chain, and H3 table. So for a typical game partition, `pd[0].first_sector * 0x8000` would be 0x0F820000, not 0x0F800000.
+
+Wii partition data is stored decrypted and with hashes removed. For each 0x8000 bytes on the disc, 0x7C00 bytes are stored in the WIA file (prior to compression). If the hashes are desired, the reading program must first recalculate the hashes as done when creating a Wii disc image from scratch (see https://wiibrew.org/wiki/Wii_Disc), and must then apply the hash exceptions which are stored along with the data (see the `wia_except_list_t` section).
+
+|Type and name|Description|
+|--|--|
+|`u8 part_key[16]`|The title key for this partition (128-bit AES), which can be used for re-encrypting the partition data. This key can be used directly, without decrypting it using the Wii common key.|
+|`wia_part_data_t pd[2]`|To quote the wit source code: `segment 0 is small and defined for management data (boot .. fst). segment 1 takes the remaining data`. The point at which wit splits the two segments is the FST end offset rounded up to the next 2 MiB. Giving the first segment a size which is not a multiple of 2 MiB is likely a bad idea (unless the second segment has a size of 0).|
+
+## `wia_raw_data_t`
+
+This struct is used for keeping track of disc data that is not stored as `wia_part_t`. The data is stored as is (other than compression being applied).
+
+The first `wia_raw_data_t` has `raw_data_off` set to 0x80 and  `raw_data_size` set to 0x4FF80, but despite this, it actually contains 0x50000 bytes of data. (However, the first 0x80 bytes should be read from `wia_disc_t` instead.) This should be handled by rounding the offset down to the previous multiple of 0x8000 (and adding the equivalent amount to the size so that the end offset stays the same), not by special casing the first `wia_raw_data_t`.
+
+|Type and name|Description|
+|--|--|
+|`u64 raw_data_off`|The offset on the disc at which this data starts.|
+|`u64 raw_data_size`|The number of bytes on the disc covered by this struct.|
+|`u32 group_index`|The index of the first `wia_group_t` struct that points to the data covered by this struct. The other `wia_group_t` indices follow sequentially.|
+|`u32 n_groups`|The number of `wia_group_t` structs used for this data.|
+
+## `wia_group_t`
+
+This struct points directly to the actual disc data, stored compressed. The data is interpreted differently depending on whether the `wia_group_t` is referenced by a `wia_part_data_t` or a `wia_raw_data_t` (see the `wia_part_t` section for details).
+
+A `wia_group_t` normally contains `chunk_size` bytes of decompressed data (or `chunk_size / 0x8000 * 0x7C00` for Wii partition data when not counting hashes), not counting any `wia_except_list_t` structs. However, the last `wia_group_t` of a `wia_part_data_t` or `wia_raw_data_t` contains less data than that if `n_sectors * 0x8000` (for `wia_part_data_t`) or  `raw_data_size` (for `wia_raw_data_t`)  is not evenly divisible by `chunk_size`.
+
+|Type and name|Description|
+|--|--|
+|`u32 data_off4`|The offset in the file where the compressed data is stored, divided by 4.|
+|`u32 data_size`|The size of the compressed data, including any `wia_except_list_t` structs. 0 is a special case meaning that every byte of the decompressed data is `0x00` and the `wia_except_list_t` structs (if there are supposed to be any) contain 0 exceptions.|
+
+## `wia_exception_t`
+
+This struct represents a 20-byte difference between the recalculated hash data and the original hash data. (See also `wia_except_list_t` below.)
+
+When recalculating hashes for a `wia_group_t` with a size which is not evenly divisible by 2 MiB (with the size of the hashes included), the missing bytes should be treated as zeroes for the purpose of hashing. (wit's writing code seems to act as if the reading code does not assume that these missing bytes are zero, but both wit's and Dolphin's reading code treat them as zero. Dolphin's writing code assumes that the reading code treats them as zero.)
+
+wit's writing code only outputs `wia_exception_t` structs for mismatches in the actual hash data, not in the padding data (which normally only contains zeroes). Dolphin's writing code outputs `wia_exception_t` structs for both hash data and padding data. When Dolphin needs to write `wia_exception_t` structs for a padding area which is 32 bytes long, it writes one which covers the first 20 bytes of the padding area and one which covers the last 20 bytes of the padding area, generating 12 bytes of overlap between the `wia_exception_t` structs.
+
+|Type and name|Description|
+|--|--|
+|`u16 offset`|The offset among the hashes. The offsets `0x0000`-`0x0400` here map to the offsets `0x0000`-`0x0400` in the full 2 MiB of data, the offsets `0x0400`-`0x0800` here map to the offsets `0x8000`-`0x8400` in the full 2 MiB of data, and so on. The offsets start over at 0 for each new `wia_except_list_t`.|
+|`sha1_hash_t hash`|The hash that the automatically generated hash at the given offset needs to be replaced with. The replacement should happen after calculating all hashes for the current 2 MiB of data but before encrypting the hashes.|
+
+## `wia_except_list_t`
+
+Each `wia_group_t` of Wii partition data contains one or more `wia_except_list_t` structs before the actual data, one for each 2 MiB of data in the `wia_group_t`. The number of `wia_except_list_t` structs per `wia_group_t` is always `chunk_size / 0x200000`, even for a `wia_group_t` which contains less data than normal due to it being at the end of a partition.
+
+For memory management reasons, programs which read WIA files might place a limit on how many exceptions there can be in a `wia_except_list_t`. Dolphin's reading code has a limit of 52×64=3328 (unless the compression method is NONE or PURGE, in which case there is no limit), which is enough to cover all hashes and all padding. wit's reading code seems to be written as if 47×64=3008 is the maximum it needs to be able to handle, which is enough to cover all hashes but not any padding. However, because wit allocates more memory than needed, it seems to be possible to exceed 3008 by some amount without problems. It should be safe for writing code to assume that reading code can handle at least 3328 exceptions per `wia_except_list_t`.
+
+|Type and name|Description|
+|--|--|
+|`u16 n_exceptions`|The number of `wia_exception_t` structs.|
+|`wia_exception_t exception[n_exceptions]`|Each `wia_exception_t` describes one difference between the hashes obtained by hashing the partition data and the original hashes.|
+
+Somewhat ironically, there are exceptions to how `wia_except_list_t` structs are handled:
+
+ - For the compression method PURGE, the `wia_except_list_t` structs are stored uncompressed (in other words, before the first `wia_segment_t`). For BZIP2, LZMA and LZMA2, they are compressed along with the rest of the data.
+ - For the compression methods NONE and PURGE, if the end offset of the last ``wia_except_list_t`` is not evenly divisible by 4, padding is inserted after it so that the data afterwards will start at a 4 byte boundary. This padding is not inserted for the other compression methods.
+
+## `wia_segment_t`
+
+This struct is used by the simple compression method PURGE, which stores runs of zeroes efficiently and stores other data as is.
+
+|Type and name|Description|
+|--|--|
+|`u32 offset`|The offset of `data` within the decompressed data. (Any `wia_except_list_t` structs are not counted as part of the decompressed data.)|
+|`u32 size`|The number of bytes in `data`.|
+|`u8 data[size]`|Data.|
+
+Each PURGE chunk contains zero or more `wia_segment_t` structs stored in order of ascending `offset`, followed by a SHA-1 hash (0x14 bytes) of the `wia_except_list_t` structs (if any) and the `wia_segment_t` structs. Bytes in the decompressed data that are not covered by any `wia_segment_t` struct are set to `0x00`.
+
+# RVZ file format description
+
+RVZ is a file format which is closely based on WIA. The differences are as follows:
+
+* Zstandard has been added as a compression method. `compression` in `wia_disc_t` is set to 5 when Zstandard is used, and there is no compressor specific data. `compr_level` in `wia_disc_t` should be treated as signed instead of unsigned because Zstandard supports negative compression levels.
+* PURGE has been removed as a compression method.
+* Chunk sizes smaller than 2 MiB are supported. The following applies when using a chunk size smaller than 2 MiB:
+    * The chunk size must be at least 32 KiB and must be a power of two. (Just like with WIA, sizes larger than 2 MiB do not have to be a power of two, they just have to be an integer multiple of 2 MiB.)
+    * For Wii partition data, each chunk contains one `wia_except_list_t` which contains exceptions for that chunk (and no other chunks). Offset 0 refers to the first hash of the current chunk, not the first hash of the full 2 MiB of data.
+* The `wia_group_t` struct has been expanded. See the `rvz_group_t` section below.
+* Pseudorandom padding data is stored losslessly using an encoding scheme described in the *RVZ packing* section below.
+
+## `rvz_group_t`
+
+Compared to `wia_group_t`, `rvz_group_t` changes the meaning of the most significant bit of `data_size` and adds one additional attribute.
+
+"Compressed data" below means the data as it is stored in the file. When compression is disabled, this "compressed data" is actually not compressed.
+
+|Type and name|Description|
+|--|--|
+|`u32 data_off4`|The offset in the file where the compressed data is stored, divided by 4.|
+|`u32 data_size`|The most significant bit is 1 if the data is compressed using the compression method indicated in `wia_disc_t`, and 0 if it is not compressed. The lower 31 bits are the size of the compressed data, including any `wia_except_list_t` structs. The lower 31 bits being 0 is a special case meaning that every byte of the decompressed and unpacked data is `0x00` and the `wia_except_list_t` structs (if there are supposed to be any) contain 0 exceptions.|
+|`u32 rvz_packed_size`|The size after decompressing but before decoding the RVZ packing. If this is 0, RVZ packing is not used for this group.|
+
+## RVZ packing
+
+The RVZ packing encoding scheme can be applied to `wia_group_t` data, with any bzip2/LZMA/Zstandard compression being applied on top of it. (In other words, when reading an RVZ file, bzip2/LZMA/Zstandard decompression is done before decoding the RVZ packing.) RVZ packed data can be decoded as follows:
+
+1. Read 4 bytes of data and interpret it as a 32-bit unsigned big endian integer. Call this `size`.
+2. If the most significant bit of `size` is not set, read `size` bytes and output them unchanged. If the most significant bit of `size` is set, unset the most significant bit of `size`, then read 68 bytes of PRNG seed data and output `size` bytes using the PRNG algorithm described below.
+3. Repeat until all input has been read.
+
+### PRNG algorithm
+
+The PRNG algorithm used for generating padding data on GameCube and Wii discs is a Lagged Fibonacci generator with the parameters f = xor, j = 32, k = 521.
+
+Start by allocating a buffer of 521 32-bit words.
+
+```
+u32 buffer[521];
+```
+
+Copy the 68 bytes (17 words) of seed data into the start of the buffer. This seed data is stored in big endian in RVZ files, so remember to byteswap each word if the system is not big endian. Then, use the following code to fill the remaining part of the buffer:
+
+```
+for (size_t i = 17; i < 521; i++)
+    buffer[i] = (buffer[i - 17] << 23) ^ (buffer[i - 16] >> 9) ^ buffer[i - 1];
+```
+
+The following code is used for advancing the state of the PRNG by a full buffer length. You must run it 4 times before you can start outputting data, and must then run it once after every 521 words of data you output.
+
+```
+for (size_t i = 0; i < 32; i++)
+    buffer[i] ^= buffer[i + 521 - 32];
+
+for (size_t i = 32; i < 521; i++)
+    buffer[i] ^= buffer[i - 32];
+```
+
+After running the above code 4 times, you are ready to output data from the buffer -- but only if the offset (relative to the start of the disc for `wia_raw_data_t` and relative to the start of the partition data for `wia_part_t`) at which you are outputting data is evenly divisible by 32 KiB. Otherwise, you first have to advance the state of the PRNG by `offset % 0x8000` bytes. Please note that the hashes are not counted in the offset for `wia_part_t`, yet the number is still 32 KiB and not 31 KiB.
+
+To finally output a word of data from the buffer, use the following code:
+
+```
+u8* out;
+u32* buffer_ptr;
+
+/* ... */
+
+*(out++) = *buffer_ptr >> 24;
+*(out++) = *buffer_ptr >> 18;  // NB: 18, not 16
+*(out++) = *buffer_ptr >> 8;
+*(out++) = *buffer_ptr;
+
+buffer_ptr++;
+```