From 2d410ddf4d9c0109d64fdf3319efeb9e6cc0bce1 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Mon, 13 May 2019 18:51:02 -0400 Subject: bcat: Implement DeliveryCacheProgressImpl structure Huge thanks to lioncash for re-ing this for me. --- src/core/hle/service/bcat/backend/backend.cpp | 88 ++++++++++++++- src/core/hle/service/bcat/backend/backend.h | 97 ++++++++++++++++- src/core/hle/service/bcat/backend/boxcat.cpp | 147 +++++++++++++++++++++----- src/core/hle/service/bcat/backend/boxcat.h | 6 +- src/core/hle/service/bcat/module.cpp | 56 ++-------- 5 files changed, 310 insertions(+), 84 deletions(-) (limited to 'src/core/hle') diff --git a/src/core/hle/service/bcat/backend/backend.cpp b/src/core/hle/service/bcat/backend/backend.cpp index 9a67da2ef..e389ad568 100644 --- a/src/core/hle/service/bcat/backend/backend.cpp +++ b/src/core/hle/service/bcat/backend/backend.cpp @@ -4,10 +4,90 @@ #include "common/hex_util.h" #include "common/logging/log.h" +#include "core/core.h" +#include "core/hle/lock.h" #include "core/hle/service/bcat/backend/backend.h" namespace Service::BCAT { +ProgressServiceBackend::ProgressServiceBackend(std::string event_name) : impl{} { + auto& kernel{Core::System::GetInstance().Kernel()}; + event = Kernel::WritableEvent::CreateEventPair( + kernel, Kernel::ResetType::OneShot, "ProgressServiceBackend:UpdateEvent:" + event_name); +} + +Kernel::SharedPtr ProgressServiceBackend::GetEvent() { + return event.readable; +} + +DeliveryCacheProgressImpl& ProgressServiceBackend::GetImpl() { + return impl; +} + +void ProgressServiceBackend::SetNeedHLELock(bool need) { + need_hle_lock = need; +} + +void ProgressServiceBackend::SetTotalSize(u64 size) { + impl.total_bytes = size; + SignalUpdate(); +} + +void ProgressServiceBackend::StartConnecting() { + impl.status = DeliveryCacheProgressImpl::Status::Connecting; + SignalUpdate(); +} + +void ProgressServiceBackend::StartProcessingDataList() { + impl.status = DeliveryCacheProgressImpl::Status::ProcessingDataList; + SignalUpdate(); +} + +void ProgressServiceBackend::StartDownloadingFile(std::string_view dir_name, + std::string_view file_name, u64 file_size) { + impl.status = DeliveryCacheProgressImpl::Status::Downloading; + impl.current_downloaded_bytes = 0; + impl.current_total_bytes = file_size; + std::memcpy(impl.current_directory.data(), dir_name.data(), std::min(dir_name.size(), 0x31ull)); + std::memcpy(impl.current_file.data(), file_name.data(), std::min(file_name.size(), 0x31ull)); + SignalUpdate(); +} + +void ProgressServiceBackend::UpdateFileProgress(u64 downloaded) { + impl.current_downloaded_bytes = downloaded; + SignalUpdate(); +} + +void ProgressServiceBackend::FinishDownloadingFile() { + impl.total_downloaded_bytes += impl.current_total_bytes; + SignalUpdate(); +} + +void ProgressServiceBackend::CommitDirectory(std::string_view dir_name) { + impl.status = DeliveryCacheProgressImpl::Status::Committing; + impl.current_file.fill(0); + impl.current_downloaded_bytes = 0; + impl.current_total_bytes = 0; + std::memcpy(impl.current_directory.data(), dir_name.data(), std::min(dir_name.size(), 0x31ull)); + SignalUpdate(); +} + +void ProgressServiceBackend::FinishDownload(ResultCode result) { + impl.total_downloaded_bytes = impl.total_bytes; + impl.status = DeliveryCacheProgressImpl::Status::Done; + impl.result = result; + SignalUpdate(); +} + +void ProgressServiceBackend::SignalUpdate() const { + if (need_hle_lock) { + std::lock_guard lock(HLE::g_hle_lock); + event.writable->Signal(); + } else { + event.writable->Signal(); + } +} + Backend::Backend(DirectoryGetter getter) : dir_getter(std::move(getter)) {} Backend::~Backend() = default; @@ -16,20 +96,20 @@ NullBackend::NullBackend(const DirectoryGetter& getter) : Backend(std::move(gett NullBackend::~NullBackend() = default; -bool NullBackend::Synchronize(TitleIDVersion title, CompletionCallback callback) { +bool NullBackend::Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) { LOG_DEBUG(Service_BCAT, "called, title_id={:016X}, build_id={:016X}", title.title_id, title.build_id); - callback(true); + progress.FinishDownload(RESULT_SUCCESS); return true; } bool NullBackend::SynchronizeDirectory(TitleIDVersion title, std::string name, - CompletionCallback callback) { + ProgressServiceBackend& progress) { LOG_DEBUG(Service_BCAT, "called, title_id={:016X}, build_id={:016X}, name={}", title.title_id, title.build_id, name); - callback(true); + progress.FinishDownload(RESULT_SUCCESS); return true; } diff --git a/src/core/hle/service/bcat/backend/backend.h b/src/core/hle/service/bcat/backend/backend.h index 5b4118814..50973a13a 100644 --- a/src/core/hle/service/bcat/backend/backend.h +++ b/src/core/hle/service/bcat/backend/backend.h @@ -8,10 +8,14 @@ #include #include "common/common_types.h" #include "core/file_sys/vfs_types.h" +#include "core/hle/kernel/readable_event.h" +#include "core/hle/kernel/writable_event.h" +#include "core/hle/result.h" namespace Service::BCAT { -using CompletionCallback = std::function; +struct DeliveryCacheProgressImpl; + using DirectoryGetter = std::function; using Passphrase = std::array; @@ -20,33 +24,116 @@ struct TitleIDVersion { u64 build_id; }; +using DirectoryName = std::array; +using FileName = std::array; + +struct DeliveryCacheProgressImpl { + enum class Status : s32 { + None = 0x0, + Queued = 0x1, + Connecting = 0x2, + ProcessingDataList = 0x3, + Downloading = 0x4, + Committing = 0x5, + Done = 0x9, + }; + + Status status; + ResultCode result = RESULT_SUCCESS; + DirectoryName current_directory; + FileName current_file; + s64 current_downloaded_bytes; ///< Bytes downloaded on current file. + s64 current_total_bytes; ///< Bytes total on current file. + s64 total_downloaded_bytes; ///< Bytes downloaded on overall download. + s64 total_bytes; ///< Bytes total on overall download. + INSERT_PADDING_BYTES( + 0x198); ///< Appears to be unused in official code, possibly reserved for future use. +}; +static_assert(sizeof(DeliveryCacheProgressImpl) == 0x200, + "DeliveryCacheProgressImpl has incorrect size."); + +// A class to manage the signalling to the game about BCAT download progress. +// Some of this class is implemented in module.cpp to avoid exposing the implementation structure. +class ProgressServiceBackend { + friend class IBcatService; + + ProgressServiceBackend(std::string event_name); + + Kernel::SharedPtr GetEvent(); + DeliveryCacheProgressImpl& GetImpl(); + +public: + // Clients should call this with true if any of the functions are going to be called from a + // non-HLE thread and this class need to lock the hle mutex. (default is false) + void SetNeedHLELock(bool need); + + // Sets the number of bytes total in the entire download. + void SetTotalSize(u64 size); + + // Notifies the application that the backend has started connecting to the server. + void StartConnecting(); + // Notifies the application that the backend has begun accumulating and processing metadata. + void StartProcessingDataList(); + + // Notifies the application that a file is starting to be downloaded. + void StartDownloadingFile(std::string_view dir_name, std::string_view file_name, u64 file_size); + // Updates the progress of the current file to the size passed. + void UpdateFileProgress(u64 downloaded); + // Notifies the application that the current file has completed download. + void FinishDownloadingFile(); + + // Notifies the application that all files in this directory have completed and are being + // finalized. + void CommitDirectory(std::string_view dir_name); + + // Notifies the application that the operation completed with result code result. + void FinishDownload(ResultCode result); + +private: + void SignalUpdate() const; + + DeliveryCacheProgressImpl impl; + Kernel::EventPair event; + bool need_hle_lock = false; +}; + +// A class representing an abstract backend for BCAT functionality. class Backend { public: explicit Backend(DirectoryGetter getter); virtual ~Backend(); - virtual bool Synchronize(TitleIDVersion title, CompletionCallback callback) = 0; + // Called when the backend is needed to synchronize the data for the game with title ID and + // version in title. A ProgressServiceBackend object is provided to alert the application of + // status. + virtual bool Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) = 0; + // Very similar to Synchronize, but only for the directory provided. Backends should not alter + // the data for any other directories. virtual bool SynchronizeDirectory(TitleIDVersion title, std::string name, - CompletionCallback callback) = 0; + ProgressServiceBackend& progress) = 0; + // Removes all cached data associated with title id provided. virtual bool Clear(u64 title_id) = 0; + // Sets the BCAT Passphrase to be used with the associated title ID. virtual void SetPassphrase(u64 title_id, const Passphrase& passphrase) = 0; + // Gets the launch parameter used by AM associated with the title ID and version provided. virtual std::optional> GetLaunchParameter(TitleIDVersion title) = 0; protected: DirectoryGetter dir_getter; }; +// A backend of BCAT that provides no operation. class NullBackend : public Backend { public: explicit NullBackend(const DirectoryGetter& getter); ~NullBackend() override; - bool Synchronize(TitleIDVersion title, CompletionCallback callback) override; + bool Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) override; bool SynchronizeDirectory(TitleIDVersion title, std::string name, - CompletionCallback callback) override; + ProgressServiceBackend& progress) override; bool Clear(u64 title_id) override; diff --git a/src/core/hle/service/bcat/backend/boxcat.cpp b/src/core/hle/service/bcat/backend/boxcat.cpp index 31d2e045c..3754594df 100644 --- a/src/core/hle/service/bcat/backend/boxcat.cpp +++ b/src/core/hle/service/bcat/backend/boxcat.cpp @@ -14,13 +14,28 @@ #include "core/file_sys/vfs_libzip.h" #include "core/file_sys/vfs_vector.h" #include "core/frontend/applets/error.h" -#include "core/hle/lock.h" #include "core/hle/service/am/applets/applets.h" #include "core/hle/service/bcat/backend/boxcat.h" #include "core/settings.h" +namespace { + +// Prevents conflicts with windows macro called CreateFile +FileSys::VirtualFile VfsCreateFileWrap(FileSys::VirtualDir dir, std::string_view name) { + return dir->CreateFile(name); +} + +// Prevents conflicts with windows macro called DeleteFile +bool VfsDeleteFileWrap(FileSys::VirtualDir dir, std::string_view name) { + return dir->DeleteFile(name); +} + +} // Anonymous namespace + namespace Service::BCAT { +constexpr ResultCode ERROR_GENERAL_BCAT_FAILURE{ErrorModule::BCAT, 1}; + constexpr char BOXCAT_HOSTNAME[] = "api.yuzu-emu.org"; // Formatted using fmt with arg[0] = hex title id @@ -102,7 +117,68 @@ void HandleDownloadDisplayResult(DownloadResult res) { DOWNLOAD_RESULT_LOG_MESSAGES[static_cast(res)], [] {}); } -} // namespace +bool VfsRawCopyProgress(FileSys::VirtualFile src, FileSys::VirtualFile dest, + std::string_view dir_name, ProgressServiceBackend& progress, + std::size_t block_size = 0x1000) { + if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) + return false; + if (!dest->Resize(src->GetSize())) + return false; + + progress.StartDownloadingFile(dir_name, src->GetName(), src->GetSize()); + + std::vector temp(std::min(block_size, src->GetSize())); + for (std::size_t i = 0; i < src->GetSize(); i += block_size) { + const auto read = std::min(block_size, src->GetSize() - i); + + if (src->Read(temp.data(), read, i) != read) { + return false; + } + + if (dest->Write(temp.data(), read, i) != read) { + return false; + } + + progress.UpdateFileProgress(i); + } + + progress.FinishDownloadingFile(); + + return true; +} + +bool VfsRawCopyDProgressSingle(FileSys::VirtualDir src, FileSys::VirtualDir dest, + ProgressServiceBackend& progress, std::size_t block_size = 0x1000) { + if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) + return false; + + for (const auto& file : src->GetFiles()) { + const auto out_file = VfsCreateFileWrap(dest, file->GetName()); + if (!VfsRawCopyProgress(file, out_file, src->GetName(), progress, block_size)) { + return false; + } + } + progress.CommitDirectory(src->GetName()); + + return true; +} + +bool VfsRawCopyDProgress(FileSys::VirtualDir src, FileSys::VirtualDir dest, + ProgressServiceBackend& progress, std::size_t block_size = 0x1000) { + if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) + return false; + + for (const auto& dir : src->GetSubdirectories()) { + const auto out = dest->CreateSubdirectory(dir->GetName()); + if (!VfsRawCopyDProgressSingle(dir, out, progress, block_size)) { + return false; + } + } + + return true; +} + +} // Anonymous namespace class Boxcat::Client { public: @@ -194,24 +270,24 @@ Boxcat::Boxcat(DirectoryGetter getter) : Backend(std::move(getter)) {} Boxcat::~Boxcat() = default; void SynchronizeInternal(DirectoryGetter dir_getter, TitleIDVersion title, - CompletionCallback callback, std::optional dir_name = {}) { - const auto failure = [&callback] { - // Acquire the HLE mutex - std::lock_guard lock{HLE::g_hle_lock}; - callback(false); - }; + ProgressServiceBackend& progress, + std::optional dir_name = {}) { + progress.SetNeedHLELock(true); if (Settings::values.bcat_boxcat_local) { LOG_INFO(Service_BCAT, "Boxcat using local data by override, skipping download."); - // Acquire the HLE mutex - std::lock_guard lock{HLE::g_hle_lock}; - callback(true); + const auto dir = dir_getter(title.title_id); + if (dir) + progress.SetTotalSize(dir->GetSize()); + progress.FinishDownload(RESULT_SUCCESS); return; } const auto zip_path{GetZIPFilePath(title.title_id)}; Boxcat::Client client{zip_path, title.title_id, title.build_id}; + progress.StartConnecting(); + const auto res = client.DownloadDataZip(); if (res != DownloadResult::Success) { LOG_ERROR(Service_BCAT, "Boxcat synchronization failed with error '{}'!", res); @@ -221,68 +297,85 @@ void SynchronizeInternal(DirectoryGetter dir_getter, TitleIDVersion title, } HandleDownloadDisplayResult(res); - failure(); + progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE); return; } + progress.StartProcessingDataList(); + FileUtil::IOFile zip{zip_path, "rb"}; const auto size = zip.GetSize(); std::vector bytes(size); if (!zip.IsOpen() || size == 0 || zip.ReadBytes(bytes.data(), bytes.size()) != bytes.size()) { LOG_ERROR(Service_BCAT, "Boxcat failed to read ZIP file at path '{}'!", zip_path); - failure(); + progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE); return; } const auto extracted = FileSys::ExtractZIP(std::make_shared(bytes)); if (extracted == nullptr) { LOG_ERROR(Service_BCAT, "Boxcat failed to extract ZIP file!"); - failure(); + progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE); return; } if (dir_name == std::nullopt) { + progress.SetTotalSize(extracted->GetSize()); + const auto target_dir = dir_getter(title.title_id); - if (target_dir == nullptr || - !FileSys::VfsRawCopyD(extracted, target_dir, VFS_COPY_BLOCK_SIZE)) { + if (target_dir == nullptr || !VfsRawCopyDProgress(extracted, target_dir, progress)) { LOG_ERROR(Service_BCAT, "Boxcat failed to copy extracted ZIP to target directory!"); - failure(); + progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE); return; } } else { const auto target_dir = dir_getter(title.title_id); if (target_dir == nullptr) { LOG_ERROR(Service_BCAT, "Boxcat failed to get directory for title ID!"); - failure(); + progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE); return; } const auto target_sub = target_dir->GetSubdirectory(*dir_name); const auto source_sub = extracted->GetSubdirectory(*dir_name); + progress.SetTotalSize(source_sub->GetSize()); + + std::vector filenames; + { + const auto files = target_sub->GetFiles(); + std::transform(files.begin(), files.end(), std::back_inserter(filenames), + [](const auto& vfile) { return vfile->GetName(); }); + } + + for (const auto& filename : filenames) { + VfsDeleteFileWrap(target_sub, filename); + } + if (target_sub == nullptr || source_sub == nullptr || - !FileSys::VfsRawCopyD(source_sub, target_sub, VFS_COPY_BLOCK_SIZE)) { + !VfsRawCopyDProgressSingle(source_sub, target_sub, progress)) { LOG_ERROR(Service_BCAT, "Boxcat failed to copy extracted ZIP to target directory!"); - failure(); + progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE); return; } } - // Acquire the HLE mutex - std::lock_guard lock{HLE::g_hle_lock}; - callback(true); + progress.FinishDownload(RESULT_SUCCESS); } -bool Boxcat::Synchronize(TitleIDVersion title, CompletionCallback callback) { +bool Boxcat::Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) { is_syncing.exchange(true); - std::thread(&SynchronizeInternal, dir_getter, title, callback, std::nullopt).detach(); + std::thread([this, title, &progress] { SynchronizeInternal(dir_getter, title, progress); }) + .detach(); return true; } bool Boxcat::SynchronizeDirectory(TitleIDVersion title, std::string name, - CompletionCallback callback) { + ProgressServiceBackend& progress) { is_syncing.exchange(true); - std::thread(&SynchronizeInternal, dir_getter, title, callback, name).detach(); + std::thread( + [this, title, name, &progress] { SynchronizeInternal(dir_getter, title, progress, name); }) + .detach(); return true; } diff --git a/src/core/hle/service/bcat/backend/boxcat.h b/src/core/hle/service/bcat/backend/boxcat.h index 1148a4eca..601151189 100644 --- a/src/core/hle/service/bcat/backend/boxcat.h +++ b/src/core/hle/service/bcat/backend/boxcat.h @@ -21,16 +21,16 @@ struct EventStatus { /// doesn't require a switch or nintendo account. The content is controlled by the yuzu team. class Boxcat final : public Backend { friend void SynchronizeInternal(DirectoryGetter dir_getter, TitleIDVersion title, - CompletionCallback callback, + ProgressServiceBackend& progress, std::optional dir_name); public: explicit Boxcat(DirectoryGetter getter); ~Boxcat() override; - bool Synchronize(TitleIDVersion title, CompletionCallback callback) override; + bool Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) override; bool SynchronizeDirectory(TitleIDVersion title, std::string name, - CompletionCallback callback) override; + ProgressServiceBackend& progress) override; bool Clear(u64 title_id) override; diff --git a/src/core/hle/service/bcat/module.cpp b/src/core/hle/service/bcat/module.cpp index a8d545992..d5f9e9d3b 100644 --- a/src/core/hle/service/bcat/module.cpp +++ b/src/core/hle/service/bcat/module.cpp @@ -33,20 +33,6 @@ constexpr ResultCode ERROR_FAILED_CLEAR_CACHE{ErrorModule::FS, 6400}; using BCATDigest = std::array; -struct DeliveryCacheProgressImpl { - enum class Status : u8 { - Incomplete = 0x1, - Complete = 0x9, - }; - - Status status = Status::Incomplete; - INSERT_PADDING_BYTES( - 0x1FF); ///< TODO(DarkLordZach): RE this structure. It just seems to convey info about the - ///< progress of the BCAT sync, but for us just setting completion works. -}; -static_assert(sizeof(DeliveryCacheProgressImpl) == 0x200, - "DeliveryCacheProgressImpl has incorrect size."); - namespace { u64 GetCurrentBuildID() { @@ -84,19 +70,16 @@ bool VerifyNameValidInternal(Kernel::HLERequestContext& ctx, std::array name) { +bool VerifyNameValidDir(Kernel::HLERequestContext& ctx, DirectoryName name) { return VerifyNameValidInternal(ctx, name, '-'); } -bool VerifyNameValidFile(Kernel::HLERequestContext& ctx, std::array name) { +bool VerifyNameValidFile(Kernel::HLERequestContext& ctx, FileName name) { return VerifyNameValidInternal(ctx, name, '.'); } } // Anonymous namespace -using DirectoryName = std::array; -using FileName = std::array; - struct DeliveryCacheDirectoryEntry { FileName name; u64 size; @@ -162,15 +145,6 @@ public: }; // clang-format on RegisterHandlers(functions); - - auto& kernel{Core::System::GetInstance().Kernel()}; - progress.at(static_cast(SyncType::Normal)).event = - Kernel::WritableEvent::CreateEventPair(kernel, Kernel::ResetType::Sticky, - "BCAT::IDeliveryCacheProgressEvent"); - progress.at(static_cast(SyncType::Directory)).event = - Kernel::WritableEvent::CreateEventPair( - kernel, Kernel::ResetType::OneShot, - "BCAT::IDeliveryCacheProgressEvent::DirectoryName"); } private: @@ -180,24 +154,17 @@ private: Count, }; - std::function CreateCallback(SyncType type) { - return [this, type](bool success) { - auto& pair{progress.at(static_cast(type))}; - pair.impl.status = DeliveryCacheProgressImpl::Status::Complete; - pair.event.writable->Signal(); - }; - } - std::shared_ptr CreateProgressService(SyncType type) { - const auto& pair{progress.at(static_cast(type))}; - return std::make_shared(pair.event.readable, pair.impl); + auto& backend{progress.at(static_cast(type))}; + return std::make_shared(backend.GetEvent(), + backend.GetImpl()); } void RequestSyncDeliveryCache(Kernel::HLERequestContext& ctx) { LOG_DEBUG(Service_BCAT, "called"); backend.Synchronize({Core::CurrentProcess()->GetTitleID(), GetCurrentBuildID()}, - CreateCallback(SyncType::Normal)); + progress.at(static_cast(SyncType::Normal))); IPC::ResponseBuilder rb{ctx, 2, 0, 1}; rb.Push(RESULT_SUCCESS); @@ -213,7 +180,8 @@ private: LOG_DEBUG(Service_BCAT, "called, name={}", name); backend.SynchronizeDirectory({Core::CurrentProcess()->GetTitleID(), GetCurrentBuildID()}, - name, CreateCallback(SyncType::Directory)); + name, + progress.at(static_cast(SyncType::Directory))); IPC::ResponseBuilder rb{ctx, 2, 0, 1}; rb.Push(RESULT_SUCCESS); @@ -278,12 +246,10 @@ private: Backend& backend; - struct ProgressPair { - Kernel::EventPair event; - DeliveryCacheProgressImpl impl; + std::array(SyncType::Count)> progress{ + ProgressServiceBackend{"Normal"}, + ProgressServiceBackend{"Directory"}, }; - - std::array(SyncType::Count)> progress{}; }; void Module::Interface::CreateBcatService(Kernel::HLERequestContext& ctx) { -- cgit v1.2.3