CVarManager: Bring back de/serialization

Runtime/CBasics.hpp

@@ -64,7 +64,6 @@ public:
   static int RecursiveMakeDir(const char* dir);
   static void MakeDir(const char* dir);
   static int Stat(const char* path, Sstat* statOut);
-  static inline void ToLower(std::string& str) { std::transform(str.begin(), str.end(), str.begin(), ::tolower); }
 };
 
 } // namespace metaforce

Runtime/CMakeLists.txt

@@ -88,6 +88,8 @@ set(RUNTIME_SOURCES_B
         Streams/CMemoryInStream.hpp
         Streams/CZipInputStream.hpp Streams/CZipInputStream.cpp
         Streams/ContainerReaders.hpp
+        Streams/CTextInStream.hpp Streams/CTextInStream.cpp
+        Streams/CTextOutStream.hpp Streams/CTextOutStream.cpp
         CGameAllocator.hpp CGameAllocator.cpp
         CMemoryCardSys.hpp CMemoryCardSys.cpp
         CScannableObjectInfo.hpp CScannableObjectInfo.cpp

Runtime/CPlayerState.cpp

@@ -4,6 +4,7 @@
 #include <array>
 #include <cstring>
 
+#include "Runtime/CStringExtras.hpp"
 #include "Runtime/CMemoryCardSys.hpp"
 #include "Runtime/CStateManager.hpp"
 #include "Runtime/GameGlobalObjects.hpp"
@@ -437,7 +438,7 @@ CPlayerState::EItemType CPlayerState::ItemNameToType(std::string_view name) {
   }};
 
   std::string lowName{name};
-  CBasics::ToLower(lowName);
+  CStringExtras::ToLower(lowName);
 
   const auto iter = std::find_if(typeNameMap.cbegin(), typeNameMap.cend(),
                                  [&lowName](const auto& entry) { return entry.first == lowName; });

Runtime/CStringExtras.hpp

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <cctype>
 #include <string>
+#include <sstream>
 
 namespace metaforce {
 class CInputStream;
@@ -42,7 +43,71 @@ public:
     return s;
   }
 
+  static inline void ToLower(std::string& str) { std::transform(str.begin(), str.end(), str.begin(), ::tolower); }
   static std::string ReadString(CInputStream& in);
+  static inline bool ParseBool(std::string_view boolean, bool* valid) {
+    std::string val(boolean);
+    // compare must be case insensitive
+    // This is the cleanest solution since I only need to do it once
+    ToLower(val);
+
+    // Check for true first
+    if (val == "true" || val == "1" || val == "yes" || val == "on") {
+      if (valid)
+        *valid = true;
+
+      return true;
+    }
+
+    // Now false
+    if (val == "false" || val == "0" || val == "no" || val == "off") {
+      if (valid)
+        *valid = true;
+
+      return false;
+    }
+
+    // Well that could've gone better
+
+    if (valid)
+      *valid = false;
+
+    return false;
+  }
+
+  static inline std::vector<std::string>& Split(std::string_view s, char delim, std::vector<std::string>& elems) {
+    std::string tmps(s);
+    std::stringstream ss(tmps);
+    std::string item;
+
+    while (std::getline(ss, item, delim)) {
+      elems.push_back(item);
+    }
+
+    return elems;
+  }
+
+  static inline std::vector<std::string> Split(std::string_view s, char delim) {
+    std::vector<std::string> elems;
+    Split(s, delim, elems);
+    return elems;
+  }
+
+  static inline std::string LeftTrim(const std::string &s)
+  {
+    size_t start = s.find_first_not_of(" \n\r\t\f\v");
+    return (start == std::string::npos) ? "" : s.substr(start);
+  }
+
+  static inline std::string RightTrim(const std::string &s)
+  {
+    size_t end = s.find_last_not_of(" \n\r\t\f\v");
+    return (end == std::string::npos) ? "" : s.substr(0, end + 1);
+  }
+
+  static inline std::string Trim(const std::string &s) {
+    return RightTrim(LeftTrim(s));
+  }
 };
 
 } // namespace metaforce

Runtime/ConsoleVariables/CVar.cpp

@@ -1,5 +1,6 @@
 #include "Runtime/ConsoleVariables/CVar.hpp"
 #include "Runtime/CBasics.hpp"
+#include "Runtime/CStringExtras.hpp"
 
 #include <logvisor/logvisor.hpp>
 
@@ -8,56 +9,6 @@
 #include "Runtime/ConsoleVariables/CVarManager.hpp"
 
 namespace metaforce {
-namespace {
-// TODO: Move these to CBasics?
-inline bool parseBool(std::string_view boolean, bool* valid) {
-  std::string val(boolean);
-  // compare must be case insensitive
-  // This is the cleanest solution since I only need to do it once
-  CBasics::ToLower(val);
-
-  // Check for true first
-  if (val == "true" || val == "1" || val == "yes" || val == "on") {
-    if (valid)
-      *valid = true;
-
-    return true;
-  }
-
-  // Now false
-  if (val == "false" || val == "0" || val == "no" || val == "off") {
-    if (valid)
-      *valid = true;
-
-    return false;
-  }
-
-  // Well that could've gone better
-
-  if (valid)
-    *valid = false;
-
-  return false;
-}
-
-static std::vector<std::string>& split(std::string_view s, char delim, std::vector<std::string>& elems) {
-  std::string tmps(s);
-  std::stringstream ss(tmps);
-  std::string item;
-
-  while (std::getline(ss, item, delim)) {
-    elems.push_back(item);
-  }
-
-  return elems;
-}
-
-std::vector<std::string> split(std::string_view s, char delim) {
-  std::vector<std::string> elems;
-  split(s, delim, elems);
-  return elems;
-}
-}
 extern CVar* com_developer;
 extern CVar* com_enableCheats;
 
@@ -250,7 +201,7 @@ bool CVar::toBoolean(bool* isValid) const {
     return false;
   }
 
-  return parseBool(m_value, isValid);
+  return CStringExtras::ParseBool(m_value, isValid);
 }
 
 int32_t CVar::toSigned(bool* isValid) const {
@@ -494,12 +445,12 @@ bool isReal(const std::vector<std::string>& v) {
 }
 
 bool CVar::isValidInput(std::string_view input) const {
-  std::vector<std::string> parts = split(input, ' ');
+  std::vector<std::string> parts = CStringExtras::Split(input, ' ');
   char* p;
   switch (m_type) {
   case EType::Boolean: {
     bool valid = false;
-    parseBool(input, &valid);
+    CStringExtras::ParseBool(input, &valid);
     return valid;
   }
   case EType::Signed:

Runtime/ConsoleVariables/CVarManager.cpp

@@ -2,7 +2,11 @@
 
 #include "Runtime/ConsoleVariables/FileStoreManager.hpp"
 #include "Runtime/CBasics.hpp"
-
+#include "Runtime/Streams/CTextInStream.hpp"
+#include "Runtime/Streams/CTextOutStream.hpp"
+#include "Runtime/Streams/CMemoryInStream.hpp"
+#include "Runtime/Streams/CMemoryStreamOut.hpp"
+#include "Runtime/CStringExtras.hpp"
 #include <logvisor/logvisor.hpp>
 #include <algorithm>
 #include <memory>
@@ -19,8 +23,7 @@ static const std::regex cmdLineRegex(R"(\+([\w\.]+)([=])?([\/\\\s\w\.\-]+)?)");
 CVarManager* CVarManager::m_instance = nullptr;
 
 static logvisor::Module CVarLog("CVarManager");
-CVarManager::CVarManager(FileStoreManager& store, bool useBinary)
+CVarManager::CVarManager(FileStoreManager& store, bool useBinary) : m_store(store), m_useBinary(useBinary) {
-: m_store(store), m_useBinary(useBinary) {
   m_instance = this;
   com_configfile =
       newCVar("config", "File to store configuration", std::string("config"),
@@ -38,7 +41,7 @@ CVarManager::~CVarManager() {}
 
 CVar* CVarManager::registerCVar(std::unique_ptr<CVar>&& cvar) {
   std::string tmp(cvar->name());
-  CBasics::ToLower(tmp);
+  CStringExtras::ToLower(tmp);
 
   if (m_cvars.find(tmp) != m_cvars.end()) {
     return nullptr;
@@ -51,7 +54,7 @@ CVar* CVarManager::registerCVar(std::unique_ptr<CVar>&& cvar) {
 
 CVar* CVarManager::findCVar(std::string_view name) {
   std::string lower(name);
-  CBasics::ToLower(lower);
+  CStringExtras::ToLower(lower);
   auto search = m_cvars.find(lower);
   if (search == m_cvars.end())
     return nullptr;
@@ -78,15 +81,14 @@ std::vector<CVar*> CVarManager::cvars(CVar::EFlags filter) const {
 }
 
 void CVarManager::deserialize(CVar* cvar) {
-  /* Make sure we're not trying to deserialize a CVar that is invalid or not exposed, unless it's been specified on the
+  /* Make sure we're not trying to deserialize a CVar that is invalid*/
-   * command line (i.e deferred) */
   if (!cvar) {
     return;
   }
 
   /* First let's check for a deferred value */
   std::string lowName = cvar->name().data();
-  CBasics::ToLower(lowName);
+  CStringExtras::ToLower(lowName);
   if (const auto iter = m_deferedCVars.find(lowName); iter != m_deferedCVars.end()) {
     std::string val = std::move(iter->second);
     m_deferedCVars.erase(lowName);
@@ -106,103 +108,89 @@ void CVarManager::deserialize(CVar* cvar) {
   if (!cvar->isArchive() && !cvar->isInternalArchivable()) {
     return;
   }
-#if 0 // TODO: Reimplement this
   /* We were either unable to find a deferred value or got an invalid value */
-  std::string filename =
+  std::string filename = std::string(m_store.getStoreRoot()) + '/' + com_configfile->toLiteral() + ".yaml";
-      std::string(m_store.getStoreRoot()) + '/' + com_configfile->toLiteral();
+  auto container = loadCVars(filename);
-  CBascis::Sstat st;
+  auto serialized =
-
+      std::find_if(container.cbegin(), container.cend(), [&cvar](const auto& c) { return c.m_name == cvar->name(); });
-  if (m_useBinary) {
+  if (serialized != container.cend()) {
-    CVarContainer container;
+    if (cvar->m_value != serialized->m_value) {
-    filename += ".bin";
-    if (CBascis::Stat(filename.c_str(), &st) || !S_ISREG(st.st_mode))
-      return;
-    athena::io::FileReader reader(filename);
-    if (reader.isOpen())
-      container.read(reader);
-
-    if (container.cvars.size() > 0) {
-      auto serialized = std::find_if(container.cvars.begin(), container.cvars.end(),
-                                     [&cvar](const DNACVAR::CVar& c) { return c.m_name == cvar->name(); });
-
-      if (serialized != container.cvars.end()) {
-        DNACVAR::CVar& tmp = *serialized;
-
-        if (cvar->m_value != tmp.m_value) {
       CVarUnlocker lc(cvar);
-          cvar->fromLiteralToType(tmp.m_value);
+      cvar->fromLiteralToType(serialized->m_value);
       cvar->m_wasDeserialized = true;
     }
   }
-    }
-  } else {
-    filename += ".yaml";
-    if (Stat(filename.c_str(), &st) || !S_ISREG(st.st_mode))
-      return;
-    athena::io::FileReader reader(filename);
-    if (reader.isOpen()) {
-      athena::io::YAMLDocReader docReader;
-      if (docReader.parse(&reader)) {
-        std::unique_ptr<athena::io::YAMLNode> root = docReader.releaseRootNode();
-        auto serialized = std::find_if(root->m_mapChildren.begin(), root->m_mapChildren.end(),
-                                       [&cvar](const auto& c) { return c.first == cvar->name(); });
-
-        if (serialized != root->m_mapChildren.end()) {
-          const std::unique_ptr<athena::io::YAMLNode>& tmp = serialized->second;
-
-          if (cvar->m_value != tmp->m_scalarString) {
-            CVarUnlocker lc(cvar);
-            cvar->fromLiteralToType(tmp->m_scalarString);
-            cvar->m_wasDeserialized = true;
-          }
-        }
-      }
-    }
-  }
-#endif
 }
 
 void CVarManager::serialize() {
-#if 0 // TODO: reimplement this
+  std::string filename = std::string(m_store.getStoreRoot()) + '/' + com_configfile->toLiteral() + ".yaml";
-  std::string filename =
-      std::string(m_store.getStoreRoot()) + '/' + com_configfile->toLiteral();
 
-  if (m_useBinary) {
+  /* If we have an existing config load it in, so we can update it */
-    CVarContainer container;
+  auto container = loadCVars(filename);
+
+  u32 minLength = 0;
   for (const auto& pair : m_cvars) {
     const auto& cvar = pair.second;
 
     if (cvar->isArchive() || (cvar->isInternalArchivable() && cvar->wasDeserialized() && !cvar->hasDefaultValue())) {
-        container.cvars.push_back(*cvar);
+      /* Look for an existing CVar in the file... */
-      }
+      auto serialized =
-    }
+          std::find_if(container.begin(), container.end(), [&cvar](const auto& c) { return c.m_name == cvar->name(); });
-    container.cvarCount = u32(container.cvars.size());
+      if (serialized != container.end()) {
-
+        /* Found it! Update the value */
-    filename += ".bin";
+        serialized->m_value = cvar->value();
-    athena::io::FileWriter writer(filename);
-    if (writer.isOpen())
-      container.write(writer);
       } else {
-    filename += ".yaml";
+        /* Store this value as a new CVar in the config */
-
+        container.emplace_back(StoreCVar::CVar{std::string(cvar->name()), cvar->value()});
-    athena::io::FileReader r(filename);
+      }
-    athena::io::YAMLDocWriter docWriter(r.isOpen() ? &r : nullptr);
+      /* Compute length needed for this cvar */
-    r.close();
+      minLength += cvar->name().length() + cvar->value().length() + 2;
-
-    docWriter.setStyle(athena::io::YAMLNodeStyle::Block);
-    for (const auto& pair : m_cvars) {
-      const auto& cvar = pair.second;
-
-      if (cvar->isArchive() || (cvar->isInternalArchivable() && cvar->wasDeserialized() && !cvar->hasDefaultValue())) {
-        docWriter.writeString(cvar->name().data(), cvar->toLiteral());
     }
   }
 
-    athena::io::FileWriter w(filename);
+  // Allocate enough space to write all the strings with some space to spare
-    if (w.isOpen())
+  const auto requiredLen = minLength + (4 * container.size());
-      docWriter.finish(&w);
+  std::unique_ptr<u8[]> workBuf(new u8[requiredLen]);
+  CMemoryStreamOut memOut(workBuf.get(), requiredLen);
+  CTextOutStream textOut(memOut);
+  for (const auto& cvar : container) {
+    textOut.WriteString(fmt::format(FMT_STRING("{}: {}"), cvar.m_name, cvar.m_value));
   }
-#endif
+
+  auto* file = fopen(filename.c_str(), "wbe");
+  if (file != nullptr) {
+    fwrite(workBuf.get(), 1, memOut.GetWritePosition(), file);
+  }
+  fclose(file);
+}
+
+std::vector<StoreCVar::CVar> CVarManager::loadCVars(const std::string& filename) const {
+  std::vector<StoreCVar::CVar> ret;
+  CBasics::Sstat st;
+  if (CBasics::Stat(filename.c_str(), &st) == 0 && S_ISREG(st.st_mode)) {
+
+    auto* file = fopen(filename.c_str(), "rbe");
+    if (file != nullptr) {
+
+      std::unique_ptr<u8[]> inBuf(new u8[st.st_size]);
+      fread(inBuf.get(), 1, st.st_size, file);
+      fclose(file);
+      CMemoryInStream mem(inBuf.get(), st.st_size, CMemoryInStream::EOwnerShip::NotOwned);
+      CTextInStream textIn(mem, st.st_size);
+      while (!textIn.IsEOF()) {
+        auto cvString = textIn.GetNextLine();
+        if (cvString.empty()) {
+          continue;
+        }
+        auto parts = CStringExtras::Split(cvString, ':');
+        if (parts.size() < 2) {
+          continue;
+        }
+        ret.emplace_back(StoreCVar::CVar{CStringExtras::Trim(parts[0]), CStringExtras::Trim(parts[1])});
+      }
+    }
+  }
+  return ret;
 }
 
 CVarManager* CVarManager::instance() { return m_instance; }
@@ -234,7 +222,7 @@ bool CVarManager::restartRequired() const {
 void CVarManager::parseCommandLine(const std::vector<std::string>& args) {
   bool oldDeveloper = suppressDeveloper();
   std::string developerName(com_developer->name());
-  CBasics::ToLower(developerName);
+  CStringExtras::ToLower(developerName);
   for (const std::string& arg : args) {
     if (arg[0] != '+') {
       continue;
@@ -267,13 +255,13 @@ void CVarManager::parseCommandLine(const std::vector<std::string>& args) {
         cv->fromLiteralToType(cvarValue);
       }
       cv->m_wasDeserialized = true;
-      CBasics::ToLower(cvarName);
+      CStringExtras::ToLower(cvarName);
       if (developerName == cvarName)
         /* Make sure we're not overriding developer mode when we restore */
         oldDeveloper = com_developer->toBoolean();
     } else {
       /* Unable to find an existing CVar, let's defer for the time being 8 */
-      CBasics::ToLower(cvarName);
+      CStringExtras::ToLower(cvarName);
       m_deferedCVars.insert_or_assign(std::move(cvarName), std::move(cvarValue));
     }
   }
@@ -303,4 +291,4 @@ void CVarManager::proc() {
   }
 }
 
-} // namespace hecl
+} // namespace metaforce

Runtime/ConsoleVariables/CVarManager.hpp

@@ -108,6 +108,7 @@ private:
 
   std::unordered_map<std::string, std::unique_ptr<CVar>> m_cvars;
   std::unordered_map<std::string, std::string> m_deferedCVars;
+  std::vector<StoreCVar::CVar> loadCVars(const std::string& filename) const;
 };
 
 } // namespace hecl

Runtime/Streams/CMemoryStreamOut.cpp

@@ -17,6 +17,7 @@ void CMemoryStreamOut::Write(const u8* ptr, u32 len) {
 
   if (len != 0) {
     memcpy(x7c_ptr + x84_position, ptr, len);
+    x84_position += len;
   }
 }
 }

Runtime/Streams/CMemoryStreamOut.hpp

@@ -21,5 +21,6 @@ public:
   ~CMemoryStreamOut() override;
 
   void Write(const u8* ptr, u32 len) override;
+  u32 GetWritePosition() const { return x84_position; }
 };
 } // namespace metaforce

Runtime/Streams/CTextInStream.cpp

@@ -0,0 +1,24 @@
+#include "Runtime/Streams/CTextInStream.hpp"
+#include <algorithm>
+
+namespace metaforce {
+CTextInStream::CTextInStream(CInputStream& in, int len) : m_in(&in), m_len(len) {}
+
+std::string CTextInStream::GetNextLine() {
+  std::string ret;
+  while (true) {
+    auto chr = m_in->ReadChar();
+    ret += chr;
+    if (ret.back() == '\r' || ret.back() == '\n') {
+      if (ret.back() == '\r') {
+        m_in->ReadChar();
+      }
+      break;
+    }
+  }
+
+  ret.erase(std::remove(ret.begin(), ret.end(), '\r'), ret.end());
+  ret.erase(std::remove(ret.begin(), ret.end(), '\n'), ret.end());
+  return ret;
+}
+} // namespace metaforce

Runtime/Streams/CTextInStream.hpp

@@ -0,0 +1,14 @@
+#pragma once
+#include "Runtime/Streams/CInputStream.hpp"
+
+namespace metaforce {
+class CTextInStream {
+  CInputStream* m_in;
+  s32 m_len;
+public:
+  CTextInStream(CInputStream& in, int len);
+
+  bool IsEOF() { return m_in->GetReadPosition() >= m_len; }
+  std::string GetNextLine();
+};
+}

Runtime/Streams/CTextOutStream.cpp

@@ -0,0 +1,29 @@
+#include "Runtime/Streams/CTextOutStream.hpp"
+
+namespace metaforce {
+CTextOutStream::CTextOutStream(COutputStream& out) : m_out(&out) {}
+
+void CTextOutStream::WriteString(const std::string& str) { CTextOutStream::WriteString(str.c_str(), str.length()); }
+void CTextOutStream::WriteString(const char* str, u32 len) {
+  bool wroteCarriageReturn = false;
+  bool wroteLineFeed = false;
+  for (u32 i = 0; i < len; ++i) {
+    if (str[i] == '\r') {
+      wroteCarriageReturn = true;
+    } else if (str[i] == '\n' && !wroteCarriageReturn) {
+      m_out->WriteChar('\r');
+      wroteLineFeed = true;
+    }
+    m_out->WriteChar(str[i]);
+  }
+
+  /* If we didn't write either a line feed or carriage return we need to do that now */
+  if (!wroteCarriageReturn && !wroteLineFeed) {
+    m_out->WriteChar('\r');
+    m_out->WriteChar('\n');
+  }
+
+  /* Since this is a text buffer, we always want to flush after writing a string */
+  m_out->Flush();
+}
+} // namespace metaforce

Runtime/Streams/CTextOutStream.hpp

@@ -0,0 +1,14 @@
+#pragma once
+#include "Runtime/Streams/COutputStream.hpp"
+
+namespace metaforce {
+class CTextOutStream {
+  COutputStream* m_out;
+public:
+  CTextOutStream(COutputStream& out);
+
+  void WriteString(const std::string& str);
+  void WriteString(const char* str, u32 len);
+};
+}
+