feat(lobby): Add the /net chat command for the relay and connection engine settings

Core/GameEngine/CMakeLists.txt

@@ -15,6 +15,7 @@ set(GAMEENGINE_SRC
 #    Include/Common/BattleHonors.h
 #    Include/Common/BezFwdIterator.h
 #    Include/Common/BezierSegment.h
+    Include/Common/BIGFileSystem.h
 #    Include/Common/BitFlags.h
 #    Include/Common/BitFlagsIO.h
 #    Include/Common/BorderColors.h
@@ -505,6 +506,7 @@ set(GAMEENGINE_SRC
 #    Include/GameLogic/WeaponStatus.h
     Include/GameNetwork/Connection.h
     Include/GameNetwork/ConnectionManager.h
+    Include/GameNetwork/ConnectionStats.h
     Include/GameNetwork/DisconnectManager.h
     Include/GameNetwork/DownloadManager.h
     Include/GameNetwork/FileTransfer.h
@@ -644,6 +646,7 @@ set(GAMEENGINE_SRC
     Source/Common/System/ArchiveFile.cpp
     Source/Common/System/ArchiveFileSystem.cpp
     Source/Common/System/AsciiString.cpp
+    Source/Common/System/BIGFileSystem.cpp
 #    Source/Common/System/BuildAssistant.cpp
 #    Source/Common/System/CriticalSection.cpp
 #    Source/Common/System/DataChunk.cpp
@@ -1099,6 +1102,7 @@ set(GAMEENGINE_SRC
     Source/GameLogic/System/RankInfo.cpp
     Source/GameNetwork/Connection.cpp
     Source/GameNetwork/ConnectionManager.cpp
+    Source/GameNetwork/ConnectionStats.cpp
     Source/GameNetwork/DisconnectManager.cpp
     Source/GameNetwork/DownloadManager.cpp
     Source/GameNetwork/FileTransfer.cpp

Core/GameEngine/Include/Common/ArchiveFile.h

@@ -47,26 +47,35 @@ class File;
 class ArchiveFile
 {
 public:
-	ArchiveFile();
-	virtual ~ArchiveFile();
+	ArchiveFile(AsciiString name, AsciiString path);
+	~ArchiveFile();
 
-	virtual Bool					getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const = 0;	///< fill in the fileInfo struct with info about the file requested.
-	virtual File*					openFile( const Char *filename, Int access = 0) = 0;	///< Open the specified file within the archive file
-	virtual void					closeAllFiles() = 0;									///< Close all file opened in this archive file
-	virtual AsciiString		getName() = 0;												///< Returns the name of the archive file
-	virtual AsciiString		getPath() = 0;												///< Returns full path and name of archive file
-	virtual void					setSearchPriority( Int new_priority ) = 0;	///< Set this archive file's search priority
-	virtual void					close() = 0;													///< Close this archive file
+	Bool					getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const;	///< fill in the fileInfo struct with info about the file requested.
+	File*					openFile( const Char *filename, Int access = 0);	///< Open the specified file within the archive file
+	void					closeAllFiles();									///< Close all file opened in this archive file
+	AsciiString		getName();												///< Returns the name of the archive file
+	AsciiString		getPath();												///< Returns full path and name of archive file
+	void					setSearchPriority( Int new_priority );	///< Set this archive file's search priority
+	void					close();													///< Close this archive file
 	void									attachFile(File *file);
+	void									setPriority(Int priority) { m_priority = priority; }
+	Int										getPriority() const { return m_priority; }
 
 	void									getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const;
-	void									getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const;
 
 	void									addFile(const AsciiString& path, const ArchivedFileInfo *fileInfo); ///< add this file to our directory tree.
 
 protected:
+	void									getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const;
 	const ArchivedFileInfo *		getArchivedFileInfo(const AsciiString& filename) const;	///< return the ArchivedFileInfo from the directory tree.
 
+protected:
+	Int m_priority = ARCHIVE_PRIORITY_RETAIL;	///< lookup priority in the archive file system, lower wins
 	File *m_file; ///< file pointer to the archive file on disk.  Kept open so we don't have to continuously open and close the file all the time.
+	FileInfo m_fileInfo; ///< file info of the archive file on disk.
+	AsciiString m_name; ///< archive file name
+	AsciiString m_path; ///< archive file path
+
+private:
 	DetailedArchivedDirectoryInfo m_rootDirectory;
 };

Core/GameEngine/Include/Common/ArchiveFileSystem.h

@@ -53,6 +53,8 @@
 #include "Common/AsciiString.h"
 #include "Common/FileSystem.h" // for typedefs, etc.
 #include "Common/STLTypedefs.h"
+#include <string>
+#include <vector>
 
 //----------------------------------------------------------------------------
 //           Forward References
@@ -120,6 +122,31 @@ class ArchivedFileInfo
 };
 
 
+// Lookup priority of an archive, lower wins. The archive mounted first wins on equal priority.
+enum ArchivePriority
+{
+	ARCHIVE_PRIORITY_EXCLUDED = -1,			///< not mounted: a BIG of a game folder that matches user.exclude_paths
+	ARCHIVE_PRIORITY_MOD = 0,						///< -mod and -modDir, 0 to 999, a later one is lower and wins
+	ARCHIVE_PRIORITY_GAME_FOLDER = 1000,	///< BIGs of the Zero Hour folder that are not retail, in name order
+	ARCHIVE_PRIORITY_GAME_FOLDER_GENERALS = 1100,	///< BIGs of the Generals data (ZH_Generals or the Generals install) that are not retail
+	ARCHIVE_PRIORITY_USER = 100000,				///< data_packs.user of settings.json, 100000 plus the position in its include_paths
+	ARCHIVE_PRIORITY_GENERALS_ONLINE = 1000000,	///< data_packs.generals_online, 1000000 plus the position in its include_paths
+	ARCHIVE_PRIORITY_COMMUNITY_PATCH = 2000000,	///< data_packs.community_patch, 2000000 plus the position in its include_paths
+	ARCHIVE_PRIORITY_RETAIL = 3000000,		///< Zero Hour retail and official patch BIGs
+	ARCHIVE_PRIORITY_RETAIL_GENERALS = 3000100	///< Generals retail and official patch BIGs
+};
+
+// One entry of the include_paths of a data pack layer in settings.json, see ArchiveFileSystem::parseMountEntry
+struct ArchiveMountEntry
+{
+	enum Kind { KIND_BIG, KIND_BIG_GLOB };
+	enum Base { BASE_GAME, BASE_USER, BASE_ABSOLUTE };
+
+	Kind m_kind;
+	Base m_base;
+	AsciiString m_path;	///< with backslashes, relative to the base, the BASE_ABSOLUTE path is complete
+};
+
 class ArchiveFileSystem : public SubsystemInterface
 {
 public:
@@ -141,12 +168,22 @@ class ArchiveFileSystem : public SubsystemInterface
 	void					getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep).  Possibly search subdirectories.  Scans each Archive file.
 	Bool					getFileInfo(const AsciiString& filename, FileInfo *fileInfo, FileInstance instance = 0) const; ///< see FileSystem.h
 
-	virtual Bool	loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0;
+	virtual Bool	loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0;
 
-	// Unprotected this for copy-protection routines
-	ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const;
+	virtual void loadMods() = 0;
+
+	/// Parses an entry of the include_paths of a data pack layer: "game:<path>", "user:<path>" or an absolute path. A path ending in .big is one BIG,
+	/// a last component with * or ? and ending in .big is a set of BIGs, a path ending in a slash is every BIG in that folder.
+	/// Returns FALSE and a reason for an entry that is not valid.
+	static Bool parseMountEntry(const AsciiString& text, ArchiveMountEntry* entry, AsciiString* reason);
+
+	/// TRUE for the retail archives and the official patches of Zero Hour and Generals, in every language.
+	static Bool isGeneralsArchivePath(const AsciiString& path, Bool generalsFolder);	///< TRUE for the Generals archives: the Generals install folder, or a ZH_Generals folder
+
+	static Bool isRetailArchive(const AsciiString& path, Bool generalsFolder, Bool zeroHour);
 
-	void loadMods();
+	/// BIGs of the game folders that are not retail and not excluded by user.exclude_paths, which are mounted above the data pack layers
+	const std::vector<AsciiString>& getGameFolderArchives() const { return m_gameFolderArchives; }
 
 	ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory);
 
@@ -160,11 +197,21 @@ class ArchiveFileSystem : public SubsystemInterface
 		AsciiString lastToken; ///< Synonymous for file name if the search directory was a file path
 	};
 
+	ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const;
 	ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory);
 
-	virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE, Bool sortedByName = FALSE);	///< load the archive file's header information and apply it to the global archive directory tree. sortedByName inserts by archive filename order (addon number convention), ignoring overwrite.
+	virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Int priority = ARCHIVE_PRIORITY_RETAIL);	///< load the archive file's header information and apply it to the global archive directory tree. The lowest priority number wins, the first mounted wins on equal priority.
+
+	Int getNextModPriority() { return m_nextModPriority > ARCHIVE_PRIORITY_MOD ? m_nextModPriority-- : ARCHIVE_PRIORITY_MOD; }	///< -mod and -modDir: a later one wins
+	Int getGameFolderArchivePriority(const AsciiString& path, Bool generalsFolder, Bool zeroHour);	///< retail archives are the bottom, any other BIG of the game folders ranks above the data pack layers unless user.exclude_paths matches it
+
+	void mountListedArchive(const ArchiveMountEntry& entry, Int priority, const PathExcludeList* excludes);
+	void mountDataPackLayer(const char* layer, Bool enabled, const std::vector<std::string>& includePaths, const PathExcludeList* excludes, Int priority);	///< mounts the include_paths of one layer in list order, the first one wins, minus what excludes matches
 
 	ArchiveFileMap m_archiveFileMap;
+	std::vector<AsciiString> m_gameFolderArchives;
+	AsciiString m_generalsBaseSource;	///< where the Generals data came from, for the log
+	Int m_nextModPriority = ARCHIVE_PRIORITY_MOD + 999;
 	ArchivedDirectoryInfo m_rootDirectory;
 };
 

Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h → Core/GameEngine/Include/Common/BIGFileSystem.h

@@ -22,19 +22,19 @@
 //																																						//
 ////////////////////////////////////////////////////////////////////////////////
 
-//////// Win32BIGFileSystem.h ///////////////////////////
+//////// BIGFileSystem.h ///////////////////////////
 // Bryan Cleveland, August 2002
 /////////////////////////////////////////////////////////////
 
 #pragma once
 
 #include "Common/ArchiveFileSystem.h"
 
-class Win32BIGFileSystem : public ArchiveFileSystem
+class BIGFileSystem : public ArchiveFileSystem
 {
 public:
-	Win32BIGFileSystem();
-	virtual ~Win32BIGFileSystem() override;
+	BIGFileSystem();
+	virtual ~BIGFileSystem() override;
 
 	virtual void init() override;
 	virtual void update() override;
@@ -49,7 +49,9 @@ class Win32BIGFileSystem : public ArchiveFileSystem
 	virtual void closeArchiveFile(const Char *filename) override;
 	virtual void closeAllFiles() override;															///< Close all files associated with ArchiveFiles
 
-	virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override;
+	virtual Bool loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override;
+
+	virtual void loadMods() override;
 protected:
 
 };

Core/GameEngine/Include/Common/FileSystem.h

@@ -53,6 +53,9 @@
 
 #include <Utility/hash_map_adapter.h>
 
+#include <string>
+#include <vector>
+
 #include "WWLib/mutex.h"
 
 //----------------------------------------------------------------------------
@@ -109,6 +112,14 @@ typedef UnsignedByte FileInstance;
 
 struct FileInfo {
 
+	FileInfo()
+		: sizeHigh(0)
+		, sizeLow(0)
+		, timestampHigh(0)
+		, timestampLow(0)
+	{
+	}
+
 	Int64 size() const { return (Int64)sizeHigh << 32 | sizeLow; }
 	Int64 timestamp() const { return (Int64)timestampHigh << 32 | timestampLow; }
 
@@ -118,6 +129,30 @@ struct FileInfo {
 	Int timestampLow;
 };
 
+//===============================
+// PathExcludeList
+//===============================
+/**
+  * The exclude_paths of settings.json. An entry is game:<pattern>, user:<pattern> or an absolute pattern, where * matches any text
+  * including folders and ? one character. A pattern ending in a slash is a folder and everything in it. Matching ignores case and slash direction.
+  * A path is tested as a key: game:<path relative to the game folder>, user:<path relative to the user data folder> or the absolute path.
+  */
+class PathExcludeList
+{
+public:
+	static Bool parse(const std::string& text, std::string* pattern, AsciiString* reason);	///< FALSE and a reason for an entry that is not valid
+	static std::string makeKey(const Char* path);	///< key of a path that is relative to the game folder or absolute
+	static std::string makeKey(const Char* path, const Char* userDataPath);	///< same, a path inside the user data folder becomes user:<path>
+
+	void set(const std::vector<std::string>& entries);	///< entries that are not valid are dropped
+	Bool isEmpty() const { return m_patterns.empty(); }
+	Bool matches(const std::string& key) const;
+	Bool matchesPath(const Char* path) const { return !m_patterns.empty() && matches(makeKey(path)); }
+
+private:
+	std::vector<std::string> m_patterns;
+};
+
 //===============================
 // FileSystem
 //===============================
@@ -157,13 +192,21 @@ class FileSystem : public SubsystemInterface
 
 	Bool createDirectory(AsciiString directory); ///< create a directory of the given name.
 
+	/// Loose files of the game folder that match are treated as if they did not exist: not opened, not found, not listed. Files that are written and files outside the game folder are not affected.
+	void setExcludedPaths(const std::vector<std::string>& entries) { m_excludedPaths.set(entries); }
+	const PathExcludeList& getExcludedPaths() const { return m_excludedPaths; }
+
 	static AsciiString normalizePath(const AsciiString& path);	///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure.
 	static Bool isPathInDirectory(const AsciiString& testPath, const AsciiString& basePath);	///< determines if a file path is within a base path. Both paths must be absolute, but do not need to exist.
 
 	static bool removeExtension(AsciiString& path);
 	static bool removeExtension(UnicodeString& path);
 
 protected:
+	Bool isLooseFileExcluded(const Char *filename) const { return m_excludedPaths.matchesPath(filename); }
+
+	PathExcludeList m_excludedPaths;
+
 #if ENABLE_FILESYSTEM_EXISTENCE_CACHE
 	struct FileExistData
 	{

Core/GameEngine/Include/GameNetwork/Connection.h

@@ -39,11 +39,9 @@
 #include "GameNetwork/User.h"
 #include "GameNetwork/Transport.h"
 #include "GameNetwork/NetPacket.h"
-
+#include "GameNetwork/ConnectionStats.h"
 #if defined(GENERALS_ONLINE)
-#define CONNECTION_LATENCY_HISTORY_LENGTH 10
-#else
-#define CONNECTION_LATENCY_HISTORY_LENGTH 200
+#include "GameNetwork/NetworkSettings.h"
 #endif
 
 class Connection : public MemoryPoolObject
@@ -66,6 +64,13 @@ class Connection : public MemoryPoolObject
 	void setUser(User *user);
 	User *getUser();
 	void setFrameGrouping(time_t frameGrouping);
+#if defined(GENERALS_ONLINE)
+	void setNetworkSettings(const NetworkSettings &settings, Bool unreliableTransport) { m_networkSettings = settings; m_unreliableTransport = unreliableTransport; }
+#endif
+	void setSlot(Int slot) { m_slot = slot; }
+	void setShortGrouping(Bool shortGrouping);	///< Send at one frame while the run ahead is close to its floor.
+	Int getRedundancyDepth() const { return m_redundancyDepth; }
+	time_t getFrameGrouping() const { return m_frameGrouping; }	///< The send interval in use, shorter than the configured one while redundancy is on.
 
 	void sendNetCommandMsg(NetCommandMsg *msg, UnsignedByte relay);
 
@@ -80,12 +85,27 @@ class Connection : public MemoryPoolObject
 	void setQuitting();
 	Bool isQuitting() { return m_isQuitting; }
 
+	const ConnectionStats& getStats() const { return m_stats; }
+
 #if defined(RTS_DEBUG)
 	void debugPrintCommands();
 #endif
 
 protected:
-	void doRetryMetrics();
+#if defined(GENERALS_ONLINE)
+	NetworkSettings m_networkSettings;
+	Bool m_unreliableTransport;
+#endif
+
+	void resetRedundancy();
+	void updateRedundancy(time_t curtime, Bool &active, time_t &window, Int &depth);
+	Bool hasLossEvidence();	///< A packet was lost in either direction lately, or nothing says otherwise
+	Bool getOutgoingLoss(float &lossPercent, Int &longestRun);	///< FALSE when the peer sent no packet feedback
+	void noteLossEvent(time_t now, Bool burst);
+	void setRedundancyDepth(Int depth, Int cap, time_t now);
+	Bool isQueueBuilding(time_t now);	///< Round trip at 2x its low while packets are lost: a full queue
+	void applyFrameGrouping();
+	void addRedundantCommands(NetPacket *packet, time_t curtime, time_t window, Int depth);
 
 	Bool m_isQuitting;
 	UnsignedInt m_quitTime;
@@ -95,11 +115,17 @@ class Connection : public MemoryPoolObject
 
 	NetCommandList *m_netCommandList;
 	time_t m_retryTime;						///< The time between sending retry packets for this connection.  Time is in milliseconds.
-	Real m_averageLatency;			///< The average time between sending a command and receiving an ACK.
-	Real m_latencies[CONNECTION_LATENCY_HISTORY_LENGTH];	///< List of the last 100 latencies.
+	ConnectionStats m_stats;			///< Latency, jitter and loss of this connection.
+
+	Int m_redundancyDepth;					///< How many later packets each un-acked command is repeated in, 0 on a clean connection.
+	Int m_redundancyCap;						///< The deepest depth, from the longest burst to repair and the send interval.
+	time_t m_lastLossEventTime;			///< What the decay counts from.
+	time_t m_lastDepthChangeTime;		///< A burst needs a round trip to show whether the new depth helped.
+	Bool m_queueSuspected;				///< A full queue on the path, redundancy steps down instead of up.
+	Int m_slot;
 
+	Bool m_shortGrouping;
+	time_t m_configuredFrameGrouping;
 	time_t m_frameGrouping;				///< The minimum time between packet sends.
 	time_t m_lastTimeSent;				///< The time of the last packet send.
-	Int m_numRetries;							///< The number of retries for the last second.
-	time_t m_retryMetricsTime;		///< The start time of the current retry metrics thing.
 };